<commit_before>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
shifter -- shifts time of events

shifter is a utilty for shifting the time of events in an iCalendar ICS file

It defines classes_and_methods

@author:     Jofagi

@copyright:  2014 Jofagi. All rights reserved.

@license:    MIT

@contact:    github.com/Jofagi
@deffield    updated: Updated
'''

import sys
import os
import re
import datetime
import copy

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter

__all__ = []
__version__ = 0.1
__date__ = '2014-03-20'
__updated__ = '2014-03-20'

DEBUG = 1
TESTRUN = 0
PROFILE = 0

def verbose():
    return 0 < verbosityLevel
def very_verbose():
    return 1 < verbosityLevel
def extremely_verbose():
    return 2 < verbosityLevel

class CLIError(Exception):
    '''Generic exception to raise and log different fatal errors.'''
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg
    def __str__(self):
        return self.msg
    def __unicode__(self):
        return self.msg

class IcsData:
    '''Generic list of keys and values'''
    def __init__(self, lines):

        separator = re.compile(":")
        self._values = {}

        for l in lines:
            kv = re.split(separator, l, maxsplit=1)
            if len(kv) == 2:
                self._values[kv[0]] = kv[1]

                if extremely_verbose():
                    print(kv[0], "=", kv[1])

            elif 0 < verbose:
                print("line not in key:value format:", l)

    def content(self):
        lines = []
        for k, v in self._values.items():
            lines.append(k + ":" + v)
        return lines

class Event (IcsData):
    '''Calendar Event'''

    _START = "DTSTART"
    _END = "DTEND"
    _SUMMARY = "SUMMARY"

    def name(self):
        return self._values[self._SUMMARY]

    def _set_start(self, newValue = ""):
        self._values[self._START] = newValue
    def _start(self, newValue = ""):
        return self._values[self._START]

    def _set_end(self, newValue = ""):
        self._values[self._END] = newValue
    def _end(self):
        return self._values[self._END]

    def apply_delta(self, delta):
        '''Applies delta hours to start and end timestamps'''

        # todo: add support for other formats
        utcTimeFormat = "%Y%m%dT%H%M%SZ"

        try:
            oldStart = self._start()
            timestamp = datetime.datetime.strptime(oldStart, utcTimeFormat)
            self._set_start((timestamp + delta).strftime(utcTimeFormat))

            oldEnd = self._end()
            timestamp = datetime.datetime.strptime(oldEnd, utcTimeFormat)
            self._set_end((timestamp + delta).strftime(utcTimeFormat))

            if very_verbose():
                print("Start:", oldStart, "->", self._start())
                print("End:", oldEnd, "->", self._end())

            return True

        except KeyError as e:
            print("Event has no such key:", e);
            if verbose():
                print(self.content())

        except ValueError as e:
            print("Date/Time conversion error:", e)
            if verbose():
                print(self.content())

        return False

def main(argv=None): # IGNORE:C0111
    '''Command line options.'''

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Jofagi on %s.
  Copyright 2014 Jofagi. All rights reserved.

  Licensed under the MIT License
  http://opensource.org/licenses/MIT

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description = program_license,
                                formatter_class = RawDescriptionHelpFormatter)
        parser.add_argument("-v", "--verbose", dest = "verbose",
                            action = "count",
                            help = "set verbosity level [default: %(default)s]")
        parser.add_argument('-V', '--version', action = 'version',
                            version = program_version_message)
        parser.add_argument("-d", "--delta", dest = "delta", type = int,
                            help = "hours to shift by", required = True)
        parser.add_argument(dest = "inPath", metavar = "path",
                            help = "file path [default: %(default)s]")
        parser.add_argument("-o", "--output", dest = "outPath",
                            help = "output file", required = True, type = str)
        parser.add_argument("-f", "--force-overwrite", dest = "overwrite",
                            help = "overwrite output file if it exists",
                            type = bool)

        # Process arguments
        args = parser.parse_args()

        inPath = args.inPath
        outPath = args.outPath
        overwrite = (args.overwrite or False)
        delta = (args.delta or 0)

        global verbosityLevel
        verbosityLevel = (args.verbose or 0) # NoneType if no -v specified

        if verbose():
            print("Verbosity level:", verbosityLevel)
            print("Input:", inPath)
            print("Output:", outPath)
            print("Overwrite output:", overwrite)
            print("Delta:", delta, "hour(s)")

        if os.path.exists(outPath):
            if overwrite and verbose():
                print("overwriting existing output file", outPath)
            elif not overwrite:
                print("output file exists already!")
                return 1

        items = read_items(open(inPath))
        shifted = apply_delta(items, delta)
        write_items(shifted, open(outPath, 'w'))
        return 0

    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
# This prevents pydev from printing a traceback
#    except Exception as e:
#        if DEBUG or TESTRUN:
#            raise(e)
#        indent = len(program_name) * " "
#        sys.stderr.write(program_name + ": " + repr(e) + "\n")
#        sys.stderr.write(indent + "  for help use --help")
#        return 2

def read_items(file):
    '''Creates Event objects from input'''

    items = [] # stores events and other items
    eventCount = 0

    eventBegin = re.compile("BEGIN:VEVENT")
    eventEnd = re.compile("END:VEVENT")

    itemLines = []
    insideEvent = False

    # Everything that's not an event is just treated as some generic data.
    # This is necessary for writing the "unimportant" data to the output file.

    for line in file:

        line = line.rstrip("\r\n")

        if not insideEvent and re.match(eventBegin, line):
            insideEvent = True

            # treat collected lines as generic data item
            if itemLines:
                items.append(IcsData(itemLines))
                if extremely_verbose():
                    print("stored", len(itemLines), "lines of non-event data")
                itemLines.clear()

            itemLines.append(line) # BEGIN line is part of event

        elif insideEvent and re.match(eventEnd, line):
            insideEvent = False

            # treat collected lines as event item
            itemLines.append(line) # END line is part of event
            items.append(Event(itemLines))
            eventCount = eventCount + 1

            if extremely_verbose():
                print("found END of event:", len(itemLines), "lines of content")

            itemLines.clear()

        else:
            itemLines.append(line) # Just collect line for current item

    if very_verbose():
        print("found", len(items), "total items of which", eventCount,
              "are events")
    elif verbose():
        print("found", eventCount, "events")

    return items

def apply_delta(items, deltaHours):
    shiftedItems = []

    delta = datetime.timedelta(hours = deltaHours)

    for i in items:
        try:
            s = copy.deepcopy(i);
            if s.apply_delta(delta):
                shiftedItems.append(s)
            else:
                shiftedItems.append(i) # Insert unmodified object

                if very_verbose():
                    print("event not shifted:", i.content())
                elif verbose():
                    print("event not shifted")

        except AttributeError:
            # Item is not an event object and apply_delta() does not exist
            # -- this is normal
            shiftedItems.append(i)

    return shiftedItems

def write_items(items, file):
    for i in items:
        for line in i.content():
            file.write(line + '\n')

if __name__ == "__main__":
    if DEBUG:
        #sys.argv.append("-h")
        sys.argv.append("-vvv")
        sys.argv.append("-d -12")
        sys.argv.append("sample.ics")
        sys.argv.append("-o sample_shifted.ics")
    if TESTRUN:
        import doctest
        doctest.testmod()
    if PROFILE:
        import cProfile
        import pstats
        profile_filename = 'shifter_profile.txt'
        cProfile.run('main()', profile_filename)
        statsfile = open("profile_stats.txt", "wb")
        p = pstats.Stats(profile_filename, stream=statsfile)
        stats = p.strip_dirs().sort_stats('cumulative')
        stats.print_stats()
        statsfile.close()
        sys.exit(0)
    sys.exit(main())<commit_msg>storing item values in list instead of map to keep original order<commit_after>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
shifter -- shifts time of events

shifter is a utilty for shifting the time of events in an iCalendar ICS file

It defines classes_and_methods

@author:     Jofagi

@copyright:  2014 Jofagi. All rights reserved.

@license:    MIT

@contact:    github.com/Jofagi
@deffield    updated: Updated
'''

import sys
import os
import re
import datetime
import copy

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter

__all__ = []
__version__ = 0.1
__date__ = '2014-03-20'
__updated__ = '2014-03-20'

DEBUG = 1
TESTRUN = 0
PROFILE = 0

def verbose():
    return 0 < verbosityLevel
def very_verbose():
    return 1 < verbosityLevel
def extremely_verbose():
    return 2 < verbosityLevel

class CLIError(Exception):
    '''Generic exception to raise and log different fatal errors.'''
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg
    def __str__(self):
        return self.msg
    def __unicode__(self):
        return self.msg

class IcsData:
    '''Generic list of keys and values'''
    def __init__(self, lines):

        separator = re.compile(":")
        self._values = [] # must be a list so the insert order is not changed

        for l in lines:
            kv = re.split(separator, l, maxsplit=1)
            if len(kv) == 2:
                self._values.append(kv)

                if extremely_verbose():
                    print(kv[0], "=", kv[1])

            elif 0 < verbose:
                print("line not in key:value format:", l)

    def content(self):
        lines = []
        for entry in self._values:
            lines.append(entry[0] + ":" + entry[1])
        return lines

class Event (IcsData):
    '''Calendar Event'''

    _START = "DTSTART"
    _END = "DTEND"
    _SUMMARY = "SUMMARY"

    def name(self):
        return self._value(self._SUMMARY)

    def _value(self, needle):
        for k, v in self._values:
            if needle == k:
                return v

    def _index(self, needle):
        idx = 0
        for k, v in self._values:
            if needle == k:
                break
            idx += 1
        return idx

    def _set_start(self, newValue = ""):
        self._values[self._index(self._START)] = newValue
    def _start(self):
        return self._value(self._START)

    def _set_end(self, newValue = ""):
        self._values[self._index(self._END)] = newValue
    def _end(self):
        return self._value(self._END)

    def apply_delta(self, delta):
        '''Applies delta hours to start and end timestamps'''

        # todo: add support for other formats
        utcTimeFormat = "%Y%m%dT%H%M%SZ"

        try:
            oldStart = self._start()
            timestamp = datetime.datetime.strptime(oldStart, utcTimeFormat)
            self._set_start((timestamp + delta).strftime(utcTimeFormat))

            oldEnd = self._end()
            timestamp = datetime.datetime.strptime(oldEnd, utcTimeFormat)
            self._set_end((timestamp + delta).strftime(utcTimeFormat))

            if very_verbose():
                print("Start:", oldStart, "->", self._start())
                print("End:", oldEnd, "->", self._end())

            return True

        except KeyError as e:
            print("Event has no such key:", e);
            if verbose():
                print(self.content())

        except ValueError as e:
            print("Date/Time conversion error:", e)
            if verbose():
                print(self.content())

        return False

def main(argv=None): # IGNORE:C0111
    '''Command line options.'''

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Jofagi on %s.
  Copyright 2014 Jofagi. All rights reserved.

  Licensed under the MIT License
  http://opensource.org/licenses/MIT

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description = program_license,
                                formatter_class = RawDescriptionHelpFormatter)
        parser.add_argument("-v", "--verbose", dest = "verbose",
                            action = "count",
                            help = "set verbosity level [default: %(default)s]")
        parser.add_argument('-V', '--version', action = 'version',
                            version = program_version_message)
        parser.add_argument("-d", "--delta", dest = "delta", type = int,
                            help = "hours to shift by", required = True)
        parser.add_argument(dest = "inPath", metavar = "path",
                            help = "file path [default: %(default)s]")
        parser.add_argument("-o", "--output", dest = "outPath",
                            help = "output file", required = True, type = str)
        parser.add_argument("-f", "--force-overwrite", dest = "overwrite",
                            help = "overwrite output file if it exists",
                            type = bool)

        # Process arguments
        args = parser.parse_args()

        inPath = args.inPath
        outPath = args.outPath
        overwrite = (args.overwrite or False)
        delta = (args.delta or 0)

        global verbosityLevel
        verbosityLevel = (args.verbose or 0) # NoneType if no -v specified

        if verbose():
            print("Verbosity level:", verbosityLevel)
            print("Input:", inPath)
            print("Output:", outPath)
            print("Overwrite output:", overwrite)
            print("Delta:", delta, "hour(s)")

        if os.path.exists(outPath):
            if overwrite and verbose():
                print("overwriting existing output file", outPath)
            elif not overwrite:
                print("output file exists already!")
                return 1

        items = read_items(open(inPath))
        shifted = apply_delta(items, delta)
        write_items(shifted, open(outPath, 'w'))
        return 0

    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
# This prevents pydev from printing a traceback
#    except Exception as e:
#        if DEBUG or TESTRUN:
#            raise(e)
#        indent = len(program_name) * " "
#        sys.stderr.write(program_name + ": " + repr(e) + "\n")
#        sys.stderr.write(indent + "  for help use --help")
#        return 2

def read_items(file):
    '''Creates Event objects from input'''

    items = [] # stores events and other items
    eventCount = 0

    eventBegin = re.compile("BEGIN:VEVENT")
    eventEnd = re.compile("END:VEVENT")

    itemLines = []
    insideEvent = False

    # Everything that's not an event is just treated as some generic data.
    # This is necessary for writing the "unimportant" data to the output file.

    for line in file:

        line = line.rstrip("\r\n")

        if not insideEvent and re.match(eventBegin, line):
            insideEvent = True

            # treat collected lines as generic data item
            if itemLines:
                items.append(IcsData(itemLines))
                if extremely_verbose():
                    print("stored", len(itemLines), "lines of non-event data")
                itemLines.clear()

            itemLines.append(line) # BEGIN line is part of event

        elif insideEvent and re.match(eventEnd, line):
            insideEvent = False

            # treat collected lines as event item
            itemLines.append(line) # END line is part of event
            items.append(Event(itemLines))
            eventCount = eventCount + 1

            if extremely_verbose():
                print("found END of event:", len(itemLines), "lines of content")

            itemLines.clear()

        else:
            itemLines.append(line) # Just collect line for current item

    if itemLines:
        items.append(IcsData(itemLines))
        if extremely_verbose():
            print("stored", len(itemLines), "lines of non-event data at end")

    if very_verbose():
        print("found", len(items), "total items of which", eventCount,
              "are events")
    elif verbose():
        print("found", eventCount, "events")

    return items

def apply_delta(items, deltaHours):
    shiftedItems = []

    delta = datetime.timedelta(hours = deltaHours)

    for i in items:
        try:
            s = copy.deepcopy(i);
            if s.apply_delta(delta):
                shiftedItems.append(s)
            else:
                shiftedItems.append(i) # Insert unmodified object

                if very_verbose():
                    print("event not shifted:", i.content())
                elif verbose():
                    print("event not shifted")

        except AttributeError:
            # Item is not an event object and apply_delta() does not exist
            # -- this is normal
            shiftedItems.append(i)

    return shiftedItems

def write_items(items, file):
    for i in items:
        for line in i.content():
            file.write(line + '\n')

if __name__ == "__main__":
    if DEBUG:
        #sys.argv.append("-h")
        sys.argv.append("-vvv")
        sys.argv.append("-d -12")
        sys.argv.append("sample.ics")
        sys.argv.append("-o sample_shifted.ics")
    if TESTRUN:
        import doctest
        doctest.testmod()
    if PROFILE:
        import cProfile
        import pstats
        profile_filename = 'shifter_profile.txt'
        cProfile.run('main()', profile_filename)
        statsfile = open("profile_stats.txt", "wb")
        p = pstats.Stats(profile_filename, stream=statsfile)
        stats = p.strip_dirs().sort_stats('cumulative')
        stats.print_stats()
        statsfile.close()
        sys.exit(0)
    sys.exit(main())<|endoftext|>


<commit_before># Copyright (c) 2012 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

"""Package defining common functionality for SMTP queuing. According to the
SMTP RFCs, a server must persistently store a message before telling the client
it accepts the message. Rather than being a traditional queue data structure,
SMTP queues will retry messages after certain periods of time.

"""

from __future__ import absolute_import

import time
from itertools import imap, izip, repeat, chain

import gevent
from gevent import Greenlet
from gevent.event import Event
from gevent.coros import Semaphore
from gevent.pool import Pool

from slimta.logging import log_exception
from slimta.core import SlimtaError
from slimta.relay import PermanentRelayError, TransientRelayError
from slimta.smtp.reply import Reply
from slimta.bounce import Bounce
from slimta.policy import QueuePolicy

__all__ = ['QueueError', 'Queue', 'QueueStorage']

class QueueError(SlimtaError):
    """Base exception for errors in the queue package."""
    pass

class QueueStorage(object):
    """Base class to show the interface that classes must implement to be a
    storage mechanism for :class:`Queue`.

    """
    def __init__(self):
        pass

    def write(self, envelope, timestamp):
        """Writes the given envelope to storage, along with the timestamp of
        its next delivery attempt. The number of delivery attempts asociated
        with the message should start at zero.

        :param envelope: |Envelope| object to write.
        :param timestamp: Timestamp of message's next delivery attempt.
        :returns: Unique identifier string for the message in queue.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def set_timestamp(self, id, timestamp):
        """Sets a new timestamp for the message's next delivery attempt.

        :param id: The unique identifier string for the message.
        :param timestamp: The new delivery attempt timestamp.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def increment_attempts(self, id):
        """Increments the number of delivery attempts associated with the
        message.

        :param id: The unique identifier string for the message.
        :returns: The new number of message delivery attempts.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def load(self):
        """Loads all queued messages from the storage engine, such that the
        :class:`Queue` can be aware of all upcoming delivery attempts.

        :returns: Iterable that yields tuples of the form ``(timestamp, id)``
                  for each message in storage.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def get(self, id):
        """Returns the |Envelope| object associated with the given unique
        identifier sring.

        :param id: The unique identfier string of the requested |Envelope|.
        :returns: Tuple with the |Envelope| object and number of attempts.
        :raises: :class:`KeyError`, :class:`QueueError`

        """
        raise NotImplementedError()

    def remove(self, id):
        """Removes the |Envelope| associated with the given unique identifier
        string from queue storage. This is typically used when the message has
        been successfully delivered or delivery has permanently failed.

        :param id: The unique identfier string of the |Envelope| to remove.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def notify(self, id):
        """If messages are not received in the same process as queue delivery,
        the storage mechanism needs a way to notify the queue delivery process
        that a new message has been stored and is ready to deliver.

        :param id: The unique identfier string of the |Envelope| that has
                   already been stored and needs its first delivery attempt.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

    def wait(self, timeout=None):
        """If messages are not being delivered from the same process in which
        they were received, the storage mechanism needs a way to wait until it
        is notified that a new message has been stored.

        :param timeout: The length of time in seconds to wait for new messages.
        :returns: The unique identifier string of a new message in storage, or
                  ``None`` if nothing was received before the timeout.
        :raises: :class:`QueueError`.

        """
        raise NotImplementedError()

class Queue(Greenlet):
    """Manages the queue of |Envelope| objects waiting for delivery. This is
    not a standard FIFO queue, a message's place in the queue depends entirely
    on the timestamp of its next delivery attempt.

    :param store: Object implementing :class:`QueueStorage`.
    :param relay: |Relay| object used to attempt message deliveries.
    :param backoff: Function that, given an |Envelope| and number of delivery
                    attempts, will return the number of seconds before the next
                    attempt. If it returns ``None``, the message will be
                    permanently failed. The default backoff function simply
                    returns ``None`` and messages are never retried.
    :param bounce_factory: Function that produces a |Bounce| object given the
                           same parameters as the |Bounce| constructor. If the
                           function returns ``None``, no bounce is delivered.
                           By default, a new |Bounce| is created in every case.
    :param store_pool: Number of simultaneous operations performable against
                       the ``store`` object. Default is unlimited.
    :param relay_pool: Number of simultaneous operations performable against
                       the ``relay`` object. Default is unlimited.

    """

    def __init__(self, store, relay, backoff=None, bounce_factory=None,
                 store_pool=None, relay_pool=None):
        super(Queue, self).__init__()
        self.store = store
        self.relay = relay
        self.backoff = backoff or self._default_backoff
        self.bounce_factory = bounce_factory or Bounce
        self.wake = Event()
        self.queued = []
        self.queued_lock = Semaphore(1)
        self.queue_policies = []
        self._use_pool('store_pool', store_pool)
        self._use_pool('relay_pool', relay_pool)

    def add_policy(self, policy):
        """Adds a |QueuePolicy| to be executed before messages are persisted
        to storage.

        :param policy: |QueuePolicy| object to execute.

        """
        if isinstance(policy, QueuePolicy):
            self.queue_policies.append(policy)
        else:
            raise TypeError('Argument not a QueuePolicy.')

    @staticmethod
    def _default_backoff(envelope, attempts):
        pass

    def _run_policies(self, envelope):
        results = [envelope]

        def recurse(current, i):
            try:
                policy = self.queue_policies[i]
            except IndexError:
                return
            ret = policy.apply(current)
            if ret:
                results.remove(current)
                results.extend(ret)
                for env in ret:
                    recurse(env, i+1)
            else:
                recurse(current, i+1)
        recurse(envelope, 0)
        return results

    def _use_pool(self, attr, pool):
        if pool is None:
            pass
        elif isinstance(pool, Pool):
            setattr(self, attr, pool)
        else:
            setattr(self, attr, Pool(pool))

    def _pool_run(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', None)
        if pool:
            ret = pool.spawn(func, *args, **kwargs)
            return ret.get()
        else:
            return func(*args, **kwargs)

    def _pool_imap(self, which, func, *iterables):
        pool = getattr(self, which+'_pool', gevent)
        threads = imap(pool.spawn, repeat(func), *iterables)
        ret = []
        for thread in threads:
            thread.join()
            ret.append(thread.exception or thread.value)
        return ret

    def _pool_spawn(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', gevent)
        return pool.spawn(func, *args, **kwargs)

    def _add_queued(self, entry):
        for i, info in enumerate(self.queued):
            if info[0] > entry[0]:  # [0] is the timestamp.
                self.queued.insert(i, entry)
                break
        else:
            self.queued.append(entry)
        self.wake.set()

    def enqueue(self, envelope):
        """Drops a new message in the queue for delivery. The first delivery
        attempt is made immediately (depending on relay pool availability).
        This method is not typically called directly, |Edge| objects use it
        when they receive new messages.

        :param envelope: |Envelope| object to enqueue.
        :returns: Zipped list of envelopes and their respective queue IDs (or
                  thrown :exc:`QueueError` objects).

        """
        now = time.time()
        envelopes = self._run_policies(envelope)
        ids = self._pool_imap('store', self.store.write, envelopes,
                              repeat(now))
        results = zip(envelopes, ids)
        for env, id in results:
            if not isinstance(id, BaseException):
                self._pool_spawn('relay', self._attempt, id, env, 0)
            elif not isinstance(id, QueueError):
                raise id  # Re-raise exceptions that are not QueueError.
        return results

    def _load_all(self):
        for entry in self.store.load():
            self._add_queued(entry)

    def _bounce(self, envelope, reply):
        bounce = self.bounce_factory(envelope, reply)
        if bounce:
            return self.enqueue(bounce)

    def _perm_fail(self, id, envelope, reply):
        self._pool_spawn('store', self.store.remove, id)
        if envelope.sender:  # Can't bounce to null-sender.
            self._pool_spawn('bounce', self._bounce, envelope, reply)

    def _retry_later(self, id, envelope, reply):
        attempts = self.store.increment_attempts(id)
        wait = self.backoff(envelope, attempts)
        if wait is None:
            reply.message += ' (Too many retries)'
            self._perm_fail(id, envelope, reply)
        else:
            when = time.time() + wait
            self.store.set_timestamp(id, when)
            self._add_queued((when, id))

    def _attempt(self, id, envelope, attempts):
        try:
            self.relay._attempt(envelope, attempts)
        except TransientRelayError as e:
            self._pool_spawn('store', self._retry_later, id, envelope, e.reply)
        except PermanentRelayError as e:
            self._perm_fail(id, envelope, e.reply)
        except Exception as e:
            log_exception(__name__)
            reply = Reply('450', '4.0.0 Unhandled delivery error: '+str(e))
            self._pool_spawn('store', self._retry_later, id, envelope, reply)
            raise
        else:
            self._pool_spawn('store', self.store.remove, id)

    def _dequeue(self, id):
        envelope, attempts = self.store.get(id)
        self._pool_spawn('relay', self._attempt, id, envelope, attempts)

    def _check_ready(self, now):
        last_i = 0
        for i, entry in enumerate(self.queued):
            timestamp, id = entry
            if now >= timestamp:
                self._pool_spawn('store', self._dequeue, id)
                last_i = i+1
            else:
                break
        if last_i > 0:
            self.queued = self.queued[last_i:]

    def _wait_ready(self, now):
        try:
            first = self.queued[0]
        except IndexError:
            self.wake.wait()
            self.wake.clear()
            return
        first_timestamp = first[0]
        if first_timestamp > now:
            self.wake.wait(first_timestamp-now)
            self.wake.clear()

    def flush(self):
        """Attempts to immediately flush all messages waiting in the queue,
        regardless of their retry timers.

        ***Note:*** This can be a very expensive operation, use with care.

        """
        self.wake.set()
        self.wake.clear()
        self.queued_lock.acquire()
        try:
            for entry in self.queued:
                self._pool_spawn('store', self._dequeue, entry[1])
            self.queued = []
        finally:
            self.queued_lock.release()

    def _run(self):
        self._pool_spawn('store', self._load_all)
        while True:
            self.queued_lock.acquire()
            try:
                now = time.time()
                self._check_ready(now)
                self._wait_ready(now)
            finally:
                self.queued_lock.release()

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<commit_msg>fixed some typos<commit_after># Copyright (c) 2012 Ian C. Good
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

"""Package defining common functionality for SMTP queuing. According to the
SMTP RFCs, a server must persistently store a message before telling the client
it accepts the message. Rather than being a traditional queue data structure,
SMTP queues will retry messages after certain periods of time.

"""

from __future__ import absolute_import

import time
from itertools import imap, izip, repeat, chain

import gevent
from gevent import Greenlet
from gevent.event import Event
from gevent.coros import Semaphore
from gevent.pool import Pool

from slimta.logging import log_exception
from slimta.core import SlimtaError
from slimta.relay import PermanentRelayError, TransientRelayError
from slimta.smtp.reply import Reply
from slimta.bounce import Bounce
from slimta.policy import QueuePolicy

__all__ = ['QueueError', 'Queue', 'QueueStorage']

class QueueError(SlimtaError):
    """Base exception for errors in the queue package."""
    pass

class QueueStorage(object):
    """Base class to show the interface that classes must implement to be a
    storage mechanism for :class:`Queue`.

    """
    def __init__(self):
        pass

    def write(self, envelope, timestamp):
        """Writes the given envelope to storage, along with the timestamp of
        its next delivery attempt. The number of delivery attempts asociated
        with the message should start at zero.

        :param envelope: |Envelope| object to write.
        :param timestamp: Timestamp of message's next delivery attempt.
        :returns: Unique identifier string for the message in queue.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def set_timestamp(self, id, timestamp):
        """Sets a new timestamp for the message's next delivery attempt.

        :param id: The unique identifier string for the message.
        :param timestamp: The new delivery attempt timestamp.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def increment_attempts(self, id):
        """Increments the number of delivery attempts associated with the
        message.

        :param id: The unique identifier string for the message.
        :returns: The new number of message delivery attempts.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def load(self):
        """Loads all queued messages from the storage engine, such that the
        :class:`Queue` can be aware of all upcoming delivery attempts.

        :returns: Iterable that yields tuples of the form ``(timestamp, id)``
                  for each message in storage.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def get(self, id):
        """Returns the |Envelope| object associated with the given unique
        identifier sring.

        :param id: The unique identifier string of the requested |Envelope|.
        :returns: Tuple with the |Envelope| object and number of attempts.
        :raises: :class:`KeyError`, :class:`QueueError`

        """
        raise NotImplementedError()

    def remove(self, id):
        """Removes the |Envelope| associated with the given unique identifier
        string from queue storage. This is typically used when the message has
        been successfully delivered or delivery has permanently failed.

        :param id: The unique identifier string of the |Envelope| to remove.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def notify(self, id):
        """If messages are not received in the same process as queue delivery,
        the storage mechanism needs a way to notify the queue delivery process
        that a new message has been stored and is ready to deliver.

        :param id: The unique identifier string of the |Envelope| that has
                   already been stored and needs its first delivery attempt.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

    def wait(self, timeout=None):
        """If messages are not being delivered from the same process in which
        they were received, the storage mechanism needs a way to wait until it
        is notified that a new message has been stored.

        :param timeout: The length of time in seconds to wait for new messages.
        :returns: The unique identifier string of a new message in storage, or
                  ``None`` if nothing was received before the timeout.
        :raises: :class:`QueueError`

        """
        raise NotImplementedError()

class Queue(Greenlet):
    """Manages the queue of |Envelope| objects waiting for delivery. This is
    not a standard FIFO queue, a message's place in the queue depends entirely
    on the timestamp of its next delivery attempt.

    :param store: Object implementing :class:`QueueStorage`.
    :param relay: |Relay| object used to attempt message deliveries.
    :param backoff: Function that, given an |Envelope| and number of delivery
                    attempts, will return the number of seconds before the next
                    attempt. If it returns ``None``, the message will be
                    permanently failed. The default backoff function simply
                    returns ``None`` and messages are never retried.
    :param bounce_factory: Function that produces a |Bounce| object given the
                           same parameters as the |Bounce| constructor. If the
                           function returns ``None``, no bounce is delivered.
                           By default, a new |Bounce| is created in every case.
    :param store_pool: Number of simultaneous operations performable against
                       the ``store`` object. Default is unlimited.
    :param relay_pool: Number of simultaneous operations performable against
                       the ``relay`` object. Default is unlimited.

    """

    def __init__(self, store, relay, backoff=None, bounce_factory=None,
                 store_pool=None, relay_pool=None):
        super(Queue, self).__init__()
        self.store = store
        self.relay = relay
        self.backoff = backoff or self._default_backoff
        self.bounce_factory = bounce_factory or Bounce
        self.wake = Event()
        self.queued = []
        self.queued_lock = Semaphore(1)
        self.queue_policies = []
        self._use_pool('store_pool', store_pool)
        self._use_pool('relay_pool', relay_pool)

    def add_policy(self, policy):
        """Adds a |QueuePolicy| to be executed before messages are persisted
        to storage.

        :param policy: |QueuePolicy| object to execute.

        """
        if isinstance(policy, QueuePolicy):
            self.queue_policies.append(policy)
        else:
            raise TypeError('Argument not a QueuePolicy.')

    @staticmethod
    def _default_backoff(envelope, attempts):
        pass

    def _run_policies(self, envelope):
        results = [envelope]

        def recurse(current, i):
            try:
                policy = self.queue_policies[i]
            except IndexError:
                return
            ret = policy.apply(current)
            if ret:
                results.remove(current)
                results.extend(ret)
                for env in ret:
                    recurse(env, i+1)
            else:
                recurse(current, i+1)
        recurse(envelope, 0)
        return results

    def _use_pool(self, attr, pool):
        if pool is None:
            pass
        elif isinstance(pool, Pool):
            setattr(self, attr, pool)
        else:
            setattr(self, attr, Pool(pool))

    def _pool_run(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', None)
        if pool:
            ret = pool.spawn(func, *args, **kwargs)
            return ret.get()
        else:
            return func(*args, **kwargs)

    def _pool_imap(self, which, func, *iterables):
        pool = getattr(self, which+'_pool', gevent)
        threads = imap(pool.spawn, repeat(func), *iterables)
        ret = []
        for thread in threads:
            thread.join()
            ret.append(thread.exception or thread.value)
        return ret

    def _pool_spawn(self, which, func, *args, **kwargs):
        pool = getattr(self, which+'_pool', gevent)
        return pool.spawn(func, *args, **kwargs)

    def _add_queued(self, entry):
        for i, info in enumerate(self.queued):
            if info[0] > entry[0]:  # [0] is the timestamp.
                self.queued.insert(i, entry)
                break
        else:
            self.queued.append(entry)
        self.wake.set()

    def enqueue(self, envelope):
        """Drops a new message in the queue for delivery. The first delivery
        attempt is made immediately (depending on relay pool availability).
        This method is not typically called directly, |Edge| objects use it
        when they receive new messages.

        :param envelope: |Envelope| object to enqueue.
        :returns: Zipped list of envelopes and their respective queue IDs (or
                  thrown :exc:`QueueError` objects).

        """
        now = time.time()
        envelopes = self._run_policies(envelope)
        ids = self._pool_imap('store', self.store.write, envelopes,
                              repeat(now))
        results = zip(envelopes, ids)
        for env, id in results:
            if not isinstance(id, BaseException):
                self._pool_spawn('relay', self._attempt, id, env, 0)
            elif not isinstance(id, QueueError):
                raise id  # Re-raise exceptions that are not QueueError.
        return results

    def _load_all(self):
        for entry in self.store.load():
            self._add_queued(entry)

    def _bounce(self, envelope, reply):
        bounce = self.bounce_factory(envelope, reply)
        if bounce:
            return self.enqueue(bounce)

    def _perm_fail(self, id, envelope, reply):
        self._pool_spawn('store', self.store.remove, id)
        if envelope.sender:  # Can't bounce to null-sender.
            self._pool_spawn('bounce', self._bounce, envelope, reply)

    def _retry_later(self, id, envelope, reply):
        attempts = self.store.increment_attempts(id)
        wait = self.backoff(envelope, attempts)
        if wait is None:
            reply.message += ' (Too many retries)'
            self._perm_fail(id, envelope, reply)
        else:
            when = time.time() + wait
            self.store.set_timestamp(id, when)
            self._add_queued((when, id))

    def _attempt(self, id, envelope, attempts):
        try:
            self.relay._attempt(envelope, attempts)
        except TransientRelayError as e:
            self._pool_spawn('store', self._retry_later, id, envelope, e.reply)
        except PermanentRelayError as e:
            self._perm_fail(id, envelope, e.reply)
        except Exception as e:
            log_exception(__name__)
            reply = Reply('450', '4.0.0 Unhandled delivery error: '+str(e))
            self._pool_spawn('store', self._retry_later, id, envelope, reply)
            raise
        else:
            self._pool_spawn('store', self.store.remove, id)

    def _dequeue(self, id):
        envelope, attempts = self.store.get(id)
        self._pool_spawn('relay', self._attempt, id, envelope, attempts)

    def _check_ready(self, now):
        last_i = 0
        for i, entry in enumerate(self.queued):
            timestamp, id = entry
            if now >= timestamp:
                self._pool_spawn('store', self._dequeue, id)
                last_i = i+1
            else:
                break
        if last_i > 0:
            self.queued = self.queued[last_i:]

    def _wait_ready(self, now):
        try:
            first = self.queued[0]
        except IndexError:
            self.wake.wait()
            self.wake.clear()
            return
        first_timestamp = first[0]
        if first_timestamp > now:
            self.wake.wait(first_timestamp-now)
            self.wake.clear()

    def flush(self):
        """Attempts to immediately flush all messages waiting in the queue,
        regardless of their retry timers.

        ***Note:*** This can be a very expensive operation, use with care.

        """
        self.wake.set()
        self.wake.clear()
        self.queued_lock.acquire()
        try:
            for entry in self.queued:
                self._pool_spawn('store', self._dequeue, entry[1])
            self.queued = []
        finally:
            self.queued_lock.release()

    def _run(self):
        self._pool_spawn('store', self._load_all)
        while True:
            self.queued_lock.acquire()
            try:
                now = time.time()
                self._check_ready(now)
                self._wait_ready(now)
            finally:
                self.queued_lock.release()

# vim:et:fdm=marker:sts=4:sw=4:ts=4
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
import os
import logging

import click
from dateutil.parser import parse as parse_date
from path import Path
import yaml

from housekeeper.cli.utils import run_orabort
from .migrate import migrate_root
from .utils import get_rundir
from . import api, Asset, AnalysisRun, Sample, Case

RUN_STATUSES = ['analyzed', 'compiled', 'delivered', 'archived', 'cleanedup']
SAMPLE_STATUSES = ['received', 'sequenced', 'confirmed']
CASE_STATUS_MAP = {'analyzed': 'analyze', 'delivered': 'deliver',
                   'archived': 'archive'}

log = logging.getLogger(__name__)

@click.command()
@click.option('-c', '--case', help='filter by case')
@click.option('-s', '--sample', help='filter by sample')
@click.option('-i', '--infer-case', is_flag=True,
              help='filter by related case for a sample')
@click.option('-t', '--category', help='filter by asset category')
@click.option('-a', '--all', 'all_runs', is_flag=True, default=False,
              help='get assets from ALL case runs, not only latest')
@click.pass_context
def get(context, case, sample, infer_case, category, all_runs):
    """Ask Housekeeper for a file."""
    api.manager(context.obj['database'])
    if infer_case:
        sample_obj = api.sample(sample)
        if sample_obj is None:
            log.warn('sorry, sample not found')
            context.abort()
        case = sample_obj.case_id
        sample = None
    if not all_runs and case:
        # get assets only from latest run
        latest_run = api.runs(case_name=case).first()
        if latest_run is None:
            log.error("no analysis run found: %s", case)
            context.abort()
        run_id = latest_run.id
    else:
        # get assets from all runs
        run_id = None
    assets = api.assets(case_name=case, sample=sample, category=category,
                        run_id=run_id)
    paths = [asset.path for asset in assets]
    output = ' '.join(paths)
    click.echo(output)

@click.command()
@click.option('-b', '--before', help='list runs before a date')
@click.option('-a', '--after', help='list runs after a date')
@click.option('--archived/--no-archived', is_flag=True,
              help='list archived/not archived runs')
@click.option('--compiled/--no-compiled', is_flag=True,
              help='list compiled/not compiled runs')
@click.option('--cleaned/--no-cleaned', is_flag=True,
              help='list cleaned up/not cleaned up runs')
@click.option('--to-clean', is_flag=True,
              help='list runs ready to be cleaned up')
@click.option('-o', '--output', type=click.Choice(['root', '', 'sample', 'case']),
              default='case', help='what to output to console')
@click.option('-l', '--limit', default=20)
@click.argument('case_id', required=False)
@click.pass_context
def runs(context, before, after, archived, compiled, cleaned, to_clean,
         limit, output, case_id):
    """List runs loaded in the database."""
    api.manager(context.obj['database'])
    root_path = context.obj['root']
    before_date = parse_date(before) if before else None
    after_date = parse_date(after) if after else None
    run_q = api.runs(case_name=case_id, before=before_date, after=after_date,
                     archived=archived, compiled=compiled, cleaned=cleaned,
                     to_clean=to_clean).limit(limit)
    if run_q.first() is None:
        log.error("no runs found")
        context.abort()
    for run_obj in run_q:
        if output == 'root':
            run_root = get_rundir(root_path, run_obj.case.name, run_obj)
            click.echo(run_root)
        elif output == 'sample':
            for sample in run_obj.samples:
                click.echo(sample.name)
        else:
            click.echo("{} - {}".format(run_obj.case.name,
                                        run_obj.analyzed_at.date()))

@click.command()
@click.argument('asset_path')
@click.option('-s', '--short', is_flag=True, default=False,
              help='Print only the filename in the sha1sum file')
@click.option('-d', '--dash', is_flag=True, default=False,
              help='Print a dash instead of the filename')
@click.pass_context
def getsha1(context, asset_path, short, dash):
    """Ask Housekeeper for the sha1sum of a file."""
    api.manager(context.obj['database'])
    checksum = api.sha1(asset_path)
    checksum_of = asset_path
    if short:
        checksum_of = os.path.basename(asset_path)
    if dash:
        checksum_of = '-'

    # for some reason we need two spaces
    click.echo("  ".join((checksum, checksum_of)))

@click.command()
@click.option('-d', '--days', default=30, help='days to postpone clean up')
@click.argument('case_name')
@click.pass_context
def postpone(context, days, case_name):
    """Ask Housekeeper for a file."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(case_name)
    api.postpone(run_obj, time=datetime.timedelta(days=days))
    manager.commit()
    click.echo("analysis will be archived on: {}"
               .format(run_obj.will_cleanup_at))

@click.command()
@click.option('-y', '--yes', is_flag=True, help="skip manual confirmation")
@click.option('-d', '--date', help="date of the particular run")
@click.argument('case_name')
@click.pass_context
def delete(context, date, yes, case_name):
    """Delete an analysis run and files."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(context, case_name, date)
    root_path = context.obj['root']
    run_root = get_rundir(root_path, case_name, run_obj)
    click.echo("you are about to delete: {}".format(run_root))
    if yes or click.confirm('Are you sure?'):
        api.delete(root_path, run_obj)
        manager.commit()

@click.command()
@click.option('-f', '--force', is_flag=True,
              help='auto confirm and override sanity checks')
@click.option('-d', '--date', help="date of the particular run")
@click.argument('case_name')
@click.pass_context
def clean(context, force, date, case_name):
    """Clean up files for an analysis."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(context, case_name, date)
    if force or click.confirm('Are you sure?'):
        api.clean_up(context.obj['root'], run_obj, force=force)
        manager.commit()

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-s', '--since', help='consider runs since date')
@click.option('--older', is_flag=True, help='reverse order of results')
@click.option('-c', '--category', default='bcf-raw',
              help='category of asset to return')
@click.pass_context
def ls(context, limit, offset, since, older, category):
    """List files from recently added runs."""
    api.manager(context.obj['database'])
    if category == 'samples':
        query = api.samples().join(Sample.run)
    elif category == 'cases':
        query = AnalysisRun.query
    else:
        query = api.assets(category=category).join(Asset.run)

    if older:
        query = query.order_by(AnalysisRun.analyzed_at)
    else:
        query = query.order_by(AnalysisRun.analyzed_at.desc())

    if since:
        date_obj = build_date(since) if since else None
        if older:
            query = query.filter(AnalysisRun.analyzed_at <= date_obj)
        else:
            query = query.filter(AnalysisRun.analyzed_at >= date_obj)
    else:
        query = query.offset(offset).limit(limit)

    if query.first() is None:
        log.warn('sorry, no matching assets found')
        context.abort()

    if category == 'samples':
        # we only consider unqiue samples
        output = (sample.lims_id for sample in query)
    elif category == 'cases':
        output = set(run.case.name for run in query)
    else:
        # we only consider files from the latest run per case
        cases = set()
        output = []
        for asset in query:
            if asset.run.case_id not in cases:
                output.append(asset.path)
                cases.add(asset.run.case_id)
    click.echo(" ".join(output))

@click.command()
@click.option('-y', '--yes', is_flag=True, help="answer 'yes' to confirmations")
@click.option('-o', '--only-db', is_flag=True, help='skip migrating assets')
@click.argument('new_root', type=click.Path())
@click.pass_context
def migrate(context, yes, only_db, new_root):
    """Migrate all assets from one root dir to another."""
    manager = api.manager(context.obj['database'])
    migrate_root(manager, context.obj['root'], new_root, only_db=only_db)

    if 'config' in context.obj:
        config_path = Path(context.obj['config'])
        if yes or click.confirm("update config file?"):
            # update the root in the config
            with config_path.open('w') as out_handle:
                dump = yaml.dump(context.obj, default_flow_style=False,
                                 allow_unicode=True)
                out_handle.write(dump.decode('utf-8'))

@click.command()
@click.option('-s', '--sample-status', type=click.Choice(SAMPLE_STATUSES),
              help='update sample status date')
@click.option('-r', '--run-status', type=click.Choice(RUN_STATUSES),
              help='update run status date')
@click.option('-c', '--custom-status', help='custom run status')
@click.option('-d', '--date', help='custom date')
@click.option('-n', '--now', help='set date to "now"')
@click.option('-rd', '--run-date', help='date of a particular run')
@click.argument('identifier')
@click.pass_context
def status(context, date, now, run_date, sample_status, run_status,
           custom_status, identifier):
    """Mark dates for resources."""
    manager = api.manager(context.obj['database'])

    if date:
        status_date = parse_date(date)
    elif now:
        status_date = datetime.datetime.now()
    else:
        status_date = None

    if sample_status:
        model_obj = api.sample(identifier)
        if model_obj is None:
            click.echo("can't find sample")
            context.abort()
        status_type = sample_status
    elif run_status:
        model_obj = run_orabort(context, identifier, run_date)
        status_type = run_status

    if custom_status:
        if status_date is None:
            current_date = model_obj.custom_date(custom_status)
        else:
            model_obj.set_custom_date(custom_status, status_date)
            status_field = custom_status
    else:
        status_field = "{}_at".format(status_type)
        if current_date is None:
            current_date = getattr(model_obj, status_field)
        else:
            setattr(model_obj, status_field, status_date)

    if status_date is None:
        if current_date is None:
            context.abort()
        else:
            click.echo(current_date)
    else:
        log.info("updating %s -> %s", status_field, status_date)
        manager.commit()

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-m', '--missing', type=click.Choice(CASE_STATUS_MAP.keys()))
@click.option('-r', '--ready', is_flag=True, help='check if samples are sequenced')
@click.pass_context
def cases(context, limit, offset, missing, ready):
    """Display information about cases."""
    api.manager(context.obj['database'])
    query = api.cases(status_to=CASE_STATUS_MAP[missing] if missing else None)
    for case in query.offset(offset).limit(limit):
        if ready:
            case_samples = api.samples(customer=case.customer,
                                       family_id=case.family_id)
            if ((case_samples.first() is None) or
                    (not all(sample.sequenced_at for sample in case_samples))):
                log.debug("skipping case, samples not sequenced")
                continue
        click.echo(case.name)

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-c', '--case', help='return samples related to a case')
@click.option('-m', '--missing', type=click.Choice(SAMPLE_STATUSES))
@click.pass_context
def samples(context, limit, offset, case, missing):
    """Display information about samples."""
    api.manager(context.obj['database'])
    query = api.samples()

    if case:
        query = (query.join(Sample.runs)
                      .join(AnalysisRun.case)
                      .filter(Case.name == case))
    if missing == 'received':
        query = query.filter(Sample.received_at == None)
    elif missing == 'sequenced':
        query = query.filter(Sample.sequenced_at == None)
    elif missing == 'confirmed':
        query = query.filter(Sample.confirmed_at == None)

    for sample in query.offset(offset).limit(limit):
        click.echo(sample.lims_id)

@click.command('add-sample')
@click.option('-d', '--date', help='date received for sample')
@click.argument('customer')
@click.argument('family_id')
@click.argument('lims_id')
@click.pass_context
def add_sample(context, date, customer, family_id, lims_id):
    """Add a new sample to the database."""
    manager = api.manager(context.obj['database'])
    existing_sample = Sample.query.filter_by(lims_id=lims_id).first()
    if existing_sample:
        log.error("sample already exists: %s", lims_id)
        context.abort()
    new_sample = Sample(lims_id=lims_id, customer=customer, family_id=family_id)
    if date:
        new_sample.received_at = parse_date(date)
    manager.add_commit(new_sample)
    log.info("added new sample: %s", lims_id)

    # also add an empty case if it doesn't already exist
    if api.case(new_sample.case_id) is None:
        new_case = Case(name=new_sample.case_id,
                        customer=new_sample.customer,
                        family_id=new_sample.family_id)
        manager.add_commit(new_case)
        log.info("added new case: %s", new_case.name)
    else:
        log.debug("case already exists: %s", new_sample.case_id)

def build_date(date_str):
    """Parse date out of string."""
    return datetime.date(*map(int, date_str.split('-')))
<commit_msg>fix issue with status command<commit_after># -*- coding: utf-8 -*-
import datetime
import os
import logging

import click
from dateutil.parser import parse as parse_date
from path import Path
import yaml

from housekeeper.cli.utils import run_orabort
from .migrate import migrate_root
from .utils import get_rundir
from . import api, Asset, AnalysisRun, Sample, Case

RUN_STATUSES = ['analyzed', 'compiled', 'delivered', 'archived', 'cleanedup']
SAMPLE_STATUSES = ['received', 'sequenced', 'confirmed']
CASE_STATUS_MAP = {'analyzed': 'analyze', 'delivered': 'deliver',
                   'archived': 'archive'}

log = logging.getLogger(__name__)

@click.command()
@click.option('-c', '--case', help='filter by case')
@click.option('-s', '--sample', help='filter by sample')
@click.option('-i', '--infer-case', is_flag=True,
              help='filter by related case for a sample')
@click.option('-t', '--category', help='filter by asset category')
@click.option('-a', '--all', 'all_runs', is_flag=True, default=False,
              help='get assets from ALL case runs, not only latest')
@click.pass_context
def get(context, case, sample, infer_case, category, all_runs):
    """Ask Housekeeper for a file."""
    api.manager(context.obj['database'])
    if infer_case:
        sample_obj = api.sample(sample)
        if sample_obj is None:
            log.warn('sorry, sample not found')
            context.abort()
        case = sample_obj.case_id
        sample = None
    if not all_runs and case:
        # get assets only from latest run
        latest_run = api.runs(case_name=case).first()
        if latest_run is None:
            log.error("no analysis run found: %s", case)
            context.abort()
        run_id = latest_run.id
    else:
        # get assets from all runs
        run_id = None
    assets = api.assets(case_name=case, sample=sample, category=category,
                        run_id=run_id)
    paths = [asset.path for asset in assets]
    output = ' '.join(paths)
    click.echo(output)

@click.command()
@click.option('-b', '--before', help='list runs before a date')
@click.option('-a', '--after', help='list runs after a date')
@click.option('--archived/--no-archived', is_flag=True,
              help='list archived/not archived runs')
@click.option('--compiled/--no-compiled', is_flag=True,
              help='list compiled/not compiled runs')
@click.option('--cleaned/--no-cleaned', is_flag=True,
              help='list cleaned up/not cleaned up runs')
@click.option('--to-clean', is_flag=True,
              help='list runs ready to be cleaned up')
@click.option('-o', '--output', type=click.Choice(['root', '', 'sample', 'case']),
              default='case', help='what to output to console')
@click.option('-l', '--limit', default=20)
@click.argument('case_id', required=False)
@click.pass_context
def runs(context, before, after, archived, compiled, cleaned, to_clean,
         limit, output, case_id):
    """List runs loaded in the database."""
    api.manager(context.obj['database'])
    root_path = context.obj['root']
    before_date = parse_date(before) if before else None
    after_date = parse_date(after) if after else None
    run_q = api.runs(case_name=case_id, before=before_date, after=after_date,
                     archived=archived, compiled=compiled, cleaned=cleaned,
                     to_clean=to_clean).limit(limit)
    if run_q.first() is None:
        log.error("no runs found")
        context.abort()
    for run_obj in run_q:
        if output == 'root':
            run_root = get_rundir(root_path, run_obj.case.name, run_obj)
            click.echo(run_root)
        elif output == 'sample':
            for sample in run_obj.samples:
                click.echo(sample.name)
        else:
            click.echo("{} - {}".format(run_obj.case.name,
                                        run_obj.analyzed_at.date()))

@click.command()
@click.argument('asset_path')
@click.option('-s', '--short', is_flag=True, default=False,
              help='Print only the filename in the sha1sum file')
@click.option('-d', '--dash', is_flag=True, default=False,
              help='Print a dash instead of the filename')
@click.pass_context
def getsha1(context, asset_path, short, dash):
    """Ask Housekeeper for the sha1sum of a file."""
    api.manager(context.obj['database'])
    checksum = api.sha1(asset_path)
    checksum_of = asset_path
    if short:
        checksum_of = os.path.basename(asset_path)
    if dash:
        checksum_of = '-'

    # for some reason we need two spaces
    click.echo("  ".join((checksum, checksum_of)))

@click.command()
@click.option('-d', '--days', default=30, help='days to postpone clean up')
@click.argument('case_name')
@click.pass_context
def postpone(context, days, case_name):
    """Ask Housekeeper for a file."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(case_name)
    api.postpone(run_obj, time=datetime.timedelta(days=days))
    manager.commit()
    click.echo("analysis will be archived on: {}"
               .format(run_obj.will_cleanup_at))

@click.command()
@click.option('-y', '--yes', is_flag=True, help="skip manual confirmation")
@click.option('-d', '--date', help="date of the particular run")
@click.argument('case_name')
@click.pass_context
def delete(context, date, yes, case_name):
    """Delete an analysis run and files."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(context, case_name, date)
    root_path = context.obj['root']
    run_root = get_rundir(root_path, case_name, run_obj)
    click.echo("you are about to delete: {}".format(run_root))
    if yes or click.confirm('Are you sure?'):
        api.delete(root_path, run_obj)
        manager.commit()

@click.command()
@click.option('-f', '--force', is_flag=True,
              help='auto confirm and override sanity checks')
@click.option('-d', '--date', help="date of the particular run")
@click.argument('case_name')
@click.pass_context
def clean(context, force, date, case_name):
    """Clean up files for an analysis."""
    manager = api.manager(context.obj['database'])
    run_obj = run_orabort(context, case_name, date)
    if force or click.confirm('Are you sure?'):
        api.clean_up(context.obj['root'], run_obj, force=force)
        manager.commit()

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-s', '--since', help='consider runs since date')
@click.option('--older', is_flag=True, help='reverse order of results')
@click.option('-c', '--category', default='bcf-raw',
              help='category of asset to return')
@click.pass_context
def ls(context, limit, offset, since, older, category):
    """List files from recently added runs."""
    api.manager(context.obj['database'])
    if category == 'samples':
        query = api.samples().join(Sample.run)
    elif category == 'cases':
        query = AnalysisRun.query
    else:
        query = api.assets(category=category).join(Asset.run)

    if older:
        query = query.order_by(AnalysisRun.analyzed_at)
    else:
        query = query.order_by(AnalysisRun.analyzed_at.desc())

    if since:
        date_obj = build_date(since) if since else None
        if older:
            query = query.filter(AnalysisRun.analyzed_at <= date_obj)
        else:
            query = query.filter(AnalysisRun.analyzed_at >= date_obj)
    else:
        query = query.offset(offset).limit(limit)

    if query.first() is None:
        log.warn('sorry, no matching assets found')
        context.abort()

    if category == 'samples':
        # we only consider unqiue samples
        output = (sample.lims_id for sample in query)
    elif category == 'cases':
        output = set(run.case.name for run in query)
    else:
        # we only consider files from the latest run per case
        cases = set()
        output = []
        for asset in query:
            if asset.run.case_id not in cases:
                output.append(asset.path)
                cases.add(asset.run.case_id)
    click.echo(" ".join(output))

@click.command()
@click.option('-y', '--yes', is_flag=True, help="answer 'yes' to confirmations")
@click.option('-o', '--only-db', is_flag=True, help='skip migrating assets')
@click.argument('new_root', type=click.Path())
@click.pass_context
def migrate(context, yes, only_db, new_root):
    """Migrate all assets from one root dir to another."""
    manager = api.manager(context.obj['database'])
    migrate_root(manager, context.obj['root'], new_root, only_db=only_db)

    if 'config' in context.obj:
        config_path = Path(context.obj['config'])
        if yes or click.confirm("update config file?"):
            # update the root in the config
            with config_path.open('w') as out_handle:
                dump = yaml.dump(context.obj, default_flow_style=False,
                                 allow_unicode=True)
                out_handle.write(dump.decode('utf-8'))

@click.command()
@click.option('-s', '--sample-status', type=click.Choice(SAMPLE_STATUSES),
              help='update sample status date')
@click.option('-r', '--run-status', type=click.Choice(RUN_STATUSES),
              help='update run status date')
@click.option('-c', '--custom-status', help='custom run status')
@click.option('-d', '--date', help='custom date')
@click.option('-n', '--now', help='set date to "now"')
@click.option('-rd', '--run-date', help='date of a particular run')
@click.argument('identifier')
@click.pass_context
def status(context, date, now, run_date, sample_status, run_status,
           custom_status, identifier):
    """Mark dates for resources."""
    manager = api.manager(context.obj['database'])

    # determine which date to set: custom or now!
    if date:
        status_date = parse_date(date)
    elif now:
        status_date = datetime.datetime.now()
    else:
        status_date = None

    if sample_status:
        # update a date for a sample
        model_obj = api.sample(identifier)
        if model_obj is None:
            click.echo("can't find sample")
            context.abort()
        # types are predefined; no need to check!
        status_type = sample_status
    elif run_status:
        # update a date for an analysis run
        model_obj = run_orabort(context, identifier, run_date)
        # types are predefined as well
        status_type = run_status

    if custom_status:
        if status_date is None:
            # show a custom date for an analysis run
            current_date = model_obj.custom_date(custom_status)
        else:
            # update a custom date for a run
            model_obj.set_custom_date(custom_status, status_date)
            status_field = custom_status
    else:
        status_field = "{}_at".format(status_type)
        if status_date is None:
            # show a predfined date for a model
            current_date = getattr(model_obj, status_field)
        else:
            # update a predefined date on a model
            setattr(model_obj, status_field, status_date)

    if status_date is None:
        if current_date is None:
            context.abort()
        else:
            click.echo(current_date)
    else:
        log.info("updating %s -> %s", status_field, status_date)
        manager.commit()

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-m', '--missing', type=click.Choice(CASE_STATUS_MAP.keys()))
@click.option('-r', '--ready', is_flag=True, help='check if samples are sequenced')
@click.pass_context
def cases(context, limit, offset, missing, ready):
    """Display information about cases."""
    api.manager(context.obj['database'])
    query = api.cases(status_to=CASE_STATUS_MAP[missing] if missing else None)
    for case in query.offset(offset).limit(limit):
        if ready:
            case_samples = api.samples(customer=case.customer,
                                       family_id=case.family_id)
            if ((case_samples.first() is None) or
                    (not all(sample.sequenced_at for sample in case_samples))):
                log.debug("skipping case, samples not sequenced")
                continue
        click.echo(case.name)

@click.command()
@click.option('-l', '--limit', default=20, help='limit number of results')
@click.option('-o', '--offset', default=0, help='skip initial results')
@click.option('-c', '--case', help='return samples related to a case')
@click.option('-m', '--missing', type=click.Choice(SAMPLE_STATUSES))
@click.pass_context
def samples(context, limit, offset, case, missing):
    """Display information about samples."""
    api.manager(context.obj['database'])
    query = api.samples()

    if case:
        query = (query.join(Sample.runs)
                      .join(AnalysisRun.case)
                      .filter(Case.name == case))
    if missing == 'received':
        query = query.filter(Sample.received_at == None)
    elif missing == 'sequenced':
        query = query.filter(Sample.sequenced_at == None)
    elif missing == 'confirmed':
        query = query.filter(Sample.confirmed_at == None)

    for sample in query.offset(offset).limit(limit):
        click.echo(sample.lims_id)

@click.command('add-sample')
@click.option('-d', '--date', help='date received for sample')
@click.argument('customer')
@click.argument('family_id')
@click.argument('lims_id')
@click.pass_context
def add_sample(context, date, customer, family_id, lims_id):
    """Add a new sample to the database."""
    manager = api.manager(context.obj['database'])
    existing_sample = Sample.query.filter_by(lims_id=lims_id).first()
    if existing_sample:
        log.error("sample already exists: %s", lims_id)
        context.abort()
    new_sample = Sample(lims_id=lims_id, customer=customer, family_id=family_id)
    if date:
        new_sample.received_at = parse_date(date)
    manager.add_commit(new_sample)
    log.info("added new sample: %s", lims_id)

    # also add an empty case if it doesn't already exist
    if api.case(new_sample.case_id) is None:
        new_case = Case(name=new_sample.case_id,
                        customer=new_sample.customer,
                        family_id=new_sample.family_id)
        manager.add_commit(new_case)
        log.info("added new case: %s", new_case.name)
    else:
        log.debug("case already exists: %s", new_sample.case_id)

def build_date(date_str):
    """Parse date out of string."""
    return datetime.date(*map(int, date_str.split('-')))
<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
shifter -- shifts time of events

shifter is a utilty for shifting the time of events in an iCalendar ICS file

It defines classes_and_methods

@author:     Jofagi

@copyright:  2014 Jofagi. All rights reserved.

@license:    MIT

@contact:    github.com/Jofagi
@deffield    updated: Updated
'''

import sys
import os
import re

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter

__all__ = []
__version__ = 0.1
__date__ = '2014-03-20'
__updated__ = '2014-03-20'

DEBUG = 1
TESTRUN = 0
PROFILE = 0

def verbose():
    return 0 < verbosityLevel
def veryVerbose():
    return 1 < verbosityLevel
def extremelyVerbose():
    return 2 < verbosityLevel

class CLIError(Exception):
    '''Generic exception to raise and log different fatal errors.'''
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg
    def __str__(self):
        return self.msg
    def __unicode__(self):
        return self.msg

class IcsData:
    '''Generic list of keys and values'''
    def __init__(self, lines):

        separator = re.compile(":")
        self._values = {}

        for l in lines:
            kv = re.split(separator, l, maxsplit=1)
            if len(kv) == 2:
                self._values[kv[0]] = kv[1]

                if extremelyVerbose():
                    print(kv[0],"=", kv[1])

            elif 0 < verbose:
                print("line not in key:value format:", l)

    def content(self):
        lines = []
        for k, v in self._values.items():
            lines.append(k + ":" + v)

class Event (IcsData):
    '''Calendar Event'''

    def name(self):
        return self._values["SUMMARY"]

    def start(self):
        return self._values["DTSTART"]

    def end(self):
        return self._values["DTEND"]

def main(argv=None): # IGNORE:C0111
    '''Command line options.'''

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Jofagi on %s.
  Copyright 2014 Jofagi. All rights reserved.

  Licensed under the MIT License
  http://opensource.org/licenses/MIT

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
        parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]")
        parser.add_argument('-V', '--version', action='version', version=program_version_message)
        parser.add_argument(dest="path", help="file path [default: %(default)s]", metavar="path")
        parser.add_argument("-d", "--delta", dest="delta", help="hours to shift by", required=True, type=int)

        # Process arguments
        args = parser.parse_args()

        path = args.path
        delta = (args.delta or 0)

        global verbosityLevel
        verbosityLevel = (args.verbose or 0) # NoneType if no -v specified

        if verbose():
            print("Verbosity level:", verbosityLevel)
            print("Path:", path)
            print("Delta:", delta, "hour(s)")

        return shift(path, delta)

    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
# This prevents pydev from printing a traceback
#    except Exception as e:
#        if DEBUG or TESTRUN:
#            raise(e)
#        indent = len(program_name) * " "
#        sys.stderr.write(program_name + ": " + repr(e) + "\n")
#        sys.stderr.write(indent + "  for help use --help")
#        return 2

def readItems(file):
    '''Creates Event objects from input'''

    items = [] # stores events and other items
    eventCount = 0

    eventBegin = re.compile("BEGIN:VEVENT")
    eventEnd = re.compile("END:VEVENT")

    itemLines = []
    insideEvent = False

    # Everything that's not an event is just treated as some generic data.
    # This is necessary for writing the "unimportant" data to the output file.

    for line in file:

        if not insideEvent and re.match(eventBegin, line):
            insideEvent = True

            # treat collected lines as generic data item
            if 0 < len(itemLines):
                items.append(IcsData(itemLines))
                itemLines.clear()

            itemLines.append(line) # BEGIN line is part of event

        elif insideEvent and re.match(eventEnd, line):
            insideEvent = False

            # treat collected lines as event item
            itemLines.append(line) # END line is part of event
            items.append(Event(itemLines))
            eventCount = eventCount + 1

            if extremelyVerbose():
                print("found END of event:", len(itemLines), "lines of content")

            itemLines.clear()

        else:
            itemLines.append(line) # Just collect line for current item

    if verbose():
        print("found", eventCount, "events")
    elif veryVerbose():
        print("found", len(items), "total items of which", eventCount, "are events")

    return items

def shift(path, delta):
    try:
        items = readItems(open(path))

    except OSError as e:
        print(e)
        return 1

    return 0

if __name__ == "__main__":
    if DEBUG:
        #sys.argv.append("-h")
        sys.argv.append("-vvv")
        sys.argv.append("-d -1")
        sys.argv.append("sample.ics")
    if TESTRUN:
        import doctest
        doctest.testmod()
    if PROFILE:
        import cProfile
        import pstats
        profile_filename = 'shifter_profile.txt'
        cProfile.run('main()', profile_filename)
        statsfile = open("profile_stats.txt", "wb")
        p = pstats.Stats(profile_filename, stream=statsfile)
        stats = p.strip_dirs().sort_stats('cumulative')
        stats.print_stats()
        statsfile.close()
        sys.exit(0)
    sys.exit(main())<commit_msg>fixed IcsData.content() and started with applyDelta()<commit_after>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
shifter -- shifts time of events

shifter is a utilty for shifting the time of events in an iCalendar ICS file

It defines classes_and_methods

@author:     Jofagi

@copyright:  2014 Jofagi. All rights reserved.

@license:    MIT

@contact:    github.com/Jofagi
@deffield    updated: Updated
'''

import sys
import os
import re

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter

__all__ = []
__version__ = 0.1
__date__ = '2014-03-20'
__updated__ = '2014-03-20'

DEBUG = 1
TESTRUN = 0
PROFILE = 0

def verbose():
    return 0 < verbosityLevel
def veryVerbose():
    return 1 < verbosityLevel
def extremelyVerbose():
    return 2 < verbosityLevel

class CLIError(Exception):
    '''Generic exception to raise and log different fatal errors.'''
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg
    def __str__(self):
        return self.msg
    def __unicode__(self):
        return self.msg

class IcsData:
    '''Generic list of keys and values'''
    def __init__(self, lines):

        separator = re.compile(":")
        self._values = {}

        for l in lines:
            kv = re.split(separator, l, maxsplit=1)
            if len(kv) == 2:
                self._values[kv[0]] = kv[1]

                if extremelyVerbose():
                    print(kv[0],"=", kv[1])

            elif 0 < verbose:
                print("line not in key:value format:", l)

    def content(self):
        lines = []
        for k, v in self._values.items():
            lines.append(k + ":" + v)
        return lines

class Event (IcsData):
    '''Calendar Event'''

    def name(self):
        return self._values["SUMMARY"]

    def start(self):
        return self._values["DTSTART"]

    def end(self):
        return self._values["DTEND"]

def main(argv=None): # IGNORE:C0111
    '''Command line options.'''

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Jofagi on %s.
  Copyright 2014 Jofagi. All rights reserved.

  Licensed under the MIT License
  http://opensource.org/licenses/MIT

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
        parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]")
        parser.add_argument('-V', '--version', action='version', version=program_version_message)
        parser.add_argument(dest="path", help="file path [default: %(default)s]", metavar="path")
        parser.add_argument("-d", "--delta", dest="delta", help="hours to shift by", required=True, type=int)

        # Process arguments
        args = parser.parse_args()

        path = args.path
        delta = (args.delta or 0)

        global verbosityLevel
        verbosityLevel = (args.verbose or 0) # NoneType if no -v specified

        if verbose():
            print("Verbosity level:", verbosityLevel)
            print("Path:", path)
            print("Delta:", delta, "hour(s)")

        return shift(path, delta)

    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
# This prevents pydev from printing a traceback
#    except Exception as e:
#        if DEBUG or TESTRUN:
#            raise(e)
#        indent = len(program_name) * " "
#        sys.stderr.write(program_name + ": " + repr(e) + "\n")
#        sys.stderr.write(indent + "  for help use --help")
#        return 2

def readItems(file):
    '''Creates Event objects from input'''

    items = [] # stores events and other items
    eventCount = 0

    eventBegin = re.compile("BEGIN:VEVENT")
    eventEnd = re.compile("END:VEVENT")

    itemLines = []
    insideEvent = False

    # Everything that's not an event is just treated as some generic data.
    # This is necessary for writing the "unimportant" data to the output file.

    for line in file:

        if not insideEvent and re.match(eventBegin, line):
            insideEvent = True

            # treat collected lines as generic data item
            if 0 < len(itemLines):
                items.append(IcsData(itemLines))
                itemLines.clear()

            itemLines.append(line) # BEGIN line is part of event

        elif insideEvent and re.match(eventEnd, line):
            insideEvent = False

            # treat collected lines as event item
            itemLines.append(line) # END line is part of event
            items.append(Event(itemLines))
            eventCount = eventCount + 1

            if extremelyVerbose():
                print("found END of event:", len(itemLines), "lines of content")

            itemLines.clear()

        else:
            itemLines.append(line) # Just collect line for current item

    if verbose():
        print("found", eventCount, "events")
    elif veryVerbose():
        print("found", len(items), "total items of which", eventCount, "are events")

    return items

def applyDelta(items, delta):
    shiftedItems = []

    for i in items:
        try:
            startTime = i.start()
            endTime = i.end()

        except AttributeError:
            pass # Item is not an event object - this is normal
        except KeyError as e:
            # It's and event but has no such key - this is bad
            print("Event has no such key:", e);
            if verbose():
                print(i.content());

    return shiftedItems

def shift(path, delta):
    try:
        items = readItems(open(path))
        items = applyDelta(items, delta)

    except OSError as e:
        print(e)
        return 1

    return 0

if __name__ == "__main__":
    if DEBUG:
        #sys.argv.append("-h")
        sys.argv.append("-vvv")
        sys.argv.append("-d -1")
        sys.argv.append("sample.ics")
    if TESTRUN:
        import doctest
        doctest.testmod()
    if PROFILE:
        import cProfile
        import pstats
        profile_filename = 'shifter_profile.txt'
        cProfile.run('main()', profile_filename)
        statsfile = open("profile_stats.txt", "wb")
        p = pstats.Stats(profile_filename, stream=statsfile)
        stats = p.strip_dirs().sort_stats('cumulative')
        stats.print_stats()
        statsfile.close()
        sys.exit(0)
    sys.exit(main())<|endoftext|>