<commit_before>#!/bin/env python3
from pyquery import PyQuery as pq
from pathlib import Path
import os
import argparse
import re
import requests

def lead0(num, max):
    return str(num).zfill(len(str(max)))

def terminal_size():
    try:
        import fcntl
        import termios
        import struct

        th, tw, hp, wp = struct.unpack('HHHH', fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)))
    except (IOError, ModuleNotFoundError):
        th, tw = 80, 200
    return tw, th

def printLine(msg='', noNewLine=False):
    terminalWidth = terminal_size()[0]
    spaces = terminalWidth - len(msg)

    if noNewLine:
        if args.verbose:
            print(' ' + msg + (' ' * (spaces - 1)))
        else:
            print(msg + (' ' * spaces), end='\r')
    else:
        print(msg + (' ' * spaces))

def check_path(path, slash=True, fat=False):
    evil_chars = []
    if slash:
        evil_chars.append('/')
    if fat:
        evil_chars += ['?', '<', '>', '\\', ':', '*', '|', '"', '^']
    return ''.join([char for char in path if char not in evil_chars])

# parse input and settup help
parser = argparse.ArgumentParser(description='Downloads Comics from \'https://tapas.io\'.\nIf folder of downloaded comic is found, it will only update (can be disabled with -f/--force).', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('url', metavar='URL/name', type=str, nargs='+',
                    help='URL or URL name to comic\nGo to the comic you want to download (any page)\nRightclick on the comic name in the upper left corner and select "Copy linkaddress" (Or similar) or just use the name behind series in the url\nExamples: https://tapas.io/series/Erma, RavenWolf, ...')
parser.add_argument('-f', '--force', action="store_true", help='Disables updater.')
parser.add_argument('-v', '--verbose', action="store_true", help='Enables verbose mode.')
parser.add_argument('-c', '--restrict-characters', action="store_true", help='Removes \'? < > \\ : * | " ^\' from file names')
parser.add_argument('-o', '--output-dir', type=str, nargs='?', default="", dest='baseDir', metavar='C:\\',
                    help='Output directory where comics should be placed.\nIf left blank, the script folder will be used.')

args = parser.parse_args()

basePath = ""
if (args.baseDir):
    basePath = Path(args.baseDir)

for urlCount, url in enumerate(args.url):
    # check url/name
    if re.match(r'^https://tapas\.io/series/.+$', url):
        urlName = url[url.rindex('/') + 1:]
    else:
        urlName = url

    printLine('Loading ' + urlName + '...', True)

    # Get comic start page and test if comic exsists
    pageReqest = requests.get('https://tapas.io/series/' + urlName, headers={'user-agent': 'tapas-dl'})

    if pageReqest.status_code != 200:
        printLine('Error: Comic "{}" not found\n'.format(urlName))
        break

    page = pq(pageReqest.text)

    name = page('.center-info__title.center-info__title--small').text()

    author = page('div.viewer-section.viewer-section--episode a.name').text()

    seriesId = page('.subscribe-btn').attr('data-id')

    if len(page('.row-item--info > img')) > 0:
        headerSrc = page('.row-item--info > img').attr('src')
    else:
        headerSrc = None

    data = []
    page = pq(requests.get(f'https://tapas.io/series/{seriesId}/episodes?page=1&sort=OLDEST&max_limit=99999999',  # It's over 9000! But I love that they forgot to limit the max_limit, because that means I don't have to bother with pagination ^^
                           headers={'user-agent': 'tapas-dl'}).json()['data']['body'])
    for episode in page('[data-permalink*="/episode/"]'):
        data.append({'id': int(episode.attrib['data-permalink'][episode.attrib['data-permalink'].rfind('/') + 1:])})

    printLine('{} [{}] ({} pages):'.format(name, urlName, len(data)))

    # Check if folder exsists, if not create it
    printLine('Checking folder...', True)
    # If the user specified a base output directory, prepend that on our folder
    savePath = '{} [{}]'.format(name, urlName)
    if (basePath != ""):
        savePath = os.path.join(basePath, savePath)
        printLine('Full path is: ' + str(savePath))
    if os.path.isdir(savePath) and not args.force:
        printLine('Found directory, only updating (use -f/--force to disable)')

        filesInDir = list(os.scandir(savePath))

        fileNames = []
        for fileInDir in filesInDir:
            fileNames.append(fileInDir.name)
        fileNames.sort()

        imgOffset = len(fileNames)

        if imgOffset > 1:
            lastFile = fileNames[-1]
            lastPageId = int(lastFile[lastFile.rindex('#') + 1:lastFile.rindex('.')])

            pageOffset = next(i for i, page in enumerate(data) if page['id'] == lastPageId) + 1

            data = data[pageOffset:]
        else:
            pageOffset = 0
    else:
        if not os.path.isdir(savePath):
            os.mkdir(savePath)
            printLine('Creating folder...', True)

        pageOffset = 0
        imgOffset = 0

    # Download header
    if True not in [file.starts_with('-1 - header.') for file in os.scandir(savePath)]:
        printLine('Downloading header...', True)

        if headerSrc is not None:
            with open(os.path.join(savePath, '-1 - header.{}'.format(headerSrc[headerSrc.rindex('.') + 1:])), 'wb') as f:
                f.write(requests.get(headerSrc).content)

            printLine('Downloaded header')
        else:
            printLine('Header not found')

    # Check if series is comic or novel
    if len(pq(f'https://tapas.io/episode/{data[0]["id"]}', headers={'user-agent': 'tapas-dl'})('.content__img.js-lazy')) > 0:
        printLine('Detected comic')
        # Get images from page from JS api
        allImgCount = 0
        for pageCount, pageData in enumerate(data):

            # Test whether the page we have in mind is reachable
            pageReqest = requests.get(f'https://tapas.io/episode/{pageData["id"]}', headers={'user-agent': 'tapas-dl'})
            if pageReqest.status_code != 200:
                # This page was unavailable. Let the user know and add a single dummy image entry.
                # (We will check for this when we go to download images.)
                printLine('Error: "{}" page {}/{} not found. Page Request yielded: {} (Early Access page?)'.format(urlName, pageCount + pageOffset, len(data) + pageOffset, str(pageReqest.status_code)), True)

                pageData['title'] = "PageUnavailable"
                pageData['imgs'] = []
                pageData['imgs'].append("PageUnavailable")

            else:
                # If the page did not yield an access error, go ahead an scrape for image entries.
                pageHtml = pq(f'https://tapas.io/episode/{pageData["id"]}', headers={'user-agent': 'tapas-dl'})

                printLine('Downloaded image data from {} images (pages {}/{})...'.format(allImgCount, pageCount + pageOffset, len(data) + pageOffset), True)

                pageData['title'] = pageHtml('.info__title').text()

                pageData['imgs'] = []
                for img in pageHtml('.content__img'):
                    pageData['imgs'].append(pq(img).attr('data-src'))

                    allImgCount += 1

        # Download images
        imgCount = 0
        for pageCount, pageData in enumerate(data):

            for imgOfPageCount, img in enumerate(pageData['imgs']):

                # Check if the first image entry is the fummy text that indicates the page was unavailable when we tried to scrape it.
                if pageData['imgs'][0] != "PageUnavailable":
                    # If the entry isn't a dummy entry, go ahead and download the images it contains.
                    with open(os.path.join(savePath, check_path('{} - {} - {} - {} - #{}.{}'.format(lead0(imgCount + imgOffset, allImgCount + imgOffset), lead0(pageCount + pageOffset, len(pageData) + pageOffset),
                                                                                lead0(imgOfPageCount, len(pageData['imgs'])), pageData['title'], pageData['id'], img[img.rindex('.') + 1:]),
                                                                fat=args.restrict_characters)), 'wb') as f:
                        f.write(requests.get(img).content)

                    imgCount += 1

                    printLine('Downloaded image {}/{} from page {}/{} ({}/{} images)...'.format(imgOfPageCount + 1, len(pageData['imgs']), pageCount + pageOffset, len(data) + pageOffset, imgCount + imgOffset, allImgCount + imgOffset), True)
                else:
                    # If the entry was a dummy entry, skip it and let the user know.
                    printLine('Error: No images downloaded from page {}/{}.'.format(pageCount + pageOffset, len(data) + pageOffset), True)

        if data != []:
            printLine('Downloaded {} of {} images'.format(imgCount, allImgCount))
        else:
            printLine('Nothing to do')

        if urlCount + 1 != len(args.url):
            printLine()
    else:
        printLine('Detected novel')

        from ebooklib import epub

        # download/create epub
        book = epub.EpubBook()

        customCss = None

        # Add meta data
        book.set_identifier(urlName)
        book.set_title(name)
        book.set_language('en')

        book.add_author(author)

        header_name = os.path.join(savePath, list(filter(re.compile(r'.+header\..+').match, os.listdir(savePath)))[0])
        book.set_cover("cover.jpg", open(header_name, 'rb').read())

        book.toc = []
        book.spine = ['cover']

        # create about page
        chapter = epub.EpubHtml(title='about', file_name='about.xhtml')
        chapter.content = f'
About

Title: {name}

Author: {book.metadata["http://purl.org/dc/elements/1.1/"]["creator"][0][0]}

Source: {"https://tapas.io/series/" + urlName}
'

        book.add_item(chapter)
        book.spine.append(chapter)

        # Append nav page
        book.spine.append('nav')

        # create chapters
        for pageCount, pageData in enumerate(data):
            printLine('Downloaded page {}/{}...'.format(pageCount + 1, len(data)), True)

            pagePq = pq(url='https://tapas.io/episode/' + str(pageData['id']), headers={'user-agent': 'tapas-dl'})
            pageTitle = pagePq('.viewer__header > .title').text()

            pageHtml = f'
{pageTitle}
'
            for p in pagePq('article.viewer__body > p'):
                p = pq(p)
                if p.text() is not None:
                    pageHtml += '
' + p.text() + '
'

            chapter = epub.EpubHtml(title=pageTitle, file_name=str(pageData['id']) + '.xhtml')
            chapter.content = pageHtml

            book.add_item(chapter)

            # define Table Of Contents
            book.toc.append(epub.Link(str(pageData['id']) + '.xhtml', pageTitle, str(pageData['id'])))

            # basic spine
            book.spine.append(chapter)

        # add default NCX and Nav file
        book.add_item(epub.EpubNcx())
        book.add_item(epub.EpubNav())

        # add CSS
        style = ''
        nav_css = epub.EpubItem(uid="style_nav", file_name="style/nav.css", media_type="text/css", content=style)
        book.add_item(nav_css)

        # write to the file
        epub.write_epub(os.path.join('/'.join(savePath.split('/')[:-1]), check_path(f'{name}.epub', fat=args.restrict_characters)), book)

        # remove tmp folder
        for file in os.listdir(savePath):
            os.remove(os.path.join(savePath, file))
        os.removedirs(savePath)
<commit_msg>Fix typo (#26)<commit_after>#!/bin/env python3
from pyquery import PyQuery as pq
from pathlib import Path
import os
import argparse
import re
import requests

def lead0(num, max):
    return str(num).zfill(len(str(max)))

def terminal_size():
    try:
        import fcntl
        import termios
        import struct

        th, tw, hp, wp = struct.unpack('HHHH', fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)))
    except (IOError, ModuleNotFoundError):
        th, tw = 80, 200
    return tw, th

def printLine(msg='', noNewLine=False):
    terminalWidth = terminal_size()[0]
    spaces = terminalWidth - len(msg)

    if noNewLine:
        if args.verbose:
            print(' ' + msg + (' ' * (spaces - 1)))
        else:
            print(msg + (' ' * spaces), end='\r')
    else:
        print(msg + (' ' * spaces))

def check_path(path, slash=True, fat=False):
    evil_chars = []
    if slash:
        evil_chars.append('/')
    if fat:
        evil_chars += ['?', '<', '>', '\\', ':', '*', '|', '"', '^']
    return ''.join([char for char in path if char not in evil_chars])

# parse input and settup help
parser = argparse.ArgumentParser(description='Downloads Comics from \'https://tapas.io\'.\nIf folder of downloaded comic is found, it will only update (can be disabled with -f/--force).', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('url', metavar='URL/name', type=str, nargs='+',
                    help='URL or URL name to comic\nGo to the comic you want to download (any page)\nRightclick on the comic name in the upper left corner and select "Copy linkaddress" (Or similar) or just use the name behind series in the url\nExamples: https://tapas.io/series/Erma, RavenWolf, ...')
parser.add_argument('-f', '--force', action="store_true", help='Disables updater.')
parser.add_argument('-v', '--verbose', action="store_true", help='Enables verbose mode.')
parser.add_argument('-c', '--restrict-characters', action="store_true", help='Removes \'? < > \\ : * | " ^\' from file names')
parser.add_argument('-o', '--output-dir', type=str, nargs='?', default="", dest='baseDir', metavar='C:\\',
                    help='Output directory where comics should be placed.\nIf left blank, the script folder will be used.')

args = parser.parse_args()

basePath = ""
if (args.baseDir):
    basePath = Path(args.baseDir)

for urlCount, url in enumerate(args.url):
    # check url/name
    if re.match(r'^https://tapas\.io/series/.+$', url):
        urlName = url[url.rindex('/') + 1:]
    else:
        urlName = url

    printLine('Loading ' + urlName + '...', True)

    # Get comic start page and test if comic exsists
    pageReqest = requests.get('https://tapas.io/series/' + urlName, headers={'user-agent': 'tapas-dl'})

    if pageReqest.status_code != 200:
        printLine('Error: Comic "{}" not found\n'.format(urlName))
        break

    page = pq(pageReqest.text)

    name = page('.center-info__title.center-info__title--small').text()

    author = page('div.viewer-section.viewer-section--episode a.name').text()

    seriesId = page('.subscribe-btn').attr('data-id')

    if len(page('.row-item--info > img')) > 0:
        headerSrc = page('.row-item--info > img').attr('src')
    else:
        headerSrc = None

    data = []
    page = pq(requests.get(f'https://tapas.io/series/{seriesId}/episodes?page=1&sort=OLDEST&max_limit=99999999',  # It's over 9000! But I love that they forgot to limit the max_limit, because that means I don't have to bother with pagination ^^
                           headers={'user-agent': 'tapas-dl'}).json()['data']['body'])
    for episode in page('[data-permalink*="/episode/"]'):
        data.append({'id': int(episode.attrib['data-permalink'][episode.attrib['data-permalink'].rfind('/') + 1:])})

    printLine('{} [{}] ({} pages):'.format(name, urlName, len(data)))

    # Check if folder exsists, if not create it
    printLine('Checking folder...', True)
    # If the user specified a base output directory, prepend that on our folder
    savePath = '{} [{}]'.format(name, urlName)
    if (basePath != ""):
        savePath = os.path.join(basePath, savePath)
        printLine('Full path is: ' + str(savePath))
    if os.path.isdir(savePath) and not args.force:
        printLine('Found directory, only updating (use -f/--force to disable)')

        filesInDir = list(os.scandir(savePath))

        fileNames = []
        for fileInDir in filesInDir:
            fileNames.append(fileInDir.name)
        fileNames.sort()

        imgOffset = len(fileNames)

        if imgOffset > 1:
            lastFile = fileNames[-1]
            lastPageId = int(lastFile[lastFile.rindex('#') + 1:lastFile.rindex('.')])

            pageOffset = next(i for i, page in enumerate(data) if page['id'] == lastPageId) + 1

            data = data[pageOffset:]
        else:
            pageOffset = 0
    else:
        if not os.path.isdir(savePath):
            os.mkdir(savePath)
            printLine('Creating folder...', True)

        pageOffset = 0
        imgOffset = 0

    # Download header
    if True not in [file.name.startswith('-1 - header.') for file in os.scandir(savePath)]:
        printLine('Downloading header...', True)

        if headerSrc is not None:
            with open(os.path.join(savePath, '-1 - header.{}'.format(headerSrc[headerSrc.rindex('.') + 1:])), 'wb') as f:
                f.write(requests.get(headerSrc).content)

            printLine('Downloaded header')
        else:
            printLine('Header not found')

    # Check if series is comic or novel
    if len(pq(f'https://tapas.io/episode/{data[0]["id"]}', headers={'user-agent': 'tapas-dl'})('.content__img.js-lazy')) > 0:
        printLine('Detected comic')
        # Get images from page from JS api
        allImgCount = 0
        for pageCount, pageData in enumerate(data):

            # Test whether the page we have in mind is reachable
            pageReqest = requests.get(f'https://tapas.io/episode/{pageData["id"]}', headers={'user-agent': 'tapas-dl'})
            if pageReqest.status_code != 200:
                # This page was unavailable. Let the user know and add a single dummy image entry.
                # (We will check for this when we go to download images.)
                printLine('Error: "{}" page {}/{} not found. Page Request yielded: {} (Early Access page?)'.format(urlName, pageCount + pageOffset, len(data) + pageOffset, str(pageReqest.status_code)), True)

                pageData['title'] = "PageUnavailable"
                pageData['imgs'] = []
                pageData['imgs'].append("PageUnavailable")

            else:
                # If the page did not yield an access error, go ahead an scrape for image entries.
                pageHtml = pq(f'https://tapas.io/episode/{pageData["id"]}', headers={'user-agent': 'tapas-dl'})

                printLine('Downloaded image data from {} images (pages {}/{})...'.format(allImgCount, pageCount + pageOffset, len(data) + pageOffset), True)

                pageData['title'] = pageHtml('.info__title').text()

                pageData['imgs'] = []
                for img in pageHtml('.content__img'):
                    pageData['imgs'].append(pq(img).attr('data-src'))

                    allImgCount += 1

        # Download images
        imgCount = 0
        for pageCount, pageData in enumerate(data):

            for imgOfPageCount, img in enumerate(pageData['imgs']):

                # Check if the first image entry is the fummy text that indicates the page was unavailable when we tried to scrape it.
                if pageData['imgs'][0] != "PageUnavailable":
                    # If the entry isn't a dummy entry, go ahead and download the images it contains.
                    with open(os.path.join(savePath, check_path('{} - {} - {} - {} - #{}.{}'.format(lead0(imgCount + imgOffset, allImgCount + imgOffset), lead0(pageCount + pageOffset, len(pageData) + pageOffset),
                                                                                lead0(imgOfPageCount, len(pageData['imgs'])), pageData['title'], pageData['id'], img[img.rindex('.') + 1:]),
                                                                fat=args.restrict_characters)), 'wb') as f:
                        f.write(requests.get(img).content)

                    imgCount += 1

                    printLine('Downloaded image {}/{} from page {}/{} ({}/{} images)...'.format(imgOfPageCount + 1, len(pageData['imgs']), pageCount + pageOffset, len(data) + pageOffset, imgCount + imgOffset, allImgCount + imgOffset), True)
                else:
                    # If the entry was a dummy entry, skip it and let the user know.
                    printLine('Error: No images downloaded from page {}/{}.'.format(pageCount + pageOffset, len(data) + pageOffset), True)

        if data != []:
            printLine('Downloaded {} of {} images'.format(imgCount, allImgCount))
        else:
            printLine('Nothing to do')

        if urlCount + 1 != len(args.url):
            printLine()
    else:
        printLine('Detected novel')

        from ebooklib import epub

        # download/create epub
        book = epub.EpubBook()

        customCss = None

        # Add meta data
        book.set_identifier(urlName)
        book.set_title(name)
        book.set_language('en')

        book.add_author(author)

        header_name = os.path.join(savePath, list(filter(re.compile(r'.+header\..+').match, os.listdir(savePath)))[0])
        book.set_cover("cover.jpg", open(header_name, 'rb').read())

        book.toc = []
        book.spine = ['cover']

        # create about page
        chapter = epub.EpubHtml(title='about', file_name='about.xhtml')
        chapter.content = f'
About

Title: {name}

Author: {book.metadata["http://purl.org/dc/elements/1.1/"]["creator"][0][0]}

Source: {"https://tapas.io/series/" + urlName}
'

        book.add_item(chapter)
        book.spine.append(chapter)

        # Append nav page
        book.spine.append('nav')

        # create chapters
        for pageCount, pageData in enumerate(data):
            printLine('Downloaded page {}/{}...'.format(pageCount + 1, len(data)), True)

            pagePq = pq(url='https://tapas.io/episode/' + str(pageData['id']), headers={'user-agent': 'tapas-dl'})
            pageTitle = pagePq('.viewer__header > .title').text()

            pageHtml = f'
{pageTitle}
'
            for p in pagePq('article.viewer__body > p'):
                p = pq(p)
                if p.text() is not None:
                    pageHtml += '
' + p.text() + '
'

            chapter = epub.EpubHtml(title=pageTitle, file_name=str(pageData['id']) + '.xhtml')
            chapter.content = pageHtml

            book.add_item(chapter)

            # define Table Of Contents
            book.toc.append(epub.Link(str(pageData['id']) + '.xhtml', pageTitle, str(pageData['id'])))

            # basic spine
            book.spine.append(chapter)

        # add default NCX and Nav file
        book.add_item(epub.EpubNcx())
        book.add_item(epub.EpubNav())

        # add CSS
        style = ''
        nav_css = epub.EpubItem(uid="style_nav", file_name="style/nav.css", media_type="text/css", content=style)
        book.add_item(nav_css)

        # write to the file
        epub.write_epub(os.path.join('/'.join(savePath.split('/')[:-1]), check_path(f'{name}.epub', fat=args.restrict_characters)), book)

        # remove tmp folder
        for file in os.listdir(savePath):
            os.remove(os.path.join(savePath, file))
        os.removedirs(savePath)
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from kombu import Connection

from st2common import log as logging
from st2common.constants.trace import TRACE_CONTEXT, TRACE_ID
from st2common.util import date as date_utils
from st2common.services.trace import add_or_update_given_trace_context
from st2common.transport import consumers, reactor
from st2common.transport import utils as transport_utils
import st2reactor.container.utils as container_utils
from st2reactor.rules.engine import RulesEngine

LOG = logging.getLogger(__name__)

RULESENGINE_WORK_Q = reactor.get_trigger_instances_queue(
    name='st2.trigger_instances_dispatch.rules_engine', routing_key='#')

class TriggerInstanceDispatcher(consumers.MessageHandler):
    message_type = dict

    def __init__(self, connection, queues):
        super(TriggerInstanceDispatcher, self).__init__(connection, queues)
        self.rules_engine = RulesEngine()

    def process(self, instance):
        trigger = instance['trigger']
        payload = instance['payload']

        trigger_instance = None
        try:
            trigger_instance = container_utils.create_trigger_instance(
                trigger,
                payload or {},
                date_utils.get_datetime_utc_now(),
                raise_on_no_trigger=True)
        except:
            # We got a trigger ref but we were unable to create a trigger instance.
            # This could be because a trigger object wasn't found in db for the ref.
            LOG.exception('Failed to create trigger_instance %s.', instance)
            return

        if trigger_instance:
            try:
                # Use trace_context from the instance and if not found create a new context
                # and use the trigger_instance.id as trace_id.
                trace_context = instance.get(TRACE_CONTEXT, {
                    TRACE_ID: 'trigger_instance-%s' % str(trigger_instance.id)
                })
                # add a trace or update an existing trace with trigger_instance
                add_or_update_given_trace_context(trace_context=trace_context,
                                                  trigger_instances=[str(trigger_instance.id)])
                self.rules_engine.handle_trigger_instance(trigger_instance)
            except:
                # This could be a large message but at least in case of an exception
                # we get to see more context.
                # Beyond this point code cannot really handle the exception anyway so
                # eating up the exception.
                LOG.exception('Failed to handle trigger_instance %s.', instance)
                return

def get_worker():
    with Connection(transport_utils.get_messaging_urls()) as conn:
        return TriggerInstanceDispatcher(conn, [RULESENGINE_WORK_Q])
<commit_msg>Account for trace_context being None in a trigger_instance.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from kombu import Connection

from st2common import log as logging
from st2common.constants.trace import TRACE_CONTEXT, TRACE_ID
from st2common.util import date as date_utils
from st2common.services.trace import add_or_update_given_trace_context
from st2common.transport import consumers, reactor
from st2common.transport import utils as transport_utils
import st2reactor.container.utils as container_utils
from st2reactor.rules.engine import RulesEngine

LOG = logging.getLogger(__name__)

RULESENGINE_WORK_Q = reactor.get_trigger_instances_queue(
    name='st2.trigger_instances_dispatch.rules_engine', routing_key='#')

class TriggerInstanceDispatcher(consumers.MessageHandler):
    message_type = dict

    def __init__(self, connection, queues):
        super(TriggerInstanceDispatcher, self).__init__(connection, queues)
        self.rules_engine = RulesEngine()

    def process(self, instance):
        trigger = instance['trigger']
        payload = instance['payload']

        trigger_instance = None
        try:
            trigger_instance = container_utils.create_trigger_instance(
                trigger,
                payload or {},
                date_utils.get_datetime_utc_now(),
                raise_on_no_trigger=True)
        except:
            # We got a trigger ref but we were unable to create a trigger instance.
            # This could be because a trigger object wasn't found in db for the ref.
            LOG.exception('Failed to create trigger_instance %s.', instance)
            return

        if trigger_instance:
            try:
                # Use trace_context from the instance and if not found create a new context
                # and use the trigger_instance.id as trace_id.
                trace_context = instance.get(TRACE_CONTEXT, None)
                if not trace_context:
                    trace_context = {
                        TRACE_ID: 'trigger_instance-%s' % str(trigger_instance.id)
                    }
                # add a trace or update an existing trace with trigger_instance
                add_or_update_given_trace_context(trace_context=trace_context,
                                                  trigger_instances=[str(trigger_instance.id)])
                self.rules_engine.handle_trigger_instance(trigger_instance)
            except:
                # This could be a large message but at least in case of an exception
                # we get to see more context.
                # Beyond this point code cannot really handle the exception anyway so
                # eating up the exception.
                LOG.exception('Failed to handle trigger_instance %s.', instance)
                return

def get_worker():
    with Connection(transport_utils.get_messaging_urls()) as conn:
        return TriggerInstanceDispatcher(conn, [RULESENGINE_WORK_Q])
<|endoftext|>


<commit_before># ! /usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "Tsartsaris Sotiris"
__copyright__ = "Copyright 2014, The TSP Project"
__credits__ = ["Tsartsaris Sotiris"]
__license__ = "APACHE 2.0"
__version__ = "1.0.1"
__maintainer__ = "Tsartsaris Sotiris"
__email__ = "info@tsartsaris.gr"
__status__ = "Development"

"""
    Provided with the initial population first we apply a fitness function
    which will be the division of the tour distance divided by the best distance
    we have after each iteration. Then we apply a roulette-wheel selection on the population
    to get from the accumulated fitness of each tour a random pick
"""

from operator import itemgetter
import numpy as np
import random
import collections

class TSPGeneticAlgo:
    def __init__(self, initial_population, city_tour_init):
        self.city_tour_init = city_tour_init  # pass the initial tour here for differences between children after crossover duplicates
        print self.city_tour_init
        self.children_dirty = []
        self.all_fitness = []
        self.groups_of_two = []
        self.total_best = 10000000000000000000000000000000000
        self.initial_population = initial_population
        self.current_best = self.current_best_score(self.initial_population)
        self.selected_population = []
        if self.current_best < self.total_best:
            self.total_best = self.current_best
        self.calculate_fitness(self.initial_population)
        self.accumulated_list = self.accumulate_fitness()
        self.roulette_wheel_selection(self.accumulated_list)
        for i in range(50):
            self.selected_population.append(self.roulette_wheel_selection(self.accumulated_list))
        self.random_pick_doubles(self.selected_population)
        self.children_dirty = self.crossover_genetic_operator(self.groups_of_two)
        self.remove_duplicate_cities(self.children_dirty)

    def fitness_function(self, city_cost):
        """
            We apply the fitness function to each distance by
            dividing the lowest with each one of them
        """
        return float(float(self.total_best) / float(city_cost))

    @staticmethod
    def current_best_score(array):
        """
            This is the lowest distance form the population
            we will use it in the fitness function
        """
        return min(i[0] for i in array)

    def calculate_fitness(self, in_list):
        """
            Given a list of distances we apply the fitness function to each
            one of them
        """
        for city_distance in in_list:
            self.all_fitness.append([self.fitness_function(city_distance[0]), city_distance[1]])

    def accumulate_fitness(self):
        """
            Given a list of fitness values, we sort them from lower to higher
            and the we accumulate those values. Last element should have a fitness of 1
        """
        sum1 = sum(pair[0] for pair in self.all_fitness)
        for value in self.all_fitness:
            value[0] = value[0] / sum1
        sorted_fit = sorted(self.all_fitness, key=itemgetter(0))
        unzipped_list = zip(*sorted_fit)
        accumulated_list = np.cumsum(unzipped_list[0])
        accumulated_fitness = zip(accumulated_list, unzipped_list[1])
        return accumulated_fitness

    @staticmethod
    def roulette_wheel_selection(accumulated_list):
        """
            iterating a range we get each time a random number from 0 to 1
            theν we iterate the sorted list of accumulated fitness values and we get
             the first higher from that number removing it from the list
        """
        random_number = random.random()
        for element in accumulated_list:
            if element[0] > random_number:
                accumulated_list.remove(element)
                return element

    def random_pick_doubles(self, in_list):
        """
            We iterate the selected population and we create groups of 2
            to make the breeding in the next step.
        """
        while in_list:
            local = random.sample(in_list, 2)
            for i in local:
                in_list.remove(i)
            self.groups_of_two.append(local)

    def crossover_genetic_operator(self, in_list):
        local_doubles = []
        local_children = []
        for double in in_list:
            if double[0][1][0]:
                local_doubles.append(double[0][1][0])
            if double[1][1][0]:
                local_doubles.append(double[1][1][0])
        local_doubles = map(None, *[iter(local_doubles)] * 2)
        for pair in local_doubles:
            ind1 = pair[0]
            ind2 = pair[1]
            size = min(len(ind1), len(ind2))
            cxpoint = random.randint(1, size - 1)
            ind1[cxpoint:], ind2[cxpoint:] = ind2[cxpoint:], ind1[cxpoint:]
            local_children.append(ind1)
            local_children.append(ind2)
        return local_children

    def remove_duplicate_cities(self, in_list):
        for dirty in in_list:
            differs = [x for x in self.city_tour_init if x not in dirty]
            print differs
            uniq = [x for x, y in collections.Counter(dirty).items() if y > 1]
            for unique in uniq:
                index = dirty.index(unique)
                dirty.pop(index)
                dirty.insert(index, differs[-1])
                differs.pop()
            print dirty<commit_msg>Generating offsprings correctly<commit_after># ! /usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "Tsartsaris Sotiris"
__copyright__ = "Copyright 2014, The TSP Project"
__credits__ = ["Tsartsaris Sotiris"]
__license__ = "APACHE 2.0"
__version__ = "1.0.1"
__maintainer__ = "Tsartsaris Sotiris"
__email__ = "info@tsartsaris.gr"
__status__ = "Development"

"""
    Provided with the initial population first we apply a fitness function
    which will be the division of the tour distance divided by the best distance
    we have after each iteration. Then we apply a roulette-wheel selection on the population
    to get from the accumulated fitness of each tour a random pick
"""

from operator import itemgetter
import numpy as np
import random
import collections

class TSPGeneticAlgo:
    def __init__(self, initial_population, city_tour_init):
        self.city_tour_init = city_tour_init  # pass the initial tour here for differences between children after crossover duplicates
        self.children_dirty = []
        self.offsprings = []
        self.all_fitness = []
        self.groups_of_two = []
        self.total_best = 10000000000000000000000000000000000
        self.initial_population = initial_population
        self.current_best = self.current_best_score(self.initial_population)
        self.selected_population = []
        if self.current_best < self.total_best:
            self.total_best = self.current_best
        self.calculate_fitness(self.initial_population)
        self.accumulated_list = self.accumulate_fitness()
        self.roulette_wheel_selection(self.accumulated_list)
        for i in range(50):
            self.selected_population.append(self.roulette_wheel_selection(self.accumulated_list))
        self.random_pick_doubles(self.selected_population)
        self.children_dirty = self.crossover_genetic_operator(self.groups_of_two)
        self.remove_duplicate_cities(self.children_dirty)

    def fitness_function(self, city_cost):
        """
            We apply the fitness function to each distance by
            dividing the lowest with each one of them
        """
        return float(float(self.total_best) / float(city_cost))

    @staticmethod
    def current_best_score(array):
        """
            This is the lowest distance form the population
            we will use it in the fitness function
        """
        return min(i[0] for i in array)

    def calculate_fitness(self, in_list):
        """
            Given a list of distances we apply the fitness function to each
            one of them
        """
        for city_distance in in_list:
            self.all_fitness.append([self.fitness_function(city_distance[0]), city_distance[1]])

    def accumulate_fitness(self):
        """
            Given a list of fitness values, we sort them from lower to higher
            and the we accumulate those values. Last element should have a fitness of 1
        """
        sum1 = sum(pair[0] for pair in self.all_fitness)
        for value in self.all_fitness:
            value[0] = value[0] / sum1
        sorted_fit = sorted(self.all_fitness, key=itemgetter(0))
        unzipped_list = zip(*sorted_fit)
        accumulated_list = np.cumsum(unzipped_list[0])
        accumulated_fitness = zip(accumulated_list, unzipped_list[1])
        return accumulated_fitness

    @staticmethod
    def roulette_wheel_selection(accumulated_list):
        """
            iterating a range we get each time a random number from 0 to 1
            then we iterate the sorted list of accumulated fitness values and we get
             the first higher from that number not removing it from the list
             allowing each chromosome to be a parent more than once
        """
        random_number = random.random()
        for element in accumulated_list:
            if element[0] > random_number:
                return element

    def random_pick_doubles(self, in_list):
        """
            We iterate the selected population and we create groups of 2
            to make the breeding in the next step.
        """
        while in_list:
            local = random.sample(in_list, 2)
            for i in local:
                in_list.remove(i)
            self.groups_of_two.append(local)

    def crossover_genetic_operator(self, in_list):
        """
            Takes the pairs of chromosomes in the list and applies a crossover
            on a random point to create offsprings.
        """
        local_doubles = []
        local_children = []
        for double in in_list:
            if isinstance(double[0][1], (list, tuple)):
                local_doubles.append(double[0][1])
            if isinstance(double[1][1], (list, tuple)):
                local_doubles.append(double[1][1])
        local_doubles = map(None, *[iter(local_doubles)] * 2)
        for pair in local_doubles:
            ind1 = pair[0]
            ind2 = pair[1]
            size = min(len(ind1), len(ind2))
            cxpoint = random.randint(1, size - 1)
            ind1[cxpoint:], ind2[cxpoint:] = ind2[cxpoint:], ind1[cxpoint:]
            local_children.append(ind1)
            local_children.append(ind2)
        return local_children

    def remove_duplicate_cities(self, in_list):
        """
            The offsprings from the crossover contain duplicate cities which must
            be removed by replacing them with cities that are not in the the child
        """
        for dirty in in_list:
            differs = [x for x in self.city_tour_init if x not in dirty]
            uniq = [x for x, y in collections.Counter(dirty).items() if y > 1]
            for unique in uniq:
                index = dirty.index(unique)
                dirty.pop(index)
                dirty.insert(index, differs[-1])
                differs.pop()
            self.offsprings.append(dirty)
        print len(self.offsprings)<|endoftext|>


<commit_before>import os

class Taxonomy:
    def get_data_dir(self):
        return os.path.join('data', self.name)
    data_dir = property(get_data_dir)

    def __init__(self):
        if not os.path.exists(self.data_dir): os.makedirs(self.data_dir)

    def download_file(self, url):
        filename = os.path.join(self.data_dir, url.split('/')[-1])

        if os.path.exists(filename):
            print 'Using existing copy of %s' % filename
        else:
            print 'Downloading %s...' % filename
            r = urllib2.urlopen(urllib2.Request(url))
            assert r.geturl() == url
            with open(filename, 'wb') as output_file:
                data = True
                while data:
                    data = r.read(1024)
                    if data: output_file.write(data)
            r.close()

        return filename

    @classmethod
    def format_rdf_string(cls, x):
        return '"%s"' % x.replace('"', '\\"')<commit_msg>Missing urllib2 import.<commit_after>import os
import urllib2

class Taxonomy:
    def get_data_dir(self):
        return os.path.join('data', self.name)
    data_dir = property(get_data_dir)

    def __init__(self):
        if not os.path.exists(self.data_dir): os.makedirs(self.data_dir)

    def download_file(self, url):
        filename = os.path.join(self.data_dir, url.split('/')[-1])

        if os.path.exists(filename):
            print 'Using existing copy of %s' % filename
        else:
            print 'Downloading %s...' % filename
            r = urllib2.urlopen(urllib2.Request(url))
            assert r.geturl() == url
            with open(filename, 'wb') as output_file:
                data = True
                while data:
                    data = r.read(1024)
                    if data: output_file.write(data)
            r.close()

        return filename

    @classmethod
    def format_rdf_string(cls, x):
        return '"%s"' % x.replace('"', '\\"')<|endoftext|>


<commit_before># Giles: 4_player_card_game_layout.py
# Copyright 2012 Phil Bordelon
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from giles.games.layout import Layout
from giles.games.playing_card import card_to_str

NORTH = "North"
EAST = "East"
SOUTH = "South"
WEST = "West"

NORTH_POINTER = "^^^^" # Due to Miniboa's caret codes.
EAST_POINTER = ">>"
SOUTH_POINTER = "vv"
WEST_POINTER = "<<"
NONE_POINTER = "++"

BLANK_ROW = "       |                    |\n"

class FourPlayerCardGameLayout(Layout):
    """A standard layout for a 4-player card game, with North, East, South,
    and West sitting at a table.
    """

    def __init__(self):

        self.north_card = None
        self.east_card = None
        self.south_card = None
        self.west_card = None
        self.last_played = None
        self.turn = None
        self.turn_pointer = NONE_POINTER
        self.update()

    def card_str(self, who):

        color_str = "^w"
        if who == self.last_played:
            color_str = "^W"

        card_str = "  "
        if who == NORTH:
            card_str = card_to_str(self.north_card)
        elif who == EAST:
            card_str = card_to_str(self.east_card)
        elif who == SOUTH:
            card_str = card_to_str(self.south_card)
        elif who == WEST:
            card_str = card_to_str(self.west_card)

        return "%s%s^~" % (color_str, card_str)

    def update(self):

        self.representation = "\n"
        if not self.turn:
            self.representation += "   A shuffled deck of playing cards lies face-down on the table.\n"
            return

        self.representation += "       .--------------------.\n"
        self.representation += "       |         ^RNN^~         |\n"
        self.representation += BLANK_ROW
        self.representation += "       |         %s         |\n" % self.card_str(NORTH)
        self.representation += BLANK_ROW
        self.representation += "       | ^MWW^~  %s  %s  %s  ^MEE^~ |\n" % (self.card_str(WEST), self.turn_pointer, self.card_str(EAST))
        self.representation += BLANK_ROW
        self.representation += "       |         %s         |\n" % self.card_str(SOUTH)
        self.representation += BLANK_ROW
        self.representation += "       |         ^RSS^~         |\n"
        self.representation += "       `--------------------'\n"

    def change_turn(self, who):

        self.turn = who
        if self.turn == NORTH:
            self.turn_pointer = NORTH_POINTER
        elif self.turn == EAST:
            self.turn_pointer = EAST_POINTER
        elif self.turn == SOUTH:
            self.turn_pointer = SOUTH_POINTER
        elif self.turn == WEST:
            self.turn_pointer = WEST_POINTER
        else:
            self.turn_pointer = NONE_POINTER

        self.update()

    def resize(self):

        # Cannot be resized.
        pass

    def place(self, who, card):

        if who == NORTH:
            self.north_card = card
        elif who == EAST:
            self.east_card = card
        elif who == SOUTH:
            self.south_card = card
        elif who == WEST:
            self.west_card = card

        self.last_played = who
        self.update()

    def move(self):

        # Cards cannot (currently) be moved.
        pass

    def clear(self):

        self.north_card = None
        self.east_card = None
        self.south_card = None
        self.west_card = None
        self.last_played = None
        self.update()

    def remove(self):

        # Cards cannot (currently) be removed; use clear() instead.
        pass
<commit_msg>4pcgl: Update comment.<commit_after># Giles: four_player_card_game_layout.py
# Copyright 2012 Phil Bordelon
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from giles.games.layout import Layout
from giles.games.playing_card import card_to_str

NORTH = "North"
EAST = "East"
SOUTH = "South"
WEST = "West"

NORTH_POINTER = "^^^^" # Due to Miniboa's caret codes.
EAST_POINTER = ">>"
SOUTH_POINTER = "vv"
WEST_POINTER = "<<"
NONE_POINTER = "++"

BLANK_ROW = "       |                    |\n"

class FourPlayerCardGameLayout(Layout):
    """A standard layout for a 4-player card game, with North, East, South,
    and West sitting at a table.
    """

    def __init__(self):

        self.north_card = None
        self.east_card = None
        self.south_card = None
        self.west_card = None
        self.last_played = None
        self.turn = None
        self.turn_pointer = NONE_POINTER
        self.update()

    def card_str(self, who):

        color_str = "^w"
        if who == self.last_played:
            color_str = "^W"

        card_str = "  "
        if who == NORTH:
            card_str = card_to_str(self.north_card)
        elif who == EAST:
            card_str = card_to_str(self.east_card)
        elif who == SOUTH:
            card_str = card_to_str(self.south_card)
        elif who == WEST:
            card_str = card_to_str(self.west_card)

        return "%s%s^~" % (color_str, card_str)

    def update(self):

        self.representation = "\n"
        if not self.turn:
            self.representation += "   A shuffled deck of playing cards lies face-down on the table.\n"
            return

        self.representation += "       .--------------------.\n"
        self.representation += "       |         ^RNN^~         |\n"
        self.representation += BLANK_ROW
        self.representation += "       |         %s         |\n" % self.card_str(NORTH)
        self.representation += BLANK_ROW
        self.representation += "       | ^MWW^~  %s  %s  %s  ^MEE^~ |\n" % (self.card_str(WEST), self.turn_pointer, self.card_str(EAST))
        self.representation += BLANK_ROW
        self.representation += "       |         %s         |\n" % self.card_str(SOUTH)
        self.representation += BLANK_ROW
        self.representation += "       |         ^RSS^~         |\n"
        self.representation += "       `--------------------'\n"

    def change_turn(self, who):

        self.turn = who
        if self.turn == NORTH:
            self.turn_pointer = NORTH_POINTER
        elif self.turn == EAST:
            self.turn_pointer = EAST_POINTER
        elif self.turn == SOUTH:
            self.turn_pointer = SOUTH_POINTER
        elif self.turn == WEST:
            self.turn_pointer = WEST_POINTER
        else:
            self.turn_pointer = NONE_POINTER

        self.update()

    def resize(self):

        # Cannot be resized.
        pass

    def place(self, who, card):

        if who == NORTH:
            self.north_card = card
        elif who == EAST:
            self.east_card = card
        elif who == SOUTH:
            self.south_card = card
        elif who == WEST:
            self.west_card = card

        self.last_played = who
        self.update()

    def move(self):

        # Cards cannot (currently) be moved.
        pass

    def clear(self):

        self.north_card = None
        self.east_card = None
        self.south_card = None
        self.west_card = None
        self.last_played = None
        self.update()

    def remove(self):

        # Cards cannot (currently) be removed; use clear() instead.
        pass
<|endoftext|>


<commit_before># Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import json

import unittest2 as unittest

from stacktester import openstack

class ServersMetadataTest(unittest.TestCase):

    def setUp(self):
        self.os = openstack.Manager()

        image_ref = self.os.config.env.image_ref
        flavor_ref = self.os.config.env.flavor_ref

        server = {
            'name' : 'testserver',
            'imageRef' : image_ref,
            'flavorRef' : flavor_ref,
            'metadata' : {
                'testEntry' : 'testValue',
            },
        }

        created_server = self.os.nova.create_server(server)
        self.server_id = created_server['id']

    def tearDown(self):
        self.os.nova.delete_server(self.server_id)

    def test_get_server_metadata(self):
        """Test that we can retrieve metadata for a server"""

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = {
            'metadata' : {
                'testEntry' : 'testValue',
            },
        }
        self.assertEqual(expected, result)

    def test_update_server_metadata(self):
        """Test that we can update metadata for a server"""

        post_metadata = {
            'metadata' : {
                'new_entry1' : 'new_value1',
                'new_entry2' : 'new_value2',
            },
        }
        post_body = json.dumps(post_metadata)

        url = '/sirvers/%s/meta' % self.server_id
        response, body = self.os.nova.request('POST', url, body=post_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = post_metadata
        expected['testEntry'] = 'testValue'
        # KNOWN-ISSUE
        #self.assertEqual(expected, result)

    def test_replace_server_metadata(self):
        """
        Test that we can update metadata for a server,
        removing unspecified entries
        """

        expected = {
            'metadata' : {
                'new_entry1' : 'new_value1',
                'new_entry2' : 'new_value2',
            },
        }

        url = '/servers/%s/meta' % self.server_id
        post_body = json.dumps(expected)
        response, body = self.os.nova.request('POST', url, body=post_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        # We want to make sure 'testEntry' was removed
        # KNOWN-ISSUE
        #self.assertEqual(expected, result)

    def test_get_server_metadata_key(self):
        """Test that we can retrieve specific metadata key for a server"""

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = {
            'meta':{
                'testEntry':'testValue',
            },
        }
        # KNOWN-ISSUE
        #self.assertDictEqual(expected, result)

    def test_add_server_metadata_key(self):
        """Test that we can add specific metadata key to a server"""

        expected_metadata = {
            'metadata' : {
                'testEntry' : 'testValue',
                'new_meta1' : 'new_value1',
            },
        }

        expected_meta = {
            'meta' : {
                'new_meta1' : 'new_value1',
            },
        }

        put_body = json.dumps(expected_meta)

        url = '/servers/%s/meta/new_meta1' % self.server_id
        response, body = self.os.nova.request('PUT', url, body=put_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)
        #result = json.loads(body)
        #self.assertDictEqual(expected_meta, result)

        # Now check all metadata to make sure the other values are there
        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        # KNOWN-ISSUE
        #result = json.loads(body)
        #self.assertDictEqual(expected_metadata, result)

    def test_update_server_metadata_key(self):
        """Test that we can update specific metadata key for a server"""

        expected_meta = {
            'meta' : {
            'testEntry' : 'testValue2',
            },
        }
        put_body = json.dumps(expected_meta)

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('PUT', url, body=put_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)
        #result = json.loads(body)
        #self.assertEqual(expected_meta, result)

        # Now check all metadata to make sure the other values are there
        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        # KNOWN-ISSUE
        #result = json.loads(body)
        #expected_metadata = {}
        #expected_metadata['metadata'] = expected_meta['meta']
        #self.assertDictEqual(expected_metadata, result)

    def test_delete_server_metadata_key(self):
        """Test that we can delete metadata for a server"""

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('DELETE', url)
        # KNOWN-ISSUE lp:804067
        #self.assertEquals(204, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEquals(200, response.status)

        result = json.loads(body)
        self.assertDictEqual({'metadata':{}}, result)
<commit_msg>fixing old typo<commit_after># Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json

import unittest2 as unittest

from stacktester import openstack

class ServersMetadataTest(unittest.TestCase):

    def setUp(self):
        self.os = openstack.Manager()

        image_ref = self.os.config.env.image_ref
        flavor_ref = self.os.config.env.flavor_ref

        server = {
            'name' : 'testserver',
            'imageRef' : image_ref,
            'flavorRef' : flavor_ref,
            'metadata' : {
                'testEntry' : 'testValue',
            },
        }

        created_server = self.os.nova.create_server(server)
        self.server_id = created_server['id']

    def tearDown(self):
        self.os.nova.delete_server(self.server_id)

    def test_get_server_metadata(self):
        """Test that we can retrieve metadata for a server"""

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = {
            'metadata' : {
                'testEntry' : 'testValue',
            },
        }
        self.assertEqual(expected, result)

    def test_update_server_metadata(self):
        """Test that we can update metadata for a server"""

        post_metadata = {
            'metadata' : {
                'new_entry1' : 'new_value1',
                'new_entry2' : 'new_value2',
            },
        }
        post_body = json.dumps(post_metadata)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('POST', url, body=post_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = post_metadata
        expected['testEntry'] = 'testValue'
        # KNOWN-ISSUE
        #self.assertEqual(expected, result)

    def test_replace_server_metadata(self):
        """
        Test that we can update metadata for a server,
        removing unspecified entries
        """

        expected = {
            'metadata' : {
                'new_entry1' : 'new_value1',
                'new_entry2' : 'new_value2',
            },
        }

        url = '/servers/%s/meta' % self.server_id
        post_body = json.dumps(expected)
        response, body = self.os.nova.request('POST', url, body=post_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        # We want to make sure 'testEntry' was removed
        # KNOWN-ISSUE
        #self.assertEqual(expected, result)

    def test_get_server_metadata_key(self):
        """Test that we can retrieve specific metadata key for a server"""

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEqual(200, response.status)

        result = json.loads(body)
        expected = {
            'meta':{
                'testEntry':'testValue',
            },
        }
        # KNOWN-ISSUE
        #self.assertDictEqual(expected, result)

    def test_add_server_metadata_key(self):
        """Test that we can add specific metadata key to a server"""

        expected_metadata = {
            'metadata' : {
                'testEntry' : 'testValue',
                'new_meta1' : 'new_value1',
            },
        }

        expected_meta = {
            'meta' : {
                'new_meta1' : 'new_value1',
            },
        }

        put_body = json.dumps(expected_meta)

        url = '/servers/%s/meta/new_meta1' % self.server_id
        response, body = self.os.nova.request('PUT', url, body=put_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)
        #result = json.loads(body)
        #self.assertDictEqual(expected_meta, result)

        # Now check all metadata to make sure the other values are there
        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        # KNOWN-ISSUE
        #result = json.loads(body)
        #self.assertDictEqual(expected_metadata, result)

    def test_update_server_metadata_key(self):
        """Test that we can update specific metadata key for a server"""

        expected_meta = {
            'meta' : {
            'testEntry' : 'testValue2',
            },
        }
        put_body = json.dumps(expected_meta)

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('PUT', url, body=put_body)
        # KNOWN-ISSUE lp:804067
        #self.assertEqual(201, response.status)
        #result = json.loads(body)
        #self.assertEqual(expected_meta, result)

        # Now check all metadata to make sure the other values are there
        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        # KNOWN-ISSUE
        #result = json.loads(body)
        #expected_metadata = {}
        #expected_metadata['metadata'] = expected_meta['meta']
        #self.assertDictEqual(expected_metadata, result)

    def test_delete_server_metadata_key(self):
        """Test that we can delete metadata for a server"""

        url = '/servers/%s/meta/testEntry' % self.server_id
        response, body = self.os.nova.request('DELETE', url)
        # KNOWN-ISSUE lp:804067
        #self.assertEquals(204, response.status)

        url = '/servers/%s/meta' % self.server_id
        response, body = self.os.nova.request('GET', url)
        self.assertEquals(200, response.status)

        result = json.loads(body)
        self.assertDictEqual({'metadata':{}}, result)
<|endoftext|>


<commit_before>
from pyquery import PyQuery as pq

def html_sanitize(x):
    root = pq("<div />")
    root.html(x)
    result = handle_element(root[0])
    new = pq(result)
    return new.html()

to_remove = set(["script", "head", "meta", "title", "link"])
to_unwrap = set(["html", "body"])

def handle_a(el, new):
    new.set("href", el.get("href", "#"))
special = {
    "a": handle_a,
}

def handle_element(el):
    if type(el) == str or type(el) == unicode:
        return [el]
    if el.tag in to_remove:
        return []
    if el.tag in to_unwrap:
        return reduce(lambda x,y: x+y, [handle_element(x) for x in children(el)])
    new = pq("<%s />" % el.tag)[0]
    for i in children(el):
        append_to(handle_element(i), new)
    if el.tag in special:
        special[el.tag](el, new)
    return [new]

def children(el):
    res = []
    if el.text is not None:
        res.append(el.text)
    for i in el.getchildren():
        res.append(i)
        if i.tail is not None:
            res.append(i.tail)
    return res

def append_to(new_ones, el):
    for i in new_ones:
        if type(i) == str or type(i) == unicode:
            children = el.getchildren()
            if len(children) == 0:
                el.text = i
            else:
                children[-1].tail = i
        else:
            el.append(i)<commit_msg>removed images<commit_after>
from pyquery import PyQuery as pq

def html_sanitize(x):
    root = pq("<div />")
    root.html(x)
    result = handle_element(root[0])
    new = pq(result)
    return new.html()

to_remove = set(["script", "head", "meta", "title", "link", "img"])
to_unwrap = set(["html", "body"])

def handle_a(el, new):
    new.set("href", el.get("href", "#"))
special = {
    "a": handle_a,
}

def handle_element(el):
    if type(el) == str or type(el) == unicode:
        return [el]
    if el.tag in to_remove:
        return []
    if el.tag in to_unwrap:
        return reduce(lambda x,y: x+y, [handle_element(x) for x in children(el)])
    new = pq("<%s />" % el.tag)[0]
    for i in children(el):
        append_to(handle_element(i), new)
    if el.tag in special:
        special[el.tag](el, new)
    return [new]

def children(el):
    res = []
    if el.text is not None:
        res.append(el.text)
    for i in el.getchildren():
        res.append(i)
        if i.tail is not None:
            res.append(i.tail)
    return res

def append_to(new_ones, el):
    for i in new_ones:
        if type(i) == str or type(i) == unicode:
            children = el.getchildren()
            if len(children) == 0:
                el.text = i
            else:
                children[-1].tail = i
        else:
            el.append(i)<|endoftext|>


<commit_before>#!/usr/bin/env python

import subprocess
import sys
import urwid as ur
import yaml
from ui.widgets import *

I_TITLEBAR = 0
I_BODY = 1
I_PREVIEW = 2
I_BUTTONS = 3
I_OUTPUT = 4

def command_form_from_file(filename):
	f = open(filename)
	raw = yaml.safe_load(f)
	f.close()
	command = raw['command']
	params = raw['params']
	form = CommandForm(raw['title'], command, params)
	return form

class CommandForm:
	def __init__(self, title, command, params=[], env={}, user=''):
		self.title = title
		self.command = command
		self.params = params
		self.env = env
		self.user = user

	def __len__(self):
		return len(self.params)

	def __str__(self):
		return self.command + ' ' +  ' '.join(['%s %s' % (v['flag'], v['value']) for v in self.params if v.has_key('value')])

	def set_values(self, pile):
		i = 0
		while i < len(self):
			v = pile[i].get_value()
			if v != None:
				self.params[i]['value'] = v
			elif self.params[i].has_key('value'):
				del(self.params[i]['value'])
			i += 1

class FormListBox(ur.ListBox):
	def __init__(self, form, separator=': '):
		self.form = form
		self.separator = separator
		self.widget_factory = WidgetFactory(separator)
		body = ur.SimpleFocusListWalker([
			ur.AttrWrap(ur.Text(form.title), 'command'),
			ur.Pile(self._get_widgets()),
			ur.AttrWrap(ur.Text(''), 'command'),
			ur.Columns([
				ur.AttrWrap(ur.Button('Execute', self.execute), 'execute'),
				ur.AttrWrap(ur.Button('Reset', self.reset), 'reset'),
				ur.AttrWrap(ur.Button('Exit', self.exit), 'exit')
				]),
			ur.AttrWrap(ur.Text(''), 'output')
			])
		super(FormListBox, self).__init__(body)
		self.update()

	def _get_widgets(self):
		return [self.widget_factory.get_widget(param) for param in self.form.params]

	def exit(self, *args):
		self.update()
		raise ur.ExitMainLoop()

	def execute(self, button):
		self.update()
		output = subprocess.Popen(str(self.form), shell=True, stdout=subprocess.PIPE).stdout.read()
		self.body[I_OUTPUT].set_text(output)

	def reset(self, button):
		[self.body[I_BODY][i].reset() for i in range(0, len(self.form))]
		self.body[I_OUTPUT].set_text('')
		self.update()

	def update(self):
		self.form.set_values(self.body[I_BODY])
		self.body[I_PREVIEW].set_text(str(self.form))

	def keypress(self, size, key):
		key = super(FormListBox, self).keypress(size, key)
		if self.focus_position == I_BODY:
			if key == 'enter':
				self.update()
				if self.focus.focus_position < len(self.form) - 1:
					self.focus.focus_position += 1
				else:
					self.focus_position = I_BUTTONS
			elif key == 'ctrl d':
				self.exit()
			else:
				return key
		else: # TODO: Handle ctrl d outside of form
			return key

def __main__():
	if len(sys.argv) != 2:
		raise ValueError("Exactly one command configuration file must be specified")
	filename = sys.argv[1]
	form = command_form_from_file(filename)
	palette = [
			('command', 'black,bold', 'yellow'),
			('output', 'black,bold', 'dark green'),
			('execute', 'white,bold', 'dark red'),
			('exit', 'white,bold', 'black'),
			('reset', 'white,bold', 'dark blue')
			]
	ur.MainLoop(FormListBox(form), palette).run()
	print form

if __name__ == '__main__':
	__main__()

<commit_msg>Minor improvements to keypress and IO handling<commit_after>#!/usr/bin/env python

import subprocess
import sys
import urwid as ur
import yaml
from ui.widgets import *

I_TITLEBAR = 0
I_BODY = 1
I_PREVIEW = 2
I_BUTTONS = 3
I_OUTPUT = 4

def command_form_from_file(filename):
	f = open(filename)
	raw = yaml.safe_load(f)
	f.close()
	command = raw['command']
	params = raw['params']
	form = CommandForm(raw['title'], command, params)
	return form

class CommandForm:
	def __init__(self, title, command, params=[], env={}, user=''):
		self.title = title
		self.command = command
		self.params = params
		self.env = env
		self.user = user

	def __len__(self):
		return len(self.params)

	def __str__(self):
		return self.command + ' ' +  ' '.join(['%s %s' % (v['flag'], v['value']) for v in self.params if v.has_key('value')])

	def set_values(self, pile):
		i = 0
		while i < len(self):
			v = pile[i].get_value()
			if v != None:
				self.params[i]['value'] = v
			elif self.params[i].has_key('value'):
				del(self.params[i]['value'])
			i += 1

class FormListBox(ur.ListBox):
	def __init__(self, form, separator=': '):
		self.form = form
		self.separator = separator
		self.widget_factory = WidgetFactory(separator)
		body = ur.SimpleFocusListWalker([
			ur.AttrWrap(ur.Text(form.title), 'command'),
			ur.Pile(self._get_widgets()),
			ur.AttrWrap(ur.Text(''), 'command'),
			ur.Columns([
				ur.AttrWrap(ur.Button('Execute', self.execute), 'execute'),
				ur.AttrWrap(ur.Button('Reset', self.reset), 'reset'),
				ur.AttrWrap(ur.Button('Exit', self.exit), 'exit')
				]),
			ur.AttrWrap(ur.Text(''), 'output')
			])
		super(FormListBox, self).__init__(body)
		self.update()

	def _get_widgets(self):
		return [self.widget_factory.get_widget(param) for param in self.form.params]

	def exit(self, *args):
		self.update()
		raise ur.ExitMainLoop()

	def execute(self, button):
		self.update()
		proc = subprocess.Popen(str(self.form), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		# TODO: Update output periodically to make more useful for long-running processes
		output = proc.communicate()
		self.body[I_OUTPUT].set_text(output[1] + output[0])

	def reset(self, button):
		[self.body[I_BODY][i].reset() for i in range(0, len(self.form))]
		self.body[I_OUTPUT].set_text('')
		self.update()

	def update(self):
		self.form.set_values(self.body[I_BODY])
		self.body[I_PREVIEW].set_text(str(self.form))

	def keypress(self, size, key):
		key = super(FormListBox, self).keypress(size, key)
		if key == 'ctrl n' or key == 'down':
			self.focus_position = self.move(1)
		elif key == 'ctrl p' or key == 'up':
			self.focus_position = self.move(-1)
		elif key == 'ctrl d':
			self.exit()
		else:
			return key

	def move(self, delta):
		self.update()
		if self.focus_position == I_BODY:
			return self._move_helper(self.focus_position, delta, len(self.form), I_BODY, I_BUTTONS)
		elif self.focus_position == I_BUTTONS:
			return self._move_helper(self.focus_position, delta, 3, I_BODY, I_BUTTONS)
		else:
			return self.focus_position

	def _move_helper(self, major_index, delta, max_minor_index, prev_major_index, next_major_index):
		minor_index = self.body[major_index].focus_position
		if minor_index + delta < 0:
			self.body[major_index].focus_position = 0
			return prev_major_index
		elif minor_index + delta >= max_minor_index:
			self.body[major_index].focus_position = max_minor_index - 1
			return next_major_index
		else:
			self.body[major_index].focus_position = minor_index + delta
			return major_index

def __main__():
	if len(sys.argv) != 2:
		raise ValueError("Exactly one command configuration file must be specified")
	filename = sys.argv[1]
	form = command_form_from_file(filename)
	palette = [
			('command', 'black,bold', 'yellow'),
			('output', 'black,bold', 'dark green'),
			('execute', 'white,bold', 'dark red'),
			('exit', 'white,bold', 'black'),
			('reset', 'white,bold', 'dark blue')
			]
	ur.MainLoop(FormListBox(form), palette).run()
	print form

if __name__ == '__main__':
	__main__()

<|endoftext|>


<commit_before>from __future__ import print_function

import argparse
import shlex
import sys
import json

from globus_sdk import TransferClient
from globus_cli.helpers import outformat_is_json, cliargs, CLIArg
from globus_cli.services.transfer.activation import autoactivate

def add_submission_id(client, datadoc):
    submission_id = client.get_submission_id().data['value']
    datadoc['submission_id'] = submission_id

def _validate_transfer_args(args, parser):
    if (args.source_path is None or args.dest_path is None) and (
            args.batch is None):
        parser.error('async-transfer requires either --source-path and '
                     '--dest-path OR --batch')
    if ((args.source_path is not None and args.batch is not None) or
            (args.dest_path is not None and args.batch is not None)):
        parser.error('async-transfer cannot take --batch in addition to '
                     '--source-path or --dest-path')

@cliargs(('Copy a file or directory from one endpoint '
          'to another as an asynchronous task'), [
    CLIArg('source-endpoint', required=True,
           help='ID of the endpoint from which to transfer'),
    CLIArg('dest-endpoint', required=True,
           help='ID of the endpoint to which to transfer'),
    CLIArg('source-path',
           help='Path to the file/dir to move on source-endpoint'),
    CLIArg('dest-path',
           help='Desired location of the file/dir on dest-endpoint'),
    CLIArg('recursive', default=False, action='store_true',
           help=('source-path and dest-path are both directories, do a '
                 'recursive dir transfer. Ignored when using --batch')),
    CLIArg('batch', default=False, action='store_true',
           help=('Accept a batch of source/dest path pairs on stdin.'))
    ], arg_validator=_validate_transfer_args)
def submit_transfer(args):
    """
    Executor for `globus transfer async-transfer`
    """
    client = TransferClient()
    autoactivate(client, args.source_endpoint, if_expires_in=60)
    autoactivate(client, args.dest_endpoint, if_expires_in=60)

    if args.batch is not None:
        def parse_batch_line(line):
            """
            Parse a line of batch input and turn it into a transfer submission
            item.
            """
            # define a parser to handle the batch line
            parser = argparse.ArgumentParser(
                description='async-transfer batch parser')
            parser.add_argument('--recursive', default=False,
                                action='store_true')
            parser.add_argument('source')
            parser.add_argument('dest')

            # do a shlex split to handle quoted paths with spaces in them
            args = parser.parse_args(shlex.split(line))

            return client.make_submit_transfer_item(
                args.source, args.dest, recursive=args.recursive)

        transfer_items = [parse_batch_line(line) for line in sys.stdin]
    else:
        transfer_items = [client.make_submit_transfer_item(
            args.source_path, args.dest_path, recursive=args.recursive)]

    datadoc = client.make_submit_transfer_data(
        args.source_endpoint, args.dest_endpoint, transfer_items,
        label='globus-cli transfer', sync_level="mtime")

    res = client.submit_transfer(datadoc)

    if outformat_is_json(args):
        print(json.dumps(res.data, indent=2))
    else:
        print(res.data['message'])

@cliargs(('Delete a file or directory from one endpoint '
          'as an asynchronous task'), [
    CLIArg('endpoint-id', required=True,
           help='ID of the endpoint from which to delete file(s)'),
    CLIArg('path', required=True, help='Path to the file/dir to delete'),
    CLIArg('recursive', default=False, action='store_true',
           help='Recursively delete dirs'),
    CLIArg('ignore-missing', default=False, action='store_true',
           help='Don\'t throw errors if the file or dir is absent')
    ])
def submit_delete(args):
    """
    Executor for `globus transfer submit-delete`
    """
    client = TransferClient()
    autoactivate(client, args.endpoint_id, if_expires_in=60)

    datadoc = {
        'DATA_TYPE': 'delete',
        'label': 'globus-cli delete',
        'endpoint': args.endpoint_id,
        'recursive': args.recursive,
        'ignore_missing': args.ignore_missing,
        'DATA': [
            {
                'DATA_TYPE': 'delete_item',
                'path': args.path
            }
        ]
    }
    add_submission_id(client, datadoc)
    res = client.submit_delete(datadoc)

    if outformat_is_json(args):
        print(json.dumps(res.data, indent=2))
    else:
        print(res.data['message'])
<commit_msg>Add sync-level to async-transfer args<commit_after>from __future__ import print_function

import argparse
import shlex
import sys
import json

from globus_sdk import TransferClient
from globus_cli.helpers import outformat_is_json, cliargs, CLIArg
from globus_cli.services.transfer.activation import autoactivate

def add_submission_id(client, datadoc):
    submission_id = client.get_submission_id().data['value']
    datadoc['submission_id'] = submission_id

def _validate_transfer_args(args, parser):
    if (args.source_path is None or args.dest_path is None) and (
            args.batch is None):
        parser.error('async-transfer requires either --source-path and '
                     '--dest-path OR --batch')
    if ((args.source_path is not None and args.batch is not None) or
            (args.dest_path is not None and args.batch is not None)):
        parser.error('async-transfer cannot take --batch in addition to '
                     '--source-path or --dest-path')

@cliargs(('Copy a file or directory from one endpoint '
          'to another as an asynchronous task'), [
    CLIArg('source-endpoint', required=True,
           help='ID of the endpoint from which to transfer'),
    CLIArg('dest-endpoint', required=True,
           help='ID of the endpoint to which to transfer'),
    CLIArg('source-path',
           help='Path to the file/dir to move on source-endpoint'),
    CLIArg('dest-path',
           help='Desired location of the file/dir on dest-endpoint'),
    CLIArg('recursive', default=False, action='store_true',
           help=('source-path and dest-path are both directories, do a '
                 'recursive dir transfer. Ignored when using --batch')),
    CLIArg('sync-level', default="mtime", type=str.lower,
           choices=("exists", "mtime", "checksum"),
           help=('How will the transfer task determine whether or not to '
                 'actually transfer a file? '
                 'EXISTS: if the dest file is absent; '
                 'MTIME: if source is newer (modififed time) than dest; '
                 'CHECKSUM: if source and dest contents differ')),
    CLIArg('batch', default=False, action='store_true',
           help=('Accept a batch of source/dest path pairs on stdin. '
                 'Skips empty lines and allows comments beginning with "#"'))
    ], arg_validator=_validate_transfer_args)
def submit_transfer(args):
    """
    Executor for `globus transfer async-transfer`
    """
    client = TransferClient()

    if args.batch is not None:
        # if input is interactive, print help to stderr
        if sys.stdin.isatty():
            print(('Enter transfers, line by line, as\n\n'
                   '    [--recursive] source-path dest-path\n\n'
                   'Lines are split with shlex in POSIX mode: '
                   'https://docs.python.org/library/shlex.html#parsing-rules\n'
                   'Terminate input with Ctrl+D or <EOF>\n'), file=sys.stderr)

        def parse_batch_line(line):
            """
            Parse a line of batch input and turn it into a transfer submission
            item.
            """
            # define a parser to handle the batch line
            parser = argparse.ArgumentParser(
                description='async-transfer batch parser')
            parser.add_argument('--recursive', default=False,
                                action='store_true')
            parser.add_argument('source')
            parser.add_argument('dest')

            # get the argument vector:
            # do a shlex split to handle quoted paths with spaces in them
            # also lets us have comments with #
            argv = shlex.split(line, comments=True)
            if not argv:
                return None

            args = parser.parse_args(argv)

            return client.make_submit_transfer_item(
                args.source, args.dest, recursive=args.recursive)

        # nested comprehensions to filter the results to remove None
        # use readlines() rather than implicit file read line looping to force
        # python to properly capture EOF (otherwise, EOF acts as a flush and
        # things get weird)
        transfer_items = [item for item in (parse_batch_line(line)
                                            for line in sys.stdin.readlines())
                          if item is not None]
    else:
        transfer_items = [client.make_submit_transfer_item(
            args.source_path, args.dest_path, recursive=args.recursive)]

    datadoc = client.make_submit_transfer_data(
        args.source_endpoint, args.dest_endpoint, transfer_items,
        label='globus-cli transfer', sync_level=args.sync_level)

    # autoactivate after parsing all args and putting things together
    autoactivate(client, args.source_endpoint, if_expires_in=60)
    autoactivate(client, args.dest_endpoint, if_expires_in=60)

    res = client.submit_transfer(datadoc)

    if outformat_is_json(args):
        print(json.dumps(res.data, indent=2))
    else:
        print(res.data['message'])

@cliargs(('Delete a file or directory from one endpoint '
          'as an asynchronous task'), [
    CLIArg('endpoint-id', required=True,
           help='ID of the endpoint from which to delete file(s)'),
    CLIArg('path', required=True, help='Path to the file/dir to delete'),
    CLIArg('recursive', default=False, action='store_true',
           help='Recursively delete dirs'),
    CLIArg('ignore-missing', default=False, action='store_true',
           help='Don\'t throw errors if the file or dir is absent')
    ])
def submit_delete(args):
    """
    Executor for `globus transfer submit-delete`
    """
    client = TransferClient()
    autoactivate(client, args.endpoint_id, if_expires_in=60)

    datadoc = {
        'DATA_TYPE': 'delete',
        'label': 'globus-cli delete',
        'endpoint': args.endpoint_id,
        'recursive': args.recursive,
        'ignore_missing': args.ignore_missing,
        'DATA': [
            {
                'DATA_TYPE': 'delete_item',
                'path': args.path
            }
        ]
    }
    add_submission_id(client, datadoc)
    res = client.submit_delete(datadoc)

    if outformat_is_json(args):
        print(json.dumps(res.data, indent=2))
    else:
        print(res.data['message'])
<|endoftext|>