<commit_before># -*- coding: utf-8 -*-
import re
import subprocess
import sys
from source import Source
from configuration import Configuration
from itertools import chain

class PDFKit(object):
    class NoExecutableError(Exception):
        """Wkhtmltopdf executable not found"""

        def __str__(self):
            #TODO insert actual path
            return ("No wkhtmltopdf executable found.\n"
                    "Please install wkhtmltopdf - https://github.com/jdpace/PDFKit/wiki/Installing-WKHTMLTOPDF")

    class ImproperSourceError(Exception):
        """Wrong source type for stylesheets"""

        def __init__(self, msg):
            self.msg = msg

        def __str__(self):
            return self.msg

    def __init__(self, url_or_file, type_, options=None, toc=None, cover=None, css=None):
        options = {} if options is None else options
        toc = {} if toc is None else toc
        #TODO rework wkhtml path set and check
        #TODO outline tests
        self.source = Source(url_or_file, type_)
        self.configuration = Configuration()
        self.options = dict()
        self.stylesheets = []

        self.wkhtmltopdf = self.configuration.wkhtmltopdf

        if self.source.isString():
            self.options.update(self._find_options_in_meta(url_or_file))

        self.options.update(options)
        self.options = self._normalize_options(self.options)
        self.toc = self._normalize_options(toc)
        self.cover = cover
        self.css = css

        try:
            with open(self.wkhtmltopdf) as f:
                pass
        except IOError:
            raise self.NoExecutableError()

    def command(self, path=None):
        args = [self.wkhtmltopdf]

        args += list(chain.from_iterable(self.options.items()))
        args = filter(None, args)

        if self.toc:
            args.append('toc')
            args += list(chain.from_iterable(self.toc.items()))
        if self.cover:
            args.append('cover')
            args.append(self.cover)

#args.append('--quiet')

        if self.source.isString() or self.css:
            args.append('-')
        else:
            if isinstance(self.source.source, str):
                args.append(self.source.to_s())
            else:
                args += self.source.source

        if path:
            args.append(path)
        else:
            args.append('-')

        #args = map(lambda x: '"%s"' % x, args)

        return args

    def to_pdf(self, path=None):
        self._append_stylesheets()

        args = self.command(path)
        #invoke = ' '.join(args)

        result = subprocess.Popen(args, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        if self.css:
            data = self._prepend_css(self.css)
            result.communicate(input=data.encode('utf-8'))
        elif self.source.isString():
            result.communicate(input=self.source.to_s().encode('utf-8'))
            #TODO wip

        #capture output of wkhtmltopdf and pass it to stdout ( can be seen only when running from console )
        if '--quiet' not in args:
            while True:
                out = result.stdout.read(1)
                if out == '' and result.poll() is not None:
                    break
                if out != '':
                    sys.stdout.write(out)
                    sys.stdout.flush()

        if path:
            text = open(path).read()

            if text == '':
                raise 'command failed: %s' % ' '.join(args)
            return text

    def to_file(self, path):
        self.to_pdf(path)

    def _normalize_options(self, options):
        normalized_options = {}

        for key, value in options.iteritems():
            normalized_key = '--%s' % self._normalize_arg(key)
            normalized_options[normalized_key] = self._normalize_value(value)

        return normalized_options

    def _normalize_arg(self, arg):
        return arg.lower()

    def _normalize_value(self, value):
        if isinstance(value, bool):
            return None
        else:
            return str(value)
            #TODO Guess this is not needed - Popen automatically quotes args
            #            value = str(value)
            #            if value.split(' ') == [value]:
            #                return value
            #            else:
            #                return '"%s"' % value

    def _style_tag_for(self, stylesheet):
        if self.source.isFile(stylesheet):
            return "" % stylesheet.read()
        else:
            return "" % stylesheet

    def _prepend_css(self, path):
        if not self.source.isFile():
                raise self.ImproperSourceError('CSS file can be prepended only to a file or to an raw HTML source')

        with open(path) as f:
            data = f.read()

        with open(self.source.to_s()) as f:
            out = f.read()

        ret = out.replace('', self._style_tag_for(data) + '')
        return ret

    def _append_stylesheets(self):
        if self.stylesheets and not self.source.isString():
            raise self.ImproperSourceError('Stylesheets may only be added to an HTML source')

        for x in self.stylesheets:
            if '' in x:
                self.source = Source(self.source.to_s.replace('', self._style_tag_for(x) + ''))
            else:
                self.source.source = self._style_tag_for(x) + self.source.source

    def _find_options_in_meta(self, content):
        if isinstance(content, file) or content.__class__.__name__ == 'StreamReaderWriter':
            content = content.read()

        found = {}

        for x in re.findall('<meta [^>]*>', content):
            if re.search('name=["\']%s' % self.configuration.meta_tag_prefix, x):
                name = re.findall('name=["\']%s([^"\']*)' % self.configuration.meta_tag_prefix, x)[0]
                found[name] = re.findall('content=["\']([^"\']*)', x)[0]

        return found<commit_msg>fix options keys parsing<commit_after># -*- coding: utf-8 -*-
import re
import subprocess
import sys
from source import Source
from configuration import Configuration
from itertools import chain

class PDFKit(object):
    class NoExecutableError(Exception):
        """Wkhtmltopdf executable not found"""

        def __str__(self):
            #TODO insert actual path
            return ("No wkhtmltopdf executable found.\n"
                    "Please install wkhtmltopdf - https://github.com/jdpace/PDFKit/wiki/Installing-WKHTMLTOPDF")

    class ImproperSourceError(Exception):
        """Wrong source type for stylesheets"""

        def __init__(self, msg):
            self.msg = msg

        def __str__(self):
            return self.msg

    def __init__(self, url_or_file, type_, options=None, toc=None, cover=None, css=None):
        options = {} if options is None else options
        toc = {} if toc is None else toc
        #TODO rework wkhtml path set and check
        #TODO outline tests
        self.source = Source(url_or_file, type_)
        self.configuration = Configuration()
        self.options = dict()
        self.stylesheets = []

        self.wkhtmltopdf = self.configuration.wkhtmltopdf

        if self.source.isString():
            self.options.update(self._find_options_in_meta(url_or_file))

        self.options.update(options)
        self.options = self._normalize_options(self.options)
        self.toc = self._normalize_options(toc)
        self.cover = cover
        self.css = css

        try:
            with open(self.wkhtmltopdf) as f:
                pass
        except IOError:
            raise self.NoExecutableError()

    def command(self, path=None):
        args = [self.wkhtmltopdf]

        args += list(chain.from_iterable(self.options.items()))
        args = filter(None, args)

        if self.toc:
            args.append('toc')
            args += list(chain.from_iterable(self.toc.items()))
        if self.cover:
            args.append('cover')
            args.append(self.cover)

#args.append('--quiet')

        if self.source.isString() or self.css:
            args.append('-')
        else:
            if isinstance(self.source.source, str):
                args.append(self.source.to_s())
            else:
                args += self.source.source

        if path:
            args.append(path)
        else:
            args.append('-')

        #args = map(lambda x: '"%s"' % x, args)

        return args

    def to_pdf(self, path=None):
        self._append_stylesheets()

        args = self.command(path)
        #invoke = ' '.join(args)

        result = subprocess.Popen(args, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        if self.css:
            data = self._prepend_css(self.css)
            result.communicate(input=data.encode('utf-8'))
        elif self.source.isString():
            result.communicate(input=self.source.to_s().encode('utf-8'))
            #TODO wip

        #capture output of wkhtmltopdf and pass it to stdout ( can be seen only when running from console )
        if '--quiet' not in args:
            while True:
                out = result.stdout.read(1)
                if out == '' and result.poll() is not None:
                    break
                if out != '':
                    sys.stdout.write(out)
                    sys.stdout.flush()

        if path:
            text = open(path).read()

            if text == '':
                raise 'command failed: %s' % ' '.join(args)
            return text

    def to_file(self, path):
        self.to_pdf(path)

    def _normalize_options(self, options):
        normalized_options = {}

        for key, value in options.iteritems():
            normalized_key = '--%s' % self._normalize_arg(key)
            normalized_options[normalized_key] = value

        return normalized_options

    def _normalize_arg(self, arg):
        return arg.lower()

    def _style_tag_for(self, stylesheet):
        if self.source.isFile(stylesheet):
            return "" % stylesheet.read()
        else:
            return "" % stylesheet

    def _prepend_css(self, path):
        if not self.source.isFile():
                raise self.ImproperSourceError('CSS file can be prepended only to a file or to an raw HTML source')

        with open(path) as f:
            data = f.read()

        with open(self.source.to_s()) as f:
            out = f.read()

        ret = out.replace('', self._style_tag_for(data) + '')
        return ret

    def _append_stylesheets(self):
        if self.stylesheets and not self.source.isString():
            raise self.ImproperSourceError('Stylesheets may only be added to an HTML source')

        for x in self.stylesheets:
            if '' in x:
                self.source = Source(self.source.to_s.replace('', self._style_tag_for(x) + ''))
            else:
                self.source.source = self._style_tag_for(x) + self.source.source

    def _find_options_in_meta(self, content):
        if isinstance(content, file) or content.__class__.__name__ == 'StreamReaderWriter':
            content = content.read()

        found = {}

        for x in re.findall('<meta [^>]*>', content):
            if re.search('name=["\']%s' % self.configuration.meta_tag_prefix, x):
                name = re.findall('name=["\']%s([^"\']*)' % self.configuration.meta_tag_prefix, x)[0]
                found[name] = re.findall('content=["\']([^"\']*)', x)[0]

        return found<|endoftext|>


<commit_before>def validator(n):
    s = str(n)
    has_double = False
    is_incrementing = True

    for i, c in enumerate(s[1:]):
        if s[i-1] == c:
            has_double = True

        if int(s[i-1]) > int(c):
            is_incrementing = False

    return has_double and is_incrementing

def test_validator():
    assert validator('111111')
    assert not validator('223450')
    assert not validator('123789')

if __name__ == '__main__':
    count = 0

    for n in range(158126, 624575):
        if validator(n):
            count += 1

    print(count)<commit_msg>That, my friends, is some fugly inefficient code that works - aoc/04<commit_after>from collections import Counter

def validator(n):
    s = str(n)
    has_double = False
    is_incrementing = True

    for i, c in enumerate(s[1:]):
        if s[i] == c:
            has_double = True

        if int(s[i]) > int(c):
            is_incrementing = False

    return has_double and is_incrementing

def validator2(n):
    s = str(n)
    has_double = False
    is_incrementing = True
    seq = 0

    for i, c in enumerate(s[1:]):
        if s[i] == c:
            seq += 1
        else:
            if seq == 1:
                has_double = True

            seq = 0

        if int(s[i]) > int(c):
            is_incrementing = False

    if seq == 1:
        has_double = True

    return has_double and is_incrementing

def test_validator():
    assert validator(111111)
    assert not validator(223450)
    assert not validator(123789)
    assert not validator(158585)

def test_validator_2():
    assert validator2(112233)
    assert not validator2(123444)
    assert validator2(111122)

if __name__ == '__main__':
    count = 0

    for n in range(158126, 624575):
        if validator(n):
            count += 1

    print(count)
    count = 0

    for n in range(158126, 624575):
        if validator2(n):
            count += 1

    print(count)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
from threading import Thread
import socket
import random
import urlparse

from django.core.files import base
from django.core.files.storage import Storage, FileSystemStorage
from django.conf import settings

from django_dust import http
from django_dust.settings import getsetting

class DistributionError(IOError):
    pass

class DistributedStorage(Storage):
    '''
    DistributedStorage saves files by copying them on several servers listed
    in settings.DUST_HOSTS.
    '''
    def __init__(self, hosts=None, use_local=None, base_url=getsetting(DUST_STORAGE_URL), **kwargs):
        super(DistributedStorage, self).__init__(**kwargs)
        if hosts is None:
            hosts = getsetting('DUST_HOSTS')
        self.hosts = hosts
        if use_local is None:
            use_local = getsetting('DUST_USE_LOCAL_FS')
        self.local_storage = use_local and FileSystemStorage(base_url=base_url, location=getsetting(DUST_STORAGE_ROOT), **kwargs)
        self.base_url = base_url
        self.transport = http.HTTPTransport(base_url=base_url)

    def _execute(self, func, name, args):
        '''
        Runs an operation (put or delete) over several hosts at once in multiple
        threads.
        '''
        def run(index, host):
            try:
                results[index] = func(host, name, *args)
            except Exception, e:
                results[index] = (e)

        # Run distribution threads keeping result of each operation in `results`.
        results = [None] * len(self.hosts)
        threads = [Thread(target=run, args=(index, h)) for index, h in enumerate(self.hosts)]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
        exceptions = []
        for host, result in zip(self.hosts, results):
            if result is None: # no errors, remember successful_host to use in retries
                successful_host = host
                break
        else:
            successful_host = None

        # All `socket.error` exceptions are not fatal meaning that a host might
        # be temporarily unavailable. Those operations are kept in a queue in
        # database to be retried later.
        # All other errors mean in most casess misconfigurations and are fatal
        # for the whole distributed operation.
        for host, result in zip(self.hosts, results):
            if isinstance(result, socket.error):
                if successful_host is not None:
                    from django_dust import retry_storage # this causes errors when imported at module level
                    retry_storage.create(
                        operation=func.__name__,
                        target_host=host,
                        source_host=successful_host,
                        filename=name,
                    )
                else:
                    exceptions.append(result)
            elif isinstance(result, Exception):
                exceptions.append(result)
        if exceptions:
            raise DistributionError(*exceptions)

    def _open(self, name, mode='rb'):
        if mode != 'rb':
            # In future when allowed to operate locally (self.local_storage)
            # all modes can be allowed. However this will require executing
            # distribution upon closing file opened for updates. This worth
            # evaluating.
            raise IOError('Illegal mode "%s". Only "rb" is supported.')
        if self.local_storage:
            return self.local_storage.open(name, mode)
        host = random.choice(self.hosts)
        return base.ContentFile(self.transport.get(host, name))

    def _save(self, name, content):
        content.seek(0)
        body = content.read()
        self._execute(self.transport.put, name, [body])
        return name

    def get_available_name(self, name):
        """
        Deletes the given file if it exists.
        """
        if self.exists(name):
            self.delete(name)
        return name

    """
    def get_available_name(self, name):
        from django_dust import retry_storage # this causes errors when imported at module level
        while self.exists(name) or retry_storage.filter_by_filename(name):
            try:
                dot_index = name.rindex('.')
            except ValueError: # filename has no dot
                name += '_'
            else:
                name = name[:dot_index] + '_' + name[dot_index:]
        return name
    """

    def path(self, name):
        if self.local_storage:
            return self.local_storage.path(name)
        return super(DistributedStorage, self).path(name)

    def delete(self, name):
        self._execute(self.transport.delete, name, [])

    def exists(self, name):
        if self.local_storage:
            return self.local_storage.exists(name)
        return self.transport.exists(random.choice(self.hosts), name)

    def listdir(self, path):
        if self.local_storage:
            return self.local_storage.listdir(path)
        raise NotImplementedError()

    def size(self, name):
        if self.local_storage:
            return self.local_storage.size(name)
        return self.transport.size(random.choice(self.hosts), name)

    def url(self, name):
        if self.base_url is None:
            raise ValueError("This file is not accessible via a URL.")
        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
<commit_msg>Fix error<commit_after># -*- coding:utf-8 -*-
from threading import Thread
import socket
import random
import urlparse

from django.core.files import base
from django.core.files.storage import Storage, FileSystemStorage
from django.conf import settings

from django_dust import http
from django_dust.settings import getsetting

class DistributionError(IOError):
    pass

class DistributedStorage(Storage):
    '''
    DistributedStorage saves files by copying them on several servers listed
    in settings.DUST_HOSTS.
    '''
    def __init__(self, hosts=None, use_local=None, base_url=getsetting('DUST_STORAGE_URL'), **kwargs):
        super(DistributedStorage, self).__init__(**kwargs)
        if hosts is None:
            hosts = getsetting('DUST_HOSTS')
        self.hosts = hosts
        if use_local is None:
            use_local = getsetting('DUST_USE_LOCAL_FS')
        self.local_storage = use_local and FileSystemStorage(base_url=base_url, location=getsetting('DUST_STORAGE_ROOT'), **kwargs)
        self.base_url = base_url
        self.transport = http.HTTPTransport(base_url=base_url)

    def _execute(self, func, name, args):
        '''
        Runs an operation (put or delete) over several hosts at once in multiple
        threads.
        '''
        def run(index, host):
            try:
                results[index] = func(host, name, *args)
            except Exception, e:
                results[index] = (e)

        # Run distribution threads keeping result of each operation in `results`.
        results = [None] * len(self.hosts)
        threads = [Thread(target=run, args=(index, h)) for index, h in enumerate(self.hosts)]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
        exceptions = []
        for host, result in zip(self.hosts, results):
            if result is None: # no errors, remember successful_host to use in retries
                successful_host = host
                break
        else:
            successful_host = None

        # All `socket.error` exceptions are not fatal meaning that a host might
        # be temporarily unavailable. Those operations are kept in a queue in
        # database to be retried later.
        # All other errors mean in most casess misconfigurations and are fatal
        # for the whole distributed operation.
        for host, result in zip(self.hosts, results):
            if isinstance(result, socket.error):
                if successful_host is not None:
                    from django_dust import retry_storage # this causes errors when imported at module level
                    retry_storage.create(
                        operation=func.__name__,
                        target_host=host,
                        source_host=successful_host,
                        filename=name,
                    )
                else:
                    exceptions.append(result)
            elif isinstance(result, Exception):
                exceptions.append(result)
        if exceptions:
            raise DistributionError(*exceptions)

    def _open(self, name, mode='rb'):
        if mode != 'rb':
            # In future when allowed to operate locally (self.local_storage)
            # all modes can be allowed. However this will require executing
            # distribution upon closing file opened for updates. This worth
            # evaluating.
            raise IOError('Illegal mode "%s". Only "rb" is supported.')
        if self.local_storage:
            return self.local_storage.open(name, mode)
        host = random.choice(self.hosts)
        return base.ContentFile(self.transport.get(host, name))

    def _save(self, name, content):
        content.seek(0)
        body = content.read()
        self._execute(self.transport.put, name, [body])
        return name

    def get_available_name(self, name):
        """
        Deletes the given file if it exists.
        """
        if self.exists(name):
            self.delete(name)
        return name

    """
    def get_available_name(self, name):
        from django_dust import retry_storage # this causes errors when imported at module level
        while self.exists(name) or retry_storage.filter_by_filename(name):
            try:
                dot_index = name.rindex('.')
            except ValueError: # filename has no dot
                name += '_'
            else:
                name = name[:dot_index] + '_' + name[dot_index:]
        return name
    """

    def path(self, name):
        if self.local_storage:
            return self.local_storage.path(name)
        return super(DistributedStorage, self).path(name)

    def delete(self, name):
        self._execute(self.transport.delete, name, [])

    def exists(self, name):
        if self.local_storage:
            return self.local_storage.exists(name)
        return self.transport.exists(random.choice(self.hosts), name)

    def listdir(self, path):
        if self.local_storage:
            return self.local_storage.listdir(path)
        raise NotImplementedError()

    def size(self, name):
        if self.local_storage:
            return self.local_storage.size(name)
        return self.transport.size(random.choice(self.hosts), name)

    def url(self, name):
        if self.base_url is None:
            raise ValueError("This file is not accessible via a URL.")
        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
<|endoftext|>


<commit_before>########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.
import os
import json
import urllib
import shutil
import urlparse
import tempfile
from mimetypes import MimeTypes

from requests_toolbelt.multipart.encoder import MultipartEncoder

from cloudify_rest_client import utils
from cloudify_rest_client.responses import ListResponse

class DeploymentUpdate(dict):

    def __init__(self, update):
        self.update(update)

    @property
    def id(self):
        """Deployment update id"""
        return self['id']

    @property
    def state(self):
        """Deployment update status"""
        return self['state']

    @property
    def deployment_id(self):
        """Deployment Id the outputs belong to."""
        return self['deployment_id']

    @property
    def steps(self):
        return self['steps']

    @property
    def execution_id(self):
        return self['execution_id']

    @property
    def created_at(self):
        return self['created_at']

class DeploymentUpdatesClient(object):

    def __init__(self, api):
        self.api = api

    def list(self, _include=None, **kwargs):
        """List deployment updates

        :param deployment_id: The deployment id (optional)
        :param kwargs: Optional filter fields. for a list of available fields
               see the REST service's models.DeploymentUpdate.fields
        """

        uri = '/deployment-updates'

        response = self.api.get(uri, params=kwargs, _include=_include)
        items = [DeploymentUpdate(item) for item in response['items']]
        return ListResponse(items, response['metadata'])

    def _update_from_blueprint(self,
                               deployment_id,
                               blueprint_path,
                               inputs=None):
        """Create a deployment update transaction for blueprint app.

        :param deployment_id: The deployment id
        :param blueprint_path: the path of the blueprint to stage
        """
        assert deployment_id

        tempdir = tempfile.mkdtemp()
        try:
            tar_path = utils.tar_blueprint(blueprint_path, tempdir)
            application_filename = os.path.basename(blueprint_path)

            return self._update_from_archive(deployment_id,
                                             tar_path,
                                             application_filename,
                                             inputs=inputs)
        finally:
            shutil.rmtree(tempdir)

    @staticmethod
    def _update_from_archive(deployment_id,
                             archive_path,
                             application_file_name=None,
                             inputs=None):
        """Create a deployment update transaction for an archived app.

        :param archive_path: the path for the archived app.
        :param application_file_name: the main blueprint filename.
        :param deployment_id: the deployment id to update.
        :return: DeploymentUpdate dict
        :rtype: DeploymentUpdate
        """
        assert deployment_id

        mime_types = MimeTypes()

        data_form = {}
        params = {}
        # all the inputs are passed through the query
        if inputs:
            inputs_file = tempfile.TemporaryFile()
            json.dump(inputs, inputs_file)
            inputs_file.seek(0)
            data_form['inputs'] = ('inputs', inputs_file, 'text/plain')

        if application_file_name:
            params['application_file_name'] = \
                urllib.quote(application_file_name)

        # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the
        # drive letter and therefore the 2nd condition is present
        if all([urlparse.urlparse(archive_path).scheme,
                not os.path.exists(archive_path)]):
            # archive location is URL
            params['blueprint_archive_url'] = archive_path
        else:
            data_form['blueprint_archive'] = (
                os.path.basename(archive_path),
                open(archive_path, 'rb'),
                # Guess the archive mime type
                mime_types.guess_type(urllib.pathname2url(archive_path)))

        return data_form, params

    def get(self, update_id, _include=None):
        """Get deployment update

        :param update_id: The update id
        """
        uri = '/deployment-updates/{0}'.format(update_id)
        response = self.api.get(uri, _include=_include)
        return DeploymentUpdate(response)

    def update(self,
               deployment_id,
               blueprint_or_archive_path,
               application_file_name=None,
               inputs=None,
               skip_install=False,
               skip_uninstall=False,
               workflow_id=None,
               force=False):

        # TODO better handle testing for a supported archive. in other commands
        # it is done in the cli part (`commands.<command_name>)
        if utils.is_supported_archive_type(blueprint_or_archive_path):
            data_form, params = \
                self._update_from_archive(deployment_id,
                                          blueprint_or_archive_path,
                                          application_file_name,
                                          inputs=inputs)
        else:
            data_form, params = \
                self._update_from_blueprint(deployment_id,
                                            blueprint_or_archive_path,
                                            inputs=inputs)

        if workflow_id:
            params['workflow_id'] = workflow_id
        if skip_install:
            params['skip_install'] = skip_install
        if skip_uninstall:
            params['skip_uninstall'] = skip_uninstall
        if force:
            params['force'] = force

        data_and_headers = {}

        if data_form:
            data = MultipartEncoder(fields=data_form)
            data_and_headers['data'] = data
            data_and_headers['headers'] = {'Content-type': data.content_type}

        uri = '/deployment-updates/{0}/update/initiate'.format(deployment_id)
        response = self.api.post(uri, params=params, **data_and_headers)

        return DeploymentUpdate(response)

    def finalize_commit(self, update_id):
        """Finalize the commiting process

        :param update_id:
        :return:
        """
        assert update_id

        uri = '/deployment-updates/{0}/update/finalize'.format(update_id)
        response = self.api.post(uri)
        return DeploymentUpdate(response)
<commit_msg>CY-228 improve deployment update behavior with blueprints<commit_after>########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import os
import json
import urllib
import shutil
import urlparse
import tempfile

from mimetypes import MimeTypes

from requests_toolbelt.multipart.encoder import MultipartEncoder

from cloudify_rest_client import utils
from cloudify_rest_client.responses import ListResponse

class DeploymentUpdate(dict):

    def __init__(self, update):
        self.update(update)

    @property
    def id(self):
        """Deployment update id"""
        return self['id']

    @property
    def state(self):
        """Deployment update status"""
        return self['state']

    @property
    def deployment_id(self):
        """Deployment Id the outputs belong to."""
        return self['deployment_id']

    @property
    def steps(self):
        return self['steps']

    @property
    def execution_id(self):
        return self['execution_id']

    @property
    def created_at(self):
        return self['created_at']

class DeploymentUpdatesClient(object):

    def __init__(self, api):
        self.api = api

    def list(self, _include=None, **kwargs):
        """List deployment updates

        :param deployment_id: The deployment id (optional)
        :param kwargs: Optional filter fields. for a list of available fields
               see the REST service's models.DeploymentUpdate.fields
        """

        uri = '/deployment-updates'

        response = self.api.get(uri, params=kwargs, _include=_include)
        items = [DeploymentUpdate(item) for item in response['items']]
        return ListResponse(items, response['metadata'])

    def _update_from_blueprint(self,
                               deployment_id,
                               blueprint_path,
                               inputs=None):
        """Create a deployment update transaction for blueprint app.

        :param deployment_id: The deployment id
        :param blueprint_path: the path of the blueprint to stage
        """
        assert deployment_id

        tempdir = tempfile.mkdtemp()
        try:
            tar_path = utils.tar_blueprint(blueprint_path, tempdir)
            application_filename = os.path.basename(blueprint_path)

            return self._update_from_archive(deployment_id,
                                             tar_path,
                                             application_filename,
                                             inputs=inputs)
        finally:
            shutil.rmtree(tempdir)

    @staticmethod
    def _update_from_archive(deployment_id,
                             archive_path,
                             application_file_name=None,
                             inputs=None):
        """Create a deployment update transaction for an archived app.

        :param archive_path: the path for the archived app.
        :param application_file_name: the main blueprint filename.
        :param deployment_id: the deployment id to update.
        :return: DeploymentUpdate dict
        :rtype: DeploymentUpdate
        """
        assert deployment_id

        mime_types = MimeTypes()

        data_form = {}
        params = {}
        # all the inputs are passed through the query
        if inputs:
            inputs_file = tempfile.TemporaryFile()
            json.dump(inputs, inputs_file)
            inputs_file.seek(0)
            data_form['inputs'] = ('inputs', inputs_file, 'text/plain')

        if application_file_name:
            params['application_file_name'] = \
                urllib.quote(application_file_name)

        # For a Windows path (e.g. "C:\aaa\bbb.zip") scheme is the
        # drive letter and therefore the 2nd condition is present
        if all([urlparse.urlparse(archive_path).scheme,
                not os.path.exists(archive_path)]):
            # archive location is URL
            params['blueprint_archive_url'] = archive_path
        else:
            data_form['blueprint_archive'] = (
                os.path.basename(archive_path),
                open(archive_path, 'rb'),
                # Guess the archive mime type
                mime_types.guess_type(urllib.pathname2url(archive_path)))

        return data_form, params

    def get(self, update_id, _include=None):
        """Get deployment update

        :param update_id: The update id
        """
        uri = '/deployment-updates/{0}'.format(update_id)
        response = self.api.get(uri, _include=_include)
        return DeploymentUpdate(response)

    def update(self,
               deployment_id,
               blueprint_or_archive_path,
               application_file_name=None,
               inputs=None,
               skip_install=False,
               skip_uninstall=False,
               workflow_id=None,
               force=False):

        # TODO better handle testing for a supported archive. in other commands
        # it is done in the cli part (`commands.<command_name>)
        if utils.is_supported_archive_type(blueprint_or_archive_path):
            data_form, params = \
                self._update_from_archive(deployment_id,
                                          blueprint_or_archive_path,
                                          application_file_name,
                                          inputs=inputs)
        else:
            data_form, params = \
                self._update_from_blueprint(deployment_id,
                                            blueprint_or_archive_path,
                                            inputs=inputs)

        if workflow_id:
            params['workflow_id'] = workflow_id
        if skip_install:
            params['skip_install'] = skip_install
        if skip_uninstall:
            params['skip_uninstall'] = skip_uninstall
        if force:
            params['force'] = force

        data_and_headers = {}

        if data_form:
            data = MultipartEncoder(fields=data_form)
            data_and_headers['data'] = data
            data_and_headers['headers'] = {'Content-type': data.content_type}

        uri = '/deployment-updates/{0}/update/initiate'.format(deployment_id)
        response = self.api.post(uri, params=params, **data_and_headers)

        return DeploymentUpdate(response)

    def update_with_existing_blueprint(self,
                                       deployment_id,
                                       blueprint_id,
                                       inputs=None,
                                       skip_install=False,
                                       skip_uninstall=False,
                                       workflow_id=None,
                                       force=False):
        data = {
            'workflow_id': workflow_id,
            'skip_install': skip_install,
            'skip_uninstall': skip_uninstall,
            'force': force,
            'blueprint_id': blueprint_id
        }
        if inputs:
            data['inputs'] = inputs
        uri = '/deployment-updates/{0}/update/initiate'.format(deployment_id)
        response = self.api.put(uri, data=data)
        return DeploymentUpdate(response)

    def finalize_commit(self, update_id):
        """Finalize the commiting process

        :param update_id:
        :return:
        """
        assert update_id

        uri = '/deployment-updates/{0}/update/finalize'.format(update_id)
        response = self.api.post(uri)
        return DeploymentUpdate(response)
<|endoftext|>


<commit_before># Lint as: python2, python3
# Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX bulk_inferrer executor."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from typing import Any, Dict, List, Mapping, Text

from absl import logging
import apache_beam as beam
import tensorflow as tf
from tfx_bsl.beam import run_inference
from tfx_bsl.proto import model_spec_pb2

from google.protobuf import json_format
from tensorflow_serving.apis import prediction_log_pb2
from tfx import types
from tfx.components.base import base_executor
from tfx.components.util import model_utils
from tfx.proto import bulk_inferrer_pb2
from tfx.types import artifact_utils
from tfx.utils import io_utils
from tfx.utils import path_utils

_PREDICTION_LOGS_DIR_NAME = 'prediction_logs'

class Executor(base_executor.BaseExecutor):
  """TFX bulk inferer executor."""

  def Do(self, input_dict: Dict[Text, List[types.Artifact]],
         output_dict: Dict[Text, List[types.Artifact]],
         exec_properties: Dict[Text, Any]) -> None:
    """Runs batch inference on a given model with given input examples.

    Args:
      input_dict: Input dict from input key to a list of Artifacts.
        - examples: examples for inference.
        - model: exported model.
        - model_blessing: model blessing result
      output_dict: Output dict from output key to a list of Artifacts.
        - output: bulk inference results.
      exec_properties: A dict of execution properties.
        - model_spec: JSON string of bulk_inferrer_pb2.ModelSpec instance.
        - data_spec: JSON string of bulk_inferrer_pb2.DataSpec instance.

    Returns:
      None
    """
    self._log_startup(input_dict, output_dict, exec_properties)

    if 'examples' not in input_dict:
      raise ValueError('\'examples\' is missing in input dict.')
    if 'inference_result' not in output_dict:
      raise ValueError('\'inference_result\' is missing in output dict.')
    output = artifact_utils.get_single_instance(output_dict['inference_result'])
    if 'model' not in input_dict:
      raise ValueError('Input models are not valid, model '
                       'need to be specified.')
    if 'model_blessing' in input_dict:
      model_blessing = artifact_utils.get_single_instance(
          input_dict['model_blessing'])
      if not model_utils.is_model_blessed(model_blessing):
        output.set_int_custom_property('inferred', 0)
        logging.info('Model on %s was not blessed', model_blessing.uri)
        return
    else:
      logging.info('Model blessing is not provided, exported model will be '
                   'used.')

    model = artifact_utils.get_single_instance(
        input_dict['model'])
    model_path = path_utils.serving_model_path(model.uri)
    logging.info('Use exported model from %s.', model_path)

    data_spec = bulk_inferrer_pb2.DataSpec()
    json_format.Parse(exec_properties['data_spec'], data_spec)
    example_uris = {}
    if data_spec.example_splits:
      for example in input_dict['examples']:
        for split in artifact_utils.decode_split_names(example.split_names):
          if split in data_spec.example_splits:
            example_uris[split] = os.path.join(example.uri, split)
    else:
      for example in input_dict['examples']:
        for split in artifact_utils.decode_split_names(example.split_names):
          example_uris[split] = os.path.join(example.uri, split)
    model_spec = bulk_inferrer_pb2.ModelSpec()
    json_format.Parse(exec_properties['model_spec'], model_spec)
    output_path = os.path.join(output.uri, _PREDICTION_LOGS_DIR_NAME)
    self._run_model_inference(model_path, example_uris, output_path,
                              model_spec)
    logging.info('BulkInferrer generates prediction log to %s', output_path)
    output.set_int_custom_property('inferred', 1)

  def _run_model_inference(self, model_path: Text,
                           example_uris: Mapping[Text, Text],
                           output_path: Text,
                           model_spec: bulk_inferrer_pb2.ModelSpec) -> None:
    """Runs model inference on given example data.

    Args:
      model_path: Path to model.
      example_uris: Mapping of example split name to example uri.
      output_path: Path to output generated prediction logs.
      model_spec: bulk_inferrer_pb2.ModelSpec instance.

    Returns:
      None
    """

    saved_model_spec = model_spec_pb2.SavedModelSpec(
        model_path=model_path,
        tag=model_spec.tag,
        signature_name=model_spec.model_signature_name)
    if getattr(model_spec_pb2, 'InferenceEndpoint', False):
      inference_endpoint = getattr(model_spec_pb2, 'InferenceEndpoint')()
    else:
      inference_endpoint = model_spec_pb2.InferenceSpecType()
    inference_endpoint.saved_model_spec.CopyFrom(saved_model_spec)
    with self._make_beam_pipeline() as pipeline:
      data_list = []
      for split, example_uri in example_uris.items():
        data = (
            pipeline | 'ReadData[{}]'.format(split) >> beam.io.ReadFromTFRecord(
                file_pattern=io_utils.all_files_pattern(example_uri)))
        data_list.append(data)
      _ = (
          [data for data in data_list]
          | 'FlattenExamples' >> beam.Flatten(pipeline=pipeline)
          | 'ParseExamples' >> beam.Map(tf.train.Example.FromString)
          | 'RunInference' >> run_inference.RunInference(inference_endpoint)
          | 'WritePredictionLogs' >> beam.io.WriteToTFRecord(
              output_path,
              file_name_suffix='.gz',
              coder=beam.coders.ProtoCoder(prediction_log_pb2.PredictionLog)))
    logging.info('Inference result written to %s.', output_path)
<commit_msg>Move RunInference API and related protos to tfx_bsl/public directory.<commit_after># Lint as: python2, python3
# Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX bulk_inferrer executor."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import importlib
import os
from typing import Any, Dict, List, Mapping, Text

from absl import logging
import apache_beam as beam
import tensorflow as tf

from google.protobuf import json_format
from tensorflow_serving.apis import prediction_log_pb2
from tfx import types
from tfx.components.base import base_executor
from tfx.components.util import model_utils
from tfx.proto import bulk_inferrer_pb2
from tfx.types import artifact_utils
from tfx.utils import io_utils
from tfx.utils import path_utils

_PREDICTION_LOGS_DIR_NAME = 'prediction_logs'

class Executor(base_executor.BaseExecutor):
  """TFX bulk inferer executor."""

  def Do(self, input_dict: Dict[Text, List[types.Artifact]],
         output_dict: Dict[Text, List[types.Artifact]],
         exec_properties: Dict[Text, Any]) -> None:
    """Runs batch inference on a given model with given input examples.

    Args:
      input_dict: Input dict from input key to a list of Artifacts.
        - examples: examples for inference.
        - model: exported model.
        - model_blessing: model blessing result
      output_dict: Output dict from output key to a list of Artifacts.
        - output: bulk inference results.
      exec_properties: A dict of execution properties.
        - model_spec: JSON string of bulk_inferrer_pb2.ModelSpec instance.
        - data_spec: JSON string of bulk_inferrer_pb2.DataSpec instance.

    Returns:
      None
    """
    self._log_startup(input_dict, output_dict, exec_properties)

    if 'examples' not in input_dict:
      raise ValueError('\'examples\' is missing in input dict.')
    if 'inference_result' not in output_dict:
      raise ValueError('\'inference_result\' is missing in output dict.')
    output = artifact_utils.get_single_instance(output_dict['inference_result'])
    if 'model' not in input_dict:
      raise ValueError('Input models are not valid, model '
                       'need to be specified.')
    if 'model_blessing' in input_dict:
      model_blessing = artifact_utils.get_single_instance(
          input_dict['model_blessing'])
      if not model_utils.is_model_blessed(model_blessing):
        output.set_int_custom_property('inferred', 0)
        logging.info('Model on %s was not blessed', model_blessing.uri)
        return
    else:
      logging.info('Model blessing is not provided, exported model will be '
                   'used.')

    model = artifact_utils.get_single_instance(
        input_dict['model'])
    model_path = path_utils.serving_model_path(model.uri)
    logging.info('Use exported model from %s.', model_path)

    data_spec = bulk_inferrer_pb2.DataSpec()
    json_format.Parse(exec_properties['data_spec'], data_spec)
    example_uris = {}
    if data_spec.example_splits:
      for example in input_dict['examples']:
        for split in artifact_utils.decode_split_names(example.split_names):
          if split in data_spec.example_splits:
            example_uris[split] = os.path.join(example.uri, split)
    else:
      for example in input_dict['examples']:
        for split in artifact_utils.decode_split_names(example.split_names):
          example_uris[split] = os.path.join(example.uri, split)
    model_spec = bulk_inferrer_pb2.ModelSpec()
    json_format.Parse(exec_properties['model_spec'], model_spec)
    output_path = os.path.join(output.uri, _PREDICTION_LOGS_DIR_NAME)
    self._run_model_inference(model_path, example_uris, output_path,
                              model_spec)
    logging.info('BulkInferrer generates prediction log to %s', output_path)
    output.set_int_custom_property('inferred', 1)

  def _run_model_inference(self, model_path: Text,
                           example_uris: Mapping[Text, Text],
                           output_path: Text,
                           model_spec: bulk_inferrer_pb2.ModelSpec) -> None:
    """Runs model inference on given example data.

    Args:
      model_path: Path to model.
      example_uris: Mapping of example split name to example uri.
      output_path: Path to output generated prediction logs.
      model_spec: bulk_inferrer_pb2.ModelSpec instance.

    Returns:
      None
    """

    try:
      from tfx_bsl.public.beam import run_inference
      from tfx_bsl.public.proto import model_spec_pb2
    except ImportError:
      # TODO(b/151468119): Remove this branch after next release.
      run_inference = importlib.import_module('tfx_bsl.beam.run_inference')
      model_spec_pb2 = importlib.import_module('tfx_bsl.proto.model_spec_pb2')
    saved_model_spec = model_spec_pb2.SavedModelSpec(
        model_path=model_path,
        tag=model_spec.tag,
        signature_name=model_spec.model_signature_name)
    # TODO(b/151468119): Remove this branch after next release.
    if getattr(model_spec_pb2, 'InferenceEndpoint', False):
      inference_endpoint = getattr(model_spec_pb2, 'InferenceEndpoint')()
    else:
      inference_endpoint = model_spec_pb2.InferenceSpecType()
    inference_endpoint.saved_model_spec.CopyFrom(saved_model_spec)
    with self._make_beam_pipeline() as pipeline:
      data_list = []
      for split, example_uri in example_uris.items():
        data = (
            pipeline | 'ReadData[{}]'.format(split) >> beam.io.ReadFromTFRecord(
                file_pattern=io_utils.all_files_pattern(example_uri)))
        data_list.append(data)
      _ = (
          [data for data in data_list]
          | 'FlattenExamples' >> beam.Flatten(pipeline=pipeline)
          | 'ParseExamples' >> beam.Map(tf.train.Example.FromString)
          | 'RunInference' >> run_inference.RunInference(inference_endpoint)
          | 'WritePredictionLogs' >> beam.io.WriteToTFRecord(
              output_path,
              file_name_suffix='.gz',
              coder=beam.coders.ProtoCoder(prediction_log_pb2.PredictionLog)))
    logging.info('Inference result written to %s.', output_path)
<|endoftext|>


<commit_before>import os.path as path
import argparse
import re
from . import __version__

def main():
    PARSER = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    # global options
    PARSER.add_argument(
        "--version", action="store_true", help="Print version number and exit"
    )

    SUBPARSERS = PARSER.add_subparsers(
        title="Sub-commands", dest="command", metavar="<command>"
    )

    # Add
    # --------------------------------------------
    parser_add = SUBPARSERS.add_parser(
        "add",
        help="Add pages from a source file to an output PDF file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_add.add_argument("dest", type=str, help="Destination PDF file")
    parser_add.add_argument("source", type=str, default=None, help="PDF source file")
    parser_add.add_argument(
        "-p",
        "--pages",
        nargs="+",
        help="list of pages to add to the output. Examples: 5; 1-9; 1-; -9",
    )
    # output
    parser_add.add_argument(
        "-o",
        "--output",
        type=str,
        default=None,
        help="name of the output file, if None the destination file will be overwritten",
    )

    # Copy
    # --------------------------------------------
    parser_copy = SUBPARSERS.add_parser(
        "copy",
        help="Copy specific pages of a PDF file in a new file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_copy.add_argument(
        "input", type=str, default=None, help="input file containing the source pages"
    )
    parser_copy.add_argument(
        "-o", "--output", type=str, default=None, help="filename of the output file"
    )
    parser_copy.add_argument(
        "-p",
        "--pages",
        dest="pages",
        type=str,
        nargs="+",
        default=1,
        help="list of pages to copy in the new file. "
        "Examples: \n"
        '"5 8 10": Pages 5, 8, 10; '
        '"1-9":    Pages 1 to 9; '
        '"5-":     Pages from 5 to last page; '
        '"-9":     Pages from beginning to 9',
    )
    parser_copy.add_argument("-y", action="store_true", help="yes to all")

    parser_insert = SUBPARSERS.add_parser(
        "insert",
        help="Insert pages of one file into another",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_merge = SUBPARSERS.add_parser(
        "merge",
        help="Merge the pages of multiple input files into one output file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_remove = SUBPARSERS.add_parser(
        "remove",
        help="Remove pages from a PDF file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_rotate = SUBPARSERS.add_parser(
        "rotate",
        help="Rotate the pages of a PDF files by 90 degrees",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_split = SUBPARSERS.add_parser(
        "split",
        help="Split a PDF file into multiple documents",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_zip = SUBPARSERS.add_parser(
        "zip",
        help="Python-like zipping (interleaving) the pages of two documents in one output file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )

    # parse arguments from command line
    ARGS = PARSER.parse_args()

    # validate command line arguments for the give sub-command
    # import packages after parsing to speed up command line responsiveness
    if ARGS.version:
        print("pdftools v{}".format(__version__))
        return

    if ARGS.command == "add":
        from pdftools.pdftools import pdf_add

        pdf_add(ARGS.dest, ARGS.source, ARGS.pages, ARGS.output)
    elif ARGS.command == "copy":
        from pdftools.pdftools import pdf_copy

        pdf_copy(ARGS.input, ARGS.output, ARGS.pages, ARGS.y)
<commit_msg>Adding `-V` as short version option<commit_after>import os.path as path
import argparse
import re
from . import __version__

def main():
    PARSER = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    # global options
    PARSER.add_argument(
        "-V", "--version", action="store_true", help="Print version number and exit"
    )

    SUBPARSERS = PARSER.add_subparsers(
        title="Sub-commands", dest="command", metavar="<command>"
    )

    # Add
    # --------------------------------------------
    parser_add = SUBPARSERS.add_parser(
        "add",
        help="Add pages from a source file to an output PDF file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_add.add_argument("dest", type=str, help="Destination PDF file")
    parser_add.add_argument("source", type=str, default=None, help="PDF source file")
    parser_add.add_argument(
        "-p",
        "--pages",
        nargs="+",
        help="list of pages to add to the output. Examples: 5; 1-9; 1-; -9",
    )
    # output
    parser_add.add_argument(
        "-o",
        "--output",
        type=str,
        default=None,
        help="name of the output file, if None the destination file will be overwritten",
    )

    # Copy
    # --------------------------------------------
    parser_copy = SUBPARSERS.add_parser(
        "copy",
        help="Copy specific pages of a PDF file in a new file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_copy.add_argument(
        "input", type=str, default=None, help="input file containing the source pages"
    )
    parser_copy.add_argument(
        "-o", "--output", type=str, default=None, help="filename of the output file"
    )
    parser_copy.add_argument(
        "-p",
        "--pages",
        dest="pages",
        type=str,
        nargs="+",
        default=1,
        help="list of pages to copy in the new file. "
        "Examples: \n"
        '"5 8 10": Pages 5, 8, 10; '
        '"1-9":    Pages 1 to 9; '
        '"5-":     Pages from 5 to last page; '
        '"-9":     Pages from beginning to 9',
    )
    parser_copy.add_argument("-y", action="store_true", help="yes to all")

    parser_insert = SUBPARSERS.add_parser(
        "insert",
        help="Insert pages of one file into another",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_merge = SUBPARSERS.add_parser(
        "merge",
        help="Merge the pages of multiple input files into one output file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_remove = SUBPARSERS.add_parser(
        "remove",
        help="Remove pages from a PDF file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_rotate = SUBPARSERS.add_parser(
        "rotate",
        help="Rotate the pages of a PDF files by 90 degrees",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_split = SUBPARSERS.add_parser(
        "split",
        help="Split a PDF file into multiple documents",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser_zip = SUBPARSERS.add_parser(
        "zip",
        help="Python-like zipping (interleaving) the pages of two documents in one output file",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )

    # parse arguments from command line
    ARGS = PARSER.parse_args()

    # validate command line arguments for the give sub-command
    # import packages after parsing to speed up command line responsiveness
    if ARGS.version:
        print("pdftools v{}".format(__version__))
        return

    if ARGS.command == "add":
        from pdftools.pdftools import pdf_add

        pdf_add(ARGS.dest, ARGS.source, ARGS.pages, ARGS.output)
    elif ARGS.command == "copy":
        from pdftools.pdftools import pdf_copy

        pdf_copy(ARGS.input, ARGS.output, ARGS.pages, ARGS.y)
<|endoftext|>


<commit_before>from pdb import set_trace

from django.core.urlresolvers import reverse
from django.conf.urls.defaults import patterns
from django.conf import settings
from django.db.models import get_model
from django.db import connection

from models import GeneralPost, published
from settings import HAS_TAGGING
from feeds import latest

qn = connection.ops.quote_name

def create_patterns(model, url_prefix=None):
    if isinstance(model, basestring):
        app_name, model_name = model.split('.')
        model = get_model(app_name, model_name)
        if model is None:
            raise Exception('Model %r not found in app %r' % (model_name, app_name))

    if url_prefix is None:
        url_prefix = ''
    else:
        if url_prefix[-1] != '/':
            url_prefix += '/'

    module_name = model._meta.module_name
    bytag_page_name = 'dzenlog-%s-bytag' % module_name
    tags_page_name = 'dzenlog-%s-tags' % module_name
    list_page_name = 'dzenlog-%s-list' % module_name
    details_page_name = 'dzenlog-%s-details' % module_name
    feeds_page_name = 'dzenlog-%s-feeds' % module_name
    feeds_bytag_page_name = 'dzenlog-%s-bytag-feeds' % module_name
    all_feeds_page_name = 'dzenlog-%s-feeds' % GeneralPost._meta.module_name

    def feeds_url(page_name):
        def func(*args,**kwargs):
            kwargs.setdefault('param', '')
            return reverse(page_name, args=args, kwargs=kwargs)
        return lambda: func

    extra_context = {
        'feeds_url': feeds_url(feeds_page_name),
        'all_feeds_url': feeds_url(all_feeds_page_name),
    }

    if HAS_TAGGING:
        def bytag_url(tag_name):
            return reverse(bytag_page_name, kwargs=dict(slug=tag_name))
        extra_context['bytag_url'] = lambda: bytag_url

    object_list = {
        'queryset': published(model._default_manager.all()),
        'template_name': 'django_dzenlog/generalpost_list.html',
        'extra_context': extra_context,
    }

    object_info = object_list.copy()
    object_info['template_name'] = 'django_dzenlog/generalpost_detail.html'

    feeds = {
        'rss': latest(model, list_page_name),
    }
    urlpatterns = patterns('django_dzenlog.views',
        (r'^%s(?P<slug>rss)(?P<param>)/$' % url_prefix, 'feed', {'feed_dict': feeds}, feeds_page_name),
    )

    if HAS_TAGGING:
        def calc_tag_cloud():
            from tagging.models import Tag, TaggedItem
            queryset = GeneralPost.objects.all()
            if model != GeneralPost:
                model_table = model._meta.db_table
                queryset = queryset.extra(
                                where=['id = %s.generalpost_ptr_id' % qn(model_table)],
                                tables=[model_table])
            ids = (obj.pk for obj in queryset)
            return Tag.objects.cloud_for_model(
                    GeneralPost,
                    min_count = getattr(settings, 'DZENLOG_TAGCLOUD_MINCOUNT', 0),
                    steps = getattr(settings, 'DZENLOG_TAGCLOUD_STEPS', 4),
                    filters = {'pk__in': ids}
                    )

        tag_cloud_data = {
            'template': 'django_dzenlog/tag_list.html',
            'extra_context': object_list['extra_context'].copy(),
        }
        tag_cloud_data['extra_context']['object_list'] = calc_tag_cloud

        urlpatterns += patterns('django.views.generic',
            (r'^%sbytag/$' % url_prefix, 'simple.direct_to_template', tag_cloud_data, tags_page_name),
        )

        bytag_object_list = object_list.copy()
        bytag_object_list['extra_context'] = object_list['extra_context'].copy()
        bytag_object_list['extra_context']['feeds_url'] = feeds_url(feeds_bytag_page_name)

        urlpatterns += patterns('django_dzenlog.views',
           (r'^%sbytag/(?P<slug>[^/]+)/$' % url_prefix, 'bytag', bytag_object_list, bytag_page_name),
        )

        urlpatterns += patterns('django_dzenlog.views',
            (r'^%sbytag/(?P<param>[^/]+)/(?P<slug>rss)/$' % url_prefix, 'feed', {'feed_dict': feeds}, feeds_bytag_page_name),
        )

    urlpatterns += patterns('django.views.generic',
        (r'^%s(?P<slug>[a-z0-9-]+)/$' % url_prefix, 'list_detail.object_detail', object_info, details_page_name),
        (r'^%s$' % url_prefix, 'list_detail.object_list', object_list, list_page_name),
    )
    return urlpatterns

urlpatterns = create_patterns('django_dzenlog.GeneralPost')
<commit_msg>Small bugfixes in the create_patterns.<commit_after>from pdb import set_trace

from django.core.urlresolvers import reverse, NoReverseMatch
from django.conf.urls.defaults import patterns
from django.conf import settings
from django.db.models import get_model
from django.db import connection

from models import GeneralPost, published
from settings import HAS_TAGGING
from feeds import latest

qn = connection.ops.quote_name

def create_patterns(model, url_prefix=None):
    if isinstance(model, basestring):
        app_name, model_name = model.split('.')
        model = get_model(app_name, model_name)
        if model is None:
            raise Exception('Model %r not found in app %r' % (model_name, app_name))

    if url_prefix is None:
        url_prefix = ''
    else:
        if url_prefix and url_prefix[-1] != '/':
            url_prefix += '/'

    module_name = model._meta.module_name
    bytag_page_name = 'dzenlog-%s-bytag' % module_name
    tags_page_name = 'dzenlog-%s-tags' % module_name
    list_page_name = 'dzenlog-%s-list' % module_name
    details_page_name = 'dzenlog-%s-details' % module_name
    feeds_page_name = 'dzenlog-%s-feeds' % module_name
    feeds_bytag_page_name = 'dzenlog-%s-bytag-feeds' % module_name
    all_feeds_page_name = 'dzenlog-%s-feeds' % GeneralPost._meta.module_name

    def feeds_url(page_name, fallback_page_name = None):
        def func(*args,**kwargs):
            kwargs.setdefault('param', '')
            try:
                return reverse(page_name, args=args, kwargs=kwargs)
            except NoReverseMatch:
                if fallback_page_name:
                    return reverse(fallback_page_name, args=args, kwargs=kwargs)
                return ''
        return lambda: func

    extra_context = {
        'feeds_url': feeds_url(feeds_page_name),
        'all_feeds_url': feeds_url(all_feeds_page_name, feeds_page_name),
    }

    if HAS_TAGGING:
        def bytag_url(tag_name):
            return reverse(bytag_page_name, kwargs=dict(slug=tag_name))
        extra_context['bytag_url'] = lambda: bytag_url

    object_list = {
        'queryset': published(model._default_manager.all()),
        'template_name': 'django_dzenlog/generalpost_list.html',
        'extra_context': extra_context,
    }

    object_info = object_list.copy()
    object_info['template_name'] = 'django_dzenlog/generalpost_detail.html'

    feeds = {
        'rss': latest(model, list_page_name),
    }
    urlpatterns = patterns('django_dzenlog.views',
        (r'^%s(?P<slug>rss)(?P<param>)/$' % url_prefix, 'feed', {'feed_dict': feeds}, feeds_page_name),
    )

    if HAS_TAGGING:
        def calc_tag_cloud():
            from tagging.models import Tag, TaggedItem
            queryset = GeneralPost.objects.all()
            if model != GeneralPost:
                model_table = model._meta.db_table
                queryset = queryset.extra(
                                where=['id = %s.generalpost_ptr_id' % qn(model_table)],
                                tables=[model_table])
            ids = (obj.pk for obj in queryset)
            return Tag.objects.cloud_for_model(
                    GeneralPost,
                    min_count = getattr(settings, 'DZENLOG_TAGCLOUD_MINCOUNT', 0),
                    steps = getattr(settings, 'DZENLOG_TAGCLOUD_STEPS', 4),
                    filters = {'pk__in': ids}
                    )

        tag_cloud_data = {
            'template': 'django_dzenlog/tag_list.html',
            'extra_context': object_list['extra_context'].copy(),
        }
        tag_cloud_data['extra_context']['object_list'] = calc_tag_cloud

        urlpatterns += patterns('django.views.generic',
            (r'^%sbytag/$' % url_prefix, 'simple.direct_to_template', tag_cloud_data, tags_page_name),
        )

        bytag_object_list = object_list.copy()
        bytag_object_list['extra_context'] = object_list['extra_context'].copy()
        bytag_object_list['extra_context']['feeds_url'] = feeds_url(feeds_bytag_page_name)

        urlpatterns += patterns('django_dzenlog.views',
           (r'^%sbytag/(?P<slug>[^/]+)/$' % url_prefix, 'bytag', bytag_object_list, bytag_page_name),
        )

        urlpatterns += patterns('django_dzenlog.views',
            (r'^%sbytag/(?P<param>[^/]+)/(?P<slug>rss)/$' % url_prefix, 'feed', {'feed_dict': feeds}, feeds_bytag_page_name),
        )

    urlpatterns += patterns('django.views.generic',
        (r'^%s(?P<slug>[a-z0-9-]+)/$' % url_prefix, 'list_detail.object_detail', object_info, details_page_name),
        (r'^%s$' % url_prefix, 'list_detail.object_list', object_list, list_page_name),
    )
    return urlpatterns

urlpatterns = create_patterns('django_dzenlog.GeneralPost')
<|endoftext|>


<commit_before>from .settings import CONNECTION_NAME

class Log4DjangoRouter(object):
    def db_for_read(self, model, **hints):
        if model._meta.app_label == 'log4django':
            return CONNECTION_NAME
        return None

    def db_for_write(self, model, **hints):
        if model._meta.app_label == 'log4django':
            return CONNECTION_NAME
        return None

    def allow_relation(self, obj1, obj2, **hints):
        if obj1._meta.app_label == 'log4django' or obj2._meta.app_label == 'log4django':
            return True
        return None

    def allow_syncdb(self, db, model):
        if db == CONNECTION_NAME:
            return model._meta.app_label == 'log4django'
        elif model._meta.app_label == 'log4django':
            return False
        return None<commit_msg>Fix syncdb in database router<commit_after>from .settings import CONNECTION_NAME

class Log4DjangoRouter(object):
    def db_for_read(self, model, **hints):
        if model._meta.app_label == 'log4django':
            return CONNECTION_NAME
        return None

    def db_for_write(self, model, **hints):
        if model._meta.app_label == 'log4django':
            return CONNECTION_NAME
        return None

    def allow_relation(self, obj1, obj2, **hints):
        if obj1._meta.app_label == 'log4django' or obj2._meta.app_label == 'log4django':
            return True
        return None

    def allow_syncdb(self, db, model):
        if db == CONNECTION_NAME and model._meta.app_label == 'log4django':
            return True
        return None<|endoftext|>


<commit_before>"""
This module contains code that helps to be compatible to python2.{6,7} and
python3.x.

If python2 support is no longer needed, the removal of this module is strongly
encouraged but may be difficult because it emulates python3 in terms of
python2.

 .. moduleauthor:: Michael Markert <markert.michael@googlemail.com>

 :copyright: PenchY Developers 2011-2012, see AUTHORS
 :license: MIT License, see LICENSE
 :license: PSF License
"""
import sys
from contextlib import contextmanager

on_python3 = sys.version_info[0] == 3

if on_python3:  # pragma: no cover
    unicode = str
    str = bytes
    from io import StringIO
    from xmlrpc.server import SimpleXMLRPCServer
else:
    str = str
    unicode = unicode
    from StringIO import StringIO
    from SimpleXMLRPCServer import SimpleXMLRPCServer

path = (str, unicode)

if sys.version_info >= (2, 7):  # pragma: no cover
    import unittest
    # avoiding AttributeErrors is quite difficult here...
    if not hasattr(unittest.TestCase, 'assertItemsEqual'):
        if hasattr(unittest.TestCase, 'assertCountEqual'):
            # monkey patch assertItemsEqual back in, has been renamed in python3.2
            unittest.TestCase.assertItemsEqual = unittest.TestCase.assertCountEqual
        else:
            unittest.TestCase.assertItemsEqual = lambda self, a, b: None
else:
    try:
        import unittest2 as unittest
    # allow that python2.6 nodes don't depend on unittest2
    # will lead to errors on running tests if except triggers
    except ImportError:  # pragma: no cover
        import unittest

# Copied from Python 2.6 contextlib, licensed under terms of PSF license
# XXX: this is only necessary for python2.6 after dropping support for python2.6
# you may want to replace this with ``with``-Statements native support for this
@contextmanager
def nested(*managers):
    """Support multiple context managers in a single with-statement.

    Code like this:

        with nested(A, B, C) as (X, Y, Z):
            <body>

    is equivalent to this:

        with A as X:
            with B as Y:
                with C as Z:
                    <body>

    """
    exits = []
    vars = []
    exc = (None, None, None)
    try:
        for mgr in managers:
            exit = mgr.__exit__
            enter = mgr.__enter__
            vars.append(enter())
            exits.append(exit)
        yield vars
    except Exception as e:
        exc = sys.exc_info()
        exception = e
    finally:
        while exits:
            exit = exits.pop()
            try:
                if exit(*exc):
                    exc = (None, None, None)
            except Exception as e:
                exc = sys.exc_info()
                exception = e
        if exc != (None, None, None):
            # Don't rely on sys.exc_info() still containing
            # the right information. Another exception may
            # have been raised and caught by an exit method
            raise exception

def write(file, string, codec='utf8'):
    """
    Write ``string`` to ``file`` encoding with codec.

    :param file: file object to write to
    :type file: file
    :param string: string to write
    :type string: str, unicode, bytes
    :param codec: how to encode unicode types
    """
    if isinstance(string, str):
        file.write(string)
    elif isinstance(string, unicode):
        file.write(string.encode(codec))

def update_hasher(hasher, string, codec='utf8'):
    """
    Update ``hasher`` with ``string``.

    :param string: string to hash
    :type string: str, unicode, bytes
    :param codec: how to encode unicode types
    :returns: updated hasher
    """
    if isinstance(string, str):
        hasher.update(string)
    elif isinstance(string, unicode):
        hasher.update(string.encode(codec))

    return hasher
<commit_msg>compat: added to_unicode function<commit_after>"""
This module contains code that helps to be compatible to python2.{6,7} and
python3.x.

If python2 support is no longer needed, the removal of this module is strongly
encouraged but may be difficult because it emulates python3 in terms of
python2.

 .. moduleauthor:: Michael Markert <markert.michael@googlemail.com>

 :copyright: PenchY Developers 2011-2012, see AUTHORS
 :license: MIT License, see LICENSE
 :license: PSF License
"""
import sys
from contextlib import contextmanager

on_python3 = sys.version_info[0] == 3

if on_python3:  # pragma: no cover
    unicode = str
    str = bytes
    from io import StringIO
    from xmlrpc.server import SimpleXMLRPCServer
else:
    str = str
    unicode = unicode
    from StringIO import StringIO
    from SimpleXMLRPCServer import SimpleXMLRPCServer

path = (str, unicode)

if sys.version_info >= (2, 7):  # pragma: no cover
    import unittest
    # avoiding AttributeErrors is quite difficult here...
    if not hasattr(unittest.TestCase, 'assertItemsEqual'):
        if hasattr(unittest.TestCase, 'assertCountEqual'):
            # monkey patch assertItemsEqual back in, has been renamed in python3.2
            unittest.TestCase.assertItemsEqual = unittest.TestCase.assertCountEqual
        else:
            unittest.TestCase.assertItemsEqual = lambda self, a, b: None
else:
    try:
        import unittest2 as unittest
    # allow that python2.6 nodes don't depend on unittest2
    # will lead to errors on running tests if except triggers
    except ImportError:  # pragma: no cover
        import unittest

# Copied from Python 2.6 contextlib, licensed under terms of PSF license
# XXX: this is only necessary for python2.6 after dropping support for python2.6
# you may want to replace this with ``with``-Statements native support for this
@contextmanager
def nested(*managers):
    """Support multiple context managers in a single with-statement.

    Code like this:

        with nested(A, B, C) as (X, Y, Z):
            <body>

    is equivalent to this:

        with A as X:
            with B as Y:
                with C as Z:
                    <body>

    """
    exits = []
    vars = []
    exc = (None, None, None)
    try:
        for mgr in managers:
            exit = mgr.__exit__
            enter = mgr.__enter__
            vars.append(enter())
            exits.append(exit)
        yield vars
    except Exception as e:
        exc = sys.exc_info()
        exception = e
    finally:
        while exits:
            exit = exits.pop()
            try:
                if exit(*exc):
                    exc = (None, None, None)
            except Exception as e:
                exc = sys.exc_info()
                exception = e
        if exc != (None, None, None):
            # Don't rely on sys.exc_info() still containing
            # the right information. Another exception may
            # have been raised and caught by an exit method
            raise exception

def write(file, string, codec='utf8'):
    """
    Write ``string`` to ``file`` encoding with codec.

    :param file: file object to write to
    :type file: file
    :param string: string to write
    :type string: str, unicode, bytes
    :param codec: how to encode unicode types
    """
    if isinstance(string, str):
        file.write(string)
    elif isinstance(string, unicode):
        file.write(string.encode(codec))

def update_hasher(hasher, string, codec='utf8'):
    """
    Update ``hasher`` with ``string``.

    :param string: string to hash
    :type string: str, unicode, bytes
    :param codec: how to encode unicode types
    :returns: updated hasher
    """
    if isinstance(string, str):
        hasher.update(string)
    elif isinstance(string, unicode):
        hasher.update(string.encode(codec))

    return hasher

def to_unicode(string):
    """
    If ``string`` is of type ``str`` it returns
    its unicode representation. Otherwise ``string``
    is not modified.

    :param string: string to decode
    :type string: str, object
    :returns: decoded string
    :rtype: unicode, object
    """
    if isinstance(string, str):
        return string.decode("iso-8859-1")
    else:
        return string
<|endoftext|>