<commit_before>"""WAL-E is a program to assist in performing PostgreSQL continuous
archiving on S3: it handles the major four operations of
arching/receiving WAL segments and archiving/receiving base hot
backups of the PostgreSQL file cluster.

"""

import argparse
import csv
import datetime
import multiprocessing
import os
import subprocess
import sys
import tempfile
import textwrap

PSQL_BIN = 'psql'
LZOP_BIN = 'lzop'
S3CMD_BIN = 's3cmd'

def psql_csv_run(sql_command, error_handler=None):
    """
    Runs psql and returns a CSVReader object from the query

    This CSVReader includes header names as the first record in all
    situations.  The output is fully buffered into Python.

    """
    csv_query = ('COPY ({query}) TO STDOUT WITH CSV HEADER;'
                 .format(query=sql_command))

    psql_proc = subprocess.Popen([PSQL_BIN, '-d', 'postgres', '-c', csv_query],
                                 stdout=subprocess.PIPE)
    stdout, stderr = psql_proc.communicate()

    if psql_proc.returncode != 0:
        if error_handler is not None:
            error_handler(psql_proc)
        else:
            assert error_handler is None
            raise Exception('Could not csv-execute "{query}" successfully'
                            .format(query=self._sqlcmd))

    # Previous code must raise any desired exceptions for non-zero
    # exit codes
    assert psql_proc.returncode == 0

    # Fake enough iterator interface to get a CSV Reader object
    # that works.
    return csv.reader(iter(stdout.strip().split('\n')))

class PgBackupStatements(object):
    """
    Contains operators to start and stop a backup on a Postgres server

    Relies on PsqlHelp for underlying mechanism.

    """

    @staticmethod
    def _dict_transform(csv_reader):
        rows = list(csv_reader)
        assert len(rows) == 2, 'Expect header row and data row'
        assert len(rows[1]) == 2, 'Expect (wal_file_name, offset) tuple'
        return dict(zip(*rows))

    @classmethod
    def run_start_backup(cls):
        """
        Connects to a server and attempts to start a hot backup

        Yields the WAL information in a dictionary for bookkeeping and
        recording.

        """
        def handler(popen):
            assert popen.returncode != 0
            raise Exception('Could not start hot backup')

        label = 'freeze_start_' + datetime.datetime.now().isoformat()

        return cls._dict_transform(psql_csv_run(
                "SELECT file_name, file_offset "
                "FROM pg_xlogfile_name_offset("
                "pg_start_backup('{0}'))".format(label),
                error_handler=handler))

    @classmethod
    def run_stop_backup(cls):
        """
        Stop a hot backup, if it was running, or error

        Return the last WAL file name and position that is required to
        gain consistency on the captured heap.

        """
        def handler(popen):
            assert popen.returncode != 0
            raise Exception('Could not stop hot backup')

        label = 'freeze_start_' + datetime.datetime.now().isoformat()

        return cls._dict_transform(psql_csv_run(
                "SELECT file_name, file_offset "
                "FROM pg_xlogfile_name_offset("
                "pg_stop_backup())", error_handler=handler))

def do_put(s3_url, path, s3cmd_config_path):
    """
    Synchronous version of the s3-upload wrapper

    Nominally intended to be used through a pool, but exposed here
    for testing and experimentation.

    """
    with tempfile.NamedTemporaryFile(mode='w') as tf:
        compression_p = subprocess.Popen(
            [LZOP_BIN, '--stdout', path], stdout=tf)
        compression_p.wait()

        if compression_p.returncode != 0:
            raise Exception(
                'Could not properly compress heap file: {path}'
                .format(path=path))

        # Not to be confused with fsync: the point is to make
        # sure any Python-buffered output is visible to other
        # processes, but *NOT* force a write to disk.
        tf.flush()

        subprocess.check_call([S3CMD_BIN, '-c', s3cmd_config_path,
                               'put', tf.name, s3_url])

    return None

class S3Backup(object):
    """
    A performs s3cmd uploads to copy a PostgreSQL cluster to S3.

    Note that this is also lzo compresses the files: thus, the number
    of pooled processes involves doing a full sequential scan of the
    uncompressed Postgres heap file that is pipelined into lzo. Once
    lzo is completely finished (necessary to have access to the file
    size) the file is sent to S3.

    TODO: Investigate an optimization to decouple the compression and
    upload steps to make sure that the most efficient possible use of
    pipelining of network and disk resources occurs.  Right now it
    possible to bounce back and forth between bottlenecking on reading
    from the database block device and subsequently the S3 sending
    steps should the processes be at the same stage of the upload
    pipeline: this can have a very negative impact on being able to
    make full use of system resources.

    Furthermore, it desirable to overflowing the page cache: having
    separate tunables for number of simultanious compression jobs
    (which occupy /tmp space and page cache) and number of uploads
    (which affect upload throughput) would help.

    """

    def __init__(self,
                 aws_access_key_id, aws_secret_access_key,
                 s3_url_prefix, pg_cluster_dir,
                 pool_size=6):
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key

        # Canonicalize the s3 prefix by stripping any trailing slash
        self.s3_url_prefix = s3_url_prefix.rstrip('/')
        self.pg_cluster_dir = pg_cluster_dir
        self.pool = multiprocessing.Pool(processes=pool_size)

    def upload_file(self, s3_url, path, s3cmd_config_path):
        """
        Asychronously uploads the path to the provided s3 url

        Returns a multiprocessing async result, which, when complete,
        will yield "None".  However, the result may also have an
        exception: this should be carefully checked for by callers to
        ensure the operation has (likely) succeeded.

        Mechanism includes lzo compression.  Because S3 requires a
        file size when starting the upload, it is necessary to buffer
        the complete compressed output in a temp file as to measure
        its size.  This is unfortunate but probably worth it because
        lzo output tends to be between 10% and 30% of the original
        heap file size.  Special effort should be made to not sync()
        to disk, so that most of the temp file mangling will occur
        in-memory in practice.

        """
        return self.pool.apply_async(do_put, [s3_url, path, s3cmd_config_path])

    def _s3_upload_pg_cluster_dir(self, start_backup_info):
        """
        Upload to s3_url_prefix from pg_cluster_dir

        This function ignores the directory pg_xlog, which contains WAL
        files and are not generally part of a base backup.

        """

        # Get a manifest of files first.
        matches = []

        def raise_walk_error(e):
            raise e

        walker = os.walk(self.pg_cluster_dir, onerror=raise_walk_error)
        for root, dirnames, filenames in walker:
            # Don't care about WAL, only heap. Also skip the textual log
            # directory.
            if 'pg_xlog' in dirnames:
                dirnames.remove('pg_xlog')

            for filename in filenames:
                matches.append(os.path.join(root, filename))

        canonical_s3_prefix = ('{0}/basebackups/base_{file_name}_{file_offset}'
                               .format(self.s3_prefix,
                                       **start_backup_info))

        # absolute upload paths are used for telling lzop what to compress
        local_abspaths = [os.path.abspath(match) for match in matches]

        # computed to subtract out extra extraneous absolute path
        # information when storing on S3
        common_local_prefix = os.path.commonprefix(local_abspaths)

        with tempfile.NamedTemporaryFile(mode='w') as s3cmd_config:
            try:
                s3cmd_config.write(textwrap.dedent("""\
                [default]
                access_key = {aws_access_key_id}
                secret_key = {aws_secret_access_key}
                """).format(aws_access_key_id=self.aws_access_key_id,
                            aws_secret_access_key=self.aws_secret_access_key))

                s3cmd_config.flush()

                uploads = []
                for local_abspath in local_abspaths:
                    remote_suffix = local_abspath[len(common_local_prefix):]
                    remote_absolute_path = '{0}/{1}.lzo'.format(
                        canonical_s3_prefix, remote_suffix)

                    uploads.append(self.upload_file(
                            remote_absolute_path, local_abspath,
                            s3cmd_config.name))

                self.pool.close()
            finally:
                # Necessary in case finally block gets hit before
                # .close()
                self.pool.close()

                while uploads:
                    # XXX: Need timeout to work around Python bug:
                    #
                    # http://bugs.python.org/issue8296
                    uploads.pop().get(1e100)

                self.pool.join()

    def database_s3_backup(self):
        """
        Wraps s3_upload_pg_cluster_dir with start/stop backup actions

        In particular there is a 'finally' block to stop the backup in
        most situations.

        """

        upload_good = False
        backup_stop_good = False
        try:
            start_backup_info = PgBackupStatements.run_start_backup()
            self._s3_upload_pg_cluster_dir(start_backup_info)
            upload_good = True
        finally:
            stop_backup_info = PgBackupStatements.run_stop_backup()
            backup_stop_good = True

        if not (upload_good and backup_stop_good):
            # NB: Other exceptions should be raised before this that
            # have more informative results, it is intended that this
            # exception never will get raised.
            raise Exception('Could not complete backup process')

def external_program_check():
    """
    Validates the existence and basic working-ness of other programs

    Implemented because it is easy to get confusing error output when
    one does not install a dependency because of the fork-worker model
    that is both necessary for throughput and makes more obscure the
    cause of failures.  This is intended to be a time and frustration
    saving measure.  This problem has confused The Author in practice
    when switching rapidly between machines.

    """

    could_not_run = []
    error_msgs = []

    def psql_err_handler(popen):
        assert popen.returncode != 0
        error_msgs.append(textwrap.fill(
                'Could not get a connection to the database: '
                'note that superuser access is required'))

        # Bogus error message that is re-caught and re-raised
        raise Exception('It is also possible that psql is not installed')

    with open(os.devnull, 'w') as nullf:
        for program in [PSQL_BIN, LZOP_BIN, S3CMD_BIN]:
            try:
                if program is PSQL_BIN:
                    psql_csv_run('SELECT 1', error_handler=psql_err_handler)
                else:
                    subprocess.call([program], stdout=nullf, stderr=nullf)
            except IOError, e:
                could_not_run.append(program)

    if could_not_run:
        error_msgs.append(textwrap.fill(
                'Could not run the following programs with exit '
                'status zero, are they installed and working? ' +
                ', '.join(could_not_run)))

    if error_msgs:
        raise Exception('\n' + '\n'.join(error_msgs))

    return None

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=__doc__)

    parser.add_argument('-k', '--aws-access-key-id',
                        help='public AWS access key. Can also be defined in an '
                        'environment variable. If both are defined, '
                        'the one defined in the programs arguments takes '
                        'precedence.')

    parser.add_argument('--s3-prefix',
                        help='S3 prefix to run all commands against.  '
                        'Can also be defined via environment variable '
                        'WALE_S3_PREFIX')

    subparsers = parser.add_subparsers(title='subcommands')

    backup_fetch_parser = subparsers.add_parser(
        'backup_fetch', help='fetch a hot backup from S3')
    backup_push_parser = subparsers.add_parser(
        'backup_push', help='pushing a fresh hot backup to S3')
    wal_fetch_parser = subparsers.add_parser(
        'wal_fetch', help='fetch a WAL file from S3')
    wal_push_parser = subparsers.add_parser(
        'wal_push', help='push a WAL file to S3')

    # backup_push operator section
    backup_push_parser.add_argument('PG_CLUSTER_DIRECTORY',
                                    help="Postgres cluster path, "
                                    "such as '/var/lib/database'")
    backup_push_parser.add_argument('--pool-size', '-p',
                                    type=int,
                                    help='Upload pooling size')

    args = parser.parse_args()

    secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
    if secret_key is None:
        print >>sys.stderr, ('Must define AWS_SECRET_ACCESS_KEY to upload '
                             'anything')
        sys.exit(1)

    s3_prefix = args.s3_prefix or os.getenv('WALE_S3_PREFIX')

    if s3_prefix is None:
        print >>sys.stderr, ('Must pass --s3-prefix or define environment '
                             'variable WALE_S3_PREFIX')
        sys.exit(1)

    if args.aws_access_key_id is None:
        aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
        if aws_access_key_id is None:
            print >>sys.stderr, ('Must define an AWS_ACCESS_KEY_ID, '
                                 'using environment variable or '
                                 '--aws_access_key_id')

    else:
        aws_access_key_id = args.aws_access_key_id

    external_program_check()

    backup = (S3Backup(aws_access_key_id, secret_key, s3_prefix,
                       args.PG_CLUSTER_DIRECTORY, pool_size=args.pool_size)
              .database_s3_backup())

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>s3_url_prefix to s3_prefix: consistency<commit_after>"""WAL-E is a program to assist in performing PostgreSQL continuous
archiving on S3: it handles the major four operations of
arching/receiving WAL segments and archiving/receiving base hot
backups of the PostgreSQL file cluster.

"""

import argparse
import csv
import datetime
import multiprocessing
import os
import subprocess
import sys
import tempfile
import textwrap

PSQL_BIN = 'psql'
LZOP_BIN = 'lzop'
S3CMD_BIN = 's3cmd'

def psql_csv_run(sql_command, error_handler=None):
    """
    Runs psql and returns a CSVReader object from the query

    This CSVReader includes header names as the first record in all
    situations.  The output is fully buffered into Python.

    """
    csv_query = ('COPY ({query}) TO STDOUT WITH CSV HEADER;'
                 .format(query=sql_command))

    psql_proc = subprocess.Popen([PSQL_BIN, '-d', 'postgres', '-c', csv_query],
                                 stdout=subprocess.PIPE)
    stdout, stderr = psql_proc.communicate()

    if psql_proc.returncode != 0:
        if error_handler is not None:
            error_handler(psql_proc)
        else:
            assert error_handler is None
            raise Exception('Could not csv-execute "{query}" successfully'
                            .format(query=self._sqlcmd))

    # Previous code must raise any desired exceptions for non-zero
    # exit codes
    assert psql_proc.returncode == 0

    # Fake enough iterator interface to get a CSV Reader object
    # that works.
    return csv.reader(iter(stdout.strip().split('\n')))

class PgBackupStatements(object):
    """
    Contains operators to start and stop a backup on a Postgres server

    Relies on PsqlHelp for underlying mechanism.

    """

    @staticmethod
    def _dict_transform(csv_reader):
        rows = list(csv_reader)
        assert len(rows) == 2, 'Expect header row and data row'
        assert len(rows[1]) == 2, 'Expect (wal_file_name, offset) tuple'
        return dict(zip(*rows))

    @classmethod
    def run_start_backup(cls):
        """
        Connects to a server and attempts to start a hot backup

        Yields the WAL information in a dictionary for bookkeeping and
        recording.

        """
        def handler(popen):
            assert popen.returncode != 0
            raise Exception('Could not start hot backup')

        label = 'freeze_start_' + datetime.datetime.now().isoformat()

        return cls._dict_transform(psql_csv_run(
                "SELECT file_name, file_offset "
                "FROM pg_xlogfile_name_offset("
                "pg_start_backup('{0}'))".format(label),
                error_handler=handler))

    @classmethod
    def run_stop_backup(cls):
        """
        Stop a hot backup, if it was running, or error

        Return the last WAL file name and position that is required to
        gain consistency on the captured heap.

        """
        def handler(popen):
            assert popen.returncode != 0
            raise Exception('Could not stop hot backup')

        label = 'freeze_start_' + datetime.datetime.now().isoformat()

        return cls._dict_transform(psql_csv_run(
                "SELECT file_name, file_offset "
                "FROM pg_xlogfile_name_offset("
                "pg_stop_backup())", error_handler=handler))

def do_put(s3_url, path, s3cmd_config_path):
    """
    Synchronous version of the s3-upload wrapper

    Nominally intended to be used through a pool, but exposed here
    for testing and experimentation.

    """
    with tempfile.NamedTemporaryFile(mode='w') as tf:
        compression_p = subprocess.Popen(
            [LZOP_BIN, '--stdout', path], stdout=tf)
        compression_p.wait()

        if compression_p.returncode != 0:
            raise Exception(
                'Could not properly compress heap file: {path}'
                .format(path=path))

        # Not to be confused with fsync: the point is to make
        # sure any Python-buffered output is visible to other
        # processes, but *NOT* force a write to disk.
        tf.flush()

        subprocess.check_call([S3CMD_BIN, '-c', s3cmd_config_path,
                               'put', tf.name, s3_url])

    return None

class S3Backup(object):
    """
    A performs s3cmd uploads to copy a PostgreSQL cluster to S3.

    Note that this is also lzo compresses the files: thus, the number
    of pooled processes involves doing a full sequential scan of the
    uncompressed Postgres heap file that is pipelined into lzo. Once
    lzo is completely finished (necessary to have access to the file
    size) the file is sent to S3.

    TODO: Investigate an optimization to decouple the compression and
    upload steps to make sure that the most efficient possible use of
    pipelining of network and disk resources occurs.  Right now it
    possible to bounce back and forth between bottlenecking on reading
    from the database block device and subsequently the S3 sending
    steps should the processes be at the same stage of the upload
    pipeline: this can have a very negative impact on being able to
    make full use of system resources.

    Furthermore, it desirable to overflowing the page cache: having
    separate tunables for number of simultanious compression jobs
    (which occupy /tmp space and page cache) and number of uploads
    (which affect upload throughput) would help.

    """

    def __init__(self,
                 aws_access_key_id, aws_secret_access_key,
                 s3_prefix, pg_cluster_dir,
                 pool_size=6):
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key

        # Canonicalize the s3 prefix by stripping any trailing slash
        self.s3_prefix = s3_prefix.rstrip('/')
        self.pg_cluster_dir = pg_cluster_dir
        self.pool = multiprocessing.Pool(processes=pool_size)

    def upload_file(self, s3_url, path, s3cmd_config_path):
        """
        Asychronously uploads the path to the provided s3 url

        Returns a multiprocessing async result, which, when complete,
        will yield "None".  However, the result may also have an
        exception: this should be carefully checked for by callers to
        ensure the operation has (likely) succeeded.

        Mechanism includes lzo compression.  Because S3 requires a
        file size when starting the upload, it is necessary to buffer
        the complete compressed output in a temp file as to measure
        its size.  This is unfortunate but probably worth it because
        lzo output tends to be between 10% and 30% of the original
        heap file size.  Special effort should be made to not sync()
        to disk, so that most of the temp file mangling will occur
        in-memory in practice.

        """
        return self.pool.apply_async(do_put, [s3_url, path, s3cmd_config_path])

    def _s3_upload_pg_cluster_dir(self, start_backup_info):
        """
        Upload to s3_url_prefix from pg_cluster_dir

        This function ignores the directory pg_xlog, which contains WAL
        files and are not generally part of a base backup.

        """

        # Get a manifest of files first.
        matches = []

        def raise_walk_error(e):
            raise e

        walker = os.walk(self.pg_cluster_dir, onerror=raise_walk_error)
        for root, dirnames, filenames in walker:
            # Don't care about WAL, only heap. Also skip the textual log
            # directory.
            if 'pg_xlog' in dirnames:
                dirnames.remove('pg_xlog')

            for filename in filenames:
                matches.append(os.path.join(root, filename))

        canonical_s3_prefix = ('{0}/basebackups/base_{file_name}_{file_offset}'
                               .format(self.s3_prefix,
                                       **start_backup_info))

        # absolute upload paths are used for telling lzop what to compress
        local_abspaths = [os.path.abspath(match) for match in matches]

        # computed to subtract out extra extraneous absolute path
        # information when storing on S3
        common_local_prefix = os.path.commonprefix(local_abspaths)

        with tempfile.NamedTemporaryFile(mode='w') as s3cmd_config:
            try:
                s3cmd_config.write(textwrap.dedent("""\
                [default]
                access_key = {aws_access_key_id}
                secret_key = {aws_secret_access_key}
                """).format(aws_access_key_id=self.aws_access_key_id,
                            aws_secret_access_key=self.aws_secret_access_key))

                s3cmd_config.flush()

                uploads = []
                for local_abspath in local_abspaths:
                    remote_suffix = local_abspath[len(common_local_prefix):]
                    remote_absolute_path = '{0}/{1}.lzo'.format(
                        canonical_s3_prefix, remote_suffix)

                    uploads.append(self.upload_file(
                            remote_absolute_path, local_abspath,
                            s3cmd_config.name))

                self.pool.close()
            finally:
                # Necessary in case finally block gets hit before
                # .close()
                self.pool.close()

                while uploads:
                    # XXX: Need timeout to work around Python bug:
                    #
                    # http://bugs.python.org/issue8296
                    uploads.pop().get(1e100)

                self.pool.join()

    def database_s3_backup(self):
        """
        Wraps s3_upload_pg_cluster_dir with start/stop backup actions

        In particular there is a 'finally' block to stop the backup in
        most situations.

        """

        upload_good = False
        backup_stop_good = False
        try:
            start_backup_info = PgBackupStatements.run_start_backup()
            self._s3_upload_pg_cluster_dir(start_backup_info)
            upload_good = True
        finally:
            stop_backup_info = PgBackupStatements.run_stop_backup()
            backup_stop_good = True

        if not (upload_good and backup_stop_good):
            # NB: Other exceptions should be raised before this that
            # have more informative results, it is intended that this
            # exception never will get raised.
            raise Exception('Could not complete backup process')

def external_program_check():
    """
    Validates the existence and basic working-ness of other programs

    Implemented because it is easy to get confusing error output when
    one does not install a dependency because of the fork-worker model
    that is both necessary for throughput and makes more obscure the
    cause of failures.  This is intended to be a time and frustration
    saving measure.  This problem has confused The Author in practice
    when switching rapidly between machines.

    """

    could_not_run = []
    error_msgs = []

    def psql_err_handler(popen):
        assert popen.returncode != 0
        error_msgs.append(textwrap.fill(
                'Could not get a connection to the database: '
                'note that superuser access is required'))

        # Bogus error message that is re-caught and re-raised
        raise Exception('It is also possible that psql is not installed')

    with open(os.devnull, 'w') as nullf:
        for program in [PSQL_BIN, LZOP_BIN, S3CMD_BIN]:
            try:
                if program is PSQL_BIN:
                    psql_csv_run('SELECT 1', error_handler=psql_err_handler)
                else:
                    subprocess.call([program], stdout=nullf, stderr=nullf)
            except IOError, e:
                could_not_run.append(program)

    if could_not_run:
        error_msgs.append(textwrap.fill(
                'Could not run the following programs with exit '
                'status zero, are they installed and working? ' +
                ', '.join(could_not_run)))

    if error_msgs:
        raise Exception('\n' + '\n'.join(error_msgs))

    return None

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=__doc__)

    parser.add_argument('-k', '--aws-access-key-id',
                        help='public AWS access key. Can also be defined in an '
                        'environment variable. If both are defined, '
                        'the one defined in the programs arguments takes '
                        'precedence.')

    parser.add_argument('--s3-prefix',
                        help='S3 prefix to run all commands against.  '
                        'Can also be defined via environment variable '
                        'WALE_S3_PREFIX')

    subparsers = parser.add_subparsers(title='subcommands')

    backup_fetch_parser = subparsers.add_parser(
        'backup_fetch', help='fetch a hot backup from S3')
    backup_push_parser = subparsers.add_parser(
        'backup_push', help='pushing a fresh hot backup to S3')
    wal_fetch_parser = subparsers.add_parser(
        'wal_fetch', help='fetch a WAL file from S3')
    wal_push_parser = subparsers.add_parser(
        'wal_push', help='push a WAL file to S3')

    # backup_push operator section
    backup_push_parser.add_argument('PG_CLUSTER_DIRECTORY',
                                    help="Postgres cluster path, "
                                    "such as '/var/lib/database'")
    backup_push_parser.add_argument('--pool-size', '-p',
                                    type=int,
                                    help='Upload pooling size')

    args = parser.parse_args()

    secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
    if secret_key is None:
        print >>sys.stderr, ('Must define AWS_SECRET_ACCESS_KEY to upload '
                             'anything')
        sys.exit(1)

    s3_prefix = args.s3_prefix or os.getenv('WALE_S3_PREFIX')

    if s3_prefix is None:
        print >>sys.stderr, ('Must pass --s3-prefix or define environment '
                             'variable WALE_S3_PREFIX')
        sys.exit(1)

    if args.aws_access_key_id is None:
        aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
        if aws_access_key_id is None:
            print >>sys.stderr, ('Must define an AWS_ACCESS_KEY_ID, '
                                 'using environment variable or '
                                 '--aws_access_key_id')

    else:
        aws_access_key_id = args.aws_access_key_id

    external_program_check()

    backup = (S3Backup(aws_access_key_id, secret_key, s3_prefix,
                       args.PG_CLUSTER_DIRECTORY, pool_size=args.pool_size)
              .database_s3_backup())

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Translate a git repository of Python code to JSON.

This script takes a git repository--either a local repository or a remote one
on github--and converts each line into a JSON object containing information
about the line, the author, and the committer.

Note that a github repository is saved to a local temporary directory for
processing and is removed when the processing is complete. Local repositories
are never deleted.

The JSON object for each line has the following form:

    {
        "repo_name": val,
        "commit_id": val,
        "author": val,
        "author_mail": val,
        "author_time": val,
        "author_timezone": val,
        "committer": val,
        "committer_mail": val,
        "committer_time": val,
        "committer_timezone": val,
        "comment": val,
        "filename": val,
        "line_num": val,
        "line": val,
    }

    Where the variables are as follows:
        repo_name (str): The name of the repository.
        commit_id (str): The commit hash.
        author (str): The author of the commit.
        author_mail (str): The author's email address.
        author_time (int): The time the commit was written in seconds
            since the unix epoch.
        author_timezone (int): The timezone of the author, as an integer.
            The two right most digits are the minute offsets, and the
            remaining leftmost digits are the hour offsets.
        committer (str): The committer of the commit.
        committer_mail (str): The committer's email address.
        committer_time (int): The time the commit was written in seconds
            since the unix epoch.
        committer_timezone (int): The timezone of the committer, in the
            same form as author_timezone.
        comment (str): The commit message.
        filename (str): The name of the file.
        line_num (int): The line number of the text in the file after the
            commit was applied.
        line (str): The text of the source line.

Examples:

    With a local repository:

        ./repo_to_json.py /path/to/local/repo/ /output/directory/

    For a github repository:

        ./repo_to_json.py username/reponame /output/directory/

    The repo_name field in the JSON object is either the basename of the local
    path to the repository ("repo" in the case of the first example) or the
    name of the github repository ("username/reponame" in the case of the
    second example). This can be overridden to use a custom name with the
    commandline flag `--repo-name` (also `-n`). This is done as follows:

        ./repo_to_json.py username/reponame /output/directory/ -n "New Name"

"""

from cd import cd
import blame_to_json as btj
import git_manager as gm
import os
import os.path
import subprocess
import sys
import errno

def get_filelist(directory):
    """Return a list of python files in a directory structure.

    Args:
        directory (str): The directory to start the search in.

    Returns:
        list of strings: A list of the locations of all the python files in the
            directory tree.
    """
    file_list = []
    for root, _, files in os.walk(directory):
        for file in files:
            if file.endswith('.py'):
                file_list.append(root + '/' + file)

    return file_list

def is_path_exist(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def process_local_repo(location, output_dir, repo_name):
    """Convert a local repository to a series of JSON objects.

    Args:
        location (str): The path to a local repository.
        output_dir (str): The path to the directory to save the output files
            to.
        repo_name (str): The name to save to the JSON objects as the repository
            name.

    Returns:
        None
    """
    with cd(location):
        is_path_exist(output_dir)
        output_file = output_dir + "/" + repo_name.replace('/', '_') + ".json"
        all_lines = []
        with open(output_file, 'w') as f:
            for file in get_filelist(location):
                for line in btj.file_to_json(file, repo_name):
                    f.write(line + "\n")

def get_local_repo_name(location):
    """The basename of a local repository.

    If a local repository is located at:

        /path/to/local/repo/

    This function will return "repo".

    This function must be called from within the repository, so using cd() to
    change the directory is advised.

    Args:
        location (str): The path to a local repository.

    Returns:
        str: The basename of the repository.

    """
    with cd(location):
        command = [
            "git",
            "rev-parse",
            "--show-toplevel",
        ]
        repo_name = subprocess.check_output(command)
        base = os.path.basename(repo_name).strip()
        return base

if __name__ == "__main__":

    import argparse
    # Set up command line flag handling
    parser = argparse.ArgumentParser(
        description="Clone and parse a git repository",
    )
    parser.add_argument(
        'repo_location',
        type=str,
        help="the location of the repository"
    )
    parser.add_argument(
        'output_directory',
        type=str,
        action="store",
        help="the directory to save the output JSON files",
    )
    parser.add_argument(
        '-n',
        '--repo-name',
        type=str,
        action="store",
        help="override the default repository name to save to the JSON file"
    )

    args = parser.parse_args()

    # If the user has passed in a repo name, override the default name
    repo_name = args.repo_name

    # If we are given a valid local path, then use that
    with gm.Repository(args.repo_location) as repo:
        if not repo_name:
            repo_name = get_local_repo_name(repo.local_location)
        process_local_repo(
            repo.local_location,
            args.output_directory,
            repo_name
        )
<commit_msg>Add lines around function per pep8<commit_after>#!/usr/bin/env python

"""Translate a git repository of Python code to JSON.

This script takes a git repository--either a local repository or a remote one
on github--and converts each line into a JSON object containing information
about the line, the author, and the committer.

Note that a github repository is saved to a local temporary directory for
processing and is removed when the processing is complete. Local repositories
are never deleted.

The JSON object for each line has the following form:

    {
        "repo_name": val,
        "commit_id": val,
        "author": val,
        "author_mail": val,
        "author_time": val,
        "author_timezone": val,
        "committer": val,
        "committer_mail": val,
        "committer_time": val,
        "committer_timezone": val,
        "comment": val,
        "filename": val,
        "line_num": val,
        "line": val,
    }

    Where the variables are as follows:
        repo_name (str): The name of the repository.
        commit_id (str): The commit hash.
        author (str): The author of the commit.
        author_mail (str): The author's email address.
        author_time (int): The time the commit was written in seconds
            since the unix epoch.
        author_timezone (int): The timezone of the author, as an integer.
            The two right most digits are the minute offsets, and the
            remaining leftmost digits are the hour offsets.
        committer (str): The committer of the commit.
        committer_mail (str): The committer's email address.
        committer_time (int): The time the commit was written in seconds
            since the unix epoch.
        committer_timezone (int): The timezone of the committer, in the
            same form as author_timezone.
        comment (str): The commit message.
        filename (str): The name of the file.
        line_num (int): The line number of the text in the file after the
            commit was applied.
        line (str): The text of the source line.

Examples:

    With a local repository:

        ./repo_to_json.py /path/to/local/repo/ /output/directory/

    For a github repository:

        ./repo_to_json.py username/reponame /output/directory/

    The repo_name field in the JSON object is either the basename of the local
    path to the repository ("repo" in the case of the first example) or the
    name of the github repository ("username/reponame" in the case of the
    second example). This can be overridden to use a custom name with the
    commandline flag `--repo-name` (also `-n`). This is done as follows:

        ./repo_to_json.py username/reponame /output/directory/ -n "New Name"

"""

from cd import cd
import blame_to_json as btj
import git_manager as gm
import os
import os.path
import subprocess
import sys
import errno

def get_filelist(directory):
    """Return a list of python files in a directory structure.

    Args:
        directory (str): The directory to start the search in.

    Returns:
        list of strings: A list of the locations of all the python files in the
            directory tree.
    """
    file_list = []
    for root, _, files in os.walk(directory):
        for file in files:
            if file.endswith('.py'):
                file_list.append(root + '/' + file)

    return file_list

def is_path_exist(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def process_local_repo(location, output_dir, repo_name):
    """Convert a local repository to a series of JSON objects.

    Args:
        location (str): The path to a local repository.
        output_dir (str): The path to the directory to save the output files
            to.
        repo_name (str): The name to save to the JSON objects as the repository
            name.

    Returns:
        None
    """
    with cd(location):
        is_path_exist(output_dir)
        output_file = output_dir + "/" + repo_name.replace('/', '_') + ".json"
        all_lines = []
        with open(output_file, 'w') as f:
            for file in get_filelist(location):
                for line in btj.file_to_json(file, repo_name):
                    f.write(line + "\n")

def get_local_repo_name(location):
    """The basename of a local repository.

    If a local repository is located at:

        /path/to/local/repo/

    This function will return "repo".

    This function must be called from within the repository, so using cd() to
    change the directory is advised.

    Args:
        location (str): The path to a local repository.

    Returns:
        str: The basename of the repository.

    """
    with cd(location):
        command = [
            "git",
            "rev-parse",
            "--show-toplevel",
        ]
        repo_name = subprocess.check_output(command)
        base = os.path.basename(repo_name).strip()
        return base

if __name__ == "__main__":

    import argparse
    # Set up command line flag handling
    parser = argparse.ArgumentParser(
        description="Clone and parse a git repository",
    )
    parser.add_argument(
        'repo_location',
        type=str,
        help="the location of the repository"
    )
    parser.add_argument(
        'output_directory',
        type=str,
        action="store",
        help="the directory to save the output JSON files",
    )
    parser.add_argument(
        '-n',
        '--repo-name',
        type=str,
        action="store",
        help="override the default repository name to save to the JSON file"
    )

    args = parser.parse_args()

    # If the user has passed in a repo name, override the default name
    repo_name = args.repo_name

    # If we are given a valid local path, then use that
    with gm.Repository(args.repo_location) as repo:
        if not repo_name:
            repo_name = get_local_repo_name(repo.local_location)
        process_local_repo(
            repo.local_location,
            args.output_directory,
            repo_name
        )
<|endoftext|>


<commit_before>"""Compile `Handlebars <http://handlebarsjs.com/>`_ templates.

This filter assumes that the ``handlebars`` executable is in the path.
Otherwise, you may define a ``HANDLEBARS_BIN`` setting.
"""

import subprocess

from webassets.exceptions import FilterError
from webassets.filter import Filter

__all__ = ('HandlebarsFilter',)

class HandlebarsFilter(Filter):

    name = 'handlebars'

    def setup(self):
        self.binary = self.get_config(
            'HANDLEBARS_BIN', require=False) or 'handlebars'
        self._paths = []

    def input(self, _in, out, source_path, output_path):
        args = [self.binary, source_path]
        proc = subprocess.Popen(
            args, stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        stdout, stderr = proc.communicate()

        if proc.returncode != 0:
            raise FilterError(('handlebars: subprocess had error: stderr=%s, '+
                               'stdout=%s, returncode=%s') % (
                                    stderr, stdout, proc.returncode))
        out.write(stdout.strip() + ';')
<commit_msg>Remove unused self._paths<commit_after>"""Compile `Handlebars <http://handlebarsjs.com/>`_ templates.

This filter assumes that the ``handlebars`` executable is in the path.
Otherwise, you may define a ``HANDLEBARS_BIN`` setting.
"""

import subprocess

from webassets.exceptions import FilterError
from webassets.filter import Filter

__all__ = ('HandlebarsFilter',)

class HandlebarsFilter(Filter):

    name = 'handlebars'

    def setup(self):
        self.binary = self.get_config(
            'HANDLEBARS_BIN', require=False) or 'handlebars'

    def input(self, _in, out, source_path, output_path):
        args = [self.binary, source_path]
        proc = subprocess.Popen(
            args, stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        stdout, stderr = proc.communicate()

        if proc.returncode != 0:
            raise FilterError(('handlebars: subprocess had error: stderr=%s, '+
                               'stdout=%s, returncode=%s') % (
                                    stderr, stdout, proc.returncode))
        out.write(stdout.strip() + ';')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2011-2016 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

__version_info__ = (1, 0, 0)
__version__ = '.'.join(map(str, __version_info__)) + 'a1'
<commit_msg>Bump WireCloud versions<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2011-2016 CoNWeT Lab., Universidad Politécnica de Madrid

# This file is part of Wirecloud.

# Wirecloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# Wirecloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with Wirecloud.  If not, see <http://www.gnu.org/licenses/>.

__version_info__ = (1, 0, 0)
__version__ = '.'.join(map(str, __version_info__))
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['mixcloud_download']

from ..common import *

def mixcloud_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    html = get_html(url, faker=True)
    title = r1(r'<meta property="og:title" content="([^"]*)"', html)
    preview_url = r1("m-preview=\"([^\"]+)\"", html)

    url = re.sub(r'previews', r'c/originals', preview_url)
    for i in range(10, 30):
        url = re.sub(r'stream[^.]*', r'stream' + str(i), url)

        try:
            type, ext, size = url_info(url)
            break
        except:
            continue

    try:
        type
    except:
        url = re.sub('c/originals', r'c/m4a/64', url)
        url = re.sub('.mp3', '.m4a', url)
        for i in range(10, 30):
            url = re.sub(r'stream[^.]*', r'stream' + str(i), url)

            try:
                type, ext, size = url_info(url)
                break
            except:
                continue

    print_info(site_info, title, type, size)
    if not info_only:
        download_urls([url], title, ext, size, output_dir, merge = merge)

site_info = "Mixcloud.com"
download = mixcloud_download
download_playlist = playlist_not_supported('mixcloud')
<commit_msg>[mixcloud] fix<commit_after>#!/usr/bin/env python

__all__ = ['mixcloud_download']

from ..common import *

def mixcloud_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    html = get_html(url, faker=True)
    title = r1(r'<meta property="og:title" content="([^"]*)"', html)
    preview_url = r1(r'm-preview=\"([^\"]+)\"', html)
    preview = r1(r'previews(.*)\.mp3$', preview_url)

    for i in range(10, 30):
        url = 'https://stream{i}.mixcloud.com/c/m4a/64{p}.m4a'.format(
            i = i,
            p = preview
        )
        try:
            mime, ext, size = url_info(url)
            break
        except: continue

    print_info(site_info, title, ext, size)
    if not info_only:
        download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)

site_info = "Mixcloud.com"
download = mixcloud_download
download_playlist = playlist_not_supported('mixcloud')
<|endoftext|>


<commit_before># vim:fileencoding=utf-8 encoding=utf-8
# Copyright (c) 2007-2008 gocept gmbh & co. kg
# See also LICENSE.txt
# $Id$

import zope.component.interfaces
import zope.i18nmessageid
import zope.interface
import zope.interface.interfaces
import zope.schema.interfaces

import zope.app.locking.interfaces

import zeit.cms.content.field
import zeit.cms.content.sources
import zeit.cms.interfaces

import zeit.cms.content.contentsource
from zeit.cms.content._bootstrapinterfaces import ICMSContentSource

_ = zope.i18nmessageid.MessageFactory('zeit.cms')

# XXX There is too much, too unordered in here, clean this up.

class IKeywordInterface(zope.interface.interfaces.IInterface):
    """The interface of the keyword interface."""

class IKeyword(zope.interface.Interface):

    code = zope.schema.TextLine()
    label = zope.schema.TextLine()

    inTaxonomy = zope.schema.Bool(
        title=_("Keyword is contained in the taxonomy"),
        default=False)

IKeyword.narrower = zope.schema.List(value_type=zope.schema.Object(IKeyword))
IKeyword.broader = value_type=zope.schema.Object(IKeyword, required=False)

class IKeywords(zope.interface.Interface):

    root = zope.schema.Object(
        IKeyword,
        title=_('Root of ontology'))

    def __getitem__(code):
        """return IKeyword with given code."""

    def find_keywords(searchterm):
        """Returns a list of keywords which contain the searchterm
           string.
        """

class ICommonMetadata(zope.interface.Interface):

    year = zope.schema.Int(
        title=_("Year"),
        min=1900,
        max=2100)

    volume = zope.schema.Int(
        title=_("Volume"),
        min=1,
        max=53)

    page = zope.schema.Int(
        title=_("Page"),
        readonly=True,
        required=False)

    ressort = zope.schema.Choice(
        title=_("Ressort"),
        source=zeit.cms.content.sources.NavigationSource())

    sub_ressort = zope.schema.TextLine(
        title=_('Sub ressort'),
        required=False)

    authors = zope.schema.Tuple(
        title=_("Authors"),
        value_type=zope.schema.TextLine(),
        required=False,
        default=(u'',))

    keywords = zope.schema.Tuple(
        title=_("Keywords"),
        required=False,
        default=(),
        value_type=zope.schema.Object(IKeyword))

    serie = zope.schema.Choice(
        title=_("Serie"),
        source=zeit.cms.content.sources.SerieSource(),
        required=False)

    copyrights = zope.schema.TextLine(
        title=_("Copyright (c)"),
        description=_("Do not enter (c)."),
        default=u"ZEIT online")

    supertitle = zope.schema.TextLine(
        title=_("Kicker"),
        description=_("Please take care of capitalisation."),
        required=False)

    byline = zope.schema.TextLine(
        title=_("By line"),
        required=False)

    title = zope.schema.TextLine(
        title=_("Title"))

    subtitle = zope.schema.Text(
        title=_("Subtitle"),
        required=False)

    teaserTitle = zope.schema.Text(
        title=_("Teaser title"),
        required=False)

    teaserText = zope.schema.Text(
        title=_("Teaser text"),
        required=False,
        max_length=170)

    shortTeaserTitle = zope.schema.Text(
        title=_("Index teaser title"),
        required=False,
        max_length=20)

    shortTeaserText = zope.schema.Text(
        title=_("Index teaser text"),
        required=False,
        max_length=50)

    vg_wort_id = zope.schema.TextLine(
        title=_('VG Wort Id'),
        required=False)

class IDAVPropertyConverter(zope.interface.Interface):
    """Parse a unicode string from a DAV property to a value and vice versa."""

    def fromProperty(value):
        """Convert property value to python value.

        returns python object represented by value.

        raises ValueError if the value could not be converted.
        raises zope.schema.ValidationError if the value could be converted but
        does not satisfy the constraints.

        """

    def toProperty(value):
        """Convert python value to DAV property value.

        returns unicode
        """

class IDAVToken(zope.interface.Interface):
    """A string representing a token that uniquely identifies a value."""

class IDAVPropertyChangedEvent(zope.component.interfaces.IObjectEvent):
    """A dav property has been changed."""

    old_value = zope.interface.Attribute("The value before the change.")
    new_value = zope.interface.Attribute("The value after the change.")

    property_namespace = zope.interface.Attribute("Webdav property namespace.")
    property_name = zope.interface.Attribute("Webdav property name.")

class DAVPropertyChangedEvent(zope.component.interfaces.ObjectEvent):
    zope.interface.implements(IDAVPropertyChangedEvent)

    def __init__(self, object, property_namespace, property_name,
                 old_value, new_value):
        self.object = object
        self.property_namespace = property_namespace
        self.property_name = property_name
        self.old_value = old_value
        self.new_value = new_value

class ITextContent(zope.interface.Interface):
    """Representing text content XXX"""

    data = zope.schema.Text(title=u"Document content")

class IXMLRepresentation(zope.interface.Interface):
    """Objects with an XML representation."""

    xml = zeit.cms.content.field.XMLTree(
        title=_("XML Source"))

class IXMLReference(zope.interface.Interface):
    """XML representation of an object reference.

    Object references are dependent on the target object. For instance a feed
    is referenced with <xi:include> while an image is referenced using <img>.

    """

    xml = zeit.cms.content.field.XMLTree(
        title=u"Reference Structure")

class IXMLSource(zope.interface.Interface):
    """str representing the xml of an object."""

class IXMLContent(zeit.cms.interfaces.ICMSContent, IXMLRepresentation):
    """Content with an XML representation.

    Usually *all* data should be stored in the XML structure *or* in
    webdav properties.

    """

    properties = zope.schema.Object(zeit.cms.interfaces.IWebDAVProperties)

class ILockInfo(zope.app.locking.interfaces.ILockInfo):
    """Extended LockInfo interface."""

    locked_until = zope.schema.Datetime(
        title=u"Locked Until",
        required=False)

class ITemplateManagerContainer(zope.app.container.interfaces.IReadContainer):
    """Container which holds all template managers."""

class ITemplateManager(zope.app.container.interfaces.IReadContainer):
    """Manages templates for a content type."""

class ITemplate(IXMLRepresentation):
    """A template for xml content types."""

    title = zope.schema.TextLine(title=_('Title'))

class IRelatedContent(zope.interface.Interface):
    """Relate other content."""

    related = zope.schema.Tuple(
        title=_("Related content"),
        description=_("Objects that are related to this object."),
        default=(),
        required=False,
        value_type=zope.schema.Choice(
            source=zeit.cms.content.contentsource.cmsContentSource))

class IDAVPropertiesInXML(zope.interface.Interface):
    """Marker interface for objects which store their webdav properties in xml.

    It is common for articles and CPs to store their webdav properties in the
    xml, too. That is in addition to the Metadata stored as webdav properties.

    """

<commit_msg>properties is not part of the IXMLContent interface for quite some time.<commit_after># vim:fileencoding=utf-8 encoding=utf-8
# Copyright (c) 2007-2008 gocept gmbh & co. kg
# See also LICENSE.txt
# $Id$

import zope.component.interfaces
import zope.i18nmessageid
import zope.interface
import zope.interface.interfaces
import zope.schema.interfaces

import zope.app.locking.interfaces

import zeit.cms.content.field
import zeit.cms.content.sources
import zeit.cms.interfaces

import zeit.cms.content.contentsource
from zeit.cms.content._bootstrapinterfaces import ICMSContentSource

_ = zope.i18nmessageid.MessageFactory('zeit.cms')

# XXX There is too much, too unordered in here, clean this up.

class IKeywordInterface(zope.interface.interfaces.IInterface):
    """The interface of the keyword interface."""

class IKeyword(zope.interface.Interface):

    code = zope.schema.TextLine()
    label = zope.schema.TextLine()

    inTaxonomy = zope.schema.Bool(
        title=_("Keyword is contained in the taxonomy"),
        default=False)

IKeyword.narrower = zope.schema.List(value_type=zope.schema.Object(IKeyword))
IKeyword.broader = value_type=zope.schema.Object(IKeyword, required=False)

class IKeywords(zope.interface.Interface):

    root = zope.schema.Object(
        IKeyword,
        title=_('Root of ontology'))

    def __getitem__(code):
        """return IKeyword with given code."""

    def find_keywords(searchterm):
        """Returns a list of keywords which contain the searchterm
           string.
        """

class ICommonMetadata(zope.interface.Interface):

    year = zope.schema.Int(
        title=_("Year"),
        min=1900,
        max=2100)

    volume = zope.schema.Int(
        title=_("Volume"),
        min=1,
        max=53)

    page = zope.schema.Int(
        title=_("Page"),
        readonly=True,
        required=False)

    ressort = zope.schema.Choice(
        title=_("Ressort"),
        source=zeit.cms.content.sources.NavigationSource())

    sub_ressort = zope.schema.TextLine(
        title=_('Sub ressort'),
        required=False)

    authors = zope.schema.Tuple(
        title=_("Authors"),
        value_type=zope.schema.TextLine(),
        required=False,
        default=(u'',))

    keywords = zope.schema.Tuple(
        title=_("Keywords"),
        required=False,
        default=(),
        value_type=zope.schema.Object(IKeyword))

    serie = zope.schema.Choice(
        title=_("Serie"),
        source=zeit.cms.content.sources.SerieSource(),
        required=False)

    copyrights = zope.schema.TextLine(
        title=_("Copyright (c)"),
        description=_("Do not enter (c)."),
        default=u"ZEIT online")

    supertitle = zope.schema.TextLine(
        title=_("Kicker"),
        description=_("Please take care of capitalisation."),
        required=False)

    byline = zope.schema.TextLine(
        title=_("By line"),
        required=False)

    title = zope.schema.TextLine(
        title=_("Title"))

    subtitle = zope.schema.Text(
        title=_("Subtitle"),
        required=False)

    teaserTitle = zope.schema.Text(
        title=_("Teaser title"),
        required=False)

    teaserText = zope.schema.Text(
        title=_("Teaser text"),
        required=False,
        max_length=170)

    shortTeaserTitle = zope.schema.Text(
        title=_("Index teaser title"),
        required=False,
        max_length=20)

    shortTeaserText = zope.schema.Text(
        title=_("Index teaser text"),
        required=False,
        max_length=50)

    vg_wort_id = zope.schema.TextLine(
        title=_('VG Wort Id'),
        required=False)

class IDAVPropertyConverter(zope.interface.Interface):
    """Parse a unicode string from a DAV property to a value and vice versa."""

    def fromProperty(value):
        """Convert property value to python value.

        returns python object represented by value.

        raises ValueError if the value could not be converted.
        raises zope.schema.ValidationError if the value could be converted but
        does not satisfy the constraints.

        """

    def toProperty(value):
        """Convert python value to DAV property value.

        returns unicode
        """

class IDAVToken(zope.interface.Interface):
    """A string representing a token that uniquely identifies a value."""

class IDAVPropertyChangedEvent(zope.component.interfaces.IObjectEvent):
    """A dav property has been changed."""

    old_value = zope.interface.Attribute("The value before the change.")
    new_value = zope.interface.Attribute("The value after the change.")

    property_namespace = zope.interface.Attribute("Webdav property namespace.")
    property_name = zope.interface.Attribute("Webdav property name.")

class DAVPropertyChangedEvent(zope.component.interfaces.ObjectEvent):
    zope.interface.implements(IDAVPropertyChangedEvent)

    def __init__(self, object, property_namespace, property_name,
                 old_value, new_value):
        self.object = object
        self.property_namespace = property_namespace
        self.property_name = property_name
        self.old_value = old_value
        self.new_value = new_value

class ITextContent(zope.interface.Interface):
    """Representing text content XXX"""

    data = zope.schema.Text(title=u"Document content")

class IXMLRepresentation(zope.interface.Interface):
    """Objects with an XML representation."""

    xml = zeit.cms.content.field.XMLTree(
        title=_("XML Source"))

class IXMLReference(zope.interface.Interface):
    """XML representation of an object reference.

    Object references are dependent on the target object. For instance a feed
    is referenced with <xi:include> while an image is referenced using <img>.

    """

    xml = zeit.cms.content.field.XMLTree(
        title=u"Reference Structure")

class IXMLSource(zope.interface.Interface):
    """str representing the xml of an object."""

class IXMLContent(zeit.cms.interfaces.ICMSContent, IXMLRepresentation):
    """Content with an XML representation."""

class ILockInfo(zope.app.locking.interfaces.ILockInfo):
    """Extended LockInfo interface."""

    locked_until = zope.schema.Datetime(
        title=u"Locked Until",
        required=False)

class ITemplateManagerContainer(zope.app.container.interfaces.IReadContainer):
    """Container which holds all template managers."""

class ITemplateManager(zope.app.container.interfaces.IReadContainer):
    """Manages templates for a content type."""

class ITemplate(IXMLRepresentation):
    """A template for xml content types."""

    title = zope.schema.TextLine(title=_('Title'))

class IRelatedContent(zope.interface.Interface):
    """Relate other content."""

    related = zope.schema.Tuple(
        title=_("Related content"),
        description=_("Objects that are related to this object."),
        default=(),
        required=False,
        value_type=zope.schema.Choice(
            source=zeit.cms.content.contentsource.cmsContentSource))

class IDAVPropertiesInXML(zope.interface.Interface):
    """Marker interface for objects which store their webdav properties in xml.

    It is common for articles and CPs to store their webdav properties in the
    xml, too. That is in addition to the Metadata stored as webdav properties.

    """

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# blogentry.py
# A controller for BlogEntry, representing every blog entries
#
# Copyright 2010 Atsushi Shibata
#
"""
A controller for BlogEntry, representing every blog entries

$Id: forms.py 649 2010-08-16 07:44:47Z ats $
"""

__author__  = 'Atsushi Shibata <shibata@webcore.co.jp>'
__docformat__ = 'plaintext'
__licence__ = 'MIT'

from datetime import datetime, timedelta

from aha.controller.decorator import authenticate, expose, cache
from aha.modelcontroller.formcontrol import FormControl
FC = FormControl
from aha import Config
config = Config()
from folder import ContainerBase, ContainerEditHandler, ContainerAddHandler
from model import Blog, BlogEntry, BlogCategory, BlogComment, BlogTrackback

class BlogentryEditHandler(ContainerEditHandler):
    FC = FormControl()

    def edit_form(self, parent = None):
        """
        A method to create edit form.
        """
        form = self.get_form('edit', self)
        cf = form['categories']
        blog = self.content.blog_object()
        cf.values = [(x.title, x.category_id)
                    for x in blog.get_categories(end = 100)]
        return form

class BlogcommentAddHandler(ContainerAddHandler):
    FC = FormControl()

    def make_form(self, controller):
        """
        A method to make form.
        """
        config = Config()
        authobj = config.auth_obj()
        user = authobj.get_user(controller)
        form = None
        if controller.content.blog_object().auth_comment and not user:
            form = controller.get_form('comment_add_auth', controller)
        if form == None:
            form = controller.get_form('comment_add', controller)

        form.action = controller.content.get_path()+'/postcomment'
        return form

    @FC.handle_state(FC.INITIAL, FC.FAILURE)
    def show_form(self, controller):
        """
        A method to show add form.
        You must override this method in your subclass.
        """
        controller.objects = controller.form.get_object_tag()
        controller.render(template = '/common/blog/commentform')

    @FC.handle_state(FC.SUCCESS)
    def process_data(self, controller):
        """
        A method to add new data by using posted values.
        You must override this method in your subclass.
        """
        from google.appengine.api import memcache
        from aha import Config
        config = Config()

        v = controller.form.values
        controller.content.add_comment(name = v['author_name'],
                        url = v.get('url', ''), email = v.get('email', ''),
                        title = v['title'], body = v['body'],
                        creator = controller.content.creator)
        memcache.flush_all()

        controller.redirect(config.site_root+controller.content.get_path())

    @FC.handle_validate(FC.INITIAL, FC.FAILURE)
    def do_validate(self, state, controller):
        """
        A validator method for edit transition
        """
        from aha import Config
        config = Config()
        authobj = config.auth_obj()
        user = authobj.get_user(controller)

        if controller.content.blog_object().auth_comment:
            # require authentication
            # when blog.auth_comment = True
            if not user:
                controller.session['referer'] = \
                config.site_root+controller.content.get_path()+'/postcomment'
                controller.session.put()
                authobj.auth_redirect(controller)

        if controller.content.blog_object().auth_comment and user:
            from aha.widget.field import HiddenField
            controller.form['author_name'].args.update(readonly = 'readonly')
            controller.form['author_name'].default = user.get('nickname', '')
            del controller.form['email']
            del controller.form['url']
            controller.form['url'] = \
                    HiddenField(name = 'url', default = user.get('url', ''))

        state = FC.INITIAL
        controller.blog = controller.content.blog_object()
        if controller.request.POST and \
            ('author_name' in controller.request.POST or
             'author_name2' in controller.request.POST):
            if not controller.request.POST:
                return FC.INITIAL
            e = controller.form.validate(dict([(x, controller.request.get(x))
                                    for x in controller.request.arguments()]))
            if e:
                # Some error occured
                return FC.FAILURE
            else:
                return FC.SUCCESS

        return FC.INITIAL

class BlogentryController(ContainerBase):
    """
    The controller for Blog Entry.
    """
    COMMENT_FC = FormControl()
    PAGE_SIZE = 20
    MODEL = BlogEntry
    INDEX_TEMPLATE = '/common/blog/entry'

    @classmethod
    def get_form(self, kind, ins = None):
        """
        A method to return form object based on given kind.
        kind must be one of 'add' or 'edit'
        """
        from aha.widget.form import Form
        from aha.widget.field import (TextField, RichText,
                                        TextArea, CheckboxGroup, SelectField)
        from formencode import validators as v

        class AddForm(Form):
            multipart = True
            form_title = 'Add New Entry'
            button_title = 'Add'
            submit = 'Save'
            name = TextField(title = 'ID', args = {'size':40},
                                validator = v.String(), required = True)
            title = TextField(title = 'Title', args = {'size':40},
                                validator = v.String(), required = True)

            body = RichText(title = 'Body',
                            args = dict(cols = 60, rows = 3, id = 'body'),
                            required = False)
            extend_body = RichText(title = 'Extended body',
                            args = dict(cols = 60, rows = 3, id = 'extended'),
                            collapsable = True)

            categories = CheckboxGroup(title = 'Category(s)',
                            values = ( ('No Category', 0), ),
                            required = True)

            accept_comment = SelectField(title = 'Comments',
                            validator = v.Int(), required = True,
                            values = ( ('Accept', 1),
                                     ('Not Accept, Show', 2),
                                     ('Not Accept, Now Show', 3) ) )
            accept_trackback = SelectField(title = 'Trackbacks',
                            validator = v.Int(), required = True,
                            values = ( ('Accept', 1), ('Not Accept, Show', 2),
                                     ('Not Accept, Now Show', 3) ) )

        class EditForm(AddForm):
            form_title = u'Edit Blog Entry'

        del EditForm['name']

        class AddCommentForm(Form):
            multipart = True
            form_title = 'Post a comment'
            button_title = 'Post a comment'
            submit = 'Post a comment'
            author_name = TextField(title = 'Your Name', args = {'size':20},
                            validator = v.String(), required = True)
            email = TextField(title = 'email', args = {'size':20},
                            validator = v.Email(), required = False)
            url = TextField(title = 'URL', args = {'size':60},
                            validator = v.URL(), required = False)
            title = TextField(title = 'title', args = {'size':60},
                            validator = v.String(), required = True)
            body = TextArea(title = 'Body',
                            args = dict(cols = 60, rows = 3, id = 'body'),
                            required = True)

        class RequireAuth(Form):
            form_title = 'Post a comment'
            button_title = 'Log in(%s) to post a comment' % config.auth_obj.TYPE
            submit = 'Log in(%s) to post a comment' % config.auth_obj.TYPE

        if kind == 'add':
            return AddForm()
        elif kind == 'edit':
            return EditForm()
        elif kind == 'comment_add':
            return AddCommentForm()
        elif kind == 'comment_add_auth':
            return RequireAuth()

    @expose
    @cache()
    def index(self):
        """
        A handler for blog entry.
        """
        self.blog = self.content.blog_object()
        self.get_comment_form = self.make_comment_form
        self.render(template = self.INDEX_TEMPLATE)

    def set_side_menu_items(self):
        """
        A method to obtain menu list.
            each items in list contains title and link in a tuple.
            if tuple has 3 items and the third one is sequence,
                it means menu has sub items.
        """
        _ = self.translate
        c = self.content
        addable = c.ADDABLE[:]
        addable.remove(BlogComment.TYPE)
        addable.remove(BlogTrackback.TYPE)
        self.side_menu_items = (
            ('/style/img/edit_icon.gif', 'Edit Entry',
                                         c.get_path()+'/edit' ),
            ('/style/img/list_icon.gif', 'Show Comment(s)',
                                         c.get_path()+'/comments' ),
            ('/style/img/list_icon.gif', 'Show Trackback(s)',
                                         c.get_path()+'/trackbacks' ),
            ('/style/img/add_icon.gif', 'Add', '#',
                [('/style/img/%s_icon.gif' % x.lower(),
                  x, (c.get_path()+'/add?type = %s')%x)
                        for x in addable],
            )
            )
        if c.get_parent():
            self.side_menu_items = (
                ('', 'Going up', c.get_parent().get_path()+'/list' ),
                )+self.side_menu_items

    edit = expose(authenticate(config.admin_auth)(BlogentryEditHandler()))

    @classmethod
    def make_add_form(cls, parent = None):
        """
        A method to create add form.
        """
        form = cls.get_form('add', self)
        cf = form['categories']
        blog = parent
        cf.values = [(x.title, x.category_id)
                    for x in blog.get_categories(end = 100)]
        form['accept_comment'].default = blog.accept_comment
        form['accept_trackback'].default = blog.accept_trackback
        return form

    #
    # comments
    #

    def make_comment_form(self):
        """
        A method to make form.
        """
        form = None
        if self.content.blog_object().auth_comment:
            form = self.get_form('comment_add_auth', self)
        if form == None:
            form = self.get_form('comment_add', self)

        form.action = self.content.get_path()+'/postcomment'
        return form

    postcomment = BlogcommentAddHandler()

def main(): pass;

if __name__ == '__main__':
    main()
<commit_msg>add functionarity to blog entry controler to determine comming request is from smartphone or not, pass value to template so that it can show different result.<commit_after># -*- coding: utf-8 -*-
#
# blogentry.py
# A controller for BlogEntry, representing every blog entries
#
# Copyright 2010 Atsushi Shibata
#
"""
A controller for BlogEntry, representing every blog entries

$Id: forms.py 649 2010-08-16 07:44:47Z ats $
"""

__author__  = 'Atsushi Shibata <shibata@webcore.co.jp>'
__docformat__ = 'plaintext'
__licence__ = 'MIT'

from datetime import datetime, timedelta

from aha.controller.decorator import authenticate, expose, cache
from aha.modelcontroller.formcontrol import FormControl
FC = FormControl
from aha import Config
config = Config()
from folder import ContainerBase, ContainerEditHandler, ContainerAddHandler
from model import Blog, BlogEntry, BlogCategory, BlogComment, BlogTrackback

from config import check_for_spartphone

class BlogentryEditHandler(ContainerEditHandler):
    FC = FormControl()

    def edit_form(self, parent = None):
        """
        A method to create edit form.
        """
        form = self.get_form('edit', self)
        cf = form['categories']
        blog = self.content.blog_object()
        cf.values = [(x.title, x.category_id)
                    for x in blog.get_categories(end = 100)]
        return form

class BlogcommentAddHandler(ContainerAddHandler):
    FC = FormControl()

    def make_form(self, controller):
        """
        A method to make form.
        """
        config = Config()
        authobj = config.auth_obj()
        user = authobj.get_user(controller)
        form = None
        if controller.content.blog_object().auth_comment and not user:
            form = controller.get_form('comment_add_auth', controller)
        if form == None:
            form = controller.get_form('comment_add', controller)

        form.action = controller.content.get_path()+'/postcomment'
        return form

    @FC.handle_state(FC.INITIAL, FC.FAILURE)
    def show_form(self, controller):
        """
        A method to show add form.
        You must override this method in your subclass.
        """
        controller.objects = controller.form.get_object_tag()
        controller.render(template = '/common/blog/commentform')

    @FC.handle_state(FC.SUCCESS)
    def process_data(self, controller):
        """
        A method to add new data by using posted values.
        You must override this method in your subclass.
        """
        from google.appengine.api import memcache
        from aha import Config
        config = Config()

        v = controller.form.values
        controller.content.add_comment(name = v['author_name'],
                        url = v.get('url', ''), email = v.get('email', ''),
                        title = v['title'], body = v['body'],
                        creator = controller.content.creator)
        memcache.flush_all()

        controller.redirect(config.site_root+controller.content.get_path())

    @FC.handle_validate(FC.INITIAL, FC.FAILURE)
    def do_validate(self, state, controller):
        """
        A validator method for edit transition
        """
        from aha import Config
        config = Config()
        authobj = config.auth_obj()
        user = authobj.get_user(controller)

        if controller.content.blog_object().auth_comment:
            # require authentication
            # when blog.auth_comment = True
            if not user:
                controller.session['referer'] = \
                config.site_root+controller.content.get_path()+'/postcomment'
                controller.session.put()
                authobj.auth_redirect(controller)

        if controller.content.blog_object().auth_comment and user:
            from aha.widget.field import HiddenField
            controller.form['author_name'].args.update(readonly = 'readonly')
            controller.form['author_name'].default = user.get('nickname', '')
            del controller.form['email']
            del controller.form['url']
            controller.form['url'] = \
                    HiddenField(name = 'url', default = user.get('url', ''))

        state = FC.INITIAL
        controller.blog = controller.content.blog_object()
        if controller.request.POST and \
            ('author_name' in controller.request.POST or
             'author_name2' in controller.request.POST):
            if not controller.request.POST:
                return FC.INITIAL
            e = controller.form.validate(dict([(x, controller.request.get(x))
                                    for x in controller.request.arguments()]))
            if e:
                # Some error occured
                return FC.FAILURE
            else:
                return FC.SUCCESS

        return FC.INITIAL

class BlogentryController(ContainerBase):
    """
    The controller for Blog Entry.
    """
    COMMENT_FC = FormControl()
    PAGE_SIZE = 20
    MODEL = BlogEntry
    INDEX_TEMPLATE = '/common/blog/entry'

    @classmethod
    def get_form(self, kind, ins = None):
        """
        A method to return form object based on given kind.
        kind must be one of 'add' or 'edit'
        """
        from aha.widget.form import Form
        from aha.widget.field import (TextField, RichText,
                                        TextArea, CheckboxGroup, SelectField)
        from formencode import validators as v

        class AddForm(Form):
            multipart = True
            form_title = 'Add New Entry'
            button_title = 'Add'
            submit = 'Save'
            name = TextField(title = 'ID', args = {'size':40},
                                validator = v.String(), required = True)
            title = TextField(title = 'Title', args = {'size':40},
                                validator = v.String(), required = True)

            body = RichText(title = 'Body',
                            args = dict(cols = 60, rows = 3, id = 'body'),
                            required = False)
            extend_body = RichText(title = 'Extended body',
                            args = dict(cols = 60, rows = 3, id = 'extended'),
                            collapsable = True)

            categories = CheckboxGroup(title = 'Category(s)',
                            values = ( ('No Category', 0), ),
                            required = True)

            accept_comment = SelectField(title = 'Comments',
                            validator = v.Int(), required = True,
                            values = ( ('Accept', 1),
                                     ('Not Accept, Show', 2),
                                     ('Not Accept, Now Show', 3) ) )
            accept_trackback = SelectField(title = 'Trackbacks',
                            validator = v.Int(), required = True,
                            values = ( ('Accept', 1), ('Not Accept, Show', 2),
                                     ('Not Accept, Now Show', 3) ) )

        class EditForm(AddForm):
            form_title = u'Edit Blog Entry'

        del EditForm['name']

        class AddCommentForm(Form):
            multipart = True
            form_title = 'Post a comment'
            button_title = 'Post a comment'
            submit = 'Post a comment'
            author_name = TextField(title = 'Your Name', args = {'size':20},
                            validator = v.String(), required = True)
            email = TextField(title = 'email', args = {'size':20},
                            validator = v.Email(), required = False)
            url = TextField(title = 'URL', args = {'size':60},
                            validator = v.URL(), required = False)
            title = TextField(title = 'title', args = {'size':60},
                            validator = v.String(), required = True)
            body = TextArea(title = 'Body',
                            args = dict(cols = 60, rows = 3, id = 'body'),
                            required = True)

        class RequireAuth(Form):
            form_title = 'Post a comment'
            button_title = 'Log in(%s) to post a comment' % config.auth_obj.TYPE
            submit = 'Log in(%s) to post a comment' % config.auth_obj.TYPE

        if kind == 'add':
            return AddForm()
        elif kind == 'edit':
            return EditForm()
        elif kind == 'comment_add':
            return AddCommentForm()
        elif kind == 'comment_add_auth':
            return RequireAuth()

    @expose
    @cache()
    def index(self):
        """
        A handler for blog entry.
        """
        self.smartphone = check_for_spartphone(self.request)
        self.blog = self.content.blog_object()
        self.get_comment_form = self.make_comment_form
        self.render(template = self.INDEX_TEMPLATE)

    def set_side_menu_items(self):
        """
        A method to obtain menu list.
            each items in list contains title and link in a tuple.
            if tuple has 3 items and the third one is sequence,
                it means menu has sub items.
        """
        _ = self.translate
        c = self.content
        addable = c.ADDABLE[:]
        addable.remove(BlogComment.TYPE)
        addable.remove(BlogTrackback.TYPE)
        self.side_menu_items = (
            ('/style/img/edit_icon.gif', 'Edit Entry',
                                         c.get_path()+'/edit' ),
            ('/style/img/list_icon.gif', 'Show Comment(s)',
                                         c.get_path()+'/comments' ),
            ('/style/img/list_icon.gif', 'Show Trackback(s)',
                                         c.get_path()+'/trackbacks' ),
            ('/style/img/add_icon.gif', 'Add', '#',
                [('/style/img/%s_icon.gif' % x.lower(),
                  x, (c.get_path()+'/add?type = %s')%x)
                        for x in addable],
            )
            )
        if c.get_parent():
            self.side_menu_items = (
                ('', 'Going up', c.get_parent().get_path()+'/list' ),
                )+self.side_menu_items

    edit = expose(authenticate(config.admin_auth)(BlogentryEditHandler()))

    @classmethod
    def make_add_form(cls, parent = None):
        """
        A method to create add form.
        """
        form = cls.get_form('add', self)
        cf = form['categories']
        blog = parent
        cf.values = [(x.title, x.category_id)
                    for x in blog.get_categories(end = 100)]
        form['accept_comment'].default = blog.accept_comment
        form['accept_trackback'].default = blog.accept_trackback
        return form

    #
    # comments
    #

    def make_comment_form(self):
        """
        A method to make form.
        """
        form = None
        if self.content.blog_object().auth_comment:
            form = self.get_form('comment_add_auth', self)
        if form == None:
            form = self.get_form('comment_add', self)

        form.action = self.content.get_path()+'/postcomment'
        return form

    postcomment = BlogcommentAddHandler()

def main(): pass;

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from pprint import pprint, pformat

from utils.functions.queries import debug_cursor_execute

def insere_hist(
    cursor,
    usuario,
    alteracao,
    ordem_producao,
    ordem_confeccao,
    pedido_destino,
    op_destino,
    oc_destino,
    dep_destino,
    grupo_destino,
    alter_destino,
    sub_destino,
    cor_destino,
):
    field = ':'.join(alteracao.keys())
    field = f":{field}:"
    olds = []
    news = []
    for key in alteracao:
        olds.append(pformat(alteracao[key]['old']))
        news.append(pformat(alteracao[key]['new']))
    old = ':'.join(olds)
    new = ':'.join(news)

    sql = f"""
        INSERT INTO PCPC_044_HIST_DUOMO (
          ORDEM_PRODUCAO
        , ORDEM_CONFECCAO
        , PEDIDO_DESTINO
        , OP_DESTINO
        , OC_DESTINO
        , DEP_DESTINO
        , GRUPO_DESTINO
        , ALTER_DESTINO
        , SUB_DESTINO
        , COR_DESTINO
        , CAMPO_ALTERADO
        , ANTIGO_VALOR
        , NOVO_VALOR
        , USUARIO_SYSTEXTIL
        )
        SELECT
          ORDEM_PRODUCAO
        , ORDEM_CONFECCAO
        , PEDIDO_DESTINO
        , OP_DESTINO
        , OC_DESTINO
        , DEP_DESTINO
        , GRUPO_DESTINO
        , ALTER_DESTINO
        , SUB_DESTINO
        , COR_DESTINO
        , '{field}'
        , '{old}'
        , '{new}'
        , '{usuario}'
        FROM PCPC_044  -- empenhos
        WHERE 1=1
          AND ORDEM_PRODUCAO = {ordem_producao}
          AND ORDEM_CONFECCAO = {ordem_confeccao}
          AND PEDIDO_DESTINO = {pedido_destino}
          AND OP_DESTINO = {op_destino}
          AND OC_DESTINO = {oc_destino}
          AND DEP_DESTINO = {dep_destino}
          AND GRUPO_DESTINO = '{grupo_destino}'
          AND ALTER_DESTINO = {alter_destino}
          AND SUB_DESTINO = '{sub_destino}'
          AND COR_DESTINO = '{cor_destino}'
    """
    try:
        debug_cursor_execute(cursor, sql)
        return True
    except Exception as e:
        return False
<commit_msg>retorna cursor.rowcount ou -1, caso exception<commit_after>from pprint import pprint, pformat

from utils.functions.queries import debug_cursor_execute

def insere_hist(
    cursor,
    usuario,
    alteracao,
    ordem_producao,
    ordem_confeccao,
    pedido_destino,
    op_destino,
    oc_destino,
    dep_destino,
    grupo_destino,
    alter_destino,
    sub_destino,
    cor_destino,
):
    field = ':'.join(alteracao.keys())
    field = f":{field}:"
    olds = []
    news = []
    for key in alteracao:
        olds.append(pformat(alteracao[key]['old']))
        news.append(pformat(alteracao[key]['new']))
    old = ':'.join(olds)
    new = ':'.join(news)

    sql = f"""
        INSERT INTO PCPC_044_HIST_DUOMO (
          ORDEM_PRODUCAO
        , ORDEM_CONFECCAO
        , PEDIDO_DESTINO
        , OP_DESTINO
        , OC_DESTINO
        , DEP_DESTINO
        , GRUPO_DESTINO
        , ALTER_DESTINO
        , SUB_DESTINO
        , COR_DESTINO
        , CAMPO_ALTERADO
        , ANTIGO_VALOR
        , NOVO_VALOR
        , USUARIO_SYSTEXTIL
        )
        SELECT
          ORDEM_PRODUCAO
        , ORDEM_CONFECCAO
        , PEDIDO_DESTINO
        , OP_DESTINO
        , OC_DESTINO
        , DEP_DESTINO
        , GRUPO_DESTINO
        , ALTER_DESTINO
        , SUB_DESTINO
        , COR_DESTINO
        , '{field}'
        , '{old}'
        , '{new}'
        , '{usuario}'
        FROM PCPC_044  -- empenhos
        WHERE 1=1
          AND ORDEM_PRODUCAO = {ordem_producao}
          AND ORDEM_CONFECCAO = {ordem_confeccao}
          AND PEDIDO_DESTINO = {pedido_destino}
          AND OP_DESTINO = {op_destino}
          AND OC_DESTINO = {oc_destino}
          AND DEP_DESTINO = {dep_destino}
          AND GRUPO_DESTINO = '{grupo_destino}'
          AND ALTER_DESTINO = {alter_destino}
          AND SUB_DESTINO = '{sub_destino}'
          AND COR_DESTINO = '{cor_destino}'
    """
    try:
        debug_cursor_execute(cursor, sql)
        return cursor.rowcount
    except Exception as e:
        return -1
<|endoftext|>


<commit_before>from contentbase.util import get_root_request
from elasticsearch.helpers import scan
from pyramid.threadlocal import get_current_request
from zope.interface import alsoProvides
from .interfaces import (
    ELASTIC_SEARCH,
    ICachedItem,
)

def includeme(config):
    from contentbase import STORAGE
    registry = config.registry
    es = registry[ELASTIC_SEARCH]
    es_index = registry.settings['contentbase.elasticsearch.index']
    wrapped_storage = registry[STORAGE]
    registry[STORAGE] = PickStorage(ElasticSearchStorage(es, es_index), wrapped_storage)

class CachedModel(object):
    def __init__(self, hit):
        self.hit = hit
        self.source = hit['_source']

    @property
    def item_type(self):
        return self.source['item_type']

    @property
    def properties(self):
        return self.source['properties']

    @property
    def propsheets(self):
        return self.source['propsheets']

    @property
    def uuid(self):
        return self.source['uuid']

    @property
    def tid(self):
        return self.source['tid']

    def invalidated(self):
        request = get_root_request()
        if request is None:
            return False
        edits = dict.get(request.session, 'edits', None)
        if edits is None:
            return False
        version = self.hit['_version']
        source = self.source
        linked_uuids = set(source['linked_uuids'])
        embedded_uuids = set(source['embedded_uuids'])
        for xid, updated, linked in edits:
            if xid < version:
                continue
            if not embedded_uuids.isdisjoint(updated):
                return True
            if not linked_uuids.isdisjoint(linked):
                return True
        return False

    def used_for(self, item):
        alsoProvides(item, ICachedItem)

class PickStorage(object):
    def __init__(self, read, write):
        self.read = read
        self.write = write

    def storage(self):
        request = get_current_request()
        if request and request.datastore == 'elasticsearch':
            return self.read
        return self.write

    def get_by_uuid(self, uuid):
        storage = self.storage()
        model = storage.get_by_uuid(uuid)
        if storage is self.read:
            if model is None or model.invalidated():
                return self.write.get_by_uuid(uuid)
        return model

    def get_by_unique_key(self, unique_key, name):
        storage = self.storage()
        model = storage.get_by_unique_key(unique_key, name)
        if storage is self.read:
            if model is None or model.invalidated():
                return self.write.get_by_unique_key(unique_key, name)
        return model

    def get_rev_links(self, model, rel, *item_types):
        return self.storage().get_rev_links(model, rel, *item_types)

    def __iter__(self, *item_types):
        return self.storage().__iter__(*item_types)

    def __len__(self, *item_types):
        return self.storage().__len__(*item_types)

    def create(self, item_type, uuid):
        return self.write.create(item_type, uuid)

    def update(self, model, properties=None, sheets=None, unique_keys=None, links=None):
        return self.write.update(model, properties, sheets, unique_keys, links)

class ElasticSearchStorage(object):
    writeable = False

    def __init__(self, es, index):
        self.es = es
        self.index = index

    def _one(self, query):
        data = self.es.search(index=self.index, body=query)
        hits = data['hits']['hits']
        if len(hits) != 1:
            return None
        model = CachedModel(hits[0])
        return model

    def get_by_uuid(self, uuid):
        query = {
            'filter': {'term': {'uuid': uuid}},
            'version': True,
        }
        return self._one(query)

    def get_by_unique_key(self, unique_key, name):
        term = 'unique_keys.' + unique_key
        query = {
            'filter': {'term': {term: name}},
            'version': True,
        }
        return self._one(query)

    def get_rev_links(self, model, rel, *item_types):
        filter_ = {'term': {'links.' + rel: str(model.uuid)}}
        if item_types:
            filter_ = {'and': [
                filter_,
                {'terms': {'item_type': item_types}},
            ]}
        query = {
            'fields': ['uuid'],
            'filter': filter_,
            'version': True,
        }
        data = self.es.search(index=self.index, body=query)
        return [
            hit['fields']['uuid'][0] for hit in data['hits']['hits']
        ]

    def __iter__(self, *item_types):
        query = {
            'fields': ['uuid'],
            'filter': {'terms': {'item_type': item_types}} if item_types else {'match_all': {}},
        }
        for hit in scan(self.es, query=query):
            yield hit['fields']['uuid'][0]

    def __len__(self, *item_types):
        query = {
            'filter': {'terms': {'item_type': item_types}} if item_types else {'match_all': {}},
        }
        return self.es.count(index=self.index, body=query)
<commit_msg>Avoid truncating reverse links to default elasticsearch result size.<commit_after>from contentbase.util import get_root_request
from elasticsearch.helpers import scan
from pyramid.threadlocal import get_current_request
from zope.interface import alsoProvides
from .interfaces import (
    ELASTIC_SEARCH,
    ICachedItem,
)

SEARCH_MAX = (2 ** 31) - 1

def includeme(config):
    from contentbase import STORAGE
    registry = config.registry
    es = registry[ELASTIC_SEARCH]
    es_index = registry.settings['contentbase.elasticsearch.index']
    wrapped_storage = registry[STORAGE]
    registry[STORAGE] = PickStorage(ElasticSearchStorage(es, es_index), wrapped_storage)

class CachedModel(object):
    def __init__(self, hit):
        self.hit = hit
        self.source = hit['_source']

    @property
    def item_type(self):
        return self.source['item_type']

    @property
    def properties(self):
        return self.source['properties']

    @property
    def propsheets(self):
        return self.source['propsheets']

    @property
    def uuid(self):
        return self.source['uuid']

    @property
    def tid(self):
        return self.source['tid']

    def invalidated(self):
        request = get_root_request()
        if request is None:
            return False
        edits = dict.get(request.session, 'edits', None)
        if edits is None:
            return False
        version = self.hit['_version']
        source = self.source
        linked_uuids = set(source['linked_uuids'])
        embedded_uuids = set(source['embedded_uuids'])
        for xid, updated, linked in edits:
            if xid < version:
                continue
            if not embedded_uuids.isdisjoint(updated):
                return True
            if not linked_uuids.isdisjoint(linked):
                return True
        return False

    def used_for(self, item):
        alsoProvides(item, ICachedItem)

class PickStorage(object):
    def __init__(self, read, write):
        self.read = read
        self.write = write

    def storage(self):
        request = get_current_request()
        if request and request.datastore == 'elasticsearch':
            return self.read
        return self.write

    def get_by_uuid(self, uuid):
        storage = self.storage()
        model = storage.get_by_uuid(uuid)
        if storage is self.read:
            if model is None or model.invalidated():
                return self.write.get_by_uuid(uuid)
        return model

    def get_by_unique_key(self, unique_key, name):
        storage = self.storage()
        model = storage.get_by_unique_key(unique_key, name)
        if storage is self.read:
            if model is None or model.invalidated():
                return self.write.get_by_unique_key(unique_key, name)
        return model

    def get_rev_links(self, model, rel, *item_types):
        return self.storage().get_rev_links(model, rel, *item_types)

    def __iter__(self, *item_types):
        return self.storage().__iter__(*item_types)

    def __len__(self, *item_types):
        return self.storage().__len__(*item_types)

    def create(self, item_type, uuid):
        return self.write.create(item_type, uuid)

    def update(self, model, properties=None, sheets=None, unique_keys=None, links=None):
        return self.write.update(model, properties, sheets, unique_keys, links)

class ElasticSearchStorage(object):
    writeable = False

    def __init__(self, es, index):
        self.es = es
        self.index = index

    def _one(self, query):
        data = self.es.search(index=self.index, body=query)
        hits = data['hits']['hits']
        if len(hits) != 1:
            return None
        model = CachedModel(hits[0])
        return model

    def get_by_uuid(self, uuid):
        query = {
            'filter': {'term': {'uuid': uuid}},
            'version': True,
        }
        return self._one(query)

    def get_by_unique_key(self, unique_key, name):
        term = 'unique_keys.' + unique_key
        query = {
            'filter': {'term': {term: name}},
            'version': True,
        }
        return self._one(query)

    def get_rev_links(self, model, rel, *item_types):
        filter_ = {'term': {'links.' + rel: str(model.uuid)}}
        if item_types:
            filter_ = {'and': [
                filter_,
                {'terms': {'item_type': item_types}},
            ]}
        query = {
            'fields': ['uuid'],
            'filter': filter_,
            'version': True,
        }
        data = self.es.search(index=self.index, body=query, size=SEARCH_MAX)
        return [
            hit['fields']['uuid'][0] for hit in data['hits']['hits']
        ]

    def __iter__(self, *item_types):
        query = {
            'fields': ['uuid'],
            'filter': {'terms': {'item_type': item_types}} if item_types else {'match_all': {}},
        }
        for hit in scan(self.es, query=query):
            yield hit['fields']['uuid'][0]

    def __len__(self, *item_types):
        query = {
            'filter': {'terms': {'item_type': item_types}} if item_types else {'match_all': {}},
        }
        return self.es.count(index=self.index, body=query)
<|endoftext|>