<commit_before>from __future__ import unicode_literals

import time
import random
import statistics
from autobahn.twisted.websocket import WebSocketClientProtocol, \
    WebSocketClientFactory

stats = {}

class MyClientProtocol(WebSocketClientProtocol):

    def onConnect(self, response):
        self.opened = time.time()
        self.sent = 0
        self.last_send = None
        self.received = 0
        self.corrupted = 0
        self.out_of_order = 0
        self.latencies = []
        self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
        stats[self.fingerprint] = {}

    def onOpen(self):
        def hello():
            if self.last_send is None:
                if self.sent >= self.factory.num_messages:
                    self.sendClose()
                    return
                self.last_send = time.time()
                self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii"))
                self.sent += 1
            else:
                # Wait for receipt of ping
                pass
            self.factory.reactor.callLater(1.0 / self.factory.message_rate, hello)
        hello()

    def onMessage(self, payload, isBinary):
        # Detect receive-before-send
        if self.last_send is None:
            self.corrupted += 1
            print("CRITICAL: Socket %s received before sending: %s" % (self.fingerprint, payload))
            return
        num, fingerprint = payload.decode("ascii").split(":")
        if fingerprint != self.fingerprint:
            self.corrupted += 1
        try:
            if int(num) != self.received:
                self.out_of_order += 1
        except ValueError:
            self.corrupted += 1
        self.latencies.append(time.time() - self.last_send)
        self.received += 1
        self.last_send = None

    def onClose(self, wasClean, code, reason):
        if hasattr(self, "sent"):
            stats[self.fingerprint] = {
                "sent": self.sent,
                "received": self.received,
                "corrupted": self.corrupted,
                "out_of_order": self.out_of_order,
                "latencies": self.latencies,
                "connect": True,
            }
        else:
            self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
            stats[self.fingerprint] = {
                "sent": 0,
                "received": 0,
                "corrupted": 0,
                "out_of_order": 0,
                "connect": False,
            }

class Benchmarker(object):
    """
    Performs benchmarks against WebSockets.
    """

    def __init__(self, url, num, concurrency, rate, messages):
        self.url = url
        self.num = num
        self.concurrency = concurrency
        self.rate = rate
        self.messages = messages
        self.factory = WebSocketClientFactory(
            args.url,
        )
        self.factory.protocol = MyClientProtocol
        self.factory.num_messages = self.messages
        self.factory.message_rate = self.rate

    def loop(self):
        self.spawn_loop()
        self.progress_loop()

    def spawn_loop(self):
        self.spawn_connections()
        reactor.callLater(0.01, self.spawn_loop)

    def progress_loop(self):
        self.print_progress()
        reactor.callLater(1, self.progress_loop)

    def spawn_connections(self):
        # Stop spawning if we did the right total number
        max_to_spawn = self.num - len(stats)
        if max_to_spawn <= 0:
            return
        # Decode connection args
        host, port = self.url.split("://")[1].split(":")
        port = int(port)
        # Only spawn enough to get up to concurrency
        open_protocols = len([x for x in stats.values() if not x])
        to_spawn = min(max(self.concurrency - open_protocols, 0), max_to_spawn)
        for _ in range(to_spawn):
            reactor.connectTCP(host, port, self.factory)

    def print_progress(self):
        open_protocols = len([x for x in stats.values() if not x])
        print("%s open, %s total" % (
            open_protocols,
            len(stats),
        ))
        if open_protocols == 0 and len(stats) >= self.num:
            reactor.stop()
            self.print_stats()

    def percentile(self, values, fraction):
        """
        Returns a percentile value (e.g. fraction = 0.95 -> 95th percentile)
        """
        values = sorted(values)
        stopat = int(len(values) * fraction)
        if stopat == len(values):
            stopat -= 1
        return values[stopat]

    def print_stats(self):
        # Collect stats together
        latencies = []
        num_good = 0
        num_incomplete = 0
        num_failed = 0
        num_corruption = 0
        num_out_of_order = 0
        for entry in stats.values():
            latencies.extend(entry.get("latencies", []))
            if not entry['connect']:
                num_failed += 1
            elif entry['sent'] != entry['received']:
                num_incomplete += 1
            elif entry['corrupted']:
                num_corruption += 1
            elif entry['out_of_order']:
                num_out_of_order += 1
            else:
                num_good += 1
        # Some analysis on latencies
        latency_mean = statistics.mean(latencies)
        latency_median = statistics.median(latencies)
        latency_stdev = statistics.stdev(latencies)
        latency_95 = self.percentile(latencies, 0.95)
        latency_99 = self.percentile(latencies, 0.99)
        # Print results
        print("-------")
        print("Sockets opened: %s" % len(stats))
        print("Latency stats: Mean %.3fs  Median %.3fs  Stdev %.3f  95%% %.3fs  95%% %.3fs" % (
            latency_mean,
            latency_median,
            latency_stdev,
            latency_95,
            latency_99,
        ))
        print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100))
        print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100))
        print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100))
        print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100))
        print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100))

if __name__ == '__main__':

    import sys
    import argparse

    from twisted.python import log
    from twisted.internet import reactor

#    log.startLogging(sys.stdout)

    parser = argparse.ArgumentParser()
    parser.add_argument("url")
    parser.add_argument("-n", "--num", type=int, default=100, help="Total number of sockets to open")
    parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once")
    parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second")
    parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close")
    args = parser.parse_args()

    benchmarker = Benchmarker(
        url=args.url,
        num=args.num,
        concurrency=args.concurrency,
        rate=args.rate,
        messages=args.messages,
    )
    benchmarker.loop()
    reactor.run()
<commit_msg>Don't let benchmarker overshoot concurrency<commit_after>from __future__ import unicode_literals

import time
import random
import statistics
from autobahn.twisted.websocket import WebSocketClientProtocol, \
    WebSocketClientFactory

stats = {}

class MyClientProtocol(WebSocketClientProtocol):

    def onConnect(self, response):
        self.opened = time.time()
        self.sent = 0
        self.last_send = None
        self.received = 0
        self.corrupted = 0
        self.out_of_order = 0
        self.latencies = []
        self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
        stats[self.fingerprint] = {}

    def onOpen(self):
        def hello():
            if self.last_send is None:
                if self.sent >= self.factory.num_messages:
                    self.sendClose()
                    return
                self.last_send = time.time()
                self.sendMessage(("%s:%s" % (self.sent, self.fingerprint)).encode("ascii"))
                self.sent += 1
            else:
                # Wait for receipt of ping
                pass
            self.factory.reactor.callLater(1.0 / self.factory.message_rate, hello)
        hello()

    def onMessage(self, payload, isBinary):
        # Detect receive-before-send
        if self.last_send is None:
            self.corrupted += 1
            print("CRITICAL: Socket %s received before sending: %s" % (self.fingerprint, payload))
            return
        num, fingerprint = payload.decode("ascii").split(":")
        if fingerprint != self.fingerprint:
            self.corrupted += 1
        try:
            if int(num) != self.received:
                self.out_of_order += 1
        except ValueError:
            self.corrupted += 1
        self.latencies.append(time.time() - self.last_send)
        self.received += 1
        self.last_send = None

    def onClose(self, wasClean, code, reason):
        if hasattr(self, "sent"):
            stats[self.fingerprint] = {
                "sent": self.sent,
                "received": self.received,
                "corrupted": self.corrupted,
                "out_of_order": self.out_of_order,
                "latencies": self.latencies,
                "connect": True,
            }
        else:
            self.fingerprint = "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for i in range(16))
            stats[self.fingerprint] = {
                "sent": 0,
                "received": 0,
                "corrupted": 0,
                "out_of_order": 0,
                "connect": False,
            }

class Benchmarker(object):
    """
    Performs benchmarks against WebSockets.
    """

    def __init__(self, url, num, concurrency, rate, messages):
        self.url = url
        self.num = num
        self.concurrency = concurrency
        self.rate = rate
        self.messages = messages
        self.factory = WebSocketClientFactory(
            args.url,
        )
        self.factory.protocol = MyClientProtocol
        self.factory.num_messages = self.messages
        self.factory.message_rate = self.rate

    def loop(self):
        self.spawn_loop()
        self.progress_loop()

    def spawn_loop(self):
        self.spawn_connections()
        reactor.callLater(0.1, self.spawn_loop)

    def progress_loop(self):
        self.print_progress()
        reactor.callLater(1, self.progress_loop)

    def spawn_connections(self):
        # Stop spawning if we did the right total number
        max_to_spawn = self.num - len(stats)
        if max_to_spawn <= 0:
            return
        # Decode connection args
        host, port = self.url.split("://")[1].split(":")
        port = int(port)
        # Only spawn enough to get up to concurrency
        open_protocols = len([x for x in stats.values() if not x])
        to_spawn = min(max(self.concurrency - open_protocols, 0), max_to_spawn)
        for _ in range(to_spawn):
            reactor.connectTCP(host, port, self.factory)

    def print_progress(self):
        open_protocols = len([x for x in stats.values() if not x])
        print("%s open, %s total" % (
            open_protocols,
            len(stats),
        ))
        if open_protocols == 0 and len(stats) >= self.num:
            reactor.stop()
            self.print_stats()

    def percentile(self, values, fraction):
        """
        Returns a percentile value (e.g. fraction = 0.95 -> 95th percentile)
        """
        values = sorted(values)
        stopat = int(len(values) * fraction)
        if stopat == len(values):
            stopat -= 1
        return values[stopat]

    def print_stats(self):
        # Collect stats together
        latencies = []
        num_good = 0
        num_incomplete = 0
        num_failed = 0
        num_corruption = 0
        num_out_of_order = 0
        for entry in stats.values():
            latencies.extend(entry.get("latencies", []))
            if not entry['connect']:
                num_failed += 1
            elif entry['sent'] != entry['received']:
                num_incomplete += 1
            elif entry['corrupted']:
                num_corruption += 1
            elif entry['out_of_order']:
                num_out_of_order += 1
            else:
                num_good += 1
        # Some analysis on latencies
        latency_mean = statistics.mean(latencies)
        latency_median = statistics.median(latencies)
        latency_stdev = statistics.stdev(latencies)
        latency_95 = self.percentile(latencies, 0.95)
        latency_99 = self.percentile(latencies, 0.99)
        # Print results
        print("-------")
        print("Sockets opened: %s" % len(stats))
        print("Latency stats: Mean %.3fs  Median %.3fs  Stdev %.3f  95%% %.3fs  95%% %.3fs" % (
            latency_mean,
            latency_median,
            latency_stdev,
            latency_95,
            latency_99,
        ))
        print("Good sockets: %s (%.2f%%)" % (num_good, (float(num_good) / len(stats))*100))
        print("Incomplete sockets: %s (%.2f%%)" % (num_incomplete, (float(num_incomplete) / len(stats))*100))
        print("Corrupt sockets: %s (%.2f%%)" % (num_corruption, (float(num_corruption) / len(stats))*100))
        print("Out of order sockets: %s (%.2f%%)" % (num_out_of_order, (float(num_out_of_order) / len(stats))*100))
        print("Failed to connect: %s (%.2f%%)" % (num_failed, (float(num_failed) / len(stats))*100))

if __name__ == '__main__':

    import sys
    import argparse

    from twisted.python import log
    from twisted.internet import reactor

#    log.startLogging(sys.stdout)

    parser = argparse.ArgumentParser()
    parser.add_argument("url")
    parser.add_argument("-n", "--num", type=int, default=100, help="Total number of sockets to open")
    parser.add_argument("-c", "--concurrency", type=int, default=10, help="Number of sockets to open at once")
    parser.add_argument("-r", "--rate", type=float, default=1, help="Number of messages to send per socket per second")
    parser.add_argument("-m", "--messages", type=int, default=5, help="Number of messages to send per socket before close")
    args = parser.parse_args()

    benchmarker = Benchmarker(
        url=args.url,
        num=args.num,
        concurrency=args.concurrency,
        rate=args.rate,
        messages=args.messages,
    )
    benchmarker.loop()
    reactor.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import datetime
import os
import shutil
import sys
import glob
from optparse import OptionParser
from subprocess import Popen, PIPE

from xhtml2pdf import pisa

def render_pdf(filename, output_dir, options):
    if options.debug:
        print 'Rendering %s' % filename
    basename = os.path.basename(filename)
    outname = '%s.pdf' % os.path.splitext(basename)[0]
    outfile = os.path.join(output_dir, outname)

    input = open(filename, 'rb')
    output = open(outfile, 'wb')

    result = pisa.pisaDocument(input, output, path=filename)

    input.close()
    output.close()

    if result.err:
        print 'Error rendering %s: %s' % (filename, result.err)
        sys.exit(1)
    return outfile

def convert_to_png(infile, output_dir, options):
    if options.debug:
        print 'Converting %s to PNG' % infile
    basename = os.path.basename(infile)
    filename = os.path.splitext(basename)[0]
    outname = '%s.page%%0d.png' % filename
    globname = '%s.page*.png' % filename
    outfile = os.path.join(output_dir, outname)
    exec_cmd(options, options.convert_cmd, '-density', '150', infile, outfile)
    outfiles = glob.glob(os.path.join(output_dir, globname))
    outfiles.sort()
    return outfiles

def create_diff_image(srcfile1, srcfile2, output_dir, options):
    if options.debug:
        print 'Creating difference image for %s and %s' % (srcfile1, srcfile2)

    outname = '%s.diff%s' % os.path.splitext(srcfile1)
    outfile = os.path.join(output_dir, outname)
    _, result = exec_cmd(options, options.compare_cmd, '-metric', 'ae', srcfile1, srcfile2, '-lowlight-color', 'white', outfile)
    diff_value = int(result.strip())
    if diff_value > 0:
        if not options.quiet:
            print 'Image %s differs from reference, value is %i' % (srcfile1, diff_value)
    return outfile, diff_value

def copy_ref_image(srcname, output_dir, options):
    if options.debug:
        print 'Copying reference image %s ' % srcname
    dstname = os.path.basename(srcname)
    dstfile = os.path.join(output_dir, '%s.ref%s' % os.path.splitext(dstname))
    shutil.copyfile(srcname, dstfile)
    return dstfile

def create_thumbnail(filename, options):
    thumbfile = '%s.thumb%s' % os.path.splitext(filename)
    if options.debug:
        print 'Creating thumbnail of %s' % filename
    exec_cmd(options, options.convert_cmd, '-resize', '20%', filename, thumbfile)
    return thumbfile

def render_file(filename, output_dir, ref_dir, options):
    if not options.quiet:
        print 'Rendering %s' % filename
    pdf = render_pdf(filename, output_dir, options)
    pngs = convert_to_png(pdf, output_dir, options)
    if options.create_reference:
        return
    thumbs = [create_thumbnail(png, options) for png in pngs]
    pages = [{'png': p, 'png_thumb': thumbs[i]}
             for i,p in enumerate(pngs)]
    diff_count = 0
    if not options.no_compare:
        for page in pages:
            refsrc = os.path.join(ref_dir, os.path.basename(page['png']))
            if not os.path.isfile(refsrc):
                print 'Reference image for %s not found!' % page['png']
                continue
            page['ref'] = copy_ref_image(refsrc, output_dir, options)
            page['ref_thumb'] = create_thumbnail(page['ref'], options)
            page['diff'], page['diff_value'] = \
                    create_diff_image(page['png'], page['ref'],
                                      output_dir, options)
            page['diff_thumb'] = create_thumbnail(page['diff'], options)
            if page['diff_value']:
                diff_count += 1
    return pdf, pages, diff_count

def exec_cmd(options, *args):
    if options.debug:
        print 'Executing %s' % ' '.join(args)
    proc = Popen(args, stdout=PIPE, stderr=PIPE)
    result = proc.communicate()
    if options.debug:
        print result[0], result[1]
    if proc.returncode:
        print 'exec error (%i): %s' % (proc.returncode, result[1])
        sys.exit(1)
    return result[0], result[1]

def create_html_file(results, template_file, output_dir, options):
    html = []
    for pdf, pages, diff_count in results:
        if options.only_errors and not diff_count:
            continue
        pdfname = os.path.basename(pdf)
        html.append('
\n'
                    '
%(pdf)s
\n'
                    % {'pdf': pdfname})
        for i, page in enumerate(pages):
            vars = dict(((k, os.path.basename(v)) for k,v in page.items()
                         if k != 'diff_value'))
            vars['diff_value'] = page['diff_value']
            vars['page'] = i+1
            if 'diff' in page:
                if vars['diff_value']:
                    vars['class'] = 'result-page-diff error'
                else:
                    if options.only_errors:
                        continue
                    vars['class'] = 'result-page-diff'
                html.append('
\n'
                            '
Page %(page)i
\n'

                            '
\n'
                            '
Difference '
                            '(Score %(diff_value)i)
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n'
                            '
Rendered
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n'
                            '
Reference
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n' % vars)
            else:
                html.append('
\n'
                           '
Page %(page)i
\n'

                           '
\n'
                           ''
                           '\n'
                           '
\n'

                           '
\n' % vars)
        html.append('
\n\n')

    now = datetime.datetime.now()
    title = 'xhtml2pdf Test Rendering Results, %s' % now.strftime('%c')
    template = open(template_file, 'rb').read()
    template = template.replace('%%TITLE%%', title)
    template = template.replace('%%RESULTS%%', '\n'.join(html))

    htmlfile = os.path.join(output_dir, 'index.html')
    outfile = open(htmlfile, 'wb')
    outfile.write(template)
    outfile.close()
    return htmlfile

def main():
    options, args = parser.parse_args()

    base_dir = os.path.abspath(os.path.join(__file__, os.pardir))
    source_dir = os.path.join(base_dir, options.source_dir)
    if options.create_reference is not None:
        output_dir = os.path.join(base_dir, options.create_reference)
    else:
        output_dir = os.path.join(base_dir, options.output_dir)
    template_file = os.path.join(base_dir, options.html_template)
    ref_dir = os.path.join(base_dir, options.ref_dir)

    if os.path.isdir(output_dir):
        shutil.rmtree(output_dir)
    os.makedirs(output_dir)

    results = []
    diff_count = 0
    if len(args) == 0:
        files = glob.glob(os.path.join(source_dir, '*.html'))
    else:
        files = [os.path.join(source_dir, arg) for arg in args]
    for filename in files:
        pdf, pages, diff = render_file(filename, output_dir, ref_dir, options)
        diff_count += diff
        results.append((pdf, pages, diff))

    num = len(results)

    if options.create_reference is not None:
        print 'Create reference for %i file%s' % (num, '' if num == 1 else 's')
    else:
        htmlfile = create_html_file(results, template_file, output_dir, options)
        if not options.quiet:
            print 'Rendered %i file%s' % (num, '' if num == 1 else 's')
            print '%i file%s differ%s from reference' % \
                    (diff_count, diff_count != 1 and 's' or '',
                     diff_count == 1 and 's' or '')
            print 'Check %s for results' % htmlfile
        if diff_count:
            sys.exit(1)

parser = OptionParser(
    usage='rendertest.py [options] [source_file] [source_file] ...',
    description='Renders a single html source file or all files in the data '
    'directory, converts them to PNG format and prepares a result '
    'HTML file for comparing the output with an expected result')
parser.add_option('-s', '--source-dir', dest='source_dir', default='data/source',
                  help=('Path to directory containing the html source files'))
parser.add_option('-o', '--output-dir', dest='output_dir', default='output',
                  help='Path to directory for output files. CAREFUL: this '
                  'directory will be deleted and recreated before rendering!')
parser.add_option('-r', '--ref-dir', dest='ref_dir', default='data/reference',
                  help='Path to directory containing the reference images '
                  'to compare the result with')
parser.add_option('-t', '--template', dest='html_template',
                  default='data/template.html', help='Name of HTML template file')
parser.add_option('-e', '--only-errors', dest='only_errors', action='store_true',
                  default=False, help='Only include images in HTML file which '
                  'differ from reference')
parser.add_option('-q', '--quiet', dest='quiet', action='store_true',
                  default=False, help='Try to be quiet')
parser.add_option('--no-compare', dest='no_compare', action='store_true',
                  default=False, help='Do not compare with reference image, '
                  'only render to png')
parser.add_option('-c', '--create-reference', dest='create_reference',
                  metavar='DIR',
                  default=None, help='Do not output anything, render source to '
                  'specified directory for reference. CAREFUL: this directory '
                 'will be deleted and recreated before rendering!')
parser.add_option('--debug', dest='debug', action='store_true',
                  default=False, help='More output for debugging')
parser.add_option('--convert-cmd', dest='convert_cmd', default='/usr/bin/convert',
                  help='Path to ImageMagick "convert" tool')
parser.add_option('--compare-cmd', dest='compare_cmd', default='/usr/bin/compare',
                  help='Path to ImageMagick "compare" tool')

if __name__ == '__main__':
    main()
<commit_msg>Small bugfix for images without ref image.<commit_after>#!/usr/bin/env python
import datetime
import os
import shutil
import sys
import glob
from optparse import OptionParser
from subprocess import Popen, PIPE

from xhtml2pdf import pisa

def render_pdf(filename, output_dir, options):
    if options.debug:
        print 'Rendering %s' % filename
    basename = os.path.basename(filename)
    outname = '%s.pdf' % os.path.splitext(basename)[0]
    outfile = os.path.join(output_dir, outname)

    input = open(filename, 'rb')
    output = open(outfile, 'wb')

    result = pisa.pisaDocument(input, output, path=filename)

    input.close()
    output.close()

    if result.err:
        print 'Error rendering %s: %s' % (filename, result.err)
        sys.exit(1)
    return outfile

def convert_to_png(infile, output_dir, options):
    if options.debug:
        print 'Converting %s to PNG' % infile
    basename = os.path.basename(infile)
    filename = os.path.splitext(basename)[0]
    outname = '%s.page%%0d.png' % filename
    globname = '%s.page*.png' % filename
    outfile = os.path.join(output_dir, outname)
    exec_cmd(options, options.convert_cmd, '-density', '150', infile, outfile)
    outfiles = glob.glob(os.path.join(output_dir, globname))
    outfiles.sort()
    return outfiles

def create_diff_image(srcfile1, srcfile2, output_dir, options):
    if options.debug:
        print 'Creating difference image for %s and %s' % (srcfile1, srcfile2)

    outname = '%s.diff%s' % os.path.splitext(srcfile1)
    outfile = os.path.join(output_dir, outname)
    _, result = exec_cmd(options, options.compare_cmd, '-metric', 'ae', srcfile1, srcfile2, '-lowlight-color', 'white', outfile)
    diff_value = int(result.strip())
    if diff_value > 0:
        if not options.quiet:
            print 'Image %s differs from reference, value is %i' % (srcfile1, diff_value)
    return outfile, diff_value

def copy_ref_image(srcname, output_dir, options):
    if options.debug:
        print 'Copying reference image %s ' % srcname
    dstname = os.path.basename(srcname)
    dstfile = os.path.join(output_dir, '%s.ref%s' % os.path.splitext(dstname))
    shutil.copyfile(srcname, dstfile)
    return dstfile

def create_thumbnail(filename, options):
    thumbfile = '%s.thumb%s' % os.path.splitext(filename)
    if options.debug:
        print 'Creating thumbnail of %s' % filename
    exec_cmd(options, options.convert_cmd, '-resize', '20%', filename, thumbfile)
    return thumbfile

def render_file(filename, output_dir, ref_dir, options):
    if not options.quiet:
        print 'Rendering %s' % filename
    pdf = render_pdf(filename, output_dir, options)
    pngs = convert_to_png(pdf, output_dir, options)
    if options.create_reference:
        return
    thumbs = [create_thumbnail(png, options) for png in pngs]
    pages = [{'png': p, 'png_thumb': thumbs[i]}
             for i,p in enumerate(pngs)]
    diff_count = 0
    if not options.no_compare:
        for page in pages:
            refsrc = os.path.join(ref_dir, os.path.basename(page['png']))
            if not os.path.isfile(refsrc):
                print 'Reference image for %s not found!' % page['png']
                continue
            page['ref'] = copy_ref_image(refsrc, output_dir, options)
            page['ref_thumb'] = create_thumbnail(page['ref'], options)
            page['diff'], page['diff_value'] = \
                    create_diff_image(page['png'], page['ref'],
                                      output_dir, options)
            page['diff_thumb'] = create_thumbnail(page['diff'], options)
            if page['diff_value']:
                diff_count += 1
    return pdf, pages, diff_count

def exec_cmd(options, *args):
    if options.debug:
        print 'Executing %s' % ' '.join(args)
    proc = Popen(args, stdout=PIPE, stderr=PIPE)
    result = proc.communicate()
    if options.debug:
        print result[0], result[1]
    if proc.returncode:
        print 'exec error (%i): %s' % (proc.returncode, result[1])
        sys.exit(1)
    return result[0], result[1]

def create_html_file(results, template_file, output_dir, options):
    html = []
    for pdf, pages, diff_count in results:
        if options.only_errors and not diff_count:
            continue
        pdfname = os.path.basename(pdf)
        html.append('
\n'
                    '
%(pdf)s
\n'
                    % {'pdf': pdfname})
        for i, page in enumerate(pages):
            vars = dict(((k, os.path.basename(v)) for k,v in page.items()
                         if k != 'diff_value'))
            vars['page'] = i+1
            if 'diff' in page:
                vars['diff_value'] = page['diff_value']
                if vars['diff_value']:
                    vars['class'] = 'result-page-diff error'
                else:
                    if options.only_errors:
                        continue
                    vars['class'] = 'result-page-diff'
                html.append('
\n'
                            '
Page %(page)i
\n'

                            '
\n'
                            '
Difference '
                            '(Score %(diff_value)i)
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n'
                            '
Rendered
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n'
                            '
Reference
\n'
                            ''
                            '\n'
                            '
\n'

                            '
\n' % vars)
            else:
                html.append('
\n'
                           '
Page %(page)i
\n'

                           '
\n'
                           ''
                           '\n'
                           '
\n'

                           '
\n' % vars)
        html.append('
\n\n')

    now = datetime.datetime.now()
    title = 'xhtml2pdf Test Rendering Results, %s' % now.strftime('%c')
    template = open(template_file, 'rb').read()
    template = template.replace('%%TITLE%%', title)
    template = template.replace('%%RESULTS%%', '\n'.join(html))

    htmlfile = os.path.join(output_dir, 'index.html')
    outfile = open(htmlfile, 'wb')
    outfile.write(template)
    outfile.close()
    return htmlfile

def main():
    options, args = parser.parse_args()

    base_dir = os.path.abspath(os.path.join(__file__, os.pardir))
    source_dir = os.path.join(base_dir, options.source_dir)
    if options.create_reference is not None:
        output_dir = os.path.join(base_dir, options.create_reference)
    else:
        output_dir = os.path.join(base_dir, options.output_dir)
    template_file = os.path.join(base_dir, options.html_template)
    ref_dir = os.path.join(base_dir, options.ref_dir)

    if os.path.isdir(output_dir):
        shutil.rmtree(output_dir)
    os.makedirs(output_dir)

    results = []
    diff_count = 0
    if len(args) == 0:
        files = glob.glob(os.path.join(source_dir, '*.html'))
    else:
        files = [os.path.join(source_dir, arg) for arg in args]
    for filename in files:
        pdf, pages, diff = render_file(filename, output_dir, ref_dir, options)
        diff_count += diff
        results.append((pdf, pages, diff))

    num = len(results)

    if options.create_reference is not None:
        print 'Create reference for %i file%s' % (num, '' if num == 1 else 's')
    else:
        htmlfile = create_html_file(results, template_file, output_dir, options)
        if not options.quiet:
            print 'Rendered %i file%s' % (num, '' if num == 1 else 's')
            print '%i file%s differ%s from reference' % \
                    (diff_count, diff_count != 1 and 's' or '',
                     diff_count == 1 and 's' or '')
            print 'Check %s for results' % htmlfile
        if diff_count:
            sys.exit(1)

parser = OptionParser(
    usage='rendertest.py [options] [source_file] [source_file] ...',
    description='Renders a single html source file or all files in the data '
    'directory, converts them to PNG format and prepares a result '
    'HTML file for comparing the output with an expected result')
parser.add_option('-s', '--source-dir', dest='source_dir', default='data/source',
                  help=('Path to directory containing the html source files'))
parser.add_option('-o', '--output-dir', dest='output_dir', default='output',
                  help='Path to directory for output files. CAREFUL: this '
                  'directory will be deleted and recreated before rendering!')
parser.add_option('-r', '--ref-dir', dest='ref_dir', default='data/reference',
                  help='Path to directory containing the reference images '
                  'to compare the result with')
parser.add_option('-t', '--template', dest='html_template',
                  default='data/template.html', help='Name of HTML template file')
parser.add_option('-e', '--only-errors', dest='only_errors', action='store_true',
                  default=False, help='Only include images in HTML file which '
                  'differ from reference')
parser.add_option('-q', '--quiet', dest='quiet', action='store_true',
                  default=False, help='Try to be quiet')
parser.add_option('--no-compare', dest='no_compare', action='store_true',
                  default=False, help='Do not compare with reference image, '
                  'only render to png')
parser.add_option('-c', '--create-reference', dest='create_reference',
                  metavar='DIR',
                  default=None, help='Do not output anything, render source to '
                  'specified directory for reference. CAREFUL: this directory '
                 'will be deleted and recreated before rendering!')
parser.add_option('--debug', dest='debug', action='store_true',
                  default=False, help='More output for debugging')
parser.add_option('--convert-cmd', dest='convert_cmd', default='/usr/bin/convert',
                  help='Path to ImageMagick "convert" tool')
parser.add_option('--compare-cmd', dest='compare_cmd', default='/usr/bin/compare',
                  help='Path to ImageMagick "compare" tool')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from flask import Flask, render_template, request, jsonify
import argparse
import datetime
import os, sys
import requests
from socket import error as SocketError
import errno

app = Flask(__name__)

@app.route("/book", methods=["GET"])
def book():
    starttime=request.args.get('starttime', '')
    endtime=request.args.get('endtime', '')
    user_name=request.args.get('user_name', '')
    user_email=request.args.get('user_email', '')
    room_name=request.args.get('room_name', '')

    return "no parameter provided to book request\n"

    data = {
            "cmd": "book",
            "data": {"starttime": starttime, "endtime": endtime, "user_name": user_name, "user_email": user_email, "room_name": room_name}
    }
    message = json.dumps(data)
    return send_message_to_queue(message)

@app.route("/dispo", methods=["GET"])
def dispo():
    key=request.args.get('key', '')
    sys.stderr.write( "key: "+str(key)+'\r\n')
    if key is not None and str(key) is not "":
        data = {
            "cmd": "dispo",
            "data": {"key": key}
        }
        message = json.dumps(data)
        return send_message_to_queue(message)
    return "no parameter provided to dispo request\n"

def on_response(ch, method, props, body):
    global corr_id
    global response
    if corr_id == props.correlation_id:
        response = body

def send_message_to_queue(message):
    global corr_id
    global response
    global connection
    global channel
    global callback_queue

    response=None
    connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq,port=rabbitmq_port,heartbeat_interval=30))
    channel = connection.channel()
    result=channel.queue_declare(exclusive=True)
    callback_queue = result.method.queue
    channel.basic_consume(on_response, no_ack=True,
                                   queue=callback_queue)
    corr_id=str(uuid.uuid4())

    response = None
    corr_id =  str(uuid.uuid4())
    channel.basic_publish(  exchange='',
                            routing_key="rpc_queue",
                            properties=pika.BasicProperties(
                                         reply_to = callback_queue,
                                         correlation_id = corr_id),
                            body=message)

    print(" [x] Sent data to RabbitMQ")

    while response is None:
        connection.process_data_events()
    print(" [x] Get response from RabbitMQ")
    return response

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser("Room Finder Dispo Service")
    parser.add_argument("-r","--rabbitmq", help="IP or hostname for rabbitmq server, e.g. 'rabbit.domain.com'.")
    parser.add_argument("-p","--port", help="tcp port for rabitmq server, e.g. '2765'.")
    args = parser.parse_args()

    rabbitmq = args.rabbitmq
    if (rabbitmq == None):
        rabbitmq = os.getenv("roomfinder_rabbitmq_server")
        if (rabbitmq == None):
            get_rabbitmq_server = raw_input("What is the rabbitmq server IP or hostname? ")
            rabbitmq = get_rabbitmq_server

    rabbitmq_port = args.port
    if (rabbitmq_port == None):
        rabbitmq_port = os.getenv("roomfinder_rabbitmq_port")
        if (rabbitmq_port == None):
            get_rabbitmq_port = raw_input("What is the rabbitmq TCP port? ")
            rabbitmq_port = get_rabbitmq_port

    try:
    	app.run(debug=True, host='0.0.0.0', port=int("5000"))
    except:
    	try:
    		app.run(debug=True, host='0.0.0.0', port=int("5000"))
    	except:
    		print "Dispo web services error"
<commit_msg>Add import json in roomfinder_dispo<commit_after>#!/usr/bin/env python2.7

import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from flask import Flask, render_template, request, jsonify
import argparse
import datetime
import os, sys
import requests
from socket import error as SocketError
import errno
import json

app = Flask(__name__)

@app.route("/book", methods=["GET"])
def book():
    starttime=request.args.get('starttime', '')
    endtime=request.args.get('endtime', '')
    user_name=request.args.get('user_name', '')
    user_email=request.args.get('user_email', '')
    room_name=request.args.get('room_name', '')

    if starttime is None or endtime is None or user_name is None or user_email is None or room_name is None:
        return "no parameter provided to book request\n"
    data = {
            "cmd": "book",
            "data": {"starttime": starttime, "endtime": endtime, "user_name": user_name, "user_email": user_email, "room_name": room_name}
    }
    message = json.dumps(data)
    return send_message_to_queue(message)

@app.route("/dispo", methods=["GET"])
def dispo():
    key=request.args.get('key', '')
    sys.stderr.write( "key: "+str(key)+'\r\n')
    if key is not None and str(key) is not "":
        data = {
            "cmd": "dispo",
            "data": {"key": key}
        }
        message = json.dumps(data)
        return send_message_to_queue(message)
    return "no parameter provided to dispo request\n"

def on_response(ch, method, props, body):
    global corr_id
    global response
    if corr_id == props.correlation_id:
        response = body

def send_message_to_queue(message):
    global corr_id
    global response
    global connection
    global channel
    global callback_queue

    response=None
    connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq,port=rabbitmq_port,heartbeat_interval=30))
    channel = connection.channel()
    result=channel.queue_declare(exclusive=True)
    callback_queue = result.method.queue
    channel.basic_consume(on_response, no_ack=True,
                                   queue=callback_queue)
    corr_id=str(uuid.uuid4())

    response = None
    corr_id =  str(uuid.uuid4())
    channel.basic_publish(  exchange='',
                            routing_key="rpc_queue",
                            properties=pika.BasicProperties(
                                         reply_to = callback_queue,
                                         correlation_id = corr_id),
                            body=message)

    print(" [x] Sent data to RabbitMQ")

    while response is None:
        connection.process_data_events()
    print(" [x] Get response from RabbitMQ")
    return response

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser("Room Finder Dispo Service")
    parser.add_argument("-r","--rabbitmq", help="IP or hostname for rabbitmq server, e.g. 'rabbit.domain.com'.")
    parser.add_argument("-p","--port", help="tcp port for rabitmq server, e.g. '2765'.")
    args = parser.parse_args()

    rabbitmq = args.rabbitmq
    if (rabbitmq == None):
        rabbitmq = os.getenv("roomfinder_rabbitmq_server")
        if (rabbitmq == None):
            get_rabbitmq_server = raw_input("What is the rabbitmq server IP or hostname? ")
            rabbitmq = get_rabbitmq_server

    rabbitmq_port = args.port
    if (rabbitmq_port == None):
        rabbitmq_port = os.getenv("roomfinder_rabbitmq_port")
        if (rabbitmq_port == None):
            get_rabbitmq_port = raw_input("What is the rabbitmq TCP port? ")
            rabbitmq_port = get_rabbitmq_port

    try:
    	app.run(debug=True, host='0.0.0.0', port=int("5000"))
    except:
    	try:
    		app.run(debug=True, host='0.0.0.0', port=int("5000"))
    	except:
    		print "Dispo web services error"
<|endoftext|>


<commit_before># pylint: disable=E0401,C0111,R0903

import os
import re
import json
import sys
import platform
import subprocess
import time

from deoplete.source.base import Base
from logging import getLogger

PY2 = int(sys.version[0]) == 2

if PY2:
    import urllib2 as request
    from urllib2 import HTTPError
else:  # Py3
    from urllib import request
    from urllib.error import HTTPError

opener = request.build_opener(request.ProxyHandler({}))
current = __file__

logger = getLogger(__name__)
windows = platform.system() == "Windows"

import_re = r'=?\s*require\(["\'"][\w\./-]*$|\s+from\s+["\'][\w\./-]*$'
import_pattern = re.compile(import_re)

class RequestError(Exception):

    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message

class Source(Base):

    def __init__(self, vim):
        super(Source, self).__init__(vim)

        self.name = 'ternjs'
        self.mark = '[ternjs]'
        self.input_pattern = (r'\.\w*$|^\s*@\w*$|' + import_re)
        self.rank = 700
        self.filetypes = ['javascript'].extend(self.vim.vars['tern#filetypes'])

        self._project_directory = None
        self.port = None
        self.localhost = (windows and "127.0.0.1") or "localhost"
        self.proc = None
        self.last_failed = 0
        self._tern_command = \
            self.vim.vars['deoplete#sources#ternjs#tern_bin'] or 'tern'
        self._tern_arguments = '--persistent'
        self._tern_timeout = 1
        self._tern_show_signature = True
        self._tern_first_request = False
        self._tern_last_length = 0
        self._trying_to_start = False

        if vim.eval('exists("g:tern_request_timeout")'):
            self._tern_timeout = float(vim.eval("g:tern_request_timeout"))

        if vim.eval('exists("g:tern_show_signature_in_pum")'):
            self._tern_show_signature = \
                vim.eval('g:tern_show_signature_in_pum') != '0'

    def __del__(self):
        self.stop_server()

    def start_server(self):
        if self._trying_to_start:
            return

        if not self._tern_command:
            return None

        if time.time() - self.last_failed < 30:
            return None

        self._trying_to_start = True
        self._search_tern_project_dir()
        env = None

        portFile = os.path.join(self._project_directory, ".tern-port")
        if os.path.isfile(portFile):
            self.port = int(open(portFile, "r").read())
            return

        if platform.system() == 'Darwin':
            env = os.environ.copy()
            env['PATH'] += ':/usr/local/bin'

        self.proc = subprocess.Popen(self._tern_command + ' ' + self._tern_arguments,
                                     cwd=self._project_directory, env=env,
                                     stdin=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT,
                                     shell=True)
        output = ""

        while True:
            line = self.proc.stdout.readline().decode("utf-8")
            if not line:
                print("Failed to start server" + (output and ":\n" + output))
                self.last_failed = time.time()
                self._trying_to_start = False
                return None

            match = re.match("Listening on port (\\d+)", line)
            if match:
                self.port = int(match.group(1))
                return
            else:
                output += line

    def stop_server(self):
        if self.proc is None:
            return

        self.proc.stdin.close()
        self.proc.wait()
        self.proc = None

    def _search_tern_project_dir(self):
        if not self._project_directory:
            directory = self.vim.eval("expand('%:p:h')")

            if PY2:
                directory = directory.decode(self.vim.eval('&encoding'))

            if not os.path.isdir(directory):
                return ''

            if directory:
                self._project_directory = directory
                while True:
                    parent = os.path.dirname(directory[:-1])

                    if not parent:
                        self._project_directory = self.vim.eval('getcwd()')
                        if PY2:
                            self._project_directory = self._project_directory.decode(self.vim.eval('&encoding'))

                        break

                    if os.path.isfile(os.path.join(directory, '.tern-project')):
                        self._project_directory = directory
                        break

                    directory = parent

    def make_request(self, doc, silent):
        payload = json.dumps(doc)
        if not PY2:
            payload = payload.encode('utf-8')
        try:
            req = opener.open("http://" + self.localhost + ":" + str(self.port) + "/", payload, self._tern_timeout)
            result = req.read()
            if not PY2:
                result = result.decode('utf-8')
            return json.loads(result)
        except HTTPError as error:
            message = error.read()
            if not PY2:
                message = message.decode('utf-8')
            if not silent:
                logger.error(message)
            return None

    def run_command(self, query, pos, fragments=True, silent=False):
        if self.port is None:
            self.start_server()

        if isinstance(query, str):
            query = {"type": query}

        doc = {"query": query, "files": []}

        file_length = len(self.vim.current.buffer)

        if not self._file_changed and self._tern_first_request:
            fname = self.relative_file()
        elif file_length > 250 and fragments:
            f = self.buffer_fragment()
            doc["files"].append(f)
            pos = {"line": pos["line"] - f["offsetLines"], "ch": pos["ch"]}
            fname = "#0"
        else:
            self._tern_first_request = True
            doc["files"].append(self.full_buffer())
            fname = "#0"

        query["file"] = fname
        query["end"] = pos
        query["lineCharPositions"] = True
        query["omitObjectPrototype"] = False
        query["sort"] = False
        data = None
        try:
            data = self.make_request(doc, silent)
            if data is None:
                return None
        except:
            pass

        if data is None:
            try:
                self.start_server()
                if self.port is None:
                    return

                data = self.make_request(doc, silent)

                if data is None:
                    return None
            except:
                pass
            # except Exception as e:
            #     if not silent:
            #         raise e

        return data

    def full_buffer(self):
        text = self.buffer_slice(self.vim.current.buffer, 0,
                                 len(self.vim.current.buffer))
        return {"type": "full",
                "name": self.relative_file(),
                "text": text}

    def buffer_slice(self, buf, pos, end):
        text = ""
        while pos < end:
            text += buf[pos] + "\n"
            pos += 1
        return text

    def relative_file(self):
        filename = self.vim.eval("expand('%:p')")
        if PY2:
            filename = filename.decode(self.vim.eval('&encoding'))

        return filename[len(self._project_directory) + 1:]

    def buffer_fragment(self):
        line = self.vim.eval("line('.')") - 1
        buffer = self.vim.current.buffer
        min_indent = None
        start = None

        for i in range(max(0, line - 50), line):
            if not re.match(".*\\bfunction\\b", buffer[i]):
                continue
            indent = len(re.match("^\\s*", buffer[i]).group(0))
            if min_indent is None or indent <= min_indent:
                min_indent = indent
                start = i

        if start is None:
            start = max(0, line - 50)

        end = min(len(buffer) - 1, line + 20)

        return {"type": "part",
                "name": self.relative_file(),
                "text": self.buffer_slice(buffer, start, end),
                "offsetLines": start}

    def completion_icon(self, type):
        _type = '(obj)'
        if type is None or type == "?":
            _type = "(?)"
        elif type.startswith("fn("):
            if (self._tern_show_signature):
                _type = type
            else:
                _type = "(fn)"
        elif type.startswith("["):
            if self._tern_show_signature:
                _type = "(" + type + ")"
            else:
                _type = "([])"
        elif type == "number":
            _type = "(num)"
        elif type == "string":
            _type = "(str)"
        elif type == "bool":
            _type = "(bool)"

        return _type

    def completation(self, pos):
        command = {
            "type": "completions",
            "types": True,
            "docs": True
        }

        data = self.run_command(command, pos)
        completions = []

        if data is not None:

            for rec in data["completions"]:
                completions.append({
                    "word": rec["name"],
                    "menu": self.completion_icon(rec.get("type")),
                    "info": self.type_doc(rec)})

        return completions

    def type_doc(self, rec):
        tp = rec.get("type")
        result = rec.get("doc", " ")
        if tp and tp != "?":
            result = tp + "\n" + result
        return result

    def get_complete_position(self, context):
        m = import_pattern.search(context['input'])
        if m:
            # need to tell from what position autocomplete as
            # needs to autocomplete from start quote return that
            return re.search(r'["\']', context['input']).start()

        m = re.search(r'\w*$', context['input'])
        return m.start() if m else -1

    def gather_candidates(self, context):
        self._file_changed = 'TextChanged' in context['event'] or \
            self._tern_last_length != len(self.vim.current.buffer)
        line = context['position'][1]
        col = context['complete_position']
        pos = {"line": line - 1, "ch": col}

        # Update autocomplete position need to send the position
        # where cursor is because the position is the start of
        # quote
        m = import_pattern.search(context['input'])
        if m:
            pos['ch'] = m.end()

        result = self.completation(pos) or []
        return result
<commit_msg>Fix filetypes<commit_after># pylint: disable=E0401,C0111,R0903

import os
import re
import json
import sys
import platform
import subprocess
import time

from deoplete.source.base import Base
from logging import getLogger

PY2 = int(sys.version[0]) == 2

if PY2:
    import urllib2 as request
    from urllib2 import HTTPError
else:  # Py3
    from urllib import request
    from urllib.error import HTTPError

opener = request.build_opener(request.ProxyHandler({}))
current = __file__

logger = getLogger(__name__)
windows = platform.system() == "Windows"

import_re = r'=?\s*require\(["\'"][\w\./-]*$|\s+from\s+["\'][\w\./-]*$'
import_pattern = re.compile(import_re)

class RequestError(Exception):

    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message

class Source(Base):

    def __init__(self, vim):
        super(Source, self).__init__(vim)

        self.name = 'ternjs'
        self.mark = '[ternjs]'
        self.input_pattern = (r'\.\w*$|^\s*@\w*$|' + import_re)
        self.rank = 700
        self.filetypes = ['javascript']
        if 'tern#filetypes' in vim.vars:
            self.filetypes.extend(vim.vars['tern#filetypes'])

        self._project_directory = None
        self.port = None
        self.localhost = (windows and "127.0.0.1") or "localhost"
        self.proc = None
        self.last_failed = 0
        self._tern_command = \
            self.vim.vars['deoplete#sources#ternjs#tern_bin'] or 'tern'
        self._tern_arguments = '--persistent'
        self._tern_timeout = 1
        self._tern_show_signature = True
        self._tern_first_request = False
        self._tern_last_length = 0
        self._trying_to_start = False

        if vim.eval('exists("g:tern_request_timeout")'):
            self._tern_timeout = float(vim.eval("g:tern_request_timeout"))

        if vim.eval('exists("g:tern_show_signature_in_pum")'):
            self._tern_show_signature = \
                vim.eval('g:tern_show_signature_in_pum') != '0'

    def __del__(self):
        self.stop_server()

    def start_server(self):
        if self._trying_to_start:
            return

        if not self._tern_command:
            return None

        if time.time() - self.last_failed < 30:
            return None

        self._trying_to_start = True
        self._search_tern_project_dir()
        env = None

        portFile = os.path.join(self._project_directory, ".tern-port")
        if os.path.isfile(portFile):
            self.port = int(open(portFile, "r").read())
            return

        if platform.system() == 'Darwin':
            env = os.environ.copy()
            env['PATH'] += ':/usr/local/bin'

        self.proc = subprocess.Popen(self._tern_command + ' ' + self._tern_arguments,
                                     cwd=self._project_directory, env=env,
                                     stdin=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT,
                                     shell=True)
        output = ""

        while True:
            line = self.proc.stdout.readline().decode("utf-8")
            if not line:
                print("Failed to start server" + (output and ":\n" + output))
                self.last_failed = time.time()
                self._trying_to_start = False
                return None

            match = re.match("Listening on port (\\d+)", line)
            if match:
                self.port = int(match.group(1))
                return
            else:
                output += line

    def stop_server(self):
        if self.proc is None:
            return

        self.proc.stdin.close()
        self.proc.wait()
        self.proc = None

    def _search_tern_project_dir(self):
        if not self._project_directory:
            directory = self.vim.eval("expand('%:p:h')")

            if PY2:
                directory = directory.decode(self.vim.eval('&encoding'))

            if not os.path.isdir(directory):
                return ''

            if directory:
                self._project_directory = directory
                while True:
                    parent = os.path.dirname(directory[:-1])

                    if not parent:
                        self._project_directory = self.vim.eval('getcwd()')
                        if PY2:
                            self._project_directory = self._project_directory.decode(self.vim.eval('&encoding'))

                        break

                    if os.path.isfile(os.path.join(directory, '.tern-project')):
                        self._project_directory = directory
                        break

                    directory = parent

    def make_request(self, doc, silent):
        payload = json.dumps(doc)
        if not PY2:
            payload = payload.encode('utf-8')
        try:
            req = opener.open("http://" + self.localhost + ":" + str(self.port) + "/", payload, self._tern_timeout)
            result = req.read()
            if not PY2:
                result = result.decode('utf-8')
            return json.loads(result)
        except HTTPError as error:
            message = error.read()
            if not PY2:
                message = message.decode('utf-8')
            if not silent:
                logger.error(message)
            return None

    def run_command(self, query, pos, fragments=True, silent=False):
        if self.port is None:
            self.start_server()

        if isinstance(query, str):
            query = {"type": query}

        doc = {"query": query, "files": []}

        file_length = len(self.vim.current.buffer)

        if not self._file_changed and self._tern_first_request:
            fname = self.relative_file()
        elif file_length > 250 and fragments:
            f = self.buffer_fragment()
            doc["files"].append(f)
            pos = {"line": pos["line"] - f["offsetLines"], "ch": pos["ch"]}
            fname = "#0"
        else:
            self._tern_first_request = True
            doc["files"].append(self.full_buffer())
            fname = "#0"

        query["file"] = fname
        query["end"] = pos
        query["lineCharPositions"] = True
        query["omitObjectPrototype"] = False
        query["sort"] = False
        data = None
        try:
            data = self.make_request(doc, silent)
            if data is None:
                return None
        except:
            pass

        if data is None:
            try:
                self.start_server()
                if self.port is None:
                    return

                data = self.make_request(doc, silent)

                if data is None:
                    return None
            except:
                pass
            # except Exception as e:
            #     if not silent:
            #         raise e

        return data

    def full_buffer(self):
        text = self.buffer_slice(self.vim.current.buffer, 0,
                                 len(self.vim.current.buffer))
        return {"type": "full",
                "name": self.relative_file(),
                "text": text}

    def buffer_slice(self, buf, pos, end):
        text = ""
        while pos < end:
            text += buf[pos] + "\n"
            pos += 1
        return text

    def relative_file(self):
        filename = self.vim.eval("expand('%:p')")
        if PY2:
            filename = filename.decode(self.vim.eval('&encoding'))

        return filename[len(self._project_directory) + 1:]

    def buffer_fragment(self):
        line = self.vim.eval("line('.')") - 1
        buffer = self.vim.current.buffer
        min_indent = None
        start = None

        for i in range(max(0, line - 50), line):
            if not re.match(".*\\bfunction\\b", buffer[i]):
                continue
            indent = len(re.match("^\\s*", buffer[i]).group(0))
            if min_indent is None or indent <= min_indent:
                min_indent = indent
                start = i

        if start is None:
            start = max(0, line - 50)

        end = min(len(buffer) - 1, line + 20)

        return {"type": "part",
                "name": self.relative_file(),
                "text": self.buffer_slice(buffer, start, end),
                "offsetLines": start}

    def completion_icon(self, type):
        _type = '(obj)'
        if type is None or type == "?":
            _type = "(?)"
        elif type.startswith("fn("):
            if (self._tern_show_signature):
                _type = type
            else:
                _type = "(fn)"
        elif type.startswith("["):
            if self._tern_show_signature:
                _type = "(" + type + ")"
            else:
                _type = "([])"
        elif type == "number":
            _type = "(num)"
        elif type == "string":
            _type = "(str)"
        elif type == "bool":
            _type = "(bool)"

        return _type

    def completation(self, pos):
        command = {
            "type": "completions",
            "types": True,
            "docs": True
        }

        data = self.run_command(command, pos)
        completions = []

        if data is not None:

            for rec in data["completions"]:
                completions.append({
                    "word": rec["name"],
                    "menu": self.completion_icon(rec.get("type")),
                    "info": self.type_doc(rec)})

        return completions

    def type_doc(self, rec):
        tp = rec.get("type")
        result = rec.get("doc", " ")
        if tp and tp != "?":
            result = tp + "\n" + result
        return result

    def get_complete_position(self, context):
        m = import_pattern.search(context['input'])
        if m:
            # need to tell from what position autocomplete as
            # needs to autocomplete from start quote return that
            return re.search(r'["\']', context['input']).start()

        m = re.search(r'\w*$', context['input'])
        return m.start() if m else -1

    def gather_candidates(self, context):
        self._file_changed = 'TextChanged' in context['event'] or \
            self._tern_last_length != len(self.vim.current.buffer)
        line = context['position'][1]
        col = context['complete_position']
        pos = {"line": line - 1, "ch": col}

        # Update autocomplete position need to send the position
        # where cursor is because the position is the start of
        # quote
        m = import_pattern.search(context['input'])
        if m:
            pos['ch'] = m.end()

        result = self.completation(pos) or []
        return result
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (c) 2013-2015 Didotech srl (<http://www.didotech.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from tools.translate import _

class stock_picking(orm.Model):
    """Modificamos la creación de factura desde albarán para incluir el comportamiento de comisiones"""

    _inherit = 'stock.picking'

    def _invoice_hook(self, cr, uid, picking_browse, invoice_id):
        '''Call after the creation of the invoice'''
        res = super(stock_picking, self)._invoice_hook(cr, uid, picking_browse, invoice_id)
        context = self.pool['res.users'].context_get(cr, uid)
        account_invoice_obj = self.pool['account.invoice']
        account_invoice_line_obj = self.pool['account.invoice.line']
        invoice = account_invoice_obj.browse(cr, uid, invoice_id, context)
        picking = self.browse(cr, uid, picking_browse.id, context)

        if not invoice.fiscal_position:
            if invoice.partner_id.property_account_position:
                invoice.write({'fiscal_position': invoice.partner_id.property_account_position.id})
        if not invoice.payment_term:
            if invoice.partner_id.property_payment_term:
                invoice.write({'payment_term': invoice.partner_id.property_payment_term.id})
        if picking.sale_id:
            for invoice in picking.sale_id.invoice_ids:
                if invoice.advance_order_id:
                    # here i need to check fiscal_position
                    if invoice.fiscal_position.split_invoice_advanced:
                        journal_ids = self.pool['account.journal'].search(cr, uid, [('type', '=', 'sale_refund')], limit=1, context=context)
                        journal_id = journal_ids and journal_ids[0] or False
                        new_invoice_id = account_invoice_obj.copy(cr, uid, invoice.id, {'type': 'out_refund', 'invoice_line': False, 'journal_id': journal_id}, context=context)
                        note = _('Invoice {name}').format(name=invoice.number)
                        for line in invoice.invoice_line:
                            line_vals = {
                                'invoice_id': new_invoice_id,
                                'price_unit': line.price_unit,
                                'sequence': 1000,
                                'note': note
                            }
                            account_invoice_line_obj.copy(cr, uid, line.id, line_vals, context=context)
                    else:
                        for line in invoice.invoice_line:
                            line_vals = {
                                'invoice_id': invoice_id,
                                'price_unit': -line.price_unit,
                                'sequence': 1000,
                            }
                            account_invoice_line_obj.copy(cr, uid, line.id, line_vals, context=context)
                        invoice.button_compute()
                    invoice.write({'advance_order_id': False})
        return res
<commit_msg>when group invoice also usa payment term<commit_after># -*- coding:utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (c) 2013-2015 Didotech srl (<http://www.didotech.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm, fields
from tools.translate import _

class stock_picking(orm.Model):
    """Modificamos la creación de factura desde albarán para incluir el comportamiento de comisiones"""

    _inherit = 'stock.picking'

    def _invoice_hook(self, cr, uid, picking_browse, invoice_id):
        '''Call after the creation of the invoice'''
        res = super(stock_picking, self)._invoice_hook(cr, uid, picking_browse, invoice_id)
        context = self.pool['res.users'].context_get(cr, uid)
        account_invoice_obj = self.pool['account.invoice']
        account_invoice_line_obj = self.pool['account.invoice.line']
        invoice = account_invoice_obj.browse(cr, uid, invoice_id, context)
        picking = self.browse(cr, uid, picking_browse.id, context)

        if not invoice.fiscal_position:
            if invoice.partner_id.property_account_position:
                invoice.write({'fiscal_position': invoice.partner_id.property_account_position.id})
        if not invoice.payment_term:
            if invoice.partner_id.property_payment_term:
                invoice.write({'payment_term': invoice.partner_id.property_payment_term.id})
        if picking.sale_id:
            for invoice in picking.sale_id.invoice_ids:
                if invoice.advance_order_id:
                    # here i need to check fiscal_position
                    if invoice.fiscal_position.split_invoice_advanced:
                        journal_ids = self.pool['account.journal'].search(cr, uid, [('type', '=', 'sale_refund')], limit=1, context=context)
                        journal_id = journal_ids and journal_ids[0] or False
                        new_invoice_id = account_invoice_obj.copy(cr, uid, invoice.id, {'type': 'out_refund', 'invoice_line': False, 'journal_id': journal_id}, context=context)
                        note = _('Invoice {name}').format(name=invoice.number)
                        for line in invoice.invoice_line:
                            line_vals = {
                                'invoice_id': new_invoice_id,
                                'price_unit': line.price_unit,
                                'sequence': 1000,
                                'note': note
                            }
                            account_invoice_line_obj.copy(cr, uid, line.id, line_vals, context=context)
                    else:
                        for line in invoice.invoice_line:
                            line_vals = {
                                'invoice_id': invoice_id,
                                'price_unit': -line.price_unit,
                                'sequence': 1000,
                            }
                            account_invoice_line_obj.copy(cr, uid, line.id, line_vals, context=context)
                        invoice.button_compute()
                    invoice.write({'advance_order_id': False})
        return res

    def _get_group_keys(self, cr, uid, partner, picking, context=None):
        res = super(stock_picking, self)._get_group_keys(cr, uid, partner, picking)
        if picking.sale_id and picking.sale_id.payment_term:
            res = '{0}_{1}'.format(res, picking.sale_id.payment_term.id)
        return res
<|endoftext|>


<commit_before># Generated by Django 2.2.24 on 2021-11-08 10:39

from django.db import migrations, connection

DEFAULT_COLLECT_TYPES = [
    {
        'en': {
            'name': 'Clothing',
            'unit': 'Bag of clothing',
            'unit_plural': 'Bags of clothing'
        },
        'nl': {
            'name': 'Kleding',
            'unit': 'Zak kleding',
            'unit_plural': 'Zakken kleding'
        },
    }, {
        'en': {
            'name': 'Laptops',
            'unit': 'Laptop',
            'unit_plural': 'Laptops'
        },
        'nl': {
            'name': 'Laptops',
            'unit': 'Laptop',
            'unit_plural': 'Laptops'
        },
    }, {
        'en': {
            'name': 'Groceries',
            'unit': 'Crate of groceries',
            'unit_plural': 'Crates of groceries'
        },
        'nl': {
            'name': 'Boodschappen',
            'unit': 'Krat boodschappen',
            'unit_plural': 'Kratten boodschappen'
        },
    },
]

def create_defaults(apps, schema_editor):

    if connection.tenant:
        CollectType = apps.get_model('collect', 'CollectType')
        CollectTypeTranslation = apps.get_model('collect', 'CollectTypeTranslation')

        for collect_type in DEFAULT_COLLECT_TYPES:
            model = CollectType.objects.create(disabled=False)

            for lang, translation in collect_type.items():
                CollectTypeTranslation.objects.create(
                    language_code=lang,
                    master=model,
                    **translation
                )

class Migration(migrations.Migration):

    dependencies = [
        ('collect', '0013_auto_20211108_1113'),
    ]

    operations = [
        migrations.RunPython(create_defaults)
    ]
<commit_msg>Try to fix migration<commit_after># Generated by Django 2.2.24 on 2021-11-08 10:39

from django.db import migrations, connection

from bluebottle.clients.utils import LocalTenant

DEFAULT_COLLECT_TYPES = [
    {
        'en': {
            'name': 'Clothing',
            'unit': 'Bag of clothing',
            'unit_plural': 'Bags of clothing'
        },
        'nl': {
            'name': 'Kleding',
            'unit': 'Zak kleding',
            'unit_plural': 'Zakken kleding'
        },
    }, {
        'en': {
            'name': 'Laptops',
            'unit': 'Laptop',
            'unit_plural': 'Laptops'
        },
        'nl': {
            'name': 'Laptops',
            'unit': 'Laptop',
            'unit_plural': 'Laptops'
        },
    }, {
        'en': {
            'name': 'Groceries',
            'unit': 'Crate of groceries',
            'unit_plural': 'Crates of groceries'
        },
        'nl': {
            'name': 'Boodschappen',
            'unit': 'Krat boodschappen',
            'unit_plural': 'Kratten boodschappen'
        },
    },
]

def create_defaults(apps, schema_editor):
    Client = apps.get_model('clients', 'Client')
    tenant = Client.objects.get(schema_name=connection.tenant.schema_name)
    with LocalTenant(tenant):
        CollectType = apps.get_model('collect', 'CollectType')
        CollectTypeTranslation = apps.get_model('collect', 'CollectTypeTranslation')

        for collect_type in DEFAULT_COLLECT_TYPES:
            model = CollectType.objects.create(disabled=False)

            for lang, translation in collect_type.items():
                CollectTypeTranslation.objects.create(
                    language_code=lang,
                    master=model,
                    **translation
                )

class Migration(migrations.Migration):

    dependencies = [
        ('collect', '0013_auto_20211108_1113'),
    ]

    operations = [
        migrations.RunPython(create_defaults)
    ]
<|endoftext|>


<commit_before>#!/usr/bin/env python
from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper
import requests

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_oidc_auth_method_config
version_added: "4.1.1"
short_description: Hashicorp Vault OIDC auth method config
description:
    - Module to configure an OIDC auth mount
options:
    mount_point:
        description:
            - name of the secret engine mount name.
        default: oidc
    default_role:
        description:
            - The default role to use if none is provided during login.
    oidc_discovery_url:
        description:
            - The OIDC Discovery URL, without any .well-known component (base path). Cannot be used with "jwks_url" or
              "jwt_validation_pubkeys".
    oidc_client_id:
        description:
            - The OAuth Client ID from the provider for OIDC roles.
    oidc_client_secret:
        description:
            - The OAuth Client Secret from the provider for OIDC roles.
    bound_issuer:
        description:
            - The value against which to match the iss claim in a JWT.
    jwks_ca_pem:
        description:
            - The CA certificate or chain of certificates, in PEM format, to use to validate connections to the JWKS
              URL. If not set, system certificates are used.
    jwks_url:
        description:
            - JWKS URL to use to authenticate signatures. Cannot be used with "oidc_discovery_url" or
              "jwt_validation_pubkeys".
    jwt_supported_algs:
        description:
            - A list of supported signing algorithms.
        default: RS256
    jwt_validation_pubkeys:
        description:
            - A list of PEM-encoded public keys to use to authenticate signatures locally. Cannot be used with
              "jwks_url" or "oidc_discovery_url".
    oidc_discovery_ca_pem:
        description:
            - The CA certificate or chain of certificates, in PEM format, to use to validate connections to the OIDC
              Discovery URL. If not set, system certificates are used.
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_oidc_auth_method_config:
        oidc_discovery_url: "https://accounts.google.com"
        oidc_client_id: "123456"
        oidc_client_secret: "123456"
        default_role: "gmail"
        verify: False
'''

def main():
    argspec = hashivault_argspec()
    argspec['bound_issuer'] = dict(required=False, type='str', default='')
    argspec['jwks_ca_pem'] = dict(required=False, type='str', default='')
    argspec['jwks_url'] = dict(required=False, type='str')
    argspec['jwt_supported_algs'] = dict(required=False, type='list', default=[])
    argspec['jwt_validation_pubkeys'] = dict(required=False, type='list', default=[])
    argspec['oidc_discovery_ca_pem'] = dict(required=False, type='str', default='')
    argspec['mount_point'] = dict(required=False, type='str', default='oidc')
    argspec['oidc_discovery_url'] = dict(required=False, type='str')
    argspec['oidc_client_id'] = dict(required=False, type='str')
    argspec['oidc_client_secret'] = dict(required=False, type='str')
    argspec['default_role'] = dict(required=False, type='str')
    supports_check_mode = True
    required_one_of = [['oidc_discovery_url', 'jwks_url']]
    module = hashivault_init(argspec, supports_check_mode, required_one_of=required_one_of)
    result = hashivault_oidc_auth_method_config(module)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_oidc_auth_method_config(module):
    params = module.params
    client = hashivault_auth_client(params)
    changed = False
    mount_point = params.get('mount_point')
    desired_state = dict()
    current_state = dict()
    exists = False

    token = params['token']
    namespace = params['namespace']
    headers = {'X-Vault-Token': token, 'X-Vault-Namespace': namespace}
    url = params['url']
    verify = params['verify']

    # do not want a trailing slash in mount_point
    if mount_point[-1]:
        mount_point = mount_point.strip('/')

    if not params.get('oidc_discovery_url'):
        desired_state['oidc_discovery_url'] = ''
    else:
        desired_state['oidc_discovery_url'] = params.get('oidc_discovery_url')
    desired_state['oidc_client_id'] = params.get('oidc_client_id')
    desired_state['default_role'] = params.get('default_role')
    desired_state['bound_issuer'] = params.get('bound_issuer')
    desired_state['jwks_ca_pem'] = params.get('jwks_ca_pem')
    if not params.get('jwks_url'):
        desired_state['jwks_url'] = ''
    else:
        desired_state['jwks_url'] = params.get('jwks_url')
    desired_state['jwt_supported_algs'] = params.get('jwt_supported_algs')
    desired_state['jwt_validation_pubkeys'] = params.get('jwt_validation_pubkeys')
    desired_state['oidc_discovery_ca_pem'] = params.get('oidc_discovery_ca_pem')

    # check if engine is enabled
    try:
        if (mount_point + "/") not in client.sys.list_auth_methods()['data'].keys():
            return {'failed': True, 'msg': 'auth mount is not enabled', 'rc': 1}
    except:
        if module.check_mode:
            changed = True
        else:
            return {'failed': True, 'msg': 'auth mount is not enabled', 'rc': 1}

    # check if any config exists
    try:
        current_state = requests.get(url + '/v1/auth/' + mount_point + '/config', verify=verify, headers=headers)
        if current_state.status_code == 404:
            changed = True
        elif current_state.status_code == 200:
            exists = True
    except:
        changed = True

    # check if current config matches desired config values, if they dont match, set changed true
    if exists:
        current_state = current_state.json()['data']
        for k, v in current_state.items():
            if k in desired_state and v != desired_state[k]:
                changed = True

    desired_state['oidc_client_secret'] = params.get('oidc_client_secret')

    # if configs dont match and checkmode is off, complete the change
    if changed and not module.check_mode:
        config_status = requests.post(url + '/v1/auth/' + mount_point + '/config', verify=verify, headers=headers,
                                      json=desired_state)
        try:
            config_status.raise_for_status()
        except:
            return {'failed': True, 'msg': config_status.text, 'rc': 1}
    return {'changed': changed}

if __name__ == '__main__':
    main()
<commit_msg>Add missing fragment to module<commit_after>#!/usr/bin/env python
from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper
import requests

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_oidc_auth_method_config
version_added: "4.1.1"
short_description: Hashicorp Vault OIDC auth method config
description:
    - Module to configure an OIDC auth mount
options:
    mount_point:
        description:
            - name of the secret engine mount name.
        default: oidc
    default_role:
        description:
            - The default role to use if none is provided during login.
    oidc_discovery_url:
        description:
            - The OIDC Discovery URL, without any .well-known component (base path). Cannot be used with "jwks_url" or
              "jwt_validation_pubkeys".
    oidc_client_id:
        description:
            - The OAuth Client ID from the provider for OIDC roles.
    oidc_client_secret:
        description:
            - The OAuth Client Secret from the provider for OIDC roles.
    bound_issuer:
        description:
            - The value against which to match the iss claim in a JWT.
    jwks_ca_pem:
        description:
            - The CA certificate or chain of certificates, in PEM format, to use to validate connections to the JWKS
              URL. If not set, system certificates are used.
    jwks_url:
        description:
            - JWKS URL to use to authenticate signatures. Cannot be used with "oidc_discovery_url" or
              "jwt_validation_pubkeys".
    jwt_supported_algs:
        description:
            - A list of supported signing algorithms.
        default: RS256
    jwt_validation_pubkeys:
        description:
            - A list of PEM-encoded public keys to use to authenticate signatures locally. Cannot be used with
              "jwks_url" or "oidc_discovery_url".
    oidc_discovery_ca_pem:
        description:
            - The CA certificate or chain of certificates, in PEM format, to use to validate connections to the OIDC
              Discovery URL. If not set, system certificates are used.
extends_documentation_fragment: hashivault
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_oidc_auth_method_config:
        oidc_discovery_url: "https://accounts.google.com"
        oidc_client_id: "123456"
        oidc_client_secret: "123456"
        default_role: "gmail"
        verify: False
'''

def main():
    argspec = hashivault_argspec()
    argspec['bound_issuer'] = dict(required=False, type='str', default='')
    argspec['jwks_ca_pem'] = dict(required=False, type='str', default='')
    argspec['jwks_url'] = dict(required=False, type='str')
    argspec['jwt_supported_algs'] = dict(required=False, type='list', default=[])
    argspec['jwt_validation_pubkeys'] = dict(required=False, type='list', default=[])
    argspec['oidc_discovery_ca_pem'] = dict(required=False, type='str', default='')
    argspec['mount_point'] = dict(required=False, type='str', default='oidc')
    argspec['oidc_discovery_url'] = dict(required=False, type='str')
    argspec['oidc_client_id'] = dict(required=False, type='str')
    argspec['oidc_client_secret'] = dict(required=False, type='str')
    argspec['default_role'] = dict(required=False, type='str')
    supports_check_mode = True
    required_one_of = [['oidc_discovery_url', 'jwks_url']]
    module = hashivault_init(argspec, supports_check_mode, required_one_of=required_one_of)
    result = hashivault_oidc_auth_method_config(module)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_oidc_auth_method_config(module):
    params = module.params
    client = hashivault_auth_client(params)
    changed = False
    mount_point = params.get('mount_point')
    desired_state = dict()
    current_state = dict()
    exists = False

    token = params['token']
    namespace = params['namespace']
    headers = {'X-Vault-Token': token, 'X-Vault-Namespace': namespace}
    url = params['url']
    verify = params['verify']

    # do not want a trailing slash in mount_point
    if mount_point[-1]:
        mount_point = mount_point.strip('/')

    if not params.get('oidc_discovery_url'):
        desired_state['oidc_discovery_url'] = ''
    else:
        desired_state['oidc_discovery_url'] = params.get('oidc_discovery_url')
    desired_state['oidc_client_id'] = params.get('oidc_client_id')
    desired_state['default_role'] = params.get('default_role')
    desired_state['bound_issuer'] = params.get('bound_issuer')
    desired_state['jwks_ca_pem'] = params.get('jwks_ca_pem')
    if not params.get('jwks_url'):
        desired_state['jwks_url'] = ''
    else:
        desired_state['jwks_url'] = params.get('jwks_url')
    desired_state['jwt_supported_algs'] = params.get('jwt_supported_algs')
    desired_state['jwt_validation_pubkeys'] = params.get('jwt_validation_pubkeys')
    desired_state['oidc_discovery_ca_pem'] = params.get('oidc_discovery_ca_pem')

    # check if engine is enabled
    try:
        if (mount_point + "/") not in client.sys.list_auth_methods()['data'].keys():
            return {'failed': True, 'msg': 'auth mount is not enabled', 'rc': 1}
    except:
        if module.check_mode:
            changed = True
        else:
            return {'failed': True, 'msg': 'auth mount is not enabled', 'rc': 1}

    # check if any config exists
    try:
        current_state = requests.get(url + '/v1/auth/' + mount_point + '/config', verify=verify, headers=headers)
        if current_state.status_code == 404:
            changed = True
        elif current_state.status_code == 200:
            exists = True
    except:
        changed = True

    # check if current config matches desired config values, if they dont match, set changed true
    if exists:
        current_state = current_state.json()['data']
        for k, v in current_state.items():
            if k in desired_state and v != desired_state[k]:
                changed = True

    desired_state['oidc_client_secret'] = params.get('oidc_client_secret')

    # if configs dont match and checkmode is off, complete the change
    if changed and not module.check_mode:
        config_status = requests.post(url + '/v1/auth/' + mount_point + '/config', verify=verify, headers=headers,
                                      json=desired_state)
        try:
            config_status.raise_for_status()
        except:
            return {'failed': True, 'msg': config_status.text, 'rc': 1}
    return {'changed': changed}

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Sumit Naiksatam, sumitnaiksatam@gmail.com, Big Switch Networks, Inc.
# @author: Sridar Kandaswamy, skandasw@cisco.com, Cisco Systems, Inc.
# @author: Dan Florea, dflorea@cisco.com, Cisco Systems, Inc.

import contextlib
import mock
from oslo.config import cfg

from neutron.agent.common import config as agent_config
from neutron.common import config as base_config
from neutron.services.firewall.agents.l3reference import firewall_l3_agent
from neutron.tests import base

class FWaasHelper(object):
    def __init__(self, host):
        pass

class FWaasAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, FWaasHelper):
    def __init__(self, conf=None):
        super(FWaasAgent, self).__init__(conf)

class TestFwaasL3AgentRpcCallback(base.BaseTestCase):
    def setUp(self):
        super(TestFwaasL3AgentRpcCallback, self).setUp()
        self.addCleanup(mock.patch.stopall)

        self.conf = cfg.ConfigOpts()
        self.conf.register_opts(base_config.core_opts)
        agent_config.register_root_helper(self.conf)
        self.conf.root_helper = 'sudo'
        self.api = FWaasAgent(self.conf)

    def test_create_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.create_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_update_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.update_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_delete_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.delete_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_invoke_driver_for_plugin_api(self):
        fake_firewall = {'id': 0, 'tenant_id': 001}
        self.api.plugin_rpc = mock.Mock()
        with contextlib.nested(
            mock.patch.object(self.api.plugin_rpc, 'get_routers'),
            mock.patch.object(self.api, '_get_router_info_list_for_tenant'),
            mock.patch.object(self.api.fwaas_driver, 'create_firewall'),
            mock.patch.object(self.api.fwplugin_rpc, 'set_firewall_status')
        ) as (
                mock_get_routers,
                mock_get_router_info_list_for_tenant,
                mock_driver_create_firewall,
                mock_set_firewall_status):

            mock_driver_create_firewall.return_value = True
            self.api.create_firewall(
                context=mock.sentinel.context,
                firewall=fake_firewall, host='host')

            mock_get_routers.assert_called_once_with(
                mock.sentinel.context)

            mock_get_router_info_list_for_tenant.assert_called_once_with(
                mock_get_routers.return_value, fake_firewall['tenant_id'])

            mock_set_firewall_status.assert_called_once_with(
                mock.sentinel.context,
                fake_firewall['id'],
                'ACTIVE')
<commit_msg>fix up inadevertant octal to make hacking pass<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Sumit Naiksatam, sumitnaiksatam@gmail.com, Big Switch Networks, Inc.
# @author: Sridar Kandaswamy, skandasw@cisco.com, Cisco Systems, Inc.
# @author: Dan Florea, dflorea@cisco.com, Cisco Systems, Inc.

import contextlib
import mock
from oslo.config import cfg

from neutron.agent.common import config as agent_config
from neutron.common import config as base_config
from neutron.services.firewall.agents.l3reference import firewall_l3_agent
from neutron.tests import base

class FWaasHelper(object):
    def __init__(self, host):
        pass

class FWaasAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, FWaasHelper):
    def __init__(self, conf=None):
        super(FWaasAgent, self).__init__(conf)

class TestFwaasL3AgentRpcCallback(base.BaseTestCase):
    def setUp(self):
        super(TestFwaasL3AgentRpcCallback, self).setUp()
        self.addCleanup(mock.patch.stopall)

        self.conf = cfg.ConfigOpts()
        self.conf.register_opts(base_config.core_opts)
        agent_config.register_root_helper(self.conf)
        self.conf.root_helper = 'sudo'
        self.api = FWaasAgent(self.conf)

    def test_create_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.create_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_update_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.update_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_delete_firewall(self):
        fake_firewall = {'id': 0}
        with mock.patch.object(
            self.api,
            '_invoke_driver_for_plugin_api'
        ) as mock_driver:
            self.assertEqual(
                self.api.delete_firewall(
                    mock.sentinel.context,
                    fake_firewall,
                    'host'),
                mock_driver.return_value)

    def test_invoke_driver_for_plugin_api(self):
        fake_firewall = {'id': 0, 'tenant_id': 1}
        self.api.plugin_rpc = mock.Mock()
        with contextlib.nested(
            mock.patch.object(self.api.plugin_rpc, 'get_routers'),
            mock.patch.object(self.api, '_get_router_info_list_for_tenant'),
            mock.patch.object(self.api.fwaas_driver, 'create_firewall'),
            mock.patch.object(self.api.fwplugin_rpc, 'set_firewall_status')
        ) as (
                mock_get_routers,
                mock_get_router_info_list_for_tenant,
                mock_driver_create_firewall,
                mock_set_firewall_status):

            mock_driver_create_firewall.return_value = True
            self.api.create_firewall(
                context=mock.sentinel.context,
                firewall=fake_firewall, host='host')

            mock_get_routers.assert_called_once_with(
                mock.sentinel.context)

            mock_get_router_info_list_for_tenant.assert_called_once_with(
                mock_get_routers.return_value, fake_firewall['tenant_id'])

            mock_set_firewall_status.assert_called_once_with(
                mock.sentinel.context,
                fake_firewall['id'],
                'ACTIVE')
<|endoftext|>