<commit_before>import difflib
import json
import os
import sys
from datetime import datetime

from django.conf import settings
from django.utils import timezone

from c3nav.mapdata.utils import json_encoder_reindent

from ..models import Package
from .const import ordered_models

class MapdataWriter:
    def __init__(self):
        self.keep = set()
        self.write = []
        self.delete = []

    def prepare_write_packages(self, prettify=False, diff=False):
        print('Writing Map Packages…')

        count = 0
        for model in ordered_models:
            for obj in model.objects.all().order_by('name').prefetch_related():
                file_path = os.path.join(obj.package.directory, obj.tofilename())
                full_file_path = os.path.join(settings.MAP_ROOT, file_path)
                self.keep.add(file_path)

                new_data = obj.tofile()
                new_data_encoded = json_encode(new_data)
                old_data = None
                old_data_encoded = None

                if os.path.isfile(full_file_path):
                    with open(full_file_path) as f:
                        old_data_encoded = f.read()
                    old_data = json.loads(old_data_encoded, parse_int=float)

                    if old_data != json.loads(new_data_encoded, parse_int=float):
                        if not diff:
                            print('- Updated: ' + file_path)
                    elif old_data_encoded != new_data_encoded:
                        if not prettify:
                            continue
                        if not diff:
                            print('- Prettified: ' + file_path)
                    else:
                        continue
                else:
                    if not diff:
                        print('- Created: ' + file_path)

                if diff:
                    sys.stdout.writelines(difflib.unified_diff(
                        [] if old_data is None else [(line + '\n') for line in old_data_encoded.split('\n')],
                        [(line + '\n') for line in new_data_encoded.split('\n')],
                        fromfiledate=timezone.make_aware(
                            datetime.fromtimestamp(0 if old_data is None else os.path.getmtime(full_file_path))
                        ).isoformat(),
                        tofiledate=timezone.now().isoformat(),
                        fromfile=file_path,
                        tofile=file_path
                    ))
                    print()

                self.write.append((file_path, new_data_encoded))
                count += 1

        # Delete old files
        for package_dir in Package.objects.all().values_list('directory', flat=True):
            for path, sub_dirs, filenames in os.walk(os.path.join(settings.MAP_ROOT, package_dir)):
                sub_dirs[:] = sorted([directory for directory in sub_dirs if not directory.startswith('.')])
                for filename in sorted(filenames):
                    if not filename.endswith('.json'):
                        continue
                    file_path = os.path.join(path[len(settings.MAP_ROOT) + 1:], filename)
                    if file_path not in self.keep:
                        if not diff:
                            print('- Deleted: ' + file_path)
                        else:
                            full_file_path = os.path.join(path, filename)
                            lines = list(open(full_file_path).readlines())
                            if not lines:
                                lines = ['\n']
                            sys.stdout.writelines(difflib.unified_diff(
                                lines,
                                [],
                                fromfiledate=timezone.make_aware(
                                    datetime.fromtimestamp(os.path.getmtime(full_file_path))
                                ).isoformat(),
                                tofiledate=timezone.make_aware(
                                    datetime.fromtimestamp(0)
                                ).isoformat(),
                                fromfile=file_path,
                                tofile=file_path
                            ))
                            print()
                        self.delete.append(file_path)

        return count

    def do_write_packages(self):
        for file_path, content in self.write:
            full_file_path = os.path.join(settings.MAP_ROOT, file_path)
            if content is not None:
                with open(full_file_path, 'w') as f:
                    f.write(content)

        for file_path in self.delete:
            full_file_path = os.path.join(settings.MAP_ROOT, file_path)
            os.remove(full_file_path)

def json_encode(data):
    return json_encoder_reindent(json.dumps, data, indent=4)+'\n'
<commit_msg>dumpmap: create directories if needed<commit_after>import difflib
import json
import os
import sys
from datetime import datetime

from django.conf import settings
from django.utils import timezone

from c3nav.mapdata.utils import json_encoder_reindent

from ..models import Package
from .const import ordered_models

class MapdataWriter:
    def __init__(self):
        self.keep = set()
        self.write = []
        self.delete = []

    def prepare_write_packages(self, prettify=False, diff=False):
        print('Writing Map Packages…')

        count = 0
        for model in ordered_models:
            for obj in model.objects.all().order_by('name').prefetch_related():
                file_path = os.path.join(obj.package.directory, obj.tofilename())
                full_file_path = os.path.join(settings.MAP_ROOT, file_path)
                self.keep.add(file_path)

                new_data = obj.tofile()
                new_data_encoded = json_encode(new_data)
                old_data = None
                old_data_encoded = None

                if os.path.isfile(full_file_path):
                    with open(full_file_path) as f:
                        old_data_encoded = f.read()
                    old_data = json.loads(old_data_encoded, parse_int=float)

                    if old_data != json.loads(new_data_encoded, parse_int=float):
                        if not diff:
                            print('- Updated: ' + file_path)
                    elif old_data_encoded != new_data_encoded:
                        if not prettify:
                            continue
                        if not diff:
                            print('- Prettified: ' + file_path)
                    else:
                        continue
                else:
                    if not diff:
                        print('- Created: ' + file_path)

                if diff:
                    sys.stdout.writelines(difflib.unified_diff(
                        [] if old_data is None else [(line + '\n') for line in old_data_encoded.split('\n')],
                        [(line + '\n') for line in new_data_encoded.split('\n')],
                        fromfiledate=timezone.make_aware(
                            datetime.fromtimestamp(0 if old_data is None else os.path.getmtime(full_file_path))
                        ).isoformat(),
                        tofiledate=timezone.now().isoformat(),
                        fromfile=file_path,
                        tofile=file_path
                    ))
                    print()

                self.write.append((file_path, new_data_encoded))
                count += 1

        # Delete old files
        for package_dir in Package.objects.all().values_list('directory', flat=True):
            for path, sub_dirs, filenames in os.walk(os.path.join(settings.MAP_ROOT, package_dir)):
                sub_dirs[:] = sorted([directory for directory in sub_dirs if not directory.startswith('.')])
                for filename in sorted(filenames):
                    if not filename.endswith('.json'):
                        continue
                    file_path = os.path.join(path[len(settings.MAP_ROOT) + 1:], filename)
                    if file_path not in self.keep:
                        if not diff:
                            print('- Deleted: ' + file_path)
                        else:
                            full_file_path = os.path.join(path, filename)
                            lines = list(open(full_file_path).readlines())
                            if not lines:
                                lines = ['\n']
                            sys.stdout.writelines(difflib.unified_diff(
                                lines,
                                [],
                                fromfiledate=timezone.make_aware(
                                    datetime.fromtimestamp(os.path.getmtime(full_file_path))
                                ).isoformat(),
                                tofiledate=timezone.make_aware(
                                    datetime.fromtimestamp(0)
                                ).isoformat(),
                                fromfile=file_path,
                                tofile=file_path
                            ))
                            print()
                        self.delete.append(file_path)

        return count

    def do_write_packages(self):
        for file_path, content in self.write:
            full_file_path = os.path.join(settings.MAP_ROOT, file_path)
            try:
                os.makedirs(os.path.join(os.path.split(full_file_path)[:-1]))
            except os.error:
                pass
            if content is not None:
                with open(full_file_path, 'w') as f:
                    f.write(content)

        for file_path in self.delete:
            full_file_path = os.path.join(settings.MAP_ROOT, file_path)
            os.remove(full_file_path)

def json_encode(data):
    return json_encoder_reindent(json.dumps, data, indent=4)+'\n'
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
import copy
from cloudify.decorators import operation
from libcloud.compute.types import Provider
from cloudstack_plugin.cloudstack_common import get_cloud_driver

__author__ = 'adaml'

def _get_server_from_context(ctx):
    server = {
        'name': ctx.node_id.replace('_', '-')
    }
    server.update(copy.deepcopy(ctx.properties['server']))
    return server

@operation
def start(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)
    #Change to debug level
    ctx.logger.info('reading server config from context')
    server_config = _get_server_from_context(ctx)

    name = server_config['name']
    image_id = server_config['image_id']
    size_name = server_config['size']
    keypair_name = server_config['keypair_name']
    #security_groups = server_config['security_groups']
    networks = server_config['networks']
    ipaddress = server_config['ip_address']

    network_list = cloud_driver.ex_list_networks()

    nets = [net for net in network_list if net.name in networks]

    for net in nets:
        ctx.logger.info('id: {0} name: {1}'.format(net.id, net.name))

    ctx.logger.info('getting required size {0}'.format(size_name))
    sizes = [size for size in cloud_driver.list_sizes() if size.name
                                                          == size_name]
    if sizes is None:
        raise RuntimeError(
            'Could not find size with name {0}'.format(size_name))
    size = sizes[0]

    ctx.logger.info('getting required image with ID {0}'.format(image_id))
    images = [template for template in cloud_driver.list_images()
              if image_id == template.id]
    if images is None:
        raise RuntimeError('Could not find image with ID {0}'.format(image_id))
    image = images[0]

    ctx.logger.info(
        "creating server vm with the following details: {0}".format(
            server_config))
    node = cloud_driver.create_node(name=name,
                                    image=image,
                                    size=size,
                                    ex_keyname=keypair_name,
                                    networks=nets,
                                    ipaddress=ipaddress)
    ctx.logger.info(
        'vm {0} was started successfully {1}'.format(
            node.name, server_config))

    ctx['instance_id'] = node.id

@operation
def delete(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx['instance_id']
    if instance_id is None:
        raise NameError('could not find node ID in runtime context: {0} '
                        .format(instance_id))

    ctx.logger.info('getting node with ID: {0} '.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        raise NameError('could not find node with ID: {0} '
                        .format(instance_id))

    ctx.logger.info('destroying vm with details: {0}'.format(node))
    cloud_driver.destroy_node(node)

@operation
def stop(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx.runtime_properties['instance_id']
    if instance_id is None:
        raise RuntimeError(
            'could not find node ID in runtime context: {0} '
            .format(instance_id))

    ctx.logger.info('getting node with ID: {0} '.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        raise RuntimeError('could not find node with ID {0}'
                           .format(instance_id))

    ctx.logger.info('stopping node with details {0}'.format(node))
    cloud_driver.ex_stop(node)

def _get_node_by_id(cloud_driver, instance_id):

    nodes = [node for node in cloud_driver.list_nodes() if instance_id
                                                          == node.id]
    if nodes is None:
        raise RuntimeError('could not find node by ID {0}'.format(instance_id))
        return None

    return nodes[0]

@operation
def get_state(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx.runtime_properties['instance_id']

    ctx.logger.info('getting node with ID {0}'.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        return False

    ctx.runtime_properties['ip'] = node.private_ips[0]
    ctx.runtime_properties['ip_address'] = node.private_ips[0]
    ctx.logger.info(
        'instance started successfully with IP {0}'
        .format(ctx.runtime_properties['ip']))
    return True
<commit_msg>fix fixed_ip ex_fixed_ip_address<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
import copy
from cloudify.decorators import operation
from libcloud.compute.types import Provider
from cloudstack_plugin.cloudstack_common import get_cloud_driver

__author__ = 'adaml'

def _get_server_from_context(ctx):
    server = {
        'name': ctx.node_id.replace('_', '-')
    }
    server.update(copy.deepcopy(ctx.properties['server']))
    return server

@operation
def start(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)
    #Change to debug level
    ctx.logger.info('reading server config from context')
    server_config = _get_server_from_context(ctx)

    name = server_config['name']
    image_id = server_config['image_id']
    size_name = server_config['size']
    keypair_name = server_config['keypair_name']
    #security_groups = server_config['security_groups']
    networks = server_config['networks']
    ipaddress = server_config['ip_address']

    network_list = cloud_driver.ex_list_networks()

    nets = [net for net in network_list if net.name in networks]

    for net in nets:
        ctx.logger.info('id: {0} name: {1}'.format(net.id, net.name))

    ctx.logger.info('getting required size {0}'.format(size_name))
    sizes = [size for size in cloud_driver.list_sizes() if size.name
                                                          == size_name]
    if sizes is None:
        raise RuntimeError(
            'Could not find size with name {0}'.format(size_name))
    size = sizes[0]

    ctx.logger.info('getting required image with ID {0}'.format(image_id))
    images = [template for template in cloud_driver.list_images()
              if image_id == template.id]
    if images is None:
        raise RuntimeError('Could not find image with ID {0}'.format(image_id))
    image = images[0]

    ctx.logger.info(
        "creating server vm with the following details: {0}".format(
            server_config))
    node = cloud_driver.create_node(name=name,
                                    image=image,
                                    size=size,
                                    ex_keyname=keypair_name,
                                    networks=nets,
                                    ex_ip_address=ipaddress)
    ctx.logger.info(
        'vm {0} was started successfully {1}'.format(
            node.name, server_config))

    ctx['instance_id'] = node.id

@operation
def delete(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx['instance_id']
    if instance_id is None:
        raise NameError('could not find node ID in runtime context: {0} '
                        .format(instance_id))

    ctx.logger.info('getting node with ID: {0} '.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        raise NameError('could not find node with ID: {0} '
                        .format(instance_id))

    ctx.logger.info('destroying vm with details: {0}'.format(node))
    cloud_driver.destroy_node(node)

@operation
def stop(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx.runtime_properties['instance_id']
    if instance_id is None:
        raise RuntimeError(
            'could not find node ID in runtime context: {0} '
            .format(instance_id))

    ctx.logger.info('getting node with ID: {0} '.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        raise RuntimeError('could not find node with ID {0}'
                           .format(instance_id))

    ctx.logger.info('stopping node with details {0}'.format(node))
    cloud_driver.ex_stop(node)

def _get_node_by_id(cloud_driver, instance_id):

    nodes = [node for node in cloud_driver.list_nodes() if instance_id
                                                          == node.id]
    if nodes is None:
        raise RuntimeError('could not find node by ID {0}'.format(instance_id))
        return None

    return nodes[0]

@operation
def get_state(ctx, **kwargs):

    ctx.logger.info("initializing {0} cloud driver"
                    .format(Provider.CLOUDSTACK))
    cloud_driver = get_cloud_driver(ctx)

    instance_id = ctx.runtime_properties['instance_id']

    ctx.logger.info('getting node with ID {0}'.format(instance_id))
    node = _get_node_by_id(cloud_driver, instance_id)
    if node is None:
        return False

    ctx.runtime_properties['ip'] = node.private_ips[0]
    ctx.runtime_properties['ip_address'] = node.private_ips[0]
    ctx.logger.info(
        'instance started successfully with IP {0}'
        .format(ctx.runtime_properties['ip']))
    return True
<|endoftext|>


<commit_before>from apsis.RandomSearchCore import RandomSearchCore
import numpy as np

lower_bound = np.zeros((3,1))
upper_bound = np.ones((3,1))

randomSearchCore = RandomSearchCore(lower_bound, upper_bound)

<commit_msg>First nose test.<commit_after><|endoftext|>


<commit_before>import subprocess
import signal

def subprocess_setup():
    # Python installs a SIGPIPE handler by default. This is usually not what
    # non-Python subprocesses expect.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)

class CmdError(RuntimeError):
    def __init__(self, command):
        self.command = command

    def __str__(self):
        if not isinstance(self.command, basestring):
            cmd = subprocess.list2cmdline(self.command)
        else:
            cmd = self.command

        return "Execution of '%s' failed" % cmd

class NotFoundError(CmdError):
    def __str__(self):
        return CmdError.__str__(self) + ": command not found"

class ExecutionError(CmdError):
    def __init__(self, command, exitcode, stdout = None, stderr = None):
        CmdError.__init__(self, command)
        self.exitcode = exitcode
        self.stdout = stdout
        self.stderr = stderr

    def __str__(self):
        message = ""
        if self.stderr:
            message += self.stderr
        if self.stdout:
            message += self.stdout
        if message:
            message = ":\n" + message
        return (CmdError.__str__(self) +
                " with exit code %s" % self.exitcode + message)

class Popen(subprocess.Popen):
    defaults = {
        "close_fds": True,
        "preexec_fn": subprocess_setup,
        #"stdout": subprocess.PIPE,
        #"stderr": subprocess.STDOUT,
        "stdin": subprocess.PIPE,
        "shell": False,
    }

    def __init__(self, *args, **kwargs):
        options = dict(self.defaults)
        options.update(kwargs)
        subprocess.Popen.__init__(self, *args, **options)

def run(cmd, input=None, **options):
    """Convenience function to run a command and return its output, raising an
    exception when the command fails"""

    if isinstance(cmd, basestring) and not "shell" in options:
        options["shell"] = True
    try:
        pipe = Popen(cmd, **options)
    except OSError, exc:
        if exc.errno == 2:
            raise NotFoundError(cmd)
        else:
            raise
    stdout, stderr = pipe.communicate(input)
    return pipe.returncode != 0
<commit_msg>lib/oe: Remove oe.process module<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import jmespath
import json
import os

from copy import deepcopy
from sys import exit
from netifaces import interfaces

from vyos.config import Config
from vyos.template import render
from vyos.util import call, cmd
from vyos.validate import is_addr_assigned
from vyos import ConfigError

default_config_data = {
    'deleted': False,
    'destination': [],
    'helper_functions': None,
    'pre_ct_helper': '',
    'pre_ct_conntrack': '',
    'out_ct_helper': '',
    'out_ct_conntrack': '',
    'source': []
}

iptables_nat_config = '/tmp/vyos-nat-rules.nft'

def _check_kmod():
    """ load required Kernel modules """
    modules = ['nft_nat', 'nft_chain_nat_ipv4']
    for module in modules:
        if not os.path.exists(f'/sys/module/{module}'):
            if call(f'modprobe {module}') != 0:
                raise ConfigError(f'Loading Kernel module {module} failed')

def get_handler(json, chain, target):
    """ Get nftable rule handler number of given chain/target combination.
    Handler is required when adding NAT/Conntrack helper targets """
    for x in json:
        if x['chain'] != chain:
            continue
        if x['target'] != target:
            continue
        return x['handle']

    return None

def parse_source_destination(conf, source_dest):
    """ Common wrapper to read in both NAT source and destination CLI """
    tmp = []
    base_level = ['nat', source_dest]
    conf.set_level(base_level)
    for number in conf.list_nodes(['rule']):
        rule = {
            'description': '',
            'dest_address': '',
            'dest_port': '',
            'disable': False,
            'exclude': False,
            'interface_in': '',
            'interface_out': '',
            'log': False,
            'protocol': '',
            'number': number,
            'source_address': '',
            'source_port': '',
            'translation_address': '',
            'translation_port': ''
        }
        conf.set_level(base_level + ['rule', number])

        if conf.exists(['description']):
            rule['description'] = conf.return_value(['description'])

        if conf.exists(['destination', 'address']):
            rule['dest_address'] = conf.return_value(['destination', 'address'])

        if conf.exists(['destination', 'port']):
            rule['dest_port'] = conf.return_value(['destination', 'port'])

        if conf.exists(['disable']):
            rule['disable'] = True

        if conf.exists(['exclude']):
            rule['exclude'] = True

        if conf.exists(['inbound-interface']):
            rule['interface_in'] = conf.return_value(['inbound-interface'])

        if conf.exists(['outbound-interface']):
            rule['interface_out'] = conf.return_value(['outbound-interface'])

        if conf.exists(['log']):
            rule['log'] = True

        if conf.exists(['protocol']):
            rule['protocol'] = conf.return_value(['protocol'])

        if conf.exists(['source', 'address']):
            rule['source_address'] = conf.return_value(['source', 'address'])

        if conf.exists(['source', 'port']):
            rule['source_port'] = conf.return_value(['source', 'port'])

        if conf.exists(['translation', 'address']):
            rule['translation_address'] = conf.return_value(['translation', 'address'])

        if conf.exists(['translation', 'port']):
            rule['translation_port'] = conf.return_value(['translation', 'port'])

        tmp.append(rule)

    return tmp

def get_config():
    nat = deepcopy(default_config_data)
    conf = Config()

    # read in current nftable (once) for further processing
    tmp = cmd('nft -j list table raw')
    nftable_json = json.loads(tmp)

    # condense the full JSON table into a list with only relevand informations
    pattern = 'nftables[?rule].rule[?expr[].jump].{chain: chain, handle: handle, target: expr[].jump.target | [0]}'
    condensed_json = jmespath.search(pattern, nftable_json)

    if not conf.exists(['nat']):
        nat['helper_functions'] = 'remove'

        # Retrieve current table handler positions
        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_HELPER')
        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK')
        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_HELPER')
        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK')

        nat['deleted'] = True

        return nat

    # check if NAT connection tracking helpers need to be set up - this has to
    # be done only once
    if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'):
        nat['helper_functions'] = 'add'

        # Retrieve current table handler positions
        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_IGNORE')
        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_PREROUTING_HOOK')
        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_IGNORE')
        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_OUTPUT_HOOK')

    # set config level for parsing in NAT configuration
    conf.set_level(['nat'])

    # use a common wrapper function to read in the source / destination
    # tree from the config - thus we do not need to replicate almost the
    # same code :-)
    for tgt in ['source', 'destination']:
        nat[tgt] = parse_source_destination(conf, tgt)

    return nat

def verify_rule(rule):
    if rule['translation_port']:
        if rule['protocol'] not in ['tcp', 'udp', 'tcp_udp']:
            proto = rule['protocol']
            raise ConfigError(f'{err_msg} ports can only be specified when protocol is "tcp", "udp" or "tcp_udp" (currently "{proto}")')

        if '/' in rule['translation_address']:
            raise ConfigError(f'{err_msg}\n' \
                             'Cannot use ports with an IPv4net type translation address as it\n' \
                             'statically maps a whole network of addresses onto another\n' \
                             'network of addresses')

def verify(nat):
    if nat['deleted']:
        # no need to verify the CLI as NAT is going to be deactivated
        return None

    if nat['helper_functions']:
        if not (nat['pre_ct_ignore'] or nat['pre_ct_conntrack'] or nat['out_ct_ignore'] or nat['out_ct_conntrack']):
            raise Exception('could not determine nftable ruleset handlers')

    for rule in nat['source']:
        interface = rule['interface_out']
        if interface and interface not in interfaces():
            print(f'NAT configuration warning: interface {interface} does not exist on this system')

        err_msg = f"Source NAT configuration error in rule {rule['number']}:"

        if not rule['interface_out']:
            raise ConfigError(f'{err_msg} outbound-interface not specified')

        if not rule['translation_address']:
            raise ConfigError(f'{err_msg} translation address not specified')
        else:
            addr = rule['translation_address']
            if addr != 'masquerade' and not is_addr_assigned(addr):
                printf(f'Warning: IP address {addr} does not exist on the system!')

        # common rule verification
        verify_rule(rule)

    for rule in nat['destination']:
        interface = rule['interface_in']
        if interface and interface not in interfaces():
            print(f'NAT configuration warning: interface {interface} does not exist on this system')

        if not rule['interface_in']:
            raise ConfigError(f'{err_msg} inbound-interface not specified')

        # common rule verification
        verify_rule(rule)

    return None

def generate(nat):
    render(iptables_nat_config, 'firewall/nftables-nat.tmpl', nat, trim_blocks=True, permission=0o755)

    return None

def apply(nat):
    cmd(f'{iptables_nat_config}')

    return None

if __name__ == '__main__':
    try:
        _check_kmod()
        c = get_config()
        verify(c)
        generate(c)
        apply(c)
    except ConfigError as e:
        print(e)
        exit(1)
<commit_msg>nat: T2198: extend verify() for destination ports<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2020 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import jmespath
import json
import os

from copy import deepcopy
from sys import exit
from netifaces import interfaces

from vyos.config import Config
from vyos.template import render
from vyos.util import call, cmd
from vyos.validate import is_addr_assigned
from vyos import ConfigError

default_config_data = {
    'deleted': False,
    'destination': [],
    'helper_functions': None,
    'pre_ct_helper': '',
    'pre_ct_conntrack': '',
    'out_ct_helper': '',
    'out_ct_conntrack': '',
    'source': []
}

iptables_nat_config = '/tmp/vyos-nat-rules.nft'

def _check_kmod():
    """ load required Kernel modules """
    modules = ['nft_nat', 'nft_chain_nat_ipv4']
    for module in modules:
        if not os.path.exists(f'/sys/module/{module}'):
            if call(f'modprobe {module}') != 0:
                raise ConfigError(f'Loading Kernel module {module} failed')

def get_handler(json, chain, target):
    """ Get nftable rule handler number of given chain/target combination.
    Handler is required when adding NAT/Conntrack helper targets """
    for x in json:
        if x['chain'] != chain:
            continue
        if x['target'] != target:
            continue
        return x['handle']

    return None

def verify_rule(rule, err_msg):
    if rule['translation_port'] or rule['dest_port']:
        if rule['protocol'] not in ['tcp', 'udp', 'tcp_udp']:
            proto = rule['protocol']
            raise ConfigError(f'{err_msg} ports can only be specified when protocol is "tcp", "udp" or "tcp_udp" (currently "{proto}")')

        if '/' in rule['translation_address']:
            raise ConfigError(f'{err_msg}\n' \
                             'Cannot use ports with an IPv4net type translation address as it\n' \
                             'statically maps a whole network of addresses onto another\n' \
                             'network of addresses')

def parse_source_destination(conf, source_dest):
    """ Common wrapper to read in both NAT source and destination CLI """
    tmp = []
    base_level = ['nat', source_dest]
    conf.set_level(base_level)
    for number in conf.list_nodes(['rule']):
        rule = {
            'description': '',
            'dest_address': '',
            'dest_port': '',
            'disable': False,
            'exclude': False,
            'interface_in': '',
            'interface_out': '',
            'log': False,
            'protocol': '',
            'number': number,
            'source_address': '',
            'source_port': '',
            'translation_address': '',
            'translation_port': ''
        }
        conf.set_level(base_level + ['rule', number])

        if conf.exists(['description']):
            rule['description'] = conf.return_value(['description'])

        if conf.exists(['destination', 'address']):
            rule['dest_address'] = conf.return_value(['destination', 'address'])

        if conf.exists(['destination', 'port']):
            rule['dest_port'] = conf.return_value(['destination', 'port'])

        if conf.exists(['disable']):
            rule['disable'] = True

        if conf.exists(['exclude']):
            rule['exclude'] = True

        if conf.exists(['inbound-interface']):
            rule['interface_in'] = conf.return_value(['inbound-interface'])

        if conf.exists(['outbound-interface']):
            rule['interface_out'] = conf.return_value(['outbound-interface'])

        if conf.exists(['log']):
            rule['log'] = True

        if conf.exists(['protocol']):
            rule['protocol'] = conf.return_value(['protocol'])

        if conf.exists(['source', 'address']):
            rule['source_address'] = conf.return_value(['source', 'address'])

        if conf.exists(['source', 'port']):
            rule['source_port'] = conf.return_value(['source', 'port'])

        if conf.exists(['translation', 'address']):
            rule['translation_address'] = conf.return_value(['translation', 'address'])

        if conf.exists(['translation', 'port']):
            rule['translation_port'] = conf.return_value(['translation', 'port'])

        tmp.append(rule)

    return tmp

def get_config():
    nat = deepcopy(default_config_data)
    conf = Config()

    # read in current nftable (once) for further processing
    tmp = cmd('nft -j list table raw')
    nftable_json = json.loads(tmp)

    # condense the full JSON table into a list with only relevand informations
    pattern = 'nftables[?rule].rule[?expr[].jump].{chain: chain, handle: handle, target: expr[].jump.target | [0]}'
    condensed_json = jmespath.search(pattern, nftable_json)

    if not conf.exists(['nat']):
        nat['helper_functions'] = 'remove'

        # Retrieve current table handler positions
        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_HELPER')
        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK')
        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_HELPER')
        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK')

        nat['deleted'] = True

        return nat

    # check if NAT connection tracking helpers need to be set up - this has to
    # be done only once
    if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'):
        nat['helper_functions'] = 'add'

        # Retrieve current table handler positions
        nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_IGNORE')
        nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'VYATTA_CT_PREROUTING_HOOK')
        nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_IGNORE')
        nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'VYATTA_CT_OUTPUT_HOOK')

    # set config level for parsing in NAT configuration
    conf.set_level(['nat'])

    # use a common wrapper function to read in the source / destination
    # tree from the config - thus we do not need to replicate almost the
    # same code :-)
    for tgt in ['source', 'destination']:
        nat[tgt] = parse_source_destination(conf, tgt)

    return nat

def verify(nat):
    if nat['deleted']:
        # no need to verify the CLI as NAT is going to be deactivated
        return None

    if nat['helper_functions']:
        if not (nat['pre_ct_ignore'] or nat['pre_ct_conntrack'] or nat['out_ct_ignore'] or nat['out_ct_conntrack']):
            raise Exception('could not determine nftable ruleset handlers')

    for rule in nat['source']:
        interface = rule['interface_out']
        err_msg = f"Source NAT configuration error in rule {rule['number']}:"

        if interface and interface not in interfaces():
            print(f'NAT configuration warning: interface {interface} does not exist on this system')

        if not rule['interface_out']:
            raise ConfigError(f'{err_msg} outbound-interface not specified')

        if not rule['translation_address']:
            raise ConfigError(f'{err_msg} translation address not specified')
        else:
            addr = rule['translation_address']
            if addr != 'masquerade' and not is_addr_assigned(addr):
                printf(f'Warning: IP address {addr} does not exist on the system!')

        # common rule verification
        verify_rule(rule, err_msg)

    for rule in nat['destination']:
        interface = rule['interface_in']
        err_msg = f"Destination NAT configuration error in rule {rule['number']}:"

        if interface and interface not in interfaces():
            print(f'NAT configuration warning: interface {interface} does not exist on this system')

        if not rule['interface_in']:
            raise ConfigError(f'{err_msg} inbound-interface not specified')

        # common rule verification
        verify_rule(rule, err_msg)

    return None

def generate(nat):
    render(iptables_nat_config, 'firewall/nftables-nat.tmpl', nat, trim_blocks=True, permission=0o755)

    return None

def apply(nat):
    cmd(f'{iptables_nat_config}')
    if os.path.isfile(iptables_nat_config):
        os.unlink(iptables_nat_config)

    return None

if __name__ == '__main__':
    try:
        _check_kmod()
        c = get_config()
        verify(c)
        generate(c)
        apply(c)
    except ConfigError as e:
        print(e)
        exit(1)
<|endoftext|>


<commit_before>"""
Test Runner class. Lets you setup testrail and run a bunch of tests one after the other
"""

import os,subprocess

class Test_Runner_Class:
    "Test Runner class"
    def __init__(self,base_url='http://qxf2.com',testrail_flag='N',browserstack_flag='N',os_name='Windows',os_version='7',browser='firefox',browser_version='33'):
        "Constructor"
        self.python_executable = "python"
        self.util_directory = os.path.abspath((os.path.dirname(__file__)))
        self.test_directory = os.path.abspath(os.path.join(os.path.dirname(__file__),'..','tests'))
        self.setup_testrail_script = os.path.join(self.util_directory,"setup_testrail.py")
        self.reset(base_url=base_url,
                   testrail_flag=testrail_flag,
                   browserstack_flag=browserstack_flag,
                   os_name=os_name,
                   os_version=os_version,
                   browser=browser,
                   browser_version=browser_version)

    def check_file_exists(self,file_path):
        "Check if the config file exists and is a file"
        file_exist_flag = True
        if os.path.exists(file_path):
            if not os.path.isfile(file_path):
                print('\n****')
                print('Script file provided is not a file: ')
                print(file_path)
                print('****')
                file_exist_flag = False
        else:
            print('\n****')
            print('Unable to locate the provided script file: ')
            print(file_path)
            print('****')
            conf_flag = False

        return file_exist_flag

    def reset(self,base_url=None,testrail_flag=None,browserstack_flag=None,os_name=None,os_version=None,browser=None,browser_version=None):
        "Reset the private variables"
        if base_url is not None:
            self.base_url = base_url
        if testrail_flag is not None:
            self.testrail_flag = testrail_flag
        if browserstack_flag is not None:
            self.browserstack_flag = browserstack_flag
        if os_name is not None:
            self.os_name = os_name
        if os_version is not None:
            self.os_version = os_version
        if browser is not None:
            self.browser = browser
        if browser_version is not None:
            self.browser_version = browser_version

    def run_test(self,test_name):
        "Run the test script with the given command line options"
        testscript_args_list = self.setup_test_script_args_list(test_name)
        self.run_script(testscript_args_list)

    def run_setup_testrail(self,test_name=None,test_run_name='',case_ids_list=None,name_override_flag=True):
        "Run the setup_testrail with given command line options"
        if self.testrail_flag.lower() == 'y':
            testrail_args_list = self.setup_testrail_args_list(test_name,test_run_name,case_ids_list,name_override_flag)
            self.run_script(testrail_args_list)

    def run_script(self,args_list):
        "Run the script on command line with given args_list"
        print("\nWill be running the following script:")
        print(' '.join(args_list))
        print("Starting..")
        subprocess.call(args_list,shell=True)
        print("Done!")

    def setup_testrail_args_list(self,test_name=None,test_run_name='',case_ids_list=None,name_override_flag=True):
        "Convert the command line arguments into list for setup_testrail.py"
        args_list = []
        #python setup_testrail.py -r test_run_name -d test_run_description
        if self.check_file_exists(self.setup_testrail_script):
            args_list = [self.python_executable,self.setup_testrail_script]
            if test_run_name != '':
                args_list.append("-r")
                args_list.append(test_run_name)
            if test_name is not None:
                args_list.append("-d")
                args_list.append(test_name)
            if name_override_flag is False:
                args_list.append("-n")
                args_list.append("N")
            if case_ids_list is not None:
                args_list.append("-c")
                case_ids_list = ','.join(case_ids_list)
                args_list.append(case_ids_list)

        return args_list

    def setup_test_script_args_list(self,test_name):
        "convert the command line arguments into list for test script"
        args_list = []
        #python test_script.py -x Y
        test_script_name = test_name + ".py"
        test_script_name = os.path.join(self.test_directory,test_script_name)
        if self.check_file_exists(test_script_name):
            args_list = [self.python_executable,test_script_name,"-b",self.browser,"-u",self.base_url,"-x",self.testrail_flag,"-s",self.browserstack_flag,"-o",self.os_version,"-v",self.browser_version,"-p",self.os_name]

        return args_list

<commit_msg>removed Test_Runner_Class.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function
from dateutil import parser
import json
import os
import pika
import rethinkdb as r
import subprocess
import sys
import time
import urllib

runs = int(os.environ.get("NONPARALLEL_JOB_RUNS", 1))
MAX_NUM_ITERATIONS = 1 # For now we'll limit the jobs to run
                       # up to "MAX_NUM_ITERATIONS" commits
                       # in the font project repos

REPO_URL = None
FAMILYNAME = None
db = None

def run(cmd):
  try:
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
  except subprocess.CalledProcessError, e:
    print ("Error: {}".format(e))
  except OSError:
    print("could not execute command '{}' !".format(cmd))

def get_filenames(fonts_dir, fonts_prefix):
  files = []
  try:
    for f in os.listdir(fonts_dir):
      pref = f[:len(fonts_prefix)]
      if f[-4:] == ".ttf" and pref == fonts_prefix:
        fullpath = fonts_dir + "/" + f
        # Do we need to escape spaces in the fullpaths here?
        files.append(fullpath)
  except OSError:
    pass
  return files

def clone(repo, clone_dir, depth=None):
  if depth is None:
    run(["git", "clone", repo, clone_dir])
  else:
    run(["git", "clone", repo, "--depth={}".format(depth), clone_dir])

def calc_font_stats(results):
  stats = {
    "Total": len(results.keys()),
    "OK": 0
  }
  for k in results.keys():
    result = results[k]['result']
    if result not in stats.keys():
      stats[result] = 1
    else:
      stats[result] += 1
  return stats

def update_global_stats(summary, stats):
  for k in stats.keys():
    if k in summary.keys():
      summary[k] += stats[k]
    else:
      summary[k] = stats[k]

def save_output_on_database(commit, output):
  data = {"commit": commit, "familyname": FAMILYNAME, "output": output}
  print ("save_output_on_database: '{}' [{}]".format(FAMILYNAME, commit))
  if db.table('fb_log').filter({"commit": commit, "familyname": FAMILYNAME}).count().run() == 0:
    db.table('fb_log').insert(data).run()
  else:
    db.table('fb_log').filter({"commit": commit, "familyname": FAMILYNAME}).update(data).run()

def save_results_on_database(f, fonts_dir, commit, i, family_stats, date):
#  print ("Invoked 'save_results_on_database' with f='{}'".format(f))
  if f[-20:] != ".ttf.fontbakery.json":
#    print("Not a report file.")
    return

  print ("Check results JSON file: {}".format(f))
  fname = f.split('.fontbakery.json')[0]
  family_stats['familyname'] = FAMILYNAME
  data = open(fonts_dir + "/" + f).read()
  results = json.loads(data)
  font_stats = calc_font_stats(results)
  update_global_stats(family_stats['summary'], font_stats)
  check_results = {
    "giturl": REPO_URL,
    "results": results,
    "commit": commit,
    "fontname": fname,
    "familyname": FAMILYNAME,
    "date": date,
    "stats": font_stats,
    "HEAD": (i==0)
  }

  if db.table('check_results').filter({"commit": commit, "fontname":fname}).count().run() == 0:
    db.table('check_results').insert(check_results).run()
  else:
    db.table('check_results').filter({"commit": commit, "fontname":fname}).update(check_results).run()

def infer_date_from_git():
  try:
    datestr = run(['git',
                   'log',
                   '-1',
                   '--pretty=format:\"%cd\"'])
    print ("datestr = {}".format(datestr))
    return parser.parse(datestr, fuzzy=True)
  except:
    print ("Failed to parse commit date string")
    return None

def save_overall_stats_to_database(commit, family_stats):
  print ("save_overall_stats_to_database:\nstats = {}".format(family_stats))
  if db.table('cached_stats').filter({"commit":commit, "giturl": REPO_URL, "familyname": FAMILYNAME}).count().run() == 0:
    db.table('cached_stats').insert(family_stats).run()
  else:
    db.table('cached_stats').filter({"commit":commit, "giturl": REPO_URL, "familyname": FAMILYNAME}).update(family_stats).run()

# Returns boolean "success"
def run_fontbakery_on_commit(fonts_dir, fonts_prefix, commit, i):
  date = infer_date_from_git()
  files = get_filenames(fonts_dir, fonts_prefix)
  if len(files) > 0:
    print ("We'll be checking these files:\n{}\n".format("\n".join(files)))
  else:
    print ("No font files were found.")
    return False

  output = run(["python", "/fontbakery-check-ttf.py", "--verbose", "--json"] + files)
  save_output_on_database(commit, output)

  family_stats = {
    "familyname": FAMILYNAME,
    "giturl": REPO_URL,
    "commit": commit,
     "date": date,
    "summary": {"OK": 0,
                "Total": 0},
    "HEAD": (i==0)
  }

  for f in os.listdir(fonts_dir):
    save_results_on_database(f, fonts_dir, commit, i, family_stats, date)

  save_overall_stats_to_database(commit, family_stats)
  return True

def checkout_and_test_git_repo(fonts_dir, fonts_prefix):
  global db
  clone(REPO_URL, "clonedir", depth=MAX_NUM_ITERATIONS)
  os.chdir("clonedir")
  run(["git", "checkout", "master"])
  print ("We're now at master branch.")

  lines = run(["git", "log", "--oneline", "."]).strip().split('\n')
  commits = ["master"] + [line.split()[0].strip() for line in lines]
  print ("The commits we'll iterate over are: {}".format(commits))

  for i, commit in enumerate(commits):
    print ("[{} of {}] Checking out commit '{}'".format(i+1, len(commits), commit))
    run(["git", "checkout", commit])
    print ("==> Running fontbakery on commit '{}'...".format(commit))
    run_fontbakery_on_commit(fonts_dir, fonts_prefix, commit, i)
  os.chdir("/")

def run_fontbakery_on_production_files():
  commit = "prod" # This is sort of a hack (but works neatly)!

  PROD_URL = "https://fonts.google.com/download?family={}".format(FAMILYNAME)
  os.mkdir("prod")
  os.chdir("prod")
  open("family.zip", "w+").write(urllib.urlopen(PROD_URL).read())
  output = run(["unzip", "family.zip"])
  print("unzip output: {}".format(output))
  files = []
  for f in os.listdir("."):
    if f[-4:] == ".ttf":
      # Do we need to escape spaces in the fullpaths here?
      files.append(f)

  if len(files) == 0:
    print ("Could not find production files for '{}'".format(FAMILYNAME))
    os.chdir("/")
    return False

  print ("We'll check the following PRODUCTION font files: {}".format(files))
  output = run(["python", "/fontbakery-check-ttf.py", "--verbose", "--json"] + files)
  save_output_on_database(commit, output)

  family_stats = {
    "familyname": FAMILYNAME,
    "giturl": REPO_URL,
    "commit": commit,
    "date": None,
    "summary": {"OK": 0,
                "Total": 0},
    "HEAD": False
  }

  for f in os.listdir("."):
    save_results_on_database(f, ".", commit, -1, family_stats, None)

  save_overall_stats_to_database(commit, family_stats)
  os.chdir("/")
  return True

connection = None
def callback(ch, method, properties, body): #pylint: disable=unused-argument
  global runs, REPO_URL, FAMILYNAME
  msg = json.loads(body)
  print("Received %r" % msg, file=sys.stderr)

  STATUS = msg["STATUS"]
  REPO_URL = msg["GIT_REPO_URL"]
  FAMILYNAME = msg["FAMILYNAME"]
  fonts_prefix = msg["FONTFILE_PREFIX"]
  fonts_dir = '.'
  if '/' in fonts_prefix:
    prefix_elements = fonts_prefix.split('/')
    fonts_prefix = prefix_elements.pop(-1)
    fonts_dir = '/'.join(prefix_elements)

  run_fontbakery_on_production_files()

  if STATUS == "OK" or STATUS == "NOTE":
    checkout_and_test_git_repo(fonts_dir, fonts_prefix)

  ch.basic_ack(delivery_tag = method.delivery_tag)
  connection.close()

  runs -= 1
  if runs == 0:
    print("Finished work. Shutting down this container instance...")
    sys.exit(0)
  else:
    print("Will fetch another workload...")

def main():
  global connection, db
  os.chdir("/")

  db_host = os.environ.get("RETHINKDB_DRIVER_SERVICE_HOST", 'db')
  r.connect(db_host, 28015).repl()
  try:
    r.db_create('fontbakery').run()
    r.db('fontbakery').table_create('fb_log').run()
    r.db('fontbakery').table_create('check_results').run()
    r.db('fontbakery').table_create('cached_stats').run()
  except:
    # OK, database and tables already exist.
    pass
  db = r.db('fontbakery')

  try:
    db.table('cached_stats').index_create('familyname').run()
  except:
    pass

  while True:
    try:
      msgqueue_host = os.environ.get("RABBITMQ_SERVICE_SERVICE_HOST", os.environ.get("BROKER"))
      connection = pika.BlockingConnection(pika.ConnectionParameters(host=msgqueue_host))
      channel = connection.channel()
      channel.queue_declare(queue='font_repo_queue', durable=True)
      print('Waiting for messages...', file=sys.stderr)
      channel.basic_qos(prefetch_count=1)
      channel.basic_consume(callback,
                            queue='font_repo_queue')
      channel.start_consuming()
    except pika.exceptions.ConnectionClosed:
      print ("RabbitMQ not ready yet.", file=sys.stderr)
      time.sleep(1)

main()

<commit_msg>updating container code to use new fontbakery syntax<commit_after>#!/usr/bin/env python
from __future__ import print_function
from dateutil import parser
import json
import os
import pika
import rethinkdb as r
import subprocess
import sys
import time
import urllib

runs = int(os.environ.get("NONPARALLEL_JOB_RUNS", 1))
MAX_NUM_ITERATIONS = 1 # For now we'll limit the jobs to run
                       # up to "MAX_NUM_ITERATIONS" commits
                       # in the font project repos

REPO_URL = None
FAMILYNAME = None
db = None

def run(cmd):
  try:
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
  except subprocess.CalledProcessError, e:
    print ("Error: {}".format(e))
  except OSError:
    print("could not execute command '{}' !".format(cmd))

def get_filenames(fonts_dir, fonts_prefix):
  files = []
  try:
    for f in os.listdir(fonts_dir):
      pref = f[:len(fonts_prefix)]
      if f[-4:] == ".ttf" and pref == fonts_prefix:
        fullpath = fonts_dir + "/" + f
        # Do we need to escape spaces in the fullpaths here?
        files.append(fullpath)
  except OSError:
    pass
  return files

def clone(repo, clone_dir, depth=None):
  if depth is None:
    run(["git", "clone", repo, clone_dir])
  else:
    run(["git", "clone", repo, "--depth={}".format(depth), clone_dir])

def calc_font_stats(results):
  stats = {
    "Total": len(results.keys()),
    "OK": 0
  }
  for k in results.keys():
    result = results[k]['result']
    if result not in stats.keys():
      stats[result] = 1
    else:
      stats[result] += 1
  return stats

def update_global_stats(summary, stats):
  for k in stats.keys():
    if k in summary.keys():
      summary[k] += stats[k]
    else:
      summary[k] = stats[k]

def save_output_on_database(commit, output):
  data = {"commit": commit, "familyname": FAMILYNAME, "output": output}
  print ("save_output_on_database: '{}' [{}]".format(FAMILYNAME, commit))
  if db.table('fb_log').filter({"commit": commit, "familyname": FAMILYNAME}).count().run() == 0:
    db.table('fb_log').insert(data).run()
  else:
    db.table('fb_log').filter({"commit": commit, "familyname": FAMILYNAME}).update(data).run()

def save_results_on_database(f, fonts_dir, commit, i, family_stats, date):
#  print ("Invoked 'save_results_on_database' with f='{}'".format(f))
  if f[-20:] != ".ttf.fontbakery.json":
#    print("Not a report file.")
    return

  print ("Check results JSON file: {}".format(f))
  fname = f.split('.fontbakery.json')[0]
  family_stats['familyname'] = FAMILYNAME
  data = open(fonts_dir + "/" + f).read()
  results = json.loads(data)
  font_stats = calc_font_stats(results)
  update_global_stats(family_stats['summary'], font_stats)
  check_results = {
    "giturl": REPO_URL,
    "results": results,
    "commit": commit,
    "fontname": fname,
    "familyname": FAMILYNAME,
    "date": date,
    "stats": font_stats,
    "HEAD": (i==0)
  }

  if db.table('check_results').filter({"commit": commit, "fontname":fname}).count().run() == 0:
    db.table('check_results').insert(check_results).run()
  else:
    db.table('check_results').filter({"commit": commit, "fontname":fname}).update(check_results).run()

def infer_date_from_git():
  try:
    datestr = run(['git',
                   'log',
                   '-1',
                   '--pretty=format:\"%cd\"'])
    print ("datestr = {}".format(datestr))
    return parser.parse(datestr, fuzzy=True)
  except:
    print ("Failed to parse commit date string")
    return None

def save_overall_stats_to_database(commit, family_stats):
  print ("save_overall_stats_to_database:\nstats = {}".format(family_stats))
  if db.table('cached_stats').filter({"commit":commit, "giturl": REPO_URL, "familyname": FAMILYNAME}).count().run() == 0:
    db.table('cached_stats').insert(family_stats).run()
  else:
    db.table('cached_stats').filter({"commit":commit, "giturl": REPO_URL, "familyname": FAMILYNAME}).update(family_stats).run()

# Returns boolean "success"
def run_fontbakery_on_commit(fonts_dir, fonts_prefix, commit, i):
  date = infer_date_from_git()
  files = get_filenames(fonts_dir, fonts_prefix)
  if len(files) > 0:
    print ("We'll be checking these files:\n{}\n".format("\n".join(files)))
  else:
    print ("No font files were found.")
    return False

  output = run(["fontbakery", "check-ttf", "--verbose", "--json"] + files)
  save_output_on_database(commit, output)

  family_stats = {
    "familyname": FAMILYNAME,
    "giturl": REPO_URL,
    "commit": commit,
    "date": date,
    "summary": {"OK": 0,
                "Total": 0},
    "HEAD": (i==0)
  }

  for f in os.listdir(fonts_dir):
    save_results_on_database(f, fonts_dir, commit, i, family_stats, date)

  save_overall_stats_to_database(commit, family_stats)
  return True

def checkout_and_test_git_repo(fonts_dir, fonts_prefix):
  global db
  clone(REPO_URL, "clonedir", depth=MAX_NUM_ITERATIONS)
  os.chdir("clonedir")
  run(["git", "checkout", "master"])
  print ("We're now at master branch.")

  lines = run(["git", "log", "--oneline", "."]).strip().split('\n')
  commits = ["master"] + [line.split()[0].strip() for line in lines]
  print ("The commits we'll iterate over are: {}".format(commits))

  for i, commit in enumerate(commits):
    print ("[{} of {}] Checking out commit '{}'".format(i+1, len(commits), commit))
    run(["git", "checkout", commit])
    print ("==> Running fontbakery on commit '{}'...".format(commit))
    run_fontbakery_on_commit(fonts_dir, fonts_prefix, commit, i)
  os.chdir("/")

def run_fontbakery_on_production_files():
  commit = "prod" # This is sort of a hack (but works neatly)!

  PROD_URL = "https://fonts.google.com/download?family={}".format(FAMILYNAME)
  os.mkdir("prod")
  os.chdir("prod")
  open("family.zip", "w+").write(urllib.urlopen(PROD_URL).read())
  output = run(["unzip", "family.zip"])
  print("unzip output: {}".format(output))
  files = []
  for f in os.listdir("."):
    if f[-4:] == ".ttf":
      # Do we need to escape spaces in the fullpaths here?
      files.append(f)

  if len(files) == 0:
    print ("Could not find production files for '{}'".format(FAMILYNAME))
    os.chdir("/")
    return False

  print ("We'll check the following PRODUCTION font files: {}".format(files))
  output = run(["fontbakery", "check-ttf", "--verbose", "--json"] + files)
  save_output_on_database(commit, output)

  family_stats = {
    "familyname": FAMILYNAME,
    "giturl": REPO_URL,
    "commit": commit,
    "date": None,
    "summary": {"OK": 0,
                "Total": 0},
    "HEAD": False
  }

  for f in os.listdir("."):
    save_results_on_database(f, ".", commit, -1, family_stats, None)

  save_overall_stats_to_database(commit, family_stats)
  os.chdir("/")
  return True

connection = None
def callback(ch, method, properties, body): #pylint: disable=unused-argument
  global runs, REPO_URL, FAMILYNAME
  msg = json.loads(body)
  print("Received %r" % msg, file=sys.stderr)

  STATUS = msg["STATUS"]
  REPO_URL = msg["GIT_REPO_URL"]
  FAMILYNAME = msg["FAMILYNAME"]
  fonts_prefix = msg["FONTFILE_PREFIX"]
  fonts_dir = '.'
  if '/' in fonts_prefix:
    prefix_elements = fonts_prefix.split('/')
    fonts_prefix = prefix_elements.pop(-1)
    fonts_dir = '/'.join(prefix_elements)

  run_fontbakery_on_production_files()

  if STATUS == "OK" or STATUS == "NOTE":
    checkout_and_test_git_repo(fonts_dir, fonts_prefix)

  ch.basic_ack(delivery_tag = method.delivery_tag)
  connection.close()

  runs -= 1
  if runs == 0:
    print("Finished work. Shutting down this container instance...")
    sys.exit(0)
  else:
    print("Will fetch another workload...")

def main():
  global connection, db
  os.chdir("/")

  db_host = os.environ.get("RETHINKDB_DRIVER_SERVICE_HOST", 'db')
  r.connect(db_host, 28015).repl()
  try:
    r.db_create('fontbakery').run()
    r.db('fontbakery').table_create('fb_log').run()
    r.db('fontbakery').table_create('check_results').run()
    r.db('fontbakery').table_create('cached_stats').run()
  except:
    # OK, database and tables already exist.
    pass
  db = r.db('fontbakery')

  try:
    db.table('cached_stats').index_create('familyname').run()
  except:
    pass

  while True:
    try:
      msgqueue_host = os.environ.get("RABBITMQ_SERVICE_SERVICE_HOST", os.environ.get("BROKER"))
      connection = pika.BlockingConnection(pika.ConnectionParameters(host=msgqueue_host))
      channel = connection.channel()
      channel.queue_declare(queue='font_repo_queue', durable=True)
      print('Waiting for messages...', file=sys.stderr)
      channel.basic_qos(prefetch_count=1)
      channel.basic_consume(callback,
                            queue='font_repo_queue')
      channel.start_consuming()
    except pika.exceptions.ConnectionClosed:
      print ("RabbitMQ not ready yet.", file=sys.stderr)
      time.sleep(1)

main()

<|endoftext|>


<commit_before>import socket

def get_remote_addr(transport):
    socket_info = transport.get_extra_info("socket")
    if socket_info is not None:
        info = socket_info.getpeername()
        family = socket_info.family
        if family in (socket.AF_INET, socket.AF_INET6):
            return (str(info[0]), int(info[1]))
        return None
    info = transport.get_extra_info("peername")
    if info is not None and isinstance(info, (list, tuple)) and len(info) == 2:
        return (str(info[0]), int(info[1]))
    return None

def get_local_addr(transport):
    socket_info = transport.get_extra_info("socket")
    if socket_info is not None:
        info = socket_info.getsockname()
        family = socket_info.family
        if family in (socket.AF_INET, socket.AF_INET6):
            return (str(info[0]), int(info[1]))
        return None
    info = transport.get_extra_info("sockname")
    if info is not None and isinstance(info, (list, tuple)) and len(info) == 2:
        return (str(info[0]), int(info[1]))
    return None

def is_ssl(transport):
    return bool(transport.get_extra_info("sslcontext"))

def get_client_addr(scope):
    client = scope.get("client")
    if not client:
        return ""
    return "%s:%d" % client

def get_path_with_query_string(scope):
    path_with_query_string = scope.get("root_path", "") + scope["path"]
    if scope["query_string"]:
        path_with_query_string = "{}?{}".format(
            path_with_query_string, scope["query_string"].decode("ascii")
        )
    return path_with_query_string
<commit_msg>More resiliant socket_info.getpeername() calling (#495)<commit_after>import socket

def get_remote_addr(transport):
    socket_info = transport.get_extra_info("socket")
    if socket_info is not None:
        try:
            info = socket_info.getpeername()
        except OSError:
            # This case appears to inconsistently occur with uvloop
            # bound to a unix domain socket.
            family = None
            info = None
        else:
            family = socket_info.family

        if family in (socket.AF_INET, socket.AF_INET6):
            return (str(info[0]), int(info[1]))
        return None
    info = transport.get_extra_info("peername")
    if info is not None and isinstance(info, (list, tuple)) and len(info) == 2:
        return (str(info[0]), int(info[1]))
    return None

def get_local_addr(transport):
    socket_info = transport.get_extra_info("socket")
    if socket_info is not None:
        info = socket_info.getsockname()
        family = socket_info.family
        if family in (socket.AF_INET, socket.AF_INET6):
            return (str(info[0]), int(info[1]))
        return None
    info = transport.get_extra_info("sockname")
    if info is not None and isinstance(info, (list, tuple)) and len(info) == 2:
        return (str(info[0]), int(info[1]))
    return None

def is_ssl(transport):
    return bool(transport.get_extra_info("sslcontext"))

def get_client_addr(scope):
    client = scope.get("client")
    if not client:
        return ""
    return "%s:%d" % client

def get_path_with_query_string(scope):
    path_with_query_string = scope.get("root_path", "") + scope["path"]
    if scope["query_string"]:
        path_with_query_string = "{}?{}".format(
            path_with_query_string, scope["query_string"].decode("ascii")
        )
    return path_with_query_string
<|endoftext|>


<commit_before>from threading import Thread
import unittest
import gc
from rodenioc import Container, RegistrationUsage, RegistrationNotFoundError

class ContainerTests(unittest.TestCase):
    def test_resolve_by_string(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)

        result = c.resolve_by_type(StringAnnotated)
        self.assertIsInstance(result, StringAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_by_names(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)

        result = c.resolve_by_type(NoAnnotated)
        self.assertIsInstance(result, NoAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_by_types(self):
        c = Container()
        c.register(Foo, Foo)
        c.register(Bar, Bar)

        result = c.resolve_by_type(TypeAnnotated)
        self.assertIsInstance(result, TypeAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_all_by_name(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)
        c.register(StringAnnotated, 'target')

        result = c.resolve_by_key('target')
        self.assertIsInstance(result, StringAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_singletons(self):
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.singleton
        c.register(Bar).usage = RegistrationUsage.singleton
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))

    def test_resolve_weak(self):
        """
            Can theoretically false negative as id can assign to new instances
            after collection
        """
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.weak_reference
        c.register(Bar).usage = RegistrationUsage.weak_reference
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))

        del result
        del result2

        gc.collect()

        self.assertNotEquals(foo_id, c.resolve_by_key('foo'))
        self.assertNotEquals(bar_id, c.resolve_by_key('bar'))

    def test_resolve_thread_local(self):
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.thread_local
        c.register(Bar).usage = RegistrationUsage.thread_local
        c.register(Bar, 'bar2').usage = RegistrationUsage.singleton
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)
        bar2_id = id(c.resolve_by_key('bar2'))

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))
        assert_not_equals = self.assertNotEquals
        assert_equals = self.assertEquals

        class ThreadWorker(Thread):
            def run(self):
                assert_not_equals(foo_id, id(c.resolve_by_key('foo')))
                assert_not_equals(bar_id, id(c.resolve_by_key('bar')))
                assert_equals(bar2_id, id(c.resolve_by_key('bar2')))

        thread = ThreadWorker()
        thread.start()
        thread.join()

    def test_resolve_defaults(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)
        c.register(DefaultNoAnnotated, 'target')
        result = c.resolve_by_key('target')

        self.assertIsInstance(result, DefaultNoAnnotated)
        self.assertEquals(1, result.default)

    def test_resolve_not_found(self):
        c = Container()
        c.register(DefaultNoAnnotated, 'target')
        with self.assertRaises(RegistrationNotFoundError):
            c.resolve_by_key('target')

    def test_resolve_instance(self):
        foo = Foo()

        c = Container()
        c.register_instance(foo, 'foo')
        c.register(Bar)
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        self.assertIsInstance(result, StringAnnotated)
        self.assertEquals(id(foo), id(result.foo))

        result2 = c.resolve_by_type(NoAnnotated)

        self.assertIsInstance(result2, NoAnnotated)
        self.assertEquals(id(foo), id(result2.foo))

class Foo:
    pass

class Bar:
    pass

class StringAnnotated:
    def __init__(self, foo: 'foo', bar: 'bar'):
        self.foo = foo
        self.bar = bar

class NoAnnotated:
    def __init__(self, foo, bar):
        self.foo = foo
        self.bar = bar

class TypeAnnotated:
    def __init__(self, foo: Foo, bar: Bar):
        self.foo = foo
        self.bar = bar

class DefaultNoAnnotated:
    def __init__(self, foo, bar, default=1):
        self.default = default
        self.foo = foo
        self.bar = bar
<commit_msg>Fix namespace<commit_after>from threading import Thread
import unittest
import gc
from simpleioc import Container, RegistrationUsage, RegistrationNotFoundError

class ContainerTests(unittest.TestCase):
    def test_resolve_by_string(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)

        result = c.resolve_by_type(StringAnnotated)
        self.assertIsInstance(result, StringAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_by_names(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)

        result = c.resolve_by_type(NoAnnotated)
        self.assertIsInstance(result, NoAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_by_types(self):
        c = Container()
        c.register(Foo, Foo)
        c.register(Bar, Bar)

        result = c.resolve_by_type(TypeAnnotated)
        self.assertIsInstance(result, TypeAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_all_by_name(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)
        c.register(StringAnnotated, 'target')

        result = c.resolve_by_key('target')
        self.assertIsInstance(result, StringAnnotated)
        self.assertIsInstance(result.bar, Bar)
        self.assertIsInstance(result.foo, Foo)

    def test_resolve_singletons(self):
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.singleton
        c.register(Bar).usage = RegistrationUsage.singleton
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))

    def test_resolve_weak(self):
        """
            Can theoretically false negative as id can assign to new instances
            after collection
        """
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.weak_reference
        c.register(Bar).usage = RegistrationUsage.weak_reference
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))

        del result
        del result2

        gc.collect()

        self.assertNotEquals(foo_id, c.resolve_by_key('foo'))
        self.assertNotEquals(bar_id, c.resolve_by_key('bar'))

    def test_resolve_thread_local(self):
        c = Container()
        c.register(Foo, 'foo').usage = RegistrationUsage.thread_local
        c.register(Bar).usage = RegistrationUsage.thread_local
        c.register(Bar, 'bar2').usage = RegistrationUsage.singleton
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        result_id = id(result)
        foo_id = id(result.foo)
        bar_id = id(result.bar)
        bar2_id = id(c.resolve_by_key('bar2'))

        result2 = c.resolve_by_key('target')

        self.assertNotEquals(result_id, id(result2))
        self.assertEquals(foo_id, id(result2.foo))
        self.assertEquals(bar_id, id(result2.bar))
        assert_not_equals = self.assertNotEquals
        assert_equals = self.assertEquals

        class ThreadWorker(Thread):
            def run(self):
                assert_not_equals(foo_id, id(c.resolve_by_key('foo')))
                assert_not_equals(bar_id, id(c.resolve_by_key('bar')))
                assert_equals(bar2_id, id(c.resolve_by_key('bar2')))

        thread = ThreadWorker()
        thread.start()
        thread.join()

    def test_resolve_defaults(self):
        c = Container()
        c.register(Foo, 'foo')
        c.register(Bar)
        c.register(DefaultNoAnnotated, 'target')
        result = c.resolve_by_key('target')

        self.assertIsInstance(result, DefaultNoAnnotated)
        self.assertEquals(1, result.default)

    def test_resolve_not_found(self):
        c = Container()
        c.register(DefaultNoAnnotated, 'target')
        with self.assertRaises(RegistrationNotFoundError):
            c.resolve_by_key('target')

    def test_resolve_instance(self):
        foo = Foo()

        c = Container()
        c.register_instance(foo, 'foo')
        c.register(Bar)
        c.register(StringAnnotated, 'target')
        result = c.resolve_by_key('target')

        self.assertIsInstance(result, StringAnnotated)
        self.assertEquals(id(foo), id(result.foo))

        result2 = c.resolve_by_type(NoAnnotated)

        self.assertIsInstance(result2, NoAnnotated)
        self.assertEquals(id(foo), id(result2.foo))

class Foo:
    pass

class Bar:
    pass

class StringAnnotated:
    def __init__(self, foo: 'foo', bar: 'bar'):
        self.foo = foo
        self.bar = bar

class NoAnnotated:
    def __init__(self, foo, bar):
        self.foo = foo
        self.bar = bar

class TypeAnnotated:
    def __init__(self, foo: Foo, bar: Bar):
        self.foo = foo
        self.bar = bar

class DefaultNoAnnotated:
    def __init__(self, foo, bar, default=1):
        self.default = default
        self.foo = foo
        self.bar = bar
<|endoftext|>


<commit_before>import httplib
import json
import re
import time
import urllib
import urlparse

from vial import vfunc, vim
from vial.utils import focus_window
from vial.widgets import make_scratch

header_regex = re.compile(r'^\+?[-\w\d]+$')
value_regex = re.compile(r'^([-_\w\d]+)(:=|=|:)(.+)$')

def send_collector(connection):
    connection._sdata = ''
    oldsend = connection.send
    def send(data):
        connection._sdata += data
        return oldsend(data)
    connection.send = send
    return connection

class Headers(object):
    def __init__(self):
        self.headers = []

    def set(self, header, value):
        self.headers = [r for r in self.headers if r[0].lower() != header.lower()]
        self.headers.append((header, value))

    def add(self, header, value):
        self.headers.append((header, value))

    def __contains__(self, header):
        return any(h.lower() == header.lower() for h, _ in self.headers)

    def pop(self, header, default=None):
        result = default
        headers = []
        for h, v in self.headers:
            if h.lower() == header.lower():
                result = v
            else:
                headers.append((h, v))
        self.headers = headers
        return result

    def iteritems(self):
        return self.headers

    def __iter__(self):
        return (h for h, _ in self.headers)

def get_headers(lines, line):
    headers = Headers()
    for l in lines[:line]:
        try:
            header, value = l.split(':', 1)
        except ValueError:
            continue

        if header_regex.match(header):
            if header[0] == '+':
                headers.add(header[1:], value.strip())
            else:
                headers.set(header, value.strip())

    return headers

def get_request(lines, line, headers):
    parts = lines[line].split()
    method = parts[0]
    url = parts[1]
    query = []
    form = []
    body = None
    for p in parts[2:]:
        m = value_regex.match(p)
        if m:
            param, op, value = m.group(1, 2, 3)
            if value == '__pwd__':
                value = vfunc.inputsecret('{}: '.format(param))
            if op == '=':
                query.append((param, value))
            elif op == ':':
                headers.set(param, value)
            elif op == ':=':
                form.append((param, value))

    if form:
        body = urllib.urlencode(form)
        headers.set('Content-Type', 'application/x-www-form-urlencoded')

    if not form:
        bodylines = []
        for l in lines[line+1:]:
            if not l:
                break
            bodylines.append(l)

        if bodylines:
            body = '\n'.join(bodylines)
            try:
                json.loads(body)
                if 'content-type' not in headers:
                    headers.set('Content-Type', 'application/json')
            except ValueError:
                pass

    return method, url, query, body

def http():
    lines = vim.current.buffer[:]
    line, _ = vim.current.window.cursor
    line -= 1

    headers = get_headers(lines, line)
    method, url, query, body = get_request(lines, line, headers)

    host = headers.pop('host', '')
    u = urlparse.urlsplit(url)
    if not u.hostname:
        u = urlparse.urlsplit(host + url)

    if u.query:
        query = urlparse.parse_qsl(u.query, True) + query

    path = u.path
    if query:
        path += '?' + urllib.urlencode(query)

    if u.scheme == 'https':
        cn = httplib.HTTPSConnection(u.hostname, u.port or 443)
    else:
        cn = httplib.HTTPConnection(u.hostname, u.port or 80)

    cn = send_collector(cn)

    start = time.time()
    cn.connect()
    ctime = int((time.time() - start) * 1000)

    cn.request(method, path, body, headers)
    response = cn.getresponse()
    rtime = int((time.time() - start) * 1000)

    cwin = vim.current.window

    win, buf = make_scratch('__vial_http_req__', title='Request')
    buf[:] = cn._sdata.splitlines()
    win.cursor = 1, 0

    win, buf = make_scratch('__vial_http_hdr__', title='Response headers')
    buf[:] = [r.rstrip('\r\n') for r in response.msg.headers]
    win.cursor = 1, 0

    content = response.read()
    try:
        jdata = json.loads(content)
        content = json.dumps(jdata, ensure_ascii=False, sort_keys=True, indent=2)
        ctype = 'json'
    except ValueError:
        ctype = 'html'

    win, buf = make_scratch('__vial_http__')
    win.options['statusline'] = 'Response: {} {} {}ms {}ms'.format(
        response.status, response.reason, rtime, ctime)
    vim.command('set filetype={}'.format(ctype))
    buf[:] = content.splitlines()
    win.cursor = 1, 0

    focus_window(cwin)
<commit_msg>Default user agent and arguments escaping<commit_after>import httplib
import json
import re
import time
import urllib
import urlparse
import shlex

from vial import vfunc, vim
from vial.utils import focus_window
from vial.widgets import make_scratch

header_regex = re.compile(r'^\+?[-\w\d]+$')
value_regex = re.compile(r'^([-_\w\d]+)(:=|=|:)(.+)$')

def send_collector(connection):
    connection._sdata = ''
    oldsend = connection.send
    def send(data):
        connection._sdata += data
        return oldsend(data)
    connection.send = send
    return connection

class Headers(object):
    def __init__(self):
        self.headers = [('User-Agent', 'vial-http')]

    def set(self, header, value):
        self.headers = [r for r in self.headers if r[0].lower() != header.lower()]
        self.headers.append((header, value))

    def add(self, header, value):
        self.headers.append((header, value))

    def __contains__(self, header):
        return any(h.lower() == header.lower() for h, _ in self.headers)

    def pop(self, header, default=None):
        result = default
        headers = []
        for h, v in self.headers:
            if h.lower() == header.lower():
                result = v
            else:
                headers.append((h, v))
        self.headers = headers
        return result

    def iteritems(self):
        return self.headers

    def __iter__(self):
        return (h for h, _ in self.headers)

def get_headers(lines, line):
    headers = Headers()
    for l in lines[:line]:
        try:
            header, value = l.split(':', 1)
        except ValueError:
            continue

        if header_regex.match(header):
            if header[0] == '+':
                headers.add(header[1:], value.strip())
            else:
                headers.set(header, value.strip())

    return headers

def get_request(lines, line, headers):
    parts = shlex.split(lines[line], True)
    method = parts[0]
    url = parts[1]
    query = []
    form = []
    body = None
    for p in parts[2:]:
        m = value_regex.match(p)
        if m:
            param, op, value = m.group(1, 2, 3)
            if value == '__pwd__':
                value = vfunc.inputsecret('{}: '.format(param))
            if op == '=':
                query.append((param, value))
            elif op == ':':
                headers.set(param, value)
            elif op == ':=':
                form.append((param, value))

    if form:
        body = urllib.urlencode(form)
        headers.set('Content-Type', 'application/x-www-form-urlencoded')

    if not form:
        bodylines = []
        for l in lines[line+1:]:
            if not l:
                break
            bodylines.append(l)

        if bodylines:
            body = '\n'.join(bodylines)
            try:
                json.loads(body)
                if 'content-type' not in headers:
                    headers.set('Content-Type', 'application/json')
            except ValueError:
                pass

    return method, url, query, body

def http():
    lines = vim.current.buffer[:]
    line, _ = vim.current.window.cursor
    line -= 1

    headers = get_headers(lines, line)
    method, url, query, body = get_request(lines, line, headers)

    host = headers.pop('host', '')
    u = urlparse.urlsplit(url)
    if not u.hostname:
        u = urlparse.urlsplit(host + url)

    if u.query:
        query = urlparse.parse_qsl(u.query, True) + query

    path = u.path
    if query:
        path += '?' + urllib.urlencode(query)

    if u.scheme == 'https':
        cn = httplib.HTTPSConnection(u.hostname, u.port or 443)
    else:
        cn = httplib.HTTPConnection(u.hostname, u.port or 80)

    cn = send_collector(cn)

    start = time.time()
    cn.connect()
    ctime = int((time.time() - start) * 1000)

    cn.request(method, path, body, headers)
    response = cn.getresponse()
    rtime = int((time.time() - start) * 1000)

    cwin = vim.current.window

    win, buf = make_scratch('__vial_http_req__', title='Request')
    buf[:] = cn._sdata.splitlines()
    win.cursor = 1, 0

    win, buf = make_scratch('__vial_http_hdr__', title='Response headers')
    buf[:] = [r.rstrip('\r\n') for r in response.msg.headers]
    win.cursor = 1, 0

    content = response.read()
    try:
        jdata = json.loads(content)
        content = json.dumps(jdata, ensure_ascii=False, sort_keys=True, indent=2)
        ctype = 'json'
    except ValueError:
        ctype = 'html'

    win, buf = make_scratch('__vial_http__')
    win.options['statusline'] = 'Response: {} {} {}ms {}ms'.format(
        response.status, response.reason, rtime, ctime)
    vim.command('set filetype={}'.format(ctype))
    buf[:] = content.splitlines()
    win.cursor = 1, 0

    focus_window(cwin)
<|endoftext|>


<commit_before>import os
from voeventparse.fixtures import __path__ as data_dir

data_dir = data_dir[0]
swift_bat_grb_pos_v2 = os.path.join(data_dir,
                                    'SWIFT_bat_position_v2.0_example.xml')
swift_xrt_pos_v1 = os.path.join(data_dir,
                                'SWIFT_xrt_position_v1.1_example.xml')
no_namespace_test_packet = os.path.join(data_dir, 'no_namespace_packet.xml')

moa_lensing_event_path = os.path.join(data_dir,
                                      'MOA_Lensing_Event_2015-07-10T14:50:54.00.xml')

gaia_alert_16aac_direct = os.path.join(data_dir, 'Gaia16aac.xml')

asassn_scraped_example = os.path.join(
    data_dir, 'ASASSN#2016-09-25.47_2016fvf_PTSS-16nqb_PS16ejf.xml')
<commit_msg>Change colons to underscores<commit_after>import os
from voeventparse.fixtures import __path__ as data_dir

data_dir = data_dir[0]
swift_bat_grb_pos_v2 = os.path.join(data_dir,
                                    'SWIFT_bat_position_v2.0_example.xml')
swift_xrt_pos_v1 = os.path.join(data_dir,
                                'SWIFT_xrt_position_v1.1_example.xml')
no_namespace_test_packet = os.path.join(data_dir, 'no_namespace_packet.xml')

moa_lensing_event_path = os.path.join(data_dir,
                                      'MOA_Lensing_Event_2015-07-10T14_50_54.00.xml')

gaia_alert_16aac_direct = os.path.join(data_dir, 'Gaia16aac.xml')

asassn_scraped_example = os.path.join(
    data_dir, 'ASASSN#2016-09-25.47_2016fvf_PTSS-16nqb_PS16ejf.xml')
<|endoftext|>