<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re
import os
from collections import namedtuple
from itertools import product

class DiffiosConfig(object):

    """DiffiosConfig prepares a Cisco IOS Config to diff.

    DiffiosConfig takes a Cisco IOS config, as a file or a
    list, extracts the device hostname, removes any invalid
    lines, such as comments, breaks the config into a
    hierarchical block structure and partitions the config
    according to a list of lines to ignore.

    Attributes:
        ignores (list): List of lines to ignore
        config (list): List of valid config lines

    Args:
        config (str|list): Path to config file, or list
            containing lines of config

    Kwargs:
        ignores (str|list): Path to ignores file, or list
            containing lines to ignore. Defaults to ignores
            file in current working directory.

    >>> config = [
    ... '!',
    ... 'hostname ROUTER',
    ... '!',
    ... 'interface FastEthernet0/1',
    ... ' description **Link to Core**',
    ... ' ip address 192.168.0.1 255.255.255.0']
    >>> ignores = [
    ... 'hostname',
    ... '^ description']
    >>> conf = DiffiosConfig(config, ignores=ignores)
    >>> conf.config
    ['hostname ROUTER', \
'interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']
    >>> conf.ignores
    ['hostname', '^ description']
    >>> conf.config_blocks
    [['hostname ROUTER'], \
['interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']]
    >>> conf.ignored
    [['hostname ROUTER'], [' description **Link to Core**']]
    >>> conf.recorded
    [['interface FastEthernet0/1', \
' ip address 192.168.0.1 255.255.255.0']]

    """

    def __init__(self, config, ignores=None):
        if ignores is None and os.path.exists(os.path.join(os.getcwd(), 'diffios_ignore')):
            ignores = os.path.join(os.getcwd(), "diffios_ignore")
        elif ignores is None:
            ignores = []
        self.config = self._config(config)
        self.ignores = self._ignores(ignores)

    @property
    def contains_variables(self, delimiter=None):
        """TODO: Docstring for contains_variables.

        Kwargs:
            delimiter (TODO): TODO

        Returns: TODO

        """
        if delimiter is None:
            delimiter = re.compile(r'{{(.+)}}')
        return any(re.search(delimiter, line) for line in self.config)

    def _config(self, data):
        """Transforms given config data into usable format,
            with only valid lines.

        Args:
            data (list|file): config as either list or file.

        Returns:
            list: The config as a list of valid lines.

        """
        return self._remove_invalid_lines(self._check_data(data))

    def _ignores(self, data):
        """Transforms given ignores data into usable format.

        Args:
            ignores (list|file): ignores as either list or file.

        Returns:
            list: Lines to ignore.

        """
        return [l.strip().lower() for l in self._check_data(data)]

    @staticmethod
    def _check_data(data):
        """Check type of data and convert it if necessary.

        Args:
            data (list|file): input data as either list or file.

        Returns:
            list: The input data as a list.

        Raises:
            RuntimeError: If file cannot be opened or given data
                is not a list or file.

        """
        if isinstance(data, list):
            return data
        elif os.path.isfile(data):
            try:
                with open(data) as fin:
                    return fin.readlines()
            except IOError:
                raise RuntimeError(("DiffiosConfig could not open '{}'".format(data)))
        else:
            raise RuntimeError(("[FATAL] DiffiosConfig() received an "
                                "invalid argument: config={}\n").format(data))

    def _remove_invalid_lines(self, config):
        """Remove invalid lines from a given config.

        Args:
            config (list): Config as a list of lines.

        Returns:
            list: Only valid lines in given config.

        """
        return [l.rstrip() for l in config if self._valid_line(l)]

    @staticmethod
    def _valid_line(line):
        """Assert whether a given line is valid.

        Lines considered invalid are those which are empty
        or begin with a '!' to indicate a comment.

        Args:
            line (str): A single line from a config.

        Returns:
            bool: True if line is valid, False if not.

        """
        lstrip = line.strip()
        return len(lstrip) > 0 and not lstrip.startswith("!") and lstrip != '^' and lstrip != '^C'

    @staticmethod
    def _group_into_blocks(config):
        """Group config into hierarchical blocks.

        Blocks are defined as a parent/child relationship,
        where children lines start with a ' ', and parents
        are the immediately preceding line that doesn't start
        with a ' '. Blocks can be single lines where there
        is no children. Returns the blocks sorted.

        Example Blocks:
            interface FastEthernet0/1
             ip address 192.168.0.1 255.255.255.0
             no shutdown

            hostname ROUTER

        Args:
            config (list): config as a list of lines.

        Returns:
            list: config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        current_group, groups = [], []
        for line in config:
            if not line.startswith(' ') and current_group:
                groups.append(current_group)
                current_group = [line]
            else:
                current_group.append(line)
        if current_group:
            groups.append(current_group)
        return sorted(groups)

    @property
    def config_blocks(self):
        """The config as sorted list of hierarchical blocks.

        Returns:
            list: self.config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        return self._group_into_blocks(self.config)

    @property
    def hostname(self):
        """The hostname of the given config.

        Returns:
            str: hostname of the given config
                or None if not found.

        """
        for line in self.config:
            if "hostname" in line.lower():
                return line.split()[1]
        return None

    def _ignore_line(self, line):
        """Check if a line should be ignored.

        Args:
            line (str): line to check

        Returns:
            bool: True if line should be ignored,
                False otherwise.

        """
        for line_to_ignore in self.ignores:
            if re.search(line_to_ignore, line.lower()):
                return True

    def _check_block(self, block):
        """Check for any lines to be ignored in a given block.

        Args:
            block (list): A single hierarchical block of config

        Returns:
            tuple: ignored and recorded (not ignored) lines in block

        """
        ignored, recorded = [], []
        for i, line in enumerate(block):
            if self._ignore_line(line) and i == 0:
                return (block, recorded)
            elif self._ignore_line(line):
                ignored.append(line)
            else:
                recorded.append(line)
        return (ignored, recorded)

    def _partition(self):
        """Partition out lines to ignore from the config.

        Returns:
            namedtuple: list of ignored lines and list of
                recorded (not ignored) lines

        """
        Partition = namedtuple("Partition", "ignored recorded")
        config_blocks = self._group_into_blocks(self.config)
        ignored, recorded = [], []
        for block in config_blocks:
            ignore, record = self._check_block(block)
            if ignore:
                ignored.append(ignore)
            if record:
                recorded.append(record)
        return Partition(ignored, recorded)

    @property
    def ignored(self):
        """Lines in the config that are being ignored.

        Returns:
            list: ignored lines

        """
        return self._partition().ignored

    @property
    def recorded(self):
        """Lines in the config that are being recorded (not ignored).

        Returns:
            list: recorded (not ignored) lines

        """
        return self._partition().recorded

class CheckConfig(object):

    """Docstring for CheckConfig. """

    def __init__(self, block, config):
        """TODO: to be defined1.

        Args:
            block (TODO): TODO
            config (TODO): TODO

        """
        self._block = block
        self._config = config
        self.present = []
        self.not_present = []
        self.parent = self._block[0]
        self.children = self._block[1:]
        self.check()

    def _parent_child_config_dict(self):
        return {el[0]: el[1:] for el in self._config}

    def check(self):
        if not self.children and self._block in self._config:
            self.present.append(self.parent)
        elif not self.children:
            self.not_present.append(self.parent)
        elif self.parent in self._parent_child_config_dict().keys():
                self.present.append(self.parent)
                config_children = self._parent_child_config_dict()[self.parent]
                for i, child in enumerate(self.children):
                    if child in config_children:
                        self.present.append(child)
                    elif i == 0:
                        self.not_present.append(self.parent)
                        self.not_present.append(child)
                    else:
                        self.not_present.append(child)

class DiffiosDiff(object):

    """Docstring for DiffiosDiff. """

    def __init__(self, baseline=None, comparison=None, ignore_file=None):
        """TODO: Docstring for __init__.

        Kwargs:
            baseline (TODO): TODO
            comparison (TODO): TODO
            ignore_file (TODO): TODO

        """
        # TODO: make it so DiffiosConf objects can be passed in also
        # TODO: confirm existence of files
        self.baseline = DiffiosConfig(baseline, ignore_file)
        self.comparison = DiffiosConfig(comparison, ignore_file)
        self.delimiter = r'{{[^{}]+}}'

    def _check_lines(self, yline, xline):
        """TODO: Docstring for _check_lines.

        Args:
            line (TODO): TODO
            vars (TODO): TODO
            indexes (TODO): TODO

        Returns: TODO

        """
        yline_split = yline.split()
        xline_split = ''.join(x for x in re.split('[{{|}}]', xline) if x).replace('  ', ' ').split()
        xy_lines_zipped = None

        if len(yline_split) != len(xline_split):
            res = []
            cur = ''
            j = 0
            for el in yline_split:
                if el == xline_split[j] and cur:
                    res.append(cur)
                    res.append(el)
                    j += 1
                    cur = ''
                elif el != xline_split[j] and cur:
                    cur += ' '
                    cur += el
                elif el != xline_split[j] and (j + 1) < len(xline_split):
                    cur = el
                    j += 1
                elif el == xline_split[j]:
                    res.append(el)
                    j += 1
                    cur = ''
                elif el != xline_split[j]:
                    cur = el
            if cur:
                res.append(cur)
            yline_split = res

        xy_lines_zipped = zip(yline_split, xline_split)
        for i, (x, y) in enumerate(list(xy_lines_zipped)):
            if x != y:
                yline_split[i] = y

        xline_without_delims = xline.replace('{{', '').replace('}}', '').replace('  ', ' ').strip()
        yline_with_vars = ' '.join(yline_split)
        return xline_without_delims == yline_with_vars

    def _baseline_var_blocks(self):
        var_blocks = []
        for block in self.baseline.recorded:
            for line in block:
                if re.search(self.delimiter, line):
                    var_blocks.append(block)
        return var_blocks

    def _translate_comparison(self):
        d = self.delimiter
        translated = self.comparison.recorded[:]
        prod = product(self._baseline_var_blocks(), self.comparison.recorded)
        filter_prod = [(x, y) for (x, y) in prod if d[0] in ''.join(x) or d[0] in ''.join(y)]
        similar = [(x, y) for (x, y) in filter_prod if x[0].split()[0] == y[0].split()[0]]
        more_similar = []
        # We need to ensure that if the parent line in a block is
        # not a variable line, that the parents of the x & y blocks
        # match fully, otherwise we get issues where different blocks
        # are matched up together as they have the same first word
        # in their parents.
        for x, y in similar:
            if d[0] not in x[0] and d[0] not in y[0] and x[0] == y[0]:
                more_similar.append((x, y))
            elif d[0] in x[0] or d[0] in y[0]:
                more_similar.append((x, y))
        for x, y in more_similar:
            y_copy = y[:]
            for xline in x:
                matches = re.findall(self.delimiter, xline)
                # if matches and len(matches) == 1:
                    # for yline in y_copy:
                        # var = matches[0]
                        # start = xline.index(d[0])
                        # end = (xline.index(d[-1]) + 2) - len(xline) or len(yline)
                        # before = yline[:start]
                        # after = yline[end:]
                        # translated_yline = '{0}{1}{2}'.format(before, var, after)
                        # if translated_yline == xline:
                            # y[y_copy.index(yline)] = xline
                if matches:
                    for yline in y_copy:
                        translated_yline = None
                        if yline.split()[0] == xline.split()[0]:
                            translated_yline = self._check_lines(yline, xline)
                        if translated_yline:
                            y[y_copy.index(yline)] = xline
            if d[0] in ''.join(y):
                try:
                    translated[translated.index(y_copy)] = y
                except ValueError:
                    pass
        return translated

    def _comparator(self, measurable, reference, translation):
        """TODO: Docstring for _comparator.

        Args:
            measurable (TODO): TODO
            reference (TODO): TODO

        Returns: TODO

        """
        head = [line[0] for line in reference]
        changes = []
        for mi, mb in enumerate(measurable):
            if len(mb) == 1 and mb not in reference:
                changes.append(translation[mi])
            elif len(mb) > 1:
                first_line = mb[0]
                if first_line in head:
                    rb = reference[head.index(first_line)]
                    additional = [first_line]
                    for mbi, line in enumerate(mb):
                        if line not in rb:
                            additional.append(translation[mi][mbi])
                    if len(additional) > 1:
                        changes.append(additional)
                else:
                    changes.append(translation[mi])
        return sorted(changes)

    @staticmethod
    def _format_changes(data):
        """TODO: Docstring for _format_changes.

        Args:
            data (TODO): TODO

        Returns: TODO

        """
        return "\n\n".join("\n".join(lines) for lines in data)

    @property
    def additional(self):
        """TODO: Docstring for additional.

        Returns: TODO

        """
        measurable = self._translate_comparison()
        reference = self.baseline.recorded
        translation = self.comparison.recorded
        return self._comparator(measurable, reference, translation)

    @property
    def missing(self):
        """TODO: Docstring for missing.

        Returns: TODO

        """
        measurable = self.baseline.recorded
        reference = self._translate_comparison()
        translation = self.baseline.recorded
        return self._comparator(measurable, reference, translation)

    @property
    def pprint_additional(self):
        """TODO: Docstring for pprint_additional.

        Returns: TODO

        """
        return self._format_changes(self.additional)

    @property
    def pprint_missing(self):
        """TODO: Docstring for pprint_missing.

        Returns: TODO

        """
        return self._format_changes(self.missing)

    def diff(self):
        """TODO: Docstring for diff.

        Returns: TODO

        """
        print("\nComparing {comparison} against baseline: {baseline}".format(
            comparison=os.path.basename(self.comparison.config),
            baseline=os.path.basename(self.baseline.config)
        ))
        print("\n[+] additional [+]\n")
        print("{}".format(self.pprint_additional()))
        print("\n[-] missing [-]\n")
        print("{}".format(self.pprint_missing()))
        print("\n--- END ---\n")
<commit_msg>remove some unused lines of code<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import re
import os
from collections import namedtuple
from itertools import product

class DiffiosConfig(object):

    """DiffiosConfig prepares a Cisco IOS Config to diff.

    DiffiosConfig takes a Cisco IOS config, as a file or a
    list, extracts the device hostname, removes any invalid
    lines, such as comments, breaks the config into a
    hierarchical block structure and partitions the config
    according to a list of lines to ignore.

    Attributes:
        ignores (list): List of lines to ignore
        config (list): List of valid config lines

    Args:
        config (str|list): Path to config file, or list
            containing lines of config

    Kwargs:
        ignores (str|list): Path to ignores file, or list
            containing lines to ignore. Defaults to ignores
            file in current working directory.

    >>> config = [
    ... '!',
    ... 'hostname ROUTER',
    ... '!',
    ... 'interface FastEthernet0/1',
    ... ' description **Link to Core**',
    ... ' ip address 192.168.0.1 255.255.255.0']
    >>> ignores = [
    ... 'hostname',
    ... '^ description']
    >>> conf = DiffiosConfig(config, ignores=ignores)
    >>> conf.config
    ['hostname ROUTER', \
'interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']
    >>> conf.ignores
    ['hostname', '^ description']
    >>> conf.config_blocks
    [['hostname ROUTER'], \
['interface FastEthernet0/1', \
' description **Link to Core**', \
' ip address 192.168.0.1 255.255.255.0']]
    >>> conf.ignored
    [['hostname ROUTER'], [' description **Link to Core**']]
    >>> conf.recorded
    [['interface FastEthernet0/1', \
' ip address 192.168.0.1 255.255.255.0']]

    """

    def __init__(self, config, ignores=None):
        if ignores is None and os.path.exists(os.path.join(os.getcwd(), 'diffios_ignore')):
            ignores = os.path.join(os.getcwd(), "diffios_ignore")
        elif ignores is None:
            ignores = []
        self.config = self._config(config)
        self.ignores = self._ignores(ignores)

    @property
    def contains_variables(self, delimiter=None):
        """TODO: Docstring for contains_variables.

        Kwargs:
            delimiter (TODO): TODO

        Returns: TODO

        """
        if delimiter is None:
            delimiter = re.compile(r'{{(.+)}}')
        return any(re.search(delimiter, line) for line in self.config)

    def _config(self, data):
        """Transforms given config data into usable format,
            with only valid lines.

        Args:
            data (list|file): config as either list or file.

        Returns:
            list: The config as a list of valid lines.

        """
        return self._remove_invalid_lines(self._check_data(data))

    def _ignores(self, data):
        """Transforms given ignores data into usable format.

        Args:
            ignores (list|file): ignores as either list or file.

        Returns:
            list: Lines to ignore.

        """
        return [l.strip().lower() for l in self._check_data(data)]

    @staticmethod
    def _check_data(data):
        """Check type of data and convert it if necessary.

        Args:
            data (list|file): input data as either list or file.

        Returns:
            list: The input data as a list.

        Raises:
            RuntimeError: If file cannot be opened or given data
                is not a list or file.

        """
        if isinstance(data, list):
            return data
        elif os.path.isfile(data):
            try:
                with open(data) as fin:
                    return fin.readlines()
            except IOError:
                raise RuntimeError(("DiffiosConfig could not open '{}'".format(data)))
        else:
            raise RuntimeError(("[FATAL] DiffiosConfig() received an "
                                "invalid argument: config={}\n").format(data))

    def _remove_invalid_lines(self, config):
        """Remove invalid lines from a given config.

        Args:
            config (list): Config as a list of lines.

        Returns:
            list: Only valid lines in given config.

        """
        return [l.rstrip() for l in config if self._valid_line(l)]

    @staticmethod
    def _valid_line(line):
        """Assert whether a given line is valid.

        Lines considered invalid are those which are empty
        or begin with a '!' to indicate a comment.

        Args:
            line (str): A single line from a config.

        Returns:
            bool: True if line is valid, False if not.

        """
        lstrip = line.strip()
        return len(lstrip) > 0 and not lstrip.startswith("!") and lstrip != '^' and lstrip != '^C'

    @staticmethod
    def _group_into_blocks(config):
        """Group config into hierarchical blocks.

        Blocks are defined as a parent/child relationship,
        where children lines start with a ' ', and parents
        are the immediately preceding line that doesn't start
        with a ' '. Blocks can be single lines where there
        is no children. Returns the blocks sorted.

        Example Blocks:
            interface FastEthernet0/1
             ip address 192.168.0.1 255.255.255.0
             no shutdown

            hostname ROUTER

        Args:
            config (list): config as a list of lines.

        Returns:
            list: config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        current_group, groups = [], []
        for line in config:
            if not line.startswith(' ') and current_group:
                groups.append(current_group)
                current_group = [line]
            else:
                current_group.append(line)
        if current_group:
            groups.append(current_group)
        return sorted(groups)

    @property
    def config_blocks(self):
        """The config as sorted list of hierarchical blocks.

        Returns:
            list: self.config as a sorted list of lists, each list
                representing a hierarchical block of config.

        """
        return self._group_into_blocks(self.config)

    @property
    def hostname(self):
        """The hostname of the given config.

        Returns:
            str: hostname of the given config
                or None if not found.

        """
        for line in self.config:
            if "hostname" in line.lower():
                return line.split()[1]
        return None

    def _ignore_line(self, line):
        """Check if a line should be ignored.

        Args:
            line (str): line to check

        Returns:
            bool: True if line should be ignored,
                False otherwise.

        """
        for line_to_ignore in self.ignores:
            if re.search(line_to_ignore, line.lower()):
                return True

    def _check_block(self, block):
        """Check for any lines to be ignored in a given block.

        Args:
            block (list): A single hierarchical block of config

        Returns:
            tuple: ignored and recorded (not ignored) lines in block

        """
        ignored, recorded = [], []
        for i, line in enumerate(block):
            if self._ignore_line(line) and i == 0:
                return (block, recorded)
            elif self._ignore_line(line):
                ignored.append(line)
            else:
                recorded.append(line)
        return (ignored, recorded)

    def _partition(self):
        """Partition out lines to ignore from the config.

        Returns:
            namedtuple: list of ignored lines and list of
                recorded (not ignored) lines

        """
        Partition = namedtuple("Partition", "ignored recorded")
        config_blocks = self._group_into_blocks(self.config)
        ignored, recorded = [], []
        for block in config_blocks:
            ignore, record = self._check_block(block)
            if ignore:
                ignored.append(ignore)
            if record:
                recorded.append(record)
        return Partition(ignored, recorded)

    @property
    def ignored(self):
        """Lines in the config that are being ignored.

        Returns:
            list: ignored lines

        """
        return self._partition().ignored

    @property
    def recorded(self):
        """Lines in the config that are being recorded (not ignored).

        Returns:
            list: recorded (not ignored) lines

        """
        return self._partition().recorded

class CheckConfig(object):

    """Docstring for CheckConfig. """

    def __init__(self, block, config):
        """TODO: to be defined1.

        Args:
            block (TODO): TODO
            config (TODO): TODO

        """
        self._block = block
        self._config = config
        self.present = []
        self.not_present = []
        self.parent = self._block[0]
        self.children = self._block[1:]
        self.check()

    def _parent_child_config_dict(self):
        return {el[0]: el[1:] for el in self._config}

    def check(self):
        if not self.children and self._block in self._config:
            self.present.append(self.parent)
        elif not self.children:
            self.not_present.append(self.parent)
        elif self.parent in self._parent_child_config_dict().keys():
                self.present.append(self.parent)
                config_children = self._parent_child_config_dict()[self.parent]
                for i, child in enumerate(self.children):
                    if child in config_children:
                        self.present.append(child)
                    elif i == 0:
                        self.not_present.append(self.parent)
                        self.not_present.append(child)
                    else:
                        self.not_present.append(child)

class DiffiosDiff(object):

    """Docstring for DiffiosDiff. """

    def __init__(self, baseline=None, comparison=None, ignore_file=None):
        """TODO: Docstring for __init__.

        Kwargs:
            baseline (TODO): TODO
            comparison (TODO): TODO
            ignore_file (TODO): TODO

        """
        # TODO: make it so DiffiosConf objects can be passed in also
        # TODO: confirm existence of files
        self.baseline = DiffiosConfig(baseline, ignore_file)
        self.comparison = DiffiosConfig(comparison, ignore_file)
        self.delimiter = r'{{[^{}]+}}'

    def _baseline_var_blocks(self):
        var_blocks = []
        for block in self.baseline.recorded:
            for line in block:
                if re.search(self.delimiter, line):
                    var_blocks.append(block)
        return var_blocks

    def _check_lines(self, yline, xline):
        """TODO: Docstring for _check_lines.

        Args:
            line (TODO): TODO
            vars (TODO): TODO
            indexes (TODO): TODO

        Returns: TODO

        """
        yline_split = yline.split()
        xline_split = ''.join(x for x in re.split('[{{|}}]', xline) if x).replace('  ', ' ').split()

        if len(yline_split) != len(xline_split):
            res = []
            cur = ''
            j = 0
            for el in yline_split:
                if el == xline_split[j] and cur:
                    res.append(cur)
                    res.append(el)
                    j += 1
                    cur = ''
                elif el != xline_split[j] and cur:
                    cur += ' '
                    cur += el
                elif el != xline_split[j] and (j + 1) < len(xline_split):
                    cur = el
                    j += 1
                elif el == xline_split[j]:
                    res.append(el)
                    j += 1
                    cur = ''
                elif el != xline_split[j]:
                    cur = el
            if cur:
                res.append(cur)
            yline_split = res

        xy_lines_zipped = zip(yline_split, xline_split)
        for i, (x, y) in enumerate(xy_lines_zipped):
            if x != y:
                yline_split[i] = y

        xline_without_delims = xline.replace('{{', '').replace('}}', '').replace('  ', ' ').strip()
        yline_with_vars = ' '.join(yline_split)
        return xline_without_delims == yline_with_vars

    def _translate_comparison(self):
        d = self.delimiter
        translated = self.comparison.recorded[:]
        prod = product(self._baseline_var_blocks(), self.comparison.recorded)
        filter_prod = [(x, y) for (x, y) in prod if d[0] in ''.join(x) or d[0] in ''.join(y)]
        similar = [(x, y) for (x, y) in filter_prod if x[0].split()[0] == y[0].split()[0]]
        more_similar = []
        # We need to ensure that if the parent line in a block is
        # not a variable line, that the parents of the x & y blocks
        # match fully, otherwise we get issues where different blocks
        # are matched up together as they have the same first word
        # in their parents.
        for x, y in similar:
            if d[0] not in x[0] and d[0] not in y[0] and x[0] == y[0]:
                more_similar.append((x, y))
            elif d[0] in x[0] or d[0] in y[0]:
                more_similar.append((x, y))
        for x, y in more_similar:
            y_copy = y[:]
            for xline in x:
                matches = re.findall(self.delimiter, xline)
                if matches:
                    for yline in y_copy:
                        translated_yline = None
                        if yline.split()[0] == xline.split()[0]:
                            translated_yline = self._check_lines(yline, xline)
                        if translated_yline:
                            y[y_copy.index(yline)] = xline
            if d[0] in ''.join(y):
                try:
                    translated[translated.index(y_copy)] = y
                except ValueError:
                    pass
        return translated

    def _comparator(self, measurable, reference, translation):
        """TODO: Docstring for _comparator.

        Args:
            measurable (TODO): TODO
            reference (TODO): TODO

        Returns: TODO

        """
        head = [line[0] for line in reference]
        changes = []
        for mi, mb in enumerate(measurable):
            if len(mb) == 1 and mb not in reference:
                changes.append(translation[mi])
            elif len(mb) > 1:
                first_line = mb[0]
                if first_line in head:
                    rb = reference[head.index(first_line)]
                    additional = [first_line]
                    for mbi, line in enumerate(mb):
                        if line not in rb:
                            additional.append(translation[mi][mbi])
                    if len(additional) > 1:
                        changes.append(additional)
                else:
                    changes.append(translation[mi])
        return sorted(changes)

    @staticmethod
    def _format_changes(data):
        """TODO: Docstring for _format_changes.

        Args:
            data (TODO): TODO

        Returns: TODO

        """
        return "\n\n".join("\n".join(lines) for lines in data)

    @property
    def additional(self):
        """TODO: Docstring for additional.

        Returns: TODO

        """
        measurable = self._translate_comparison()
        reference = self.baseline.recorded
        translation = self.comparison.recorded
        return self._comparator(measurable, reference, translation)

    @property
    def missing(self):
        """TODO: Docstring for missing.

        Returns: TODO

        """
        measurable = self.baseline.recorded
        reference = self._translate_comparison()
        translation = self.baseline.recorded
        return self._comparator(measurable, reference, translation)

    @property
    def pprint_additional(self):
        """TODO: Docstring for pprint_additional.

        Returns: TODO

        """
        return self._format_changes(self.additional)

    @property
    def pprint_missing(self):
        """TODO: Docstring for pprint_missing.

        Returns: TODO

        """
        return self._format_changes(self.missing)

    def diff(self):
        """TODO: Docstring for diff.

        Returns: TODO

        """
        print("\nComparing {comparison} against baseline: {baseline}".format(
            comparison=os.path.basename(self.comparison.config),
            baseline=os.path.basename(self.baseline.config)
        ))
        print("\n[+] additional [+]\n")
        print("{}".format(self.pprint_additional()))
        print("\n[-] missing [-]\n")
        print("{}".format(self.pprint_missing()))
        print("\n--- END ---\n")
<|endoftext|>


<commit_before>"""
A suite of tools that make creating and handling plugins
in TiddlyWeb a bit more sensible.

Essentially this is a way to raise duplication to a common
core without encumbering the TiddlyWeb core.
"""

from tiddlyweb.model.bag import Bag
from tiddlyweb.model.policy import UserRequiredError
from tiddlyweb.store import NoBagError

def entitle(title):
    """
    Decorator that sets tiddlyweb.title in environ.
    """
    def entangle(f):
        def entitle(environ, start_response, *args, **kwds):
            output = f(environ, start_response)
            environ['tiddlyweb.title'] = title
            return output
        return entitle
    return entangle

def do_html():
    """
    Decorator that makes sure we are sending text/html.
    """
    def entangle(f):
        def do_html(environ, start_response, *args, **kwds):
            output = f(environ, start_response)
            start_response('200 OK', [
                ('Content-Type', 'text/html; charset=UTF-8')
                ])
            return output
        return do_html
    return entangle

def require_role(role):
    """
    Decorator that requires the current user has role <role>.
    """
    role = unicode(role)
    def entangle(f):
        def require_role(environ, start_response, *args, **kwds):
            try:
                if role in environ['tiddlyweb.usersign']['roles']:
                    return f(environ, start_response)
                else:
                    raise(UserRequiredError, 'insufficient permissions')
            except KeyError:
                raise(UserRequiredError, 'insufficient permissions')
        return require_role
    return entangle

def require_any_user():
    """
    Decorator that requires the current user be someone other than 'GUEST'.
    """
    def entangle(f):
        def require_any_user(environ, start_response, *args, **kwds):
            try:
                if environ['tiddlyweb.usersign']['name'] == 'GUEST':
                    raise(UserRequiredError, 'user must be logged in')
                else:
                    return f(environ, start_response)
            except KeyError:
                raise(UserRequiredError, 'user must be logged in')
        return require_any_user
    return entangle

def ensure_bag(bag_name, store, policy_dict={}, description='', owner=None):
    """
    Ensure that bag with name bag_name exists in store.
    If not, create it with owner, policy and description optionally
    provided. In either case return the bag object.
    """
    bag = Bag(bag_name)
    try:
        store.get(bag)
    except NoBagError:
        bag.desc = description
        if owner:
            bag.policy.owner = owner
            bag.policy.manage = [owner]
        for key in policy_dict:
            bag.policy.__setattr__(key, policy_dict[key])
        store.put(bag)
    return bag

def replace_handler(selector, path, new_handler):
    """
    Replace an existing path handler in the selector
    map with a new handler. Usually we want to add a
    new one, but sometimes we just want to replace.
    This makes replacing easy. Courtesy of arno,
    the selector author.
    """
    for index, (regex, handler) in enumerate(selector.mappings):
        if regex.match(path) is not None:
            selector.mappings[index] = (regex, new_handler)
<commit_msg>tiddlyweb - follow the new interface for StorageInterface when handling a get(bag).<commit_after>"""
A suite of tools that make creating and handling plugins
in TiddlyWeb a bit more sensible.

Essentially this is a way to raise duplication to a common
core without encumbering the TiddlyWeb core.
"""

from tiddlyweb.model.bag import Bag
from tiddlyweb.model.policy import UserRequiredError
from tiddlyweb.store import NoBagError

def entitle(title):
    """
    Decorator that sets tiddlyweb.title in environ.
    """
    def entangle(f):
        def entitle(environ, start_response, *args, **kwds):
            output = f(environ, start_response)
            environ['tiddlyweb.title'] = title
            return output
        return entitle
    return entangle

def do_html():
    """
    Decorator that makes sure we are sending text/html.
    """
    def entangle(f):
        def do_html(environ, start_response, *args, **kwds):
            output = f(environ, start_response)
            start_response('200 OK', [
                ('Content-Type', 'text/html; charset=UTF-8')
                ])
            return output
        return do_html
    return entangle

def require_role(role):
    """
    Decorator that requires the current user has role <role>.
    """
    role = unicode(role)
    def entangle(f):
        def require_role(environ, start_response, *args, **kwds):
            try:
                if role in environ['tiddlyweb.usersign']['roles']:
                    return f(environ, start_response)
                else:
                    raise(UserRequiredError, 'insufficient permissions')
            except KeyError:
                raise(UserRequiredError, 'insufficient permissions')
        return require_role
    return entangle

def require_any_user():
    """
    Decorator that requires the current user be someone other than 'GUEST'.
    """
    def entangle(f):
        def require_any_user(environ, start_response, *args, **kwds):
            try:
                if environ['tiddlyweb.usersign']['name'] == 'GUEST':
                    raise(UserRequiredError, 'user must be logged in')
                else:
                    return f(environ, start_response)
            except KeyError:
                raise(UserRequiredError, 'user must be logged in')
        return require_any_user
    return entangle

def ensure_bag(bag_name, store, policy_dict={}, description='', owner=None):
    """
    Ensure that bag with name bag_name exists in store.
    If not, create it with owner, policy and description optionally
    provided. In either case return the bag object.
    """
    bag = Bag(bag_name)
    try:
        bag = store.get(bag)
    except NoBagError:
        bag.desc = description
        if owner:
            bag.policy.owner = owner
            bag.policy.manage = [owner]
        for key in policy_dict:
            bag.policy.__setattr__(key, policy_dict[key])
        store.put(bag)
    return bag

def replace_handler(selector, path, new_handler):
    """
    Replace an existing path handler in the selector
    map with a new handler. Usually we want to add a
    new one, but sometimes we just want to replace.
    This makes replacing easy. Courtesy of arno,
    the selector author.
    """
    for index, (regex, handler) in enumerate(selector.mappings):
        if regex.match(path) is not None:
            selector.mappings[index] = (regex, new_handler)
<|endoftext|>


<commit_before>'''Main connection to TWS client.

   Defines the EClientSocket class, which implements a socket connection to
   the TWS socket server, through which the entire API operates.
'''

__copyright__ = "Copyright (c) 2008 Kevin J Bluck"
__version__   = "$Id$"

import tws.EClientErrors as _EClientErrors
from tws import synchronized

def _requestmethod(min_server=0, has_ticker=False,
                   generic_error=_EClientErrors.TwsError(),
                   error_suffix=""):
    '''Socket request-method decorator.

       Eliminates repetitive error-checking boilerplate from request methods.
    '''
    assert type(min_server) == int
    assert type(has_ticker) == bool
    assert isinstance(generic_error, _EClientErrors.TwsError)
    assert type(error_suffix) == str

    def _decorator(method):
        assert  (__import__("inspect").getargspec(method)[0][0] == "self")
        assert  (not has_ticker) or (__import__("inspect").getargspec(method)[0][1] == "ticker_id")

        def _decorated(self, *args, **kwds):
            assert type(self) == EClientSocket

            try:
                # Socket client must be connected.
                if not self._connected:
                    self._error(_EClientErrors.NOT_CONNECTED)
                    return
                # Enforce minimum server version, if any.
                if self._server_version < min_server:
                    self._error(_EClientErrors.UPDATE_TWS)
                    return
                # Call wrapped method
                return method(self, *args, **kwds)
            # On exception, report generic error instance to EWrapper._error()
            except:
                self._error(_EClientErrors.TwsError(
                                source=generic_error,
                                id=kwds.get("ticker_id", args[0] if args else None)
                                    if has_ticker else _EClientErrors.NO_VALID_ID,
                                msg=error_suffix or method.__name__))

        return _decorated
    return _decorator

class EClientSocket(object):
    '''Socket client which connects to the TWS socket server.
    '''

    def __init__(self, wrapper, socket_factory=__import__("socket").socket):
        assert issubclass(type(wrapper), __import__("tws").EWrapper)
        assert callable(socket_factory)

        self._wrapper = wrapper
        self._reader = None
        self._connected = False
        self._server_version = 0
        self._tws_time = 0
        self._socket_factory = socket_factory

    @synchronized
    def _close(self):
        self.eDisconnect()
        self.wrapper().connectionClosed()

    def _send(self, data):
        if type(data) in (str, int, long, float):
            self._stream.write(str(data))
        elif type(data) == bool:
            self._stream.write("1" if data else "0")
        elif data is None:
            pass
        else:
            raise ValueError("Unknown data type for EClientSocket._send(): %s", type(data))
        self._stream.write(self.EOL)

    def _sendMax(self, data):
        if type(data) == int:
            self._send(data) if data != self._INT_MAX_VALUE else self._send(None)
        elif type(data) == float:
            self._send(data) if data != self._DOUBLE_MAX_VALUE else self._send(None)
        else:
            raise ValueError("Unknown data type for EClientSocket._sendMax(): %s", type(data))

    @synchronized
    def _error(self, e):
        self._wrapper.error(e)

    @classmethod
    def faMsgTypeName(cls, faDataType):
        if faDataType == cls.GROUPS:
            return "GROUPS"
        elif faDataType == cls.PROFILES:
            return "PROFILES"
        elif faDataType == cls.ALIASES:
            return "ALIASES"

        # Should never get here.
        assert False
        return ""

    # General constants
    CLIENT_VERSION = 42
    SERVER_VERSION = 38
    EOL = "\x00"
    BAG_SEC_TYPE = "BAG"

    # API tag constants
    REQ_MKT_DATA = 1
    CANCEL_MKT_DATA = 2
    PLACE_ORDER = 3
    CANCEL_ORDER = 4
    REQ_OPEN_ORDERS = 5
    REQ_ACCOUNT_DATA = 6
    REQ_EXECUTIONS = 7
    REQ_IDS = 8
    REQ_CONTRACT_DATA = 9
    REQ_MKT_DEPTH = 10
    CANCEL_MKT_DEPTH = 11
    REQ_NEWS_BULLETINS = 12
    CANCEL_NEWS_BULLETINS = 13
    SET_SERVER_LOGLEVEL = 14
    REQ_AUTO_OPEN_ORDERS = 15
    REQ_ALL_OPEN_ORDERS = 16
    REQ_MANAGED_ACCTS = 17
    REQ_FA = 18
    REPLACE_FA = 19
    REQ_HISTORICAL_DATA = 20
    EXERCISE_OPTIONS = 21
    REQ_SCANNER_SUBSCRIPTION = 22
    CANCEL_SCANNER_SUBSCRIPTION = 23
    REQ_SCANNER_PARAMETERS = 24
    CANCEL_HISTORICAL_DATA = 25
    REQ_CURRENT_TIME = 49
    REQ_REAL_TIME_BARS = 50
    CANCEL_REAL_TIME_BARS = 51
    REQ_FUNDAMENTAL_DATA = 52
    CANCEL_FUNDAMENTAL_DATA = 53
    MIN_SERVER_VER_REAL_TIME_BARS = 34
    MIN_SERVER_VER_SCALE_ORDERS = 35
    MIN_SERVER_VER_SNAPSHOT_MKT_DATA = 35
    MIN_SERVER_VER_SSHORT_COMBO_LEGS = 35
    MIN_SERVER_VER_WHAT_IF_ORDERS = 36
    MIN_SERVER_VER_CONTRACT_CONID = 37
    MIN_SERVER_VER_PTA_ORDERS = 39
    MIN_SERVER_VER_FUNDAMENTAL_DATA = 40
    MIN_SERVER_VER_UNDER_COMP = 40
    MIN_SERVER_VER_CONTRACT_DATA_CHAIN = 40
    MIN_SERVER_VER_SCALE_ORDERS2 = 40
    MIN_SERVER_VER_ALGO_ORDERS = 41
    MIN_SERVER_VER_EXECUTION_DATA_CHAIN = 42

    # Message Type name constants
    GROUPS = 1
    PROFILES = 2
    ALIASES = 3

    # Private class imports
    from tws._Util import _INT_MAX_VALUE
    from tws._Util import _DOUBLE_MAX_VALUE

    def wrapper(self):
        return self._wrapper

    def reader(self):
        return self._reader

    def isConnected(self):
        return self._connected

    def serverVersion(self):
        return self._server_version

    def TwsConnectionTime(self):
        return self._tws_time

    def checkConnected(self, host):
        assert issubclass(type(host), str) or (host == None)

        if self._connected:
            self._wrapper.error(_EClientErrors.ALREADY_CONNECTED)
            return None
        return host if host else "127.0.0.1"

    def connectionError(self):
        self._wrapper.error(_EClientErrors.CONNECT_FAIL)
        self.m_reader = None

    def createReader(self, connection, input_stream):
        assert issubclass(type(connection), type(self))
        assert hasattr(input_stream, "read")

        return __import__("tws").EReader(connection, input_stream)

    @synchronized
    def eConnect(self):
        # Trivial stub for now.
        self._connected = True

    @synchronized
    def eDisconnect(self):
        # Trivial stub for now.
        self._connected = False

del synchronized
<commit_msg>Privatized requestmethod decorator after actual use only for test purposes.<commit_after>'''Main connection to TWS client.

   Defines the EClientSocket class, which implements a socket connection to
   the TWS socket server, through which the entire API operates.
'''

__copyright__ = "Copyright (c) 2008 Kevin J Bluck"
__version__   = "$Id$"

import tws.EClientErrors as _EClientErrors
from tws import synchronized

def requestmethod(min_server=0, has_ticker=False,
                  generic_error=_EClientErrors.TwsError(),
                  error_suffix=""):
    '''Socket request-method decorator.

       Eliminates repetitive error-checking boilerplate from request methods.
    '''
    assert type(min_server) == int
    assert type(has_ticker) == bool
    assert isinstance(generic_error, _EClientErrors.TwsError)
    assert type(error_suffix) == str

    def _decorator(method):
        assert  (__import__("inspect").getargspec(method)[0][0] == "self")
        assert  (not has_ticker) or (__import__("inspect").getargspec(method)[0][1] == "ticker_id")

        def _decorated(self, *args, **kwds):
            assert type(self) == EClientSocket

            try:
                # Socket client must be connected.
                if not self._connected:
                    self._error(_EClientErrors.NOT_CONNECTED)
                    return
                # Enforce minimum server version, if any.
                if self._server_version < min_server:
                    self._error(_EClientErrors.UPDATE_TWS)
                    return
                # Call wrapped method
                return method(self, *args, **kwds)
            # On exception, report generic error instance to EWrapper._error()
            except:
                self._error(_EClientErrors.TwsError(
                                source=generic_error,
                                id=kwds.get("ticker_id", args[0] if args else None)
                                    if has_ticker else _EClientErrors.NO_VALID_ID,
                                msg=error_suffix or method.__name__))

        return _decorated
    return _decorator

class EClientSocket(object):
    '''Socket client which connects to the TWS socket server.
    '''

    def __init__(self, wrapper, socket_factory=__import__("socket").socket):
        assert issubclass(type(wrapper), __import__("tws").EWrapper)
        assert callable(socket_factory)

        self._wrapper = wrapper
        self._reader = None
        self._connected = False
        self._server_version = 0
        self._tws_time = 0
        self._socket_factory = socket_factory

    @synchronized
    def _close(self):
        self.eDisconnect()
        self.wrapper().connectionClosed()

    def _send(self, data):
        if type(data) in (str, int, long, float):
            self._stream.write(str(data))
        elif type(data) == bool:
            self._stream.write("1" if data else "0")
        elif data is None:
            pass
        else:
            raise ValueError("Unknown data type for EClientSocket._send(): %s", type(data))
        self._stream.write(self.EOL)

    def _sendMax(self, data):
        if type(data) == int:
            self._send(data) if data != self._INT_MAX_VALUE else self._send(None)
        elif type(data) == float:
            self._send(data) if data != self._DOUBLE_MAX_VALUE else self._send(None)
        else:
            raise ValueError("Unknown data type for EClientSocket._sendMax(): %s", type(data))

    @synchronized
    def _error(self, e):
        self._wrapper.error(e)

    @classmethod
    def faMsgTypeName(cls, faDataType):
        if faDataType == cls.GROUPS:
            return "GROUPS"
        elif faDataType == cls.PROFILES:
            return "PROFILES"
        elif faDataType == cls.ALIASES:
            return "ALIASES"

        # Should never get here.
        assert False
        return ""

    # General constants
    CLIENT_VERSION = 42
    SERVER_VERSION = 38
    EOL = "\x00"
    BAG_SEC_TYPE = "BAG"

    # API tag constants
    REQ_MKT_DATA = 1
    CANCEL_MKT_DATA = 2
    PLACE_ORDER = 3
    CANCEL_ORDER = 4
    REQ_OPEN_ORDERS = 5
    REQ_ACCOUNT_DATA = 6
    REQ_EXECUTIONS = 7
    REQ_IDS = 8
    REQ_CONTRACT_DATA = 9
    REQ_MKT_DEPTH = 10
    CANCEL_MKT_DEPTH = 11
    REQ_NEWS_BULLETINS = 12
    CANCEL_NEWS_BULLETINS = 13
    SET_SERVER_LOGLEVEL = 14
    REQ_AUTO_OPEN_ORDERS = 15
    REQ_ALL_OPEN_ORDERS = 16
    REQ_MANAGED_ACCTS = 17
    REQ_FA = 18
    REPLACE_FA = 19
    REQ_HISTORICAL_DATA = 20
    EXERCISE_OPTIONS = 21
    REQ_SCANNER_SUBSCRIPTION = 22
    CANCEL_SCANNER_SUBSCRIPTION = 23
    REQ_SCANNER_PARAMETERS = 24
    CANCEL_HISTORICAL_DATA = 25
    REQ_CURRENT_TIME = 49
    REQ_REAL_TIME_BARS = 50
    CANCEL_REAL_TIME_BARS = 51
    REQ_FUNDAMENTAL_DATA = 52
    CANCEL_FUNDAMENTAL_DATA = 53
    MIN_SERVER_VER_REAL_TIME_BARS = 34
    MIN_SERVER_VER_SCALE_ORDERS = 35
    MIN_SERVER_VER_SNAPSHOT_MKT_DATA = 35
    MIN_SERVER_VER_SSHORT_COMBO_LEGS = 35
    MIN_SERVER_VER_WHAT_IF_ORDERS = 36
    MIN_SERVER_VER_CONTRACT_CONID = 37
    MIN_SERVER_VER_PTA_ORDERS = 39
    MIN_SERVER_VER_FUNDAMENTAL_DATA = 40
    MIN_SERVER_VER_UNDER_COMP = 40
    MIN_SERVER_VER_CONTRACT_DATA_CHAIN = 40
    MIN_SERVER_VER_SCALE_ORDERS2 = 40
    MIN_SERVER_VER_ALGO_ORDERS = 41
    MIN_SERVER_VER_EXECUTION_DATA_CHAIN = 42

    # Message Type name constants
    GROUPS = 1
    PROFILES = 2
    ALIASES = 3

    # Private class imports
    from tws._Util import _INT_MAX_VALUE
    from tws._Util import _DOUBLE_MAX_VALUE

    def wrapper(self):
        return self._wrapper

    def reader(self):
        return self._reader

    def isConnected(self):
        return self._connected

    def serverVersion(self):
        return self._server_version

    def TwsConnectionTime(self):
        return self._tws_time

    def checkConnected(self, host):
        assert issubclass(type(host), str) or (host == None)

        if self._connected:
            self._wrapper.error(_EClientErrors.ALREADY_CONNECTED)
            return None
        return host if host else "127.0.0.1"

    def connectionError(self):
        self._wrapper.error(_EClientErrors.CONNECT_FAIL)
        self.m_reader = None

    def createReader(self, connection, input_stream):
        assert issubclass(type(connection), type(self))
        assert hasattr(input_stream, "read")

        return __import__("tws").EReader(connection, input_stream)

    @synchronized
    def eConnect(self):
        # Trivial stub for now.
        self._connected = True

    @synchronized
    def eDisconnect(self):
        # Trivial stub for now.
        self._connected = False

_requestmethod = requestmethod
del requestmethod
del synchronized
<|endoftext|>


<commit_before>import os
import yaml
from requests import get

from Tests.test_utils import print_error, get_yaml

class IntegrationValidator(object):
    """IntegrationValidator is designed to validate the correctness of the file structure we enter to content repo. And
    also try to catch possible Backward compatibility breaks due to the preformed changes.

    Attributes:
       _is_valid (bool): the attribute which saves the valid/in-valid status of the current file.
       file_path (str): the path to the file we are examining at the moment.
       current_integration (dict): Json representation of the current integration from the branch.
       old_integration (dict): Json representation of the current integration from master.
    """
    CONTENT_GIT_HUB_LINK = "https://raw.githubusercontent.com/demisto/content/master/"

    def __init__(self, file_path, check_git=True, old_file_path=None):
        self._is_valid = True

        self.file_path = file_path
        if check_git:
            self.current_integration = get_yaml(file_path)
            # The replace in the end is for Windows support
            if old_file_path:
                git_hub_path = os.path.join(self.CONTENT_GIT_HUB_LINK, old_file_path).replace("\\", "/")
                file_content = get(git_hub_path, verify=False).content
                self.old_integration = yaml.safe_load(file_content)
            else:
                try:
                    file_path_from_master = os.path.join(self.CONTENT_GIT_HUB_LINK, file_path).replace("\\", "/")
                    self.old_integration = yaml.safe_load(get(file_path_from_master, verify=False).content)
                except Exception as e:
                    print(str(e))
                    print_error("Could not find the old integration please make sure that you did not break "
                                "backward compatibility")
                    self.old_integration = None

    def is_backward_compatible(self):
        """Check whether the Integration is backward compatible or not, update the _is_valid field to determine that"""
        if not self.old_integration:
            return True

        self.is_changed_context_path()
        self.is_docker_image_changed()
        self.is_added_required_fields()
        self.is_changed_command_name_or_arg()
        self.is_there_duplicate_args()
        self.is_there_duplicate_params()

        return self._is_valid

    def is_there_duplicate_args(self):
        """Check if a command has the same arg more than once

        Returns:
            bool. True if there are duplicates, False otherwise.
        """
        commands = self.current_integration.get('script', {}).get('commands', [])
        for command in commands:
            arg_list = []
            for arg in command.get('arguments', []):
                if arg in arg_list:
                    self._is_valid = False
                    print_error("The argument '{}' of the command '{}' is duplicated in the integration '{}', "
                                "please remove one of its appearances "
                                "as we do not allow duplicates".format(arg, command['name'],
                                                                       self.current_integration.get('name')))
                else:
                    arg_list.append(arg)

        return not self._is_valid

    def is_there_duplicate_params(self):
        """Check if the integration has the same param more than once

        Returns:
            bool. True if there are duplicates, False otherwise.
        """
        configurations = self.current_integration.get('configuration', [])
        param_list = []
        for configuration_param in configurations:
            param_name = configuration_param['name']
            if param_name in param_list:
                self._is_valid = False
                print_error("The parameter '{}' of the "
                            "integration '{}' is duplicated, please remove one of its appearances as we do not "
                            "allow duplicated parameters".format(param_name,
                                                                 self.current_integration.get('name')))
            else:
                param_list.append(param_name)

        return not self._is_valid

    def _get_command_to_args(self, integration_json):
        """Get a dictionary command name to it's arguments.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. command name to a list of it's arguments.
        """
        command_to_args = {}
        commands = integration_json.get('script', {}).get('commands', [])
        for command in commands:
            command_to_args[command['name']] = {}
            for arg in command.get('arguments', []):
                command_to_args[command['name']][arg['name']] = arg.get('required', False)

        return command_to_args

    def is_subset_dictionary(self, new_dict, old_dict):
        """Check if the new dictionary is a sub set of the old dictionary.

        Args:
            new_dict (dict): current branch result from _get_command_to_args
            old_dict (dict): master branch result from _get_command_to_args

        Returns:
            bool. Whether the new dictionary is a sub set of the old dictionary.
        """
        for arg, required in old_dict.items():
            if arg not in new_dict.keys():
                return False

            if required != new_dict[arg] and new_dict[arg]:
                return False

        for arg, required in new_dict.items():
            if arg not in old_dict.keys() and required:
                return False

        return True

    def is_changed_command_name_or_arg(self):
        """Check if a command name or argument as been changed.

        Returns:
            bool. Whether a command name or argument as been changed.
        """
        current_command_to_args = self._get_command_to_args(self.current_integration)
        old_command_to_args = self._get_command_to_args(self.old_integration)

        for command, args_dict in old_command_to_args.items():
            if command not in current_command_to_args.keys() or \
                    not self.is_subset_dictionary(current_command_to_args[command], args_dict):
                print_error("Possible backwards compatibility break, You've changed the name of a command or its arg in"
                            " the file {0} please undo, the command was:\n{1}".format(self.file_path, command))
                self._is_valid = False
                return True

        return False

    def _is_sub_set(self, supposed_bigger_list, supposed_smaller_list):
        """Check if supposed_smaller_list is a subset of the supposed_bigger_list"""
        for check_item in supposed_smaller_list:
            if check_item not in supposed_bigger_list:
                return False

        return True

    def _get_command_to_context_paths(self, integration_json):
        """Get a dictionary command name to it's context paths.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. command name to a list of it's context paths.
        """
        command_to_context_list = {}
        commands = integration_json.get('script', {}).get('commands', [])
        for command in commands:
            context_list = []
            if not command.get('outputs', []):
                continue

            for output in command.get('outputs', []):
                context_list.append(output['contextPath'])

            command_to_context_list[command['name']] = sorted(context_list)

        return command_to_context_list

    def is_changed_context_path(self):
        """Check if a context path as been changed.

        Returns:
            bool. Whether a context path as been changed.
        """
        current_command_to_context_paths = self._get_command_to_context_paths(self.current_integration)
        old_command_to_context_paths = self._get_command_to_context_paths(self.old_integration)

        for old_command, old_context_paths in old_command_to_context_paths.items():
            if old_command in current_command_to_context_paths.keys() and \
                    not self._is_sub_set(current_command_to_context_paths[old_command],
                                         old_context_paths):
                print_error("Possible backwards compatibility break, You've changed the context in the file {0} please "
                            "undo, the command is:\n{1}".format(self.file_path, old_command))
                self._is_valid = False
                return True

        return False

    def _get_field_to_required_dict(self, integration_json):
        """Get a dictionary field name to its required status.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. Field name to its required status.
        """
        field_to_required = {}
        configuration = integration_json.get('configuration')
        for field in configuration:
            field_to_required[field.get('name')] = field.get('required', False)

        return field_to_required

    def is_added_required_fields(self):
        """Check if required field were added."""
        current_field_to_required = self._get_field_to_required_dict(self.current_integration)
        old_field_to_required = self._get_field_to_required_dict(self.old_integration)

        for field, required in current_field_to_required.items():
            if (field not in old_field_to_required.keys() and required) or \
                    (required and field in old_field_to_required.keys() and required != old_field_to_required[field]):
                print_error("You've added required fields in the integration "
                            "file '{}', the field is '{}'".format(self.file_path, field))
                self._is_valid = False
                return True

        return False

    def is_docker_image_changed(self):
        """Check if the Docker image was changed or not."""
        if self.old_integration.get('script', {}).get('dockerimage', "") != \
                self.current_integration.get('script', {}).get('dockerimage', ""):
            print_error("Possible backwards compatibility break, You've changed the docker for the file {}"
                        " this is not allowed.".format(self.file_path))
            self._is_valid = False
            return True

        return False
<commit_msg>Disable SSL verification warnings (#3767)<commit_after>import os
import yaml
import requests

from Tests.test_utils import print_error, get_yaml

# disable insecure warnings
requests.packages.urllib3.disable_warnings()

class IntegrationValidator(object):
    """IntegrationValidator is designed to validate the correctness of the file structure we enter to content repo. And
    also try to catch possible Backward compatibility breaks due to the preformed changes.

    Attributes:
       _is_valid (bool): the attribute which saves the valid/in-valid status of the current file.
       file_path (str): the path to the file we are examining at the moment.
       current_integration (dict): Json representation of the current integration from the branch.
       old_integration (dict): Json representation of the current integration from master.
    """
    CONTENT_GIT_HUB_LINK = "https://raw.githubusercontent.com/demisto/content/master/"

    def __init__(self, file_path, check_git=True, old_file_path=None):
        self._is_valid = True

        self.file_path = file_path
        if check_git:
            self.current_integration = get_yaml(file_path)
            # The replace in the end is for Windows support
            if old_file_path:
                git_hub_path = os.path.join(self.CONTENT_GIT_HUB_LINK, old_file_path).replace("\\", "/")
                file_content = requests.get(git_hub_path, verify=False).content
                self.old_integration = yaml.safe_load(file_content)
            else:
                try:
                    file_path_from_master = os.path.join(self.CONTENT_GIT_HUB_LINK, file_path).replace("\\", "/")
                    self.old_integration = yaml.safe_load(requests.get(file_path_from_master, verify=False).content)
                except Exception as e:
                    print(str(e))
                    print_error("Could not find the old integration please make sure that you did not break "
                                "backward compatibility")
                    self.old_integration = None

    def is_backward_compatible(self):
        """Check whether the Integration is backward compatible or not, update the _is_valid field to determine that"""
        if not self.old_integration:
            return True

        self.is_changed_context_path()
        self.is_docker_image_changed()
        self.is_added_required_fields()
        self.is_changed_command_name_or_arg()
        self.is_there_duplicate_args()
        self.is_there_duplicate_params()

        return self._is_valid

    def is_there_duplicate_args(self):
        """Check if a command has the same arg more than once

        Returns:
            bool. True if there are duplicates, False otherwise.
        """
        commands = self.current_integration.get('script', {}).get('commands', [])
        for command in commands:
            arg_list = []
            for arg in command.get('arguments', []):
                if arg in arg_list:
                    self._is_valid = False
                    print_error("The argument '{}' of the command '{}' is duplicated in the integration '{}', "
                                "please remove one of its appearances "
                                "as we do not allow duplicates".format(arg, command['name'],
                                                                       self.current_integration.get('name')))
                else:
                    arg_list.append(arg)

        return not self._is_valid

    def is_there_duplicate_params(self):
        """Check if the integration has the same param more than once

        Returns:
            bool. True if there are duplicates, False otherwise.
        """
        configurations = self.current_integration.get('configuration', [])
        param_list = []
        for configuration_param in configurations:
            param_name = configuration_param['name']
            if param_name in param_list:
                self._is_valid = False
                print_error("The parameter '{}' of the "
                            "integration '{}' is duplicated, please remove one of its appearances as we do not "
                            "allow duplicated parameters".format(param_name,
                                                                 self.current_integration.get('name')))
            else:
                param_list.append(param_name)

        return not self._is_valid

    def _get_command_to_args(self, integration_json):
        """Get a dictionary command name to it's arguments.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. command name to a list of it's arguments.
        """
        command_to_args = {}
        commands = integration_json.get('script', {}).get('commands', [])
        for command in commands:
            command_to_args[command['name']] = {}
            for arg in command.get('arguments', []):
                command_to_args[command['name']][arg['name']] = arg.get('required', False)

        return command_to_args

    def is_subset_dictionary(self, new_dict, old_dict):
        """Check if the new dictionary is a sub set of the old dictionary.

        Args:
            new_dict (dict): current branch result from _get_command_to_args
            old_dict (dict): master branch result from _get_command_to_args

        Returns:
            bool. Whether the new dictionary is a sub set of the old dictionary.
        """
        for arg, required in old_dict.items():
            if arg not in new_dict.keys():
                return False

            if required != new_dict[arg] and new_dict[arg]:
                return False

        for arg, required in new_dict.items():
            if arg not in old_dict.keys() and required:
                return False

        return True

    def is_changed_command_name_or_arg(self):
        """Check if a command name or argument as been changed.

        Returns:
            bool. Whether a command name or argument as been changed.
        """
        current_command_to_args = self._get_command_to_args(self.current_integration)
        old_command_to_args = self._get_command_to_args(self.old_integration)

        for command, args_dict in old_command_to_args.items():
            if command not in current_command_to_args.keys() or \
                    not self.is_subset_dictionary(current_command_to_args[command], args_dict):
                print_error("Possible backwards compatibility break, You've changed the name of a command or its arg in"
                            " the file {0} please undo, the command was:\n{1}".format(self.file_path, command))
                self._is_valid = False
                return True

        return False

    def _is_sub_set(self, supposed_bigger_list, supposed_smaller_list):
        """Check if supposed_smaller_list is a subset of the supposed_bigger_list"""
        for check_item in supposed_smaller_list:
            if check_item not in supposed_bigger_list:
                return False

        return True

    def _get_command_to_context_paths(self, integration_json):
        """Get a dictionary command name to it's context paths.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. command name to a list of it's context paths.
        """
        command_to_context_list = {}
        commands = integration_json.get('script', {}).get('commands', [])
        for command in commands:
            context_list = []
            if not command.get('outputs', []):
                continue

            for output in command.get('outputs', []):
                context_list.append(output['contextPath'])

            command_to_context_list[command['name']] = sorted(context_list)

        return command_to_context_list

    def is_changed_context_path(self):
        """Check if a context path as been changed.

        Returns:
            bool. Whether a context path as been changed.
        """
        current_command_to_context_paths = self._get_command_to_context_paths(self.current_integration)
        old_command_to_context_paths = self._get_command_to_context_paths(self.old_integration)

        for old_command, old_context_paths in old_command_to_context_paths.items():
            if old_command in current_command_to_context_paths.keys() and \
                    not self._is_sub_set(current_command_to_context_paths[old_command],
                                         old_context_paths):
                print_error("Possible backwards compatibility break, You've changed the context in the file {0} please "
                            "undo, the command is:\n{1}".format(self.file_path, old_command))
                self._is_valid = False
                return True

        return False

    def _get_field_to_required_dict(self, integration_json):
        """Get a dictionary field name to its required status.

        Args:
            integration_json (dict): Dictionary of the examined integration.

        Returns:
            dict. Field name to its required status.
        """
        field_to_required = {}
        configuration = integration_json.get('configuration')
        for field in configuration:
            field_to_required[field.get('name')] = field.get('required', False)

        return field_to_required

    def is_added_required_fields(self):
        """Check if required field were added."""
        current_field_to_required = self._get_field_to_required_dict(self.current_integration)
        old_field_to_required = self._get_field_to_required_dict(self.old_integration)

        for field, required in current_field_to_required.items():
            if (field not in old_field_to_required.keys() and required) or \
                    (required and field in old_field_to_required.keys() and required != old_field_to_required[field]):
                print_error("You've added required fields in the integration "
                            "file '{}', the field is '{}'".format(self.file_path, field))
                self._is_valid = False
                return True

        return False

    def is_docker_image_changed(self):
        """Check if the Docker image was changed or not."""
        if self.old_integration.get('script', {}).get('dockerimage', "") != \
                self.current_integration.get('script', {}).get('dockerimage', ""):
            print_error("Possible backwards compatibility break, You've changed the docker for the file {}"
                        " this is not allowed.".format(self.file_path))
            self._is_valid = False
            return True

        return False
<|endoftext|>


<commit_before>#!/usr/local/bin/python
#---------------------------------------------------------------------------------------------------
# Python interface to access Popularity Database. See website for API documentation
# (https://cms-popularity.cern.ch/popdb/popularity/apidoc)
#
# Use SSO cookie to avoid password
# (http://linux.web.cern.ch/linux/docs/cernssocookie.shtml)
# It is up to the caller to make sure a valid SSO cookie is obtained before any calls are made. A
# SSO cookie is valid for 24h. Requires usercert.pem and userkey.pem in ~/.globus/
#
# The API doesn't check to make sure correct values are passed or that rquired parameters are
# passed. All such checks needs to be done by the caller. All data is returned as JSON.
#
# In case of error an error message is printed to the log, currently specified by environemental
# variable INTELROCCS_LOG, and '0' is returned. User will have to check that something is returned.
# If a valid call is made but no data was found a JSON structure is still returned, it is up to
# the caller to check for actual data.
#---------------------------------------------------------------------------------------------------
import os, json, urllib, urllib2, datetime, subprocess, ConfigParser
from email.MIMEText import MIMEText
from email.MIMEMultipart import MIMEMultipart
from email.Utils import formataddr
from subprocess import Popen, PIPE

class popDbApi():
    def __init__(self):
        config = ConfigParser.RawConfigParser()
        config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
        self.fromEmail = config.items('from_email')[0]
        self.toEmails = config.items('error_emails')
        self.popDbBase = config.get('pop_db', 'base')
        self.cert = config.get('pop_db', 'certificate')
        self.key = config.get('pop_db', 'key')
        self.cookie = config.get('pop_db', 'sso_cookie')
        self.renewSsoCookie()

#===================================================================================================
#  H E L P E R S
#===================================================================================================
    def renewSsoCookie(self):
        # Will try to generate cookie 3 times before reporting an error
        for attempt in range(3):
            process = subprocess.Popen(["cern-get-sso-cookie", "--cert", self.cert, "--key", self.key, "-u", self.popDbBase, "-o", self.cookie], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
            process.communicate()[0]
            if process.returncode != 0:
                continue
            else:
                break
        else:
            self.error("Could not generate SSO cookie")
        return 0

    def call(self, url, values):
        data = urllib.urlencode(values)
        request = urllib2.Request(url, data)
        fullUrl = request.get_full_url() + request.get_data()
        strout = ""
        for attempt in range(3):
            process = subprocess.Popen(["curl", "-k", "-s", "-L", "--cookie", self.cookie, "--cookie-jar", self.cookie, fullUrl], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
            strout = process.communicate()[0]
            if process.returncode != 0:
                continue
            else:
                try:
                    jsonData = json.loads(strout)
                except ValueError:
                    continue
                break
        else:
            self.error("Pop DB call failed for url: %s %s" % (str(url), str(values)))
        return jsonData

    def error(self, e):
        title = "FATAL IntelROCCS Error -- Pop DB"
        text = "FATAL -- %s" % (str(e))
        msg = MIMEMultipart()
        msg['Subject'] = title
        msg['From'] = formataddr(self.fromEmail)
        msg['To'] = self._toStr(self.toEmails)
        msg1 = MIMEMultipart("alternative")
        msgText1 = MIMEText("
%s
" % text, "html")
        msgText2 = MIMEText(text)
        msg1.attach(msgText2)
        msg1.attach(msgText1)
        msg.attach(msg1)
        msg = msg.as_string()
        p = Popen(["/usr/sbin/sendmail", "-toi"], stdin=PIPE)
        p.communicate(msg)
        raise Exception("FATAL -- %s" % (str(e)))

    def _toStr(self, toEmails):
        names = [formataddr(email) for email in toEmails]
        return ', '.join(names)

#===================================================================================================
#  A P I   C A L L S
#===================================================================================================
    def DataTierStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("DataTierStatInTimeWindow"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DataTierStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (tstart, tstop, sitename))
        return jsonData

    def DSNameStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("DSNameStatInTimeWindow"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DSNameStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), tstart, tstop, sitename))
        return jsonData

    def DSStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("DSStatInTimeWindow"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DSStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (tstart, tstop, sitename))
        return jsonData

    def getCorruptedFiles(self, sitename='summary', orderby=''):
        values = {'sitename':sitename, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getCorruptedFiles"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getCorruptedFiles call failed for values: sitename=%s, orderby=%s\n" % (sitename, orderby))
        return jsonData

    def getDSdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getDSdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getDSdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getDSNdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getDSNdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getDSNdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getDTdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getDTdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getDTdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getSingleDNstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getSingleDNstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDNstat call failed for values: sitename=%s, aggr=%s, orderby=%s\n" % (sitename, aggr, orderby))
        return jsonData

    def getSingleDSstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getSingleDSstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDSstat call failed for values: name=%s, sitename=%s, aggr=%s, orderby=%s\n" % (name, sitename, aggr, orderby))
        return jsonData

    def getSingleDTstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getSingleDTstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDTstat call failed for values: name=%s, sitename=%s, aggr=%s, orderby=%s\n" % (name, sitename, aggr, orderby))
        return jsonData

    def getUserStat(self, tstart='', tstop='', collname='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'collname':collname, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s/?&" % ("getUserStat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getUserStat call failed for values: tstart=%s, tstop=%s, collname=%s, orderby=%s\n" % (tstart, tstop, collname, orderby))
        return jsonData
<commit_msg>Hotfix for popDB API call naming<commit_after>#!/usr/local/bin/python
#---------------------------------------------------------------------------------------------------
# Python interface to access Popularity Database. See website for API documentation
# (https://cms-popularity.cern.ch/popdb/popularity/apidoc)
#
# Use SSO cookie to avoid password
# (http://linux.web.cern.ch/linux/docs/cernssocookie.shtml)
# It is up to the caller to make sure a valid SSO cookie is obtained before any calls are made. A
# SSO cookie is valid for 24h. Requires usercert.pem and userkey.pem in ~/.globus/
#
# The API doesn't check to make sure correct values are passed or that rquired parameters are
# passed. All such checks needs to be done by the caller. All data is returned as JSON.
#
# In case of error an error message is printed to the log, currently specified by environemental
# variable INTELROCCS_LOG, and '0' is returned. User will have to check that something is returned.
# If a valid call is made but no data was found a JSON structure is still returned, it is up to
# the caller to check for actual data.
#---------------------------------------------------------------------------------------------------
import os, json, urllib, urllib2, datetime, subprocess, ConfigParser
from email.MIMEText import MIMEText
from email.MIMEMultipart import MIMEMultipart
from email.Utils import formataddr
from subprocess import Popen, PIPE

class popDbApi():
    def __init__(self):
        config = ConfigParser.RawConfigParser()
        config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
        self.fromEmail = config.items('from_email')[0]
        self.toEmails = config.items('error_emails')
        self.popDbBase = config.get('pop_db', 'base')
        self.cert = config.get('pop_db', 'certificate')
        self.key = config.get('pop_db', 'key')
        self.cookie = config.get('pop_db', 'sso_cookie')
        self.renewSsoCookie()

#===================================================================================================
#  H E L P E R S
#===================================================================================================
    def renewSsoCookie(self):
        # Will try to generate cookie 3 times before reporting an error
        for attempt in range(3):
            process = subprocess.Popen(["cern-get-sso-cookie", "--cert", self.cert, "--key", self.key, "-u", self.popDbBase, "-o", self.cookie], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
            process.communicate()[0]
            if process.returncode != 0:
                continue
            else:
                break
        else:
            self.error("Could not generate SSO cookie")
        return 0

    def call(self, url, values):
        data = urllib.urlencode(values)
        request = urllib2.Request(url, data)
        fullUrl = request.get_full_url() + request.get_data()
        strout = ""
        for attempt in range(3):
            process = subprocess.Popen(["curl", "-k", "-s", "-L", "--cookie", self.cookie, "--cookie-jar", self.cookie, fullUrl], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
            strout = process.communicate()[0]
            if process.returncode != 0:
                continue
            else:
                try:
                    jsonData = json.loads(strout)
                except ValueError:
                    continue
                break
        else:
            self.error("Pop DB call failed for url: %s %s" % (str(url), str(values)))
        return jsonData

    def error(self, e):
        title = "FATAL IntelROCCS Error -- Pop DB"
        text = "FATAL -- %s" % (str(e))
        msg = MIMEMultipart()
        msg['Subject'] = title
        msg['From'] = formataddr(self.fromEmail)
        msg['To'] = self._toStr(self.toEmails)
        msg1 = MIMEMultipart("alternative")
        msgText1 = MIMEText("
%s
" % text, "html")
        msgText2 = MIMEText(text)
        msg1.attach(msgText2)
        msg1.attach(msgText1)
        msg.attach(msg1)
        msg = msg.as_string()
        p = Popen(["/usr/sbin/sendmail", "-toi"], stdin=PIPE)
        p.communicate(msg)
        raise Exception("FATAL -- %s" % (str(e)))

    def _toStr(self, toEmails):
        names = [formataddr(email) for email in toEmails]
        return ', '.join(names)

#===================================================================================================
#  A P I   C A L L S
#===================================================================================================
    def DataTierStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("DataTierStatInTimeWindow/"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DataTierStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (tstart, tstop, sitename))
        return jsonData

    def DSNameStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("DSNameStatInTimeWindow/"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DSNameStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), tstart, tstop, sitename))
        return jsonData

    def DSStatInTimeWindow(self, tstart='', tstop='', sitename='summary'):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("DSStatInTimeWindow/"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- DSStatInTimeWindow call failed for values: tstart=%s, tstop=%s, sitename=%s\n" % (tstart, tstop, sitename))
        return jsonData

    def getCorruptedFiles(self, sitename='summary', orderby=''):
        values = {'sitename':sitename, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getCorruptedFiles/"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getCorruptedFiles call failed for values: sitename=%s, orderby=%s\n" % (sitename, orderby))
        return jsonData

    def getDSdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getDSdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getDSdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getDSNdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getDSNdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR -- getDSNdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getDTdata(self, tstart='', tstop='', sitename='summary', aggr='', n='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'sitename':sitename, 'aggr':aggr, 'n':n, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getDTdata"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getDTdata call failed for values: tstart=%s, tstop=%s, sitename=%s, aggr=%s, n=%s, orderby=%s\n" % (tstart, tstop, sitename, aggr, n, orderby))
        return jsonData

    def getSingleDNstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getSingleDNstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDNstat call failed for values: sitename=%s, aggr=%s, orderby=%s\n" % (sitename, aggr, orderby))
        return jsonData

    def getSingleDSstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getSingleDSstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDSstat call failed for values: name=%s, sitename=%s, aggr=%s, orderby=%s\n" % (name, sitename, aggr, orderby))
        return jsonData

    def getSingleDTstat(self, name='', sitename='summary', aggr='', orderby=''):
        values = {'name':name, 'sitename':sitename, 'aggr':aggr, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getSingleDTstat"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getSingleDTstat call failed for values: name=%s, sitename=%s, aggr=%s, orderby=%s\n" % (name, sitename, aggr, orderby))
        return jsonData

    def getUserStat(self, tstart='', tstop='', collname='', orderby=''):
        values = {'tstart':tstart, 'tstop':tstop, 'collname':collname, 'orderby':orderby}
        url = urllib.basejoin(self.popDbBase, "%s?&" % ("getUserStat/"))
        jsonData = self.call(url, values)
        if not jsonData:
            self.error("ERROR: getUserStat call failed for values: tstart=%s, tstop=%s, collname=%s, orderby=%s\n" % (tstart, tstop, collname, orderby))
        return jsonData
<|endoftext|>


<commit_before>import requests
import os
import datetime
from time import gmtime, strftime
from requests.auth import HTTPBasicAuth

#headers = {'accept': 'application/json;odata=verbose'}
#r = requests.get('http://dev/', auth=HTTPBasicAuth('booldevlocal\\administrator', 'Booldev1'))

#print r.json()["d"]["Title"]
#print r

def visitLinks(listOfLinks):
    for link in listOfLinks:
        print link
        try:
            response = requests.get(link['url'])
            print response.status_code
            if(response.status_code == 404):
                writeToFile(link, enterFilePath())
        except requests.exceptions.RequestException as e:
            # catastrophic error. bail.
            print e

def enterFilePath():
    filePath = os.path.expanduser('~\Desktop')
   # filePath  = raw_input("Enter desired file destination: ")
    return filePath

def writeToFile(linkObject, fileDirectory):
    # adding a as second parameter to open means append to the end of file.
    #fileObject = open('C:\Users\Administrator\Desktop\Output.txt','a')
    currentDatetime = strftime("%Y%m%d_%H-%M-%S", gmtime())
    fileObject = open(fileDirectory + '\\brokenLinks' + currentDatetime + '.txt', 'a')
    fileObject.write(linkObject['title'] + '    ' + linkObject['url'] +' \n')
    fileObject.close()

listOfLinks = [{'url':'http://google.com/lxfjdklj', 'title':'Google'}, {'url':'http://jioijoj.com', 'title':'joijoj'}, {'url':'http://svt.se', 'title':'Svt'}]

#path = enterFilePath()
#print path
#WriteToFile("Teeest!", "http://url.se", path)
visitLinks(listOfLinks)
<commit_msg>Print in columns :pizza:<commit_after>import requests
import os
import datetime
from time import gmtime, strftime
from requests.auth import HTTPBasicAuth

#headers = {'accept': 'application/json;odata=verbose'}
#r = requests.get('http://dev/', auth=HTTPBasicAuth('booldevlocal\\administrator', 'Booldev1'))

#print r

def visit_links(list_of_links):
    bad_links = []

    for link in list_of_links:
        print link
        try:
            response = requests.get(link['url'])
            print response.status_code

            if(response.status_code == 404):
                bad_links.append(link)

        except requests.exceptions.RequestException as e:
            # catastrophic error. fail.
            print e
    write_to_file(bad_links, get_file_path())

def get_file_path():
    # Dafult path to user desktop
    file_path = os.path.expanduser('~\Desktop')
    # filePath  = raw_input("Enter desired file destination: ")
    return file_path

def write_to_file(link_objects, file_directory):

    # time to make file anme unique
    current_datetime = strftime("%Y%m%d_%H-%M", gmtime())

    # adding a as second parameter to open means append to the end of file.
    file_object = open(file_directory + '\\brokenLinks' + current_datetime + '.txt', 'a')

    for link in link_objects:

        #substring title 30 characters
        title = link['title'][:29]

        #format output in columns
        file_object.write('{0:30}   {1:30}'.format(title, link['url']) + ' \n')
    file_object.close()

listOfLinks = [{'url':'http://google.com/lxfjdklj', 'title':'Google'},{'url':'http://google.com/dfdfgdfgfff', 'title':'Stina Qvartstrom is a girl on my job'}, {'url':'http://google.com/ogfjgiofjgi', 'title':'Malins Test'}, {'url':'http://jioijoj.com', 'title':'joijoj'}, {'url':'http://www.oppetarkiv.se//nostalgi', 'title':'Svt'}]

#path = enterFilePath()
#print path
#WriteToFile("Teeest!", "http://url.se", path)
visit_links(listOfLinks)
<|endoftext|>