<commit_before>"""
============
neighborlist
============
"""

import numpy as np

class NeighborList:
    """The NeighborList class

    Contains an `n x 5` array of neighbor vectors each being
    `[origin_site, target_site, i, j, k]`.

    >>> nl = NeighborList()
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
    <BLANKLINE>
    >>> nl = NeighborList([[0,1,1,0,0],[0,1,0,1,0],[0,1,0,0,1]])
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
        1 |      0      1 |   0   1   0
        2 |      0      1 |   0   0   1

    You can slice a NeighborList:

    >>> print(nl[0])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
    >>> print(nl[-1])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   0   0   1
    >>> print(nl[1:3])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   0   1   0
        1 |      0      1 |   0   0   1
    >>> print(nl[0,2])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
        1 |      0      1 |   0   0   1

    You can set a vector:

    >>> nl[0] = [5, 5, 5, 5, 5]
    >>> nl[-1] = [2, 2, 2, 2, 2]
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2

    You can add NeighborLists together:

    >>> nl2 = NeighborList([[1,1,1,0,0],[1,1,0,1,0]])
    >>> nl3 = nl + nl2
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2
        3 |      1      1 |   1   0   0
        4 |      1      1 |   0   1   0

    And finallly you can delete vectors from the NeighborList

    >>> del nl3[-1]
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2
        3 |      1      1 |   1   0   0
    >>> del nl3[1:3]
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      1      1 |   1   0   0
    """
    def __init__(self, vectors=None):
        if vectors is None:
            self._vectors = np.empty((0, 5), dtype=int)
        else:
            try:
                vectors = np.asarray(vectors, dtype=int)
                if vectors.ndim != 2 or vectors.shape[1] != 5:
                    raise ValueError
                else:
                    self._vectors = vectors
            except ValueError:
                raise ValueError("vectors must be array-like of shape (n, 5) of type int, "
                                 "[[origin_site1, target_site1, i1, j1, k1], "
                                 "[origin_site2, target_site2, i2, j2, k2]]")

    def __len__(self):
        return len(self._vectors)

    def __getitem__(self, key):
        if isinstance(key, slice):
            return NeighborList(self._vectors[key])
        elif isinstance(key, int):
            return NeighborList(self._vectors[key, None])
        else:
            try:
                key = np.asarray(key, dtype=int)
                if key.ndim == 1:
                    return NeighborList(self._vectors[key])
                else:
                    raise ValueError
            except (TypeError, ValueError):
                raise ValueError("Must index with int, slice or 1D array-like object")

    def __setitem__(self, key, vector):
        try:
            vector = np.asarray(vector, dtype=int)
            self._vectors[key] = vector
        except ValueError:
            raise ValueError("Must be array-like of length 5, [origin_site, target_site, i, j, k]")

    def __delitem__(self, key):
        self._vectors = np.delete(self._vectors, key, axis=0)

    def __add__(self, other):
        return NeighborList(np.append(self.values, other.values, axis=0))

    def __str__(self):
        return ("      |     site      |    vector\nindex | origin target |   i   j   k\n" +
                '\n'.join('{: 5} | {: 6} {: 6} | {: 3} {: 3} {: 3}'.
                          format(i, *v) for i, v in enumerate(self._vectors)))

    def __repr__(self):
        return "NeighborList(" + str(self._vectors).replace('\n', '\n             ') + ")"

    def append(self, vectors):
        """Append one or more vectors to the NeighborList

        >>> nl = NeighborList()
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
        <BLANKLINE>
        >>> nl.append([0,1,1,0,0])
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      1 |   1   0   0
        >>> nl.append([[0,1,0,1,0],[0,1,0,0,1]])
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      1 |   1   0   0
            1 |      0      1 |   0   1   0
            2 |      0      1 |   0   0   1
        """
        try:
            vectors = np.asarray(vectors, dtype=int)
            if vectors.ndim == 1:
                self._vectors = np.append(self._vectors, [vectors], axis=0)
            else:
                self._vectors = np.append(self._vectors, vectors, axis=0)
        except ValueError:
            raise ValueError("Must be a array-like of length 5, [origin_site, target_site, i, j, k]"
                             ", or shape (n, 5), [[origin_site1, target_site1, i1, j1, k1], "
                             "[origin_site2, target_site2, i2, j2, k2]]")

    @property
    def values(self):
        """Returns the neighbor vectors as a :class:`numpy.ndarray`

        This allows you to directly modify or set the vectors but be
        careful to maintain an (n, 5) array.

        :return: array of neighbor vectors
        :rtype: :class:`numpy.ndarray`

        """
        return self._vectors
<commit_msg>Add doc test to NeighborList.values<commit_after>"""
============
neighborlist
============
"""

import numpy as np

class NeighborList:
    """The NeighborList class

    Contains an `n x 5` array of neighbor vectors each being
    `[origin_site, target_site, i, j, k]`.

    >>> nl = NeighborList()
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
    <BLANKLINE>
    >>> nl = NeighborList([[0,1,1,0,0],[0,1,0,1,0],[0,1,0,0,1]])
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
        1 |      0      1 |   0   1   0
        2 |      0      1 |   0   0   1

    You can slice a NeighborList:

    >>> print(nl[0])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
    >>> print(nl[-1])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   0   0   1
    >>> print(nl[1:3])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   0   1   0
        1 |      0      1 |   0   0   1
    >>> print(nl[0,2])
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      0      1 |   1   0   0
        1 |      0      1 |   0   0   1

    You can set a vector:

    >>> nl[0] = [5, 5, 5, 5, 5]
    >>> nl[-1] = [2, 2, 2, 2, 2]
    >>> print(nl)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2

    You can add NeighborLists together:

    >>> nl2 = NeighborList([[1,1,1,0,0],[1,1,0,1,0]])
    >>> nl3 = nl + nl2
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2
        3 |      1      1 |   1   0   0
        4 |      1      1 |   0   1   0

    And finallly you can delete vectors from the NeighborList

    >>> del nl3[-1]
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      0      1 |   0   1   0
        2 |      2      2 |   2   2   2
        3 |      1      1 |   1   0   0
    >>> del nl3[1:3]
    >>> print(nl3)
          |     site      |    vector
    index | origin target |   i   j   k
        0 |      5      5 |   5   5   5
        1 |      1      1 |   1   0   0
    """
    def __init__(self, vectors=None):
        if vectors is None:
            self._vectors = np.empty((0, 5), dtype=int)
        else:
            try:
                vectors = np.asarray(vectors, dtype=int)
                if vectors.ndim != 2 or vectors.shape[1] != 5:
                    raise ValueError
                else:
                    self._vectors = vectors
            except ValueError:
                raise ValueError("vectors must be array-like of shape (n, 5) of type int, "
                                 "[[origin_site1, target_site1, i1, j1, k1], "
                                 "[origin_site2, target_site2, i2, j2, k2]]")

    def __len__(self):
        return len(self._vectors)

    def __getitem__(self, key):
        if isinstance(key, slice):
            return NeighborList(self._vectors[key])
        elif isinstance(key, int):
            return NeighborList(self._vectors[key, None])
        else:
            try:
                key = np.asarray(key, dtype=int)
                if key.ndim == 1:
                    return NeighborList(self._vectors[key])
                else:
                    raise ValueError
            except (TypeError, ValueError):
                raise ValueError("Must index with int, slice or 1D array-like object")

    def __setitem__(self, key, vector):
        try:
            vector = np.asarray(vector, dtype=int)
            self._vectors[key] = vector
        except ValueError:
            raise ValueError("Must be array-like of length 5, [origin_site, target_site, i, j, k]")

    def __delitem__(self, key):
        self._vectors = np.delete(self._vectors, key, axis=0)

    def __add__(self, other):
        return NeighborList(np.append(self.values, other.values, axis=0))

    def __str__(self):
        return ("      |     site      |    vector\nindex | origin target |   i   j   k\n" +
                '\n'.join('{: 5} | {: 6} {: 6} | {: 3} {: 3} {: 3}'.
                          format(i, *v) for i, v in enumerate(self._vectors)))

    def __repr__(self):
        return "NeighborList(" + str(self._vectors).replace('\n', '\n             ') + ")"

    def append(self, vectors):
        """Append one or more vectors to the NeighborList

        >>> nl = NeighborList()
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
        <BLANKLINE>
        >>> nl.append([0,1,1,0,0])
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      1 |   1   0   0
        >>> nl.append([[0,1,0,1,0],[0,1,0,0,1]])
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      1 |   1   0   0
            1 |      0      1 |   0   1   0
            2 |      0      1 |   0   0   1
        """
        try:
            vectors = np.asarray(vectors, dtype=int)
            if vectors.ndim == 1:
                self._vectors = np.append(self._vectors, [vectors], axis=0)
            else:
                self._vectors = np.append(self._vectors, vectors, axis=0)
        except ValueError:
            raise ValueError("Must be a array-like of length 5, [origin_site, target_site, i, j, k]"
                             ", or shape (n, 5), [[origin_site1, target_site1, i1, j1, k1], "
                             "[origin_site2, target_site2, i2, j2, k2]]")

    @property
    def values(self):
        """Returns the neighbor vectors as a :class:`numpy.ndarray`

        This allows you to directly modify or set the vectors but be
        careful to maintain an (n, 5) array.

        >>> nl = NeighborList([[0,1,1,0,0],[0,1,0,1,0],[0,1,0,0,1]])
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      1 |   1   0   0
            1 |      0      1 |   0   1   0
            2 |      0      1 |   0   0   1
        >>> nl.values
        array([[0, 1, 1, 0, 0],
               [0, 1, 0, 1, 0],
               [0, 1, 0, 0, 1]])
        >>> nl.values[:,1] = 2
        >>> print(nl)
              |     site      |    vector
        index | origin target |   i   j   k
            0 |      0      2 |   1   0   0
            1 |      0      2 |   0   1   0
            2 |      0      2 |   0   0   1

        :return: array of neighbor vectors
        :rtype: :class:`numpy.ndarray`

        """
        return self._vectors
<|endoftext|>


<commit_before>import inspect

from jedi._compatibility import builtins
from jedi import debug
from jedi.cache import underscore_memoization
from jedi.evaluate import compiled
from jedi.evaluate.compiled.fake import get_module
from jedi.parser import representation as pr
from jedi.parser.fast import FastParser
from jedi.evaluate import helpers

class InterpreterNamespace(pr.Module):
    def __init__(self, evaluator, namespace, parser_module):
        self.namespace = namespace
        self.parser_module = parser_module
        self._evaluator = evaluator

    def get_defined_names(self):
        for name in self.parser_module.get_defined_names():
            yield name
        for key, value in self.namespace.items():
            yield LazyName(self._evaluator, key, value)

    def __getattr__(self, name):
        return getattr(self.parser_module, name)

class LazyName(helpers.FakeName):
    def __init__(self, evaluator, name, value):
        super(LazyName, self).__init__(name)
        self._evaluator = evaluator
        self._value = value
        self._name = name

    @property
    @underscore_memoization
    def parent(self):
        parser_path = []
        obj = self._value
        if inspect.ismodule(obj):
            module = obj
        else:
            try:
                o = obj.__objclass__
                parser_path.append(pr.NamePart(obj.__name__, None, None))
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                parser_path.insert(0, pr.NamePart(obj.__name__, None, None))
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                module = __import__(module_name)
        raw_module = get_module(self._value)

        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            if path.endswith('.pyc'):
                # cut the `c` from `.pyc`
                with open(path[:-1]) as f:
                    mod = FastParser(f.read(), path[:-1]).module
                if not parser_path:
                    return mod
                found = self._evaluator.eval_call_path(iter(parser_path), mod, None)
                if found:
                    return found[0]
                debug.warning('Interpreter lookup for Python code failed %s',
                              mod)

        module = compiled.CompiledObject(raw_module)
        return compiled.create(self._value, module, module)

    @parent.setter
    def parent(self, value):
        """Needed because of the ``representation.Simple`` super class."""

def create(evaluator, namespace, parser_module):
    ns = InterpreterNamespace(evaluator, namespace, parser_module)
    for attr_name in pr.SCOPE_CONTENTS:
        for something in getattr(parser_module, attr_name):
            something.parent = ns
<commit_msg>fix python 3 issues with the new interpreter implementation<commit_after>import inspect
import re

from jedi._compatibility import builtins
from jedi import debug
from jedi.cache import underscore_memoization
from jedi.evaluate import compiled
from jedi.evaluate.compiled.fake import get_module
from jedi.parser import representation as pr
from jedi.parser.fast import FastParser
from jedi.evaluate import helpers

class InterpreterNamespace(pr.Module):
    def __init__(self, evaluator, namespace, parser_module):
        self.namespace = namespace
        self.parser_module = parser_module
        self._evaluator = evaluator

    def get_defined_names(self):
        for name in self.parser_module.get_defined_names():
            yield name
        for key, value in self.namespace.items():
            yield LazyName(self._evaluator, key, value)

    def __getattr__(self, name):
        return getattr(self.parser_module, name)

class LazyName(helpers.FakeName):
    def __init__(self, evaluator, name, value):
        super(LazyName, self).__init__(name)
        self._evaluator = evaluator
        self._value = value
        self._name = name

    @property
    @underscore_memoization
    def parent(self):
        parser_path = []
        obj = self._value
        if inspect.ismodule(obj):
            module = obj
        else:
            try:
                o = obj.__objclass__
                parser_path.append(pr.NamePart(obj.__name__, None, None))
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                parser_path.insert(0, pr.NamePart(obj.__name__, None, None))
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                module = __import__(module_name)
        raw_module = get_module(self._value)

        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    mod = FastParser(f.read(), path[:-1]).module
                if not parser_path:
                    return mod
                found = self._evaluator.eval_call_path(iter(parser_path), mod, None)
                if found:
                    return found[0]
                debug.warning('Interpreter lookup for Python code failed %s',
                              mod)

        module = compiled.CompiledObject(raw_module)
        return compiled.create(self._value, module, module)

    @parent.setter
    def parent(self, value):
        """Needed because of the ``representation.Simple`` super class."""

def create(evaluator, namespace, parser_module):
    ns = InterpreterNamespace(evaluator, namespace, parser_module)
    for attr_name in pr.SCOPE_CONTENTS:
        for something in getattr(parser_module, attr_name):
            something.parent = ns
<|endoftext|>


<commit_before>import os
import difflib
import pytest
from jinja2 import Template

securedrop_test_vars = pytest.securedrop_test_vars

def determine_app_ip(SystemInfo, Command):
    """
    Dumb logic to determine environment and lookup relevant app IP address
    """
    app_hostname = "app-prod"
    hostname = SystemInfo.hostname
    if "staging" in hostname:
        app_hostname = "app-staging"
    app_ip = Command.check_output("getent hosts "+app_hostname+" | awk '{ print $1 }'")
    return app_ip

# Skipping because the simpler string comparison check for entire output
# check is sufficient for mon-staging. On other hosts, though, we may need
# the more dynamic logic described below.
@pytest.mark.skip
def test_mon_iptables_rules(SystemInfo, Command, Sudo, Ansible):
    app_ip = determine_app_ip(SystemInfo, Command)

    # Build a dict of variables to pass to jinja for iptables comparison
    kwargs = dict(
        app_ip=app_ip,
        default_interface = Ansible("setup")["ansible_facts"]["ansible_default_ipv4"]["interface"],
        tor_user_id = Command.check_output("id -u debian-tor"),
        ssh_group_gid = Command.check_output("getent group ssh | cut -d: -f3"),
        postfix_user_id = Command.check_output("id -u postfix"),
        dns_server = securedrop_test_vars.dns_server)

    # Build iptables scrape cmd, purge comments + counters
    iptables = "iptables-save | sed 's/ \[[0-9]*\:[0-9]*\]//g' | egrep -v '^#'"
    iptables_file = "{}/iptables-{}.j2".format(
                                      os.path.dirname(os.path.abspath(__file__)),
                                      SystemInfo.hostname)

    # template out a local iptables jinja file
    jinja_iptables = Template(open(iptables_file,'r').read())
    iptables_expected = jinja_iptables.render(**kwargs)

    with Sudo():
        # Actually run the iptables scrape command
        iptables = Command.check_output(iptables)
        # print diff comparison (only shows up in pytests if test fails or
        # verbosity turned way up)
        for iptablesdiff in difflib.context_diff(iptables_expected.split('\n'),
                                             iptables.split('\n')):
            print(iptablesdiff)
        # Conduct the string comparison of the expected and actual iptables
        # ruleset
        assert iptables_expected == iptables

@pytest.mark.parametrize('unwanted_rule', [
    "-A INPUT -s {app_ip} -p tcp --sport 1515 -m state --state ESTABLISHED,RELATED -v ACCEPT -m comment --comment \"ossec authd rule only required for initial agent registration\"",
    "-A OUTPUT -d {app_ip} -p tcp --dport 1515 -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT -m comment --comment \"ossec authd rule only required for initial agent registration\"",
    "-A INPUT -s {app_ip} -p tcp --dport 1515 -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT",
    "-A OUTPUT -d {app_ip} -p tcp --sport 1515 -m state --state ESTABLISHED,RELATED -j ACCEPT",
])
def test_ensure_absent_iptables_rules(Command, SystemInfo, Sudo, unwanted_rule):
    """
    Ensure that unwanted iptables rules defined do not exist on the server.
    Mostly these are modified rules, so we're checking that the old versions have
    been permanently removed between upgrades.
    """
    app_ip = determine_app_ip(SystemInfo, Command)
    with Sudo():
        rule = unwanted_rule.format(app_ip=app_ip)
        assert rule not in Command.check_output("iptables-save")

def test_iptables_rules(Command, Sudo):
    """
    Ensure the correct iptables rules are checked. Using a single string
    equivalency check for the entirety of the iptables output, since
    rule order is critical. Testinfra will provide diffed output on failure.
    """
    with Sudo():
        c = Command("iptables -S")
        assert c.stdout == securedrop_test_vars.iptables_complete_ruleset
        assert c.rc == 0

@pytest.mark.parametrize('ossec_service', [
    dict(host="0.0.0.0", proto="tcp", port=22),
    dict(host="127.0.0.1", proto="tcp", port=25),
    dict(host="0.0.0.0", proto="udp", port=1514),
])
def test_listening_ports(Socket, Sudo, ossec_service):
    """
    Ensure the OSSEC-related services are listening on the
    expected sockets. Services to check include ossec, mail, and ssh.
    """
    socket = "{proto}://{host}:{port}".format(**ossec_service)
    with Sudo():
        assert Socket(socket).is_listening
<commit_msg>Removes redundant "unwanted" iptables config test<commit_after>import os
import difflib
import pytest
from jinja2 import Template

securedrop_test_vars = pytest.securedrop_test_vars

def determine_app_ip(SystemInfo, Command):
    """
    Dumb logic to determine environment and lookup relevant app IP address
    """
    app_hostname = "app-prod"
    hostname = SystemInfo.hostname
    if "staging" in hostname:
        app_hostname = "app-staging"
    app_ip = Command.check_output("getent hosts "+app_hostname+" | awk '{ print $1 }'")
    return app_ip

# Skipping because the simpler string comparison check for entire output
# check is sufficient for mon-staging. On other hosts, though, we may need
# the more dynamic logic described below.
@pytest.mark.skip
def test_mon_iptables_rules(SystemInfo, Command, Sudo, Ansible):
    app_ip = determine_app_ip(SystemInfo, Command)

    # Build a dict of variables to pass to jinja for iptables comparison
    kwargs = dict(
        app_ip=app_ip,
        default_interface = Ansible("setup")["ansible_facts"]["ansible_default_ipv4"]["interface"],
        tor_user_id = Command.check_output("id -u debian-tor"),
        ssh_group_gid = Command.check_output("getent group ssh | cut -d: -f3"),
        postfix_user_id = Command.check_output("id -u postfix"),
        dns_server = securedrop_test_vars.dns_server)

    # Build iptables scrape cmd, purge comments + counters
    iptables = "iptables-save | sed 's/ \[[0-9]*\:[0-9]*\]//g' | egrep -v '^#'"
    iptables_file = "{}/iptables-{}.j2".format(
                                      os.path.dirname(os.path.abspath(__file__)),
                                      SystemInfo.hostname)

    # template out a local iptables jinja file
    jinja_iptables = Template(open(iptables_file,'r').read())
    iptables_expected = jinja_iptables.render(**kwargs)

    with Sudo():
        # Actually run the iptables scrape command
        iptables = Command.check_output(iptables)
        # print diff comparison (only shows up in pytests if test fails or
        # verbosity turned way up)
        for iptablesdiff in difflib.context_diff(iptables_expected.split('\n'),
                                             iptables.split('\n')):
            print(iptablesdiff)
        # Conduct the string comparison of the expected and actual iptables
        # ruleset
        assert iptables_expected == iptables

def test_iptables_rules(Command, Sudo):
    """
    Ensure the correct iptables rules are checked. Using a single string
    equivalency check for the entirety of the iptables output, since
    rule order is critical. Testinfra will provide diffed output on failure.
    """
    with Sudo():
        c = Command("iptables -S")
        assert c.stdout == securedrop_test_vars.iptables_complete_ruleset
        assert c.rc == 0

@pytest.mark.parametrize('ossec_service', [
    dict(host="0.0.0.0", proto="tcp", port=22),
    dict(host="127.0.0.1", proto="tcp", port=25),
    dict(host="0.0.0.0", proto="udp", port=1514),
])
def test_listening_ports(Socket, Sudo, ossec_service):
    """
    Ensure the OSSEC-related services are listening on the
    expected sockets. Services to check include ossec, mail, and ssh.
    """
    socket = "{proto}://{host}:{port}".format(**ossec_service)
    with Sudo():
        assert Socket(socket).is_listening
<|endoftext|>


<commit_before># Add lib/ to the import path:
import sys
import os
agent_lib_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')
sys.path.insert(1, agent_lib_dir)

# stdlib
from collections import namedtuple
import urlparse
import json
import time
import sys

# 3p
import requests

# project
from checks import AgentCheck
from util import headers
import re
import storm_utils

StormConfig = namedtuple(
    'StormConfig',
    [
        'metric_prefix',
        'tags',
        'task_tags',
        'timeout',
        'executor_details_whitelist',
        'topology_timeout',
        'url',
        'topologies',
        'cache_file',
        'cache_staleness',
    ]
)

class StormRESTCheck(AgentCheck):
    class ConnectionFailure(StandardError):
        def __init__(self, url, timeout):
            self.url = url
            self.timeout = timeout

    DEFAULT_TIMEOUT = 5 # seconds
    DEFAULT_STALENESS=240 # seconds

    def instance_config(self, instance):
        url = instance.get('url')
        if url is None:
            raise Exception("An url must be specified in the instance")

        cache_file = instance.get('cache_file')
        if cache_file is None:
            raise Exception("You must specify a cache file (querying the storm API synchronously is too slow).")

        timeout = instance.get('timeout', self.DEFAULT_TIMEOUT)
        topology_timeout = instance.get('topology_timeout', timeout)
        topologies_re = re.compile(instance.get('topologies', '(.*)'))

        task_tags_from_json = {}
        tag_file = instance.get('task_tags_file', None)
        if tag_file is not None:
            with open(tag_file, 'r') as f:
                task_tags_from_json = json.load(f)
        task_tags = instance.get('task_tags', {}).copy()
        task_tags.update(task_tags_from_json)

        raw_whitelist = set(instance.get('executor_details_whitelist', []))
        executor_details_whitelist = [re.compile(regex) for regex in raw_whitelist]

        return StormConfig(
            metric_prefix=instance.get('metric_prefix', None),
            url=url,
            timeout=timeout,
            topology_timeout=topology_timeout,
            tags=instance.get('tags', []),
            executor_details_whitelist=executor_details_whitelist,
            task_tags=task_tags,
            cache_file=cache_file,
            cache_staleness=instance.get('cache_staleness', self.DEFAULT_STALENESS),
            topologies=topologies_re,
        )

    def metric(self, config, name):
        if config.metric_prefix is not None:
            return "%s.storm.rest.%s" % (config.metric_prefix, name)
        else:
            return "storm.rest.%s" % name

    def check(self, instance):
        config = self.instance_config(instance)

        check_status = AgentCheck.OK
        check_msg = 'Everything went well'
        check_tags = [
            'url:' + config.url,
        ] + config.tags

        details = None
        with open(config.cache_file, 'r') as cache_f:
            details = json.load(cache_f)

        oldest_acceptable_cache_time = time.time() - config.cache_staleness
        if details['status'] == 'success' and details['updated'] > oldest_acceptable_cache_time:
            data = details['data']
            self.report_cluster(config, data['cluster'])
            self.report_supervisors(config, data['supervisors'])
            self.report_topologies(config, data['topologies'])
            for topology_detail in data['topology_details']:
                self.report_topology(config, topology_detail['name'], topology_detail['topology'])
                for executor_details in topology_detail['component_details']:
                    self.report_executor_details(config, executor_details)
        else:
            check_status = AgentCheck.CRITICAL
            if details['status'] != 'success':
                check_msg = "Could not connect to URL %s with timeout %d" % (details['error_url'], details['error_timeout'])
            else:
                check_msg = "Cache is too stale: %d vs. expected minimum %d" % (details['updated'], oldest_acceptable_cache_time)

        self.service_check(
            self.metric(config, 'cached_data_ok'),
            check_status,
            message=check_msg,
            tags=check_tags)

    def report_cluster(self, config, cluster):
        self.gauge(self.metric(config, 'cluster.nimbus_uptime_seconds'),
                   storm_utils.translate_timespec(cluster['nimbusUptime']),
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_used_count'),
                   cluster['slotsUsed'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.supervisor_count'),
                   cluster['supervisors'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_total_count'),
                   cluster['slotsTotal'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_free_count'),
                   cluster['slotsFree'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.executors_total_count'),
                   cluster['executorsTotal'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.tasks_total_count'),
                   cluster['tasksTotal'],
                   tags=config.tags)

    def report_supervisors(self, config, resp):
        self.gauge(self.metric(config, 'supervisors_total'),
                   len(resp.get('supervisors', [])),
                   tags=config.tags)
        for supe in resp.get('supervisors', []):
            supe_tags = [
                'storm_host:' + supe.get('host'),
            ] + config.tags
            self.gauge(self.metric(config, 'supervisor.slots_total'),
                       supe.get('slotsTotal'), tags=supe_tags)
            self.gauge(self.metric(config, 'supervisor.slots_used_total'),
                       supe.get('slotsUsed'), tags=supe_tags)
            self.gauge(self.metric(config, 'supervisor.uptime_seconds'),
                       storm_utils.translate_timespec(supe.get('uptime')), tags=supe_tags)

    def _topology_name(self, config, topology):
        """Returns the name if a config's topology regex matches, None otherwise."""
        match = config.topologies.match(topology.get('name'))
        if match:
            return match.group(1)
        else:
            return None

    def report_topologies(self, config, topologies):
        """Report metadata about topologies that match the topologies regex.
        """
        for pretty_name, topo in topologies.iteritems():
            uptime = topo.get('uptime', '0s')
            tags = config.tags + [
                'storm_topology:' + pretty_name,
            ]
            self.gauge(self.metric(config, 'topologies.uptime_seconds'),
                       storm_utils.translate_timespec(uptime), tags=tags)
            self.gauge(self.metric(config, 'topologies.tasks_total'),
                       topo['tasksTotal'], tags=tags)
            self.gauge(self.metric(config, 'topologies.workers_total'),
                       topo['workersTotal'], tags=tags)
            self.gauge(self.metric(config, 'topologies.executors_total'), topo['executorsTotal'], tags=tags)

    def task_id_tags(self, config, component_type, task_id):
        return config.task_tags.get(component_type, {}).get(task_id, [])

    def report_topology(self, config, name, details):
        """
        Report statistics for a single topology's spouts and bolts.
        """
        tags = config.tags + [
            'storm_topology:' + name,
        ]
        def report_task(component_type, name, task, tags):
            self.gauge(self.metric(config, ('%s.executors_total' % component_type)),
                       task['executors'], task_tags)
            self.gauge(self.metric(config, ('%s.tasks_total' % component_type)),
                       task['tasks'], task_tags)

            self.monotonic_count(self.metric(config, ('%s.emitted_total' % component_type)),
                       task['emitted'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.transferred_total' % component_type)),
                       task['transferred'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.acked_total' % component_type)),
                       task['acked'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.failed_total' % component_type)),
                       task['failed'], task_tags)

        ## Report spouts
        for spout in details.get('spouts'):
            name = spout['spoutId']
            task_tags = tags + self.task_id_tags(config, 'spout', name) + [
                'storm_component_type:spout',
                'storm_task_id:' + name,
            ]
            report_task('spout', name, spout, task_tags)
            self.gauge(self.metric(config, 'spout.complete_latency_us'),
                       float(spout['completeLatency']), task_tags)

        for bolt in details.get('bolts'):
            name = bolt['boltId']
            task_tags = tags + self.task_id_tags(config, 'bolt', name) + [
                'storm_component_type:bolt',
                'storm_task_id:' + name,
            ]
            report_task('bolt', name, bolt, task_tags)
            if bolt['executed'] is not None:
                executed_count = bolt['executed']
            else:
                executed_count = 0
            self.monotonic_count(self.metric(config, 'bolt.executed_total'),
                       executed_count, task_tags)
            self.gauge(self.metric(config, 'bolt.execute_latency_us'),
                       float(bolt['executeLatency']), task_tags)
            self.gauge(self.metric(config, 'bolt.process_latency_us'),
                       float(bolt['processLatency']), task_tags)
            self.gauge(self.metric(config, 'bolt.capacity_percent'),
                       float(bolt['capacity']) * 100, task_tags)

    def report_executor_details(self, config, details):
        """
        Report statistics for a single topology's task ID's executors.
        """

        topology_name = self._topology_name(config, details)
        name = details['id']
        task_type = details['componentType']
        tags = config.tags + self.task_id_tags(config, task_type, name) + [
            'storm_topology:' + topology_name,
            'storm_component_type:' + task_type,
            'storm_task_id:' + details['id'],
        ]
        self.gauge(self.metric(config, 'executor.executors_total'),
                               details['executors'], tags=tags)
        self.gauge(self.metric(config, 'executor.tasks_total'),
                               details['executors'], tags=tags)

        # Embarrassingly, executorStats are undocumented in the REST
        # API docs (so we might not be allowed to rely on them). But
        # they're the only way to get some SERIOUSLY useful metrics -
        # per-host metrics, in particular.
        for executor in details['executorStats']:
            executor_tags = tags + [
                'executor_id:' + executor['id'],
                'storm_host:' + executor['host'],
                'storm_port:' + str(executor['port']),
            ]
            self.monotonic_count(self.metric(config, 'executor.emitted_total'),
                       executor.get('emitted', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.transferred_total'),
                       executor.get('transferred', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.acked_total'),
                       executor.get('acked', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.executed_total'),
                       executor.get('executed', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.failed_total'),
                       executor.get('failed', 0), tags=executor_tags)

            self.gauge(self.metric(config, 'executor.execute_latency_us'),
                       float(executor.get('executeLatency', 0)), tags=executor_tags)
            self.gauge(self.metric(config, 'executor.process_latency_us'),
                       float(executor.get('processLatency', 0)), tags=executor_tags)

            self.gauge(self.metric(config, 'executor.capacity_percent'),
                       float(executor.get('capacity', 0)) * 100, tags=executor_tags)
            self.gauge(self.metric(config, 'executor.uptime_seconds'),
                       storm_utils.translate_timespec(executor.get('uptime', '0s')), tags=executor_tags)
<commit_msg>added a lever to only use part of a task_id<commit_after># Add lib/ to the import path:
import sys
import os
agent_lib_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')
sys.path.insert(1, agent_lib_dir)

# stdlib
from collections import namedtuple
import urlparse
import json
import time
import sys

# 3p
import requests

# project
from checks import AgentCheck
from util import headers
import re
import storm_utils

StormConfig = namedtuple(
    'StormConfig',
    [
        'metric_prefix',
        'tags',
        'task_tags',
        'timeout',
        'executor_details_whitelist',
        'topology_timeout',
        'url',
        'topologies',
        'cache_file',
        'cache_staleness',
        'task_id_cleaner_regex',
    ]
)

class StormRESTCheck(AgentCheck):
    class ConnectionFailure(StandardError):
        def __init__(self, url, timeout):
            self.url = url
            self.timeout = timeout

    DEFAULT_TIMEOUT = 5 # seconds
    DEFAULT_STALENESS=240 # seconds

    def instance_config(self, instance):
        url = instance.get('url')
        if url is None:
            raise Exception("An url must be specified in the instance")

        cache_file = instance.get('cache_file')
        if cache_file is None:
            raise Exception("You must specify a cache file (querying the storm API synchronously is too slow).")

        timeout = instance.get('timeout', self.DEFAULT_TIMEOUT)
        topology_timeout = instance.get('topology_timeout', timeout)
        topologies_re = re.compile(instance.get('topologies', '(.*)'))

        task_tags_from_json = {}
        tag_file = instance.get('task_tags_file', None)
        if tag_file is not None:
            with open(tag_file, 'r') as f:
                task_tags_from_json = json.load(f)
        task_tags = instance.get('task_tags', {}).copy()
        task_tags.update(task_tags_from_json)

        raw_whitelist = set(instance.get('executor_details_whitelist', []))
        executor_details_whitelist = [re.compile(regex) for regex in raw_whitelist]

        return StormConfig(
            metric_prefix=instance.get('metric_prefix', None),
            url=url,
            timeout=timeout,
            topology_timeout=topology_timeout,
            tags=instance.get('tags', []),
            executor_details_whitelist=executor_details_whitelist,
            task_tags=task_tags,
            cache_file=cache_file,
            cache_staleness=instance.get('cache_staleness', self.DEFAULT_STALENESS),
            topologies=topologies_re,
            task_id_cleaner_regex=instance.get('task_id_cleaner_regex', None),
        )

    def metric(self, config, name):
        if config.metric_prefix is not None:
            return "%s.storm.rest.%s" % (config.metric_prefix, name)
        else:
            return "storm.rest.%s" % name

    def check(self, instance):
        config = self.instance_config(instance)

        check_status = AgentCheck.OK
        check_msg = 'Everything went well'
        check_tags = [
            'url:' + config.url,
        ] + config.tags

        details = None
        with open(config.cache_file, 'r') as cache_f:
            details = json.load(cache_f)

        oldest_acceptable_cache_time = time.time() - config.cache_staleness
        if details['status'] == 'success' and details['updated'] > oldest_acceptable_cache_time:
            data = details['data']
            self.report_cluster(config, data['cluster'])
            self.report_supervisors(config, data['supervisors'])
            self.report_topologies(config, data['topologies'])
            for topology_detail in data['topology_details']:
                self.report_topology(config, topology_detail['name'], topology_detail['topology'])
                for executor_details in topology_detail['component_details']:
                    self.report_executor_details(config, executor_details)
        else:
            check_status = AgentCheck.CRITICAL
            if details['status'] != 'success':
                check_msg = "Could not connect to URL %s with timeout %d" % (details['error_url'], details['error_timeout'])
            else:
                check_msg = "Cache is too stale: %d vs. expected minimum %d" % (details['updated'], oldest_acceptable_cache_time)

        self.service_check(
            self.metric(config, 'cached_data_ok'),
            check_status,
            message=check_msg,
            tags=check_tags)

    def report_cluster(self, config, cluster):
        self.gauge(self.metric(config, 'cluster.nimbus_uptime_seconds'),
                   storm_utils.translate_timespec(cluster['nimbusUptime']),
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_used_count'),
                   cluster['slotsUsed'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.supervisor_count'),
                   cluster['supervisors'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_total_count'),
                   cluster['slotsTotal'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.slots_free_count'),
                   cluster['slotsFree'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.executors_total_count'),
                   cluster['executorsTotal'],
                   tags=config.tags)
        self.gauge(self.metric(config, 'cluster.tasks_total_count'),
                   cluster['tasksTotal'],
                   tags=config.tags)

    def report_supervisors(self, config, resp):
        self.gauge(self.metric(config, 'supervisors_total'),
                   len(resp.get('supervisors', [])),
                   tags=config.tags)
        for supe in resp.get('supervisors', []):
            supe_tags = [
                'storm_host:' + supe.get('host'),
            ] + config.tags
            self.gauge(self.metric(config, 'supervisor.slots_total'),
                       supe.get('slotsTotal'), tags=supe_tags)
            self.gauge(self.metric(config, 'supervisor.slots_used_total'),
                       supe.get('slotsUsed'), tags=supe_tags)
            self.gauge(self.metric(config, 'supervisor.uptime_seconds'),
                       storm_utils.translate_timespec(supe.get('uptime')), tags=supe_tags)

    def _topology_name(self, config, topology):
        """Returns the name if a config's topology regex matches, None otherwise."""
        match = config.topologies.match(topology.get('name'))
        if match:
            return match.group(1)
        else:
            return None

    def report_topologies(self, config, topologies):
        """Report metadata about topologies that match the topologies regex.
        """
        for pretty_name, topo in topologies.iteritems():
            uptime = topo.get('uptime', '0s')
            tags = config.tags + [
                'storm_topology:' + pretty_name,
            ]
            self.gauge(self.metric(config, 'topologies.uptime_seconds'),
                       storm_utils.translate_timespec(uptime), tags=tags)
            self.gauge(self.metric(config, 'topologies.tasks_total'),
                       topo['tasksTotal'], tags=tags)
            self.gauge(self.metric(config, 'topologies.workers_total'),
                       topo['workersTotal'], tags=tags)
            self.gauge(self.metric(config, 'topologies.executors_total'), topo['executorsTotal'], tags=tags)

    def task_id_tags(self, config, component_type, task_id):
        cleaner_id = task_id
        if config.task_id_cleaner_regex is not None:
            match = re.search(config.task_id_cleaner_regex, cleaner_id)
            if match is not None:
                cleaner_id = match.group(1)
        return config.task_tags.get(component_type, {}).get(cleaner_id, [])

    def report_topology(self, config, name, details):
        """
        Report statistics for a single topology's spouts and bolts.
        """
        tags = config.tags + [
            'storm_topology:' + name,
        ]
        def report_task(component_type, name, task, tags):
            self.gauge(self.metric(config, ('%s.executors_total' % component_type)),
                       task['executors'], task_tags)
            self.gauge(self.metric(config, ('%s.tasks_total' % component_type)),
                       task['tasks'], task_tags)

            self.monotonic_count(self.metric(config, ('%s.emitted_total' % component_type)),
                       task['emitted'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.transferred_total' % component_type)),
                       task['transferred'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.acked_total' % component_type)),
                       task['acked'], task_tags)
            self.monotonic_count(self.metric(config, ('%s.failed_total' % component_type)),
                       task['failed'], task_tags)

        ## Report spouts
        for spout in details.get('spouts'):
            name = spout['spoutId']
            task_tags = tags + self.task_id_tags(config, 'spout', name) + [
                'storm_component_type:spout',
                'storm_task_id:' + name,
            ]
            report_task('spout', name, spout, task_tags)
            self.gauge(self.metric(config, 'spout.complete_latency_us'),
                       float(spout['completeLatency']), task_tags)

        for bolt in details.get('bolts'):
            name = bolt['boltId']
            task_tags = tags + self.task_id_tags(config, 'bolt', name) + [
                'storm_component_type:bolt',
                'storm_task_id:' + name,
            ]
            report_task('bolt', name, bolt, task_tags)
            if bolt['executed'] is not None:
                executed_count = bolt['executed']
            else:
                executed_count = 0
            self.monotonic_count(self.metric(config, 'bolt.executed_total'),
                       executed_count, task_tags)
            self.gauge(self.metric(config, 'bolt.execute_latency_us'),
                       float(bolt['executeLatency']), task_tags)
            self.gauge(self.metric(config, 'bolt.process_latency_us'),
                       float(bolt['processLatency']), task_tags)
            self.gauge(self.metric(config, 'bolt.capacity_percent'),
                       float(bolt['capacity']) * 100, task_tags)

    def report_executor_details(self, config, details):
        """
        Report statistics for a single topology's task ID's executors.
        """

        topology_name = self._topology_name(config, details)
        name = details['id']
        task_type = details['componentType']
        tags = config.tags + self.task_id_tags(config, task_type, name) + [
            'storm_topology:' + topology_name,
            'storm_component_type:' + task_type,
            'storm_task_id:' + details['id'],
        ]
        self.gauge(self.metric(config, 'executor.executors_total'),
                               details['executors'], tags=tags)
        self.gauge(self.metric(config, 'executor.tasks_total'),
                               details['executors'], tags=tags)

        # Embarrassingly, executorStats are undocumented in the REST
        # API docs (so we might not be allowed to rely on them). But
        # they're the only way to get some SERIOUSLY useful metrics -
        # per-host metrics, in particular.
        for executor in details['executorStats']:
            executor_tags = tags + [
                'executor_id:' + executor['id'],
                'storm_host:' + executor['host'],
                'storm_port:' + str(executor['port']),
            ]
            self.monotonic_count(self.metric(config, 'executor.emitted_total'),
                       executor.get('emitted', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.transferred_total'),
                       executor.get('transferred', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.acked_total'),
                       executor.get('acked', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.executed_total'),
                       executor.get('executed', 0), tags=executor_tags)
            self.monotonic_count(self.metric(config, 'executor.failed_total'),
                       executor.get('failed', 0), tags=executor_tags)

            self.gauge(self.metric(config, 'executor.execute_latency_us'),
                       float(executor.get('executeLatency', 0)), tags=executor_tags)
            self.gauge(self.metric(config, 'executor.process_latency_us'),
                       float(executor.get('processLatency', 0)), tags=executor_tags)

            self.gauge(self.metric(config, 'executor.capacity_percent'),
                       float(executor.get('capacity', 0)) * 100, tags=executor_tags)
            self.gauge(self.metric(config, 'executor.uptime_seconds'),
                       storm_utils.translate_timespec(executor.get('uptime', '0s')), tags=executor_tags)
<|endoftext|>


<commit_before># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'conditions': [
    ['sysroot!=""', {
      'variables': {
        'pkg-config': './pkg-config-wrapper "<(sysroot)"',
      },
    }, {
      'variables': {
        'pkg-config': 'pkg-config'
      },
    }],
    [ 'os_posix==1 and OS!="mac"', {
      'variables': {
        # We use our own copy of libssl3, although we still need to link against
        # the rest of NSS.
        'use_system_ssl%': 0,
      },
    }, {
      'variables': {
        'use_system_ssl%': 1,
      },
    }],
  ],

  'targets': [
    {
      'target_name': 'gtk',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gtk+-2.0 gthread-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gtk+-2.0 gthread-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gtk+-2.0 gthread-2.0)',
            ],
          },
      }],
      [ 'chromeos==1', {
        'link_settings': {
          'libraries': [ '-lXtst' ]
        }
      }]]
    },
    {
      'target_name': 'gtkprint',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gtk+-unix-print-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gtk+-unix-print-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gtk+-unix-print-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'ssl',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'conditions': [
            ['use_openssl==1', {
              'dependencies': [
                '../../third_party/openssl/openssl.gyp:openssl',
              ],
            }],
            ['use_openssl==0 and use_system_ssl==0', {
              'dependencies': [
                '../../net/third_party/nss/ssl.gyp:ssl',
                '../../third_party/zlib/zlib.gyp:zlib',
              ],
              'direct_dependent_settings': {
                'include_dirs+': [
                  # We need for our local copies of the libssl3 headers to come
                  # before other includes, as we are shadowing system headers.
                  '<(DEPTH)/net/third_party/nss/ssl',
                ],
                'cflags': [
                  '<!@(<(pkg-config) --cflags nss)',
                ],
              },
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other nss)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l nss | sed -e "s/-lssl3//")',
                ],
              },
            }],
            ['use_openssl==0 and use_system_ssl==1', {
              'direct_dependent_settings': {
                'cflags': [
                  '<!@(<(pkg-config) --cflags nss)',
                ],
                'defines': [
                  'USE_SYSTEM_SSL',
                ],
              },
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other nss)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l nss)',
                ],
              },
            }],
          ]
        }],
      ],
    },
    {
      'target_name': 'freetype2',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags freetype2)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other freetype2)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l freetype2)',
            ],
          },
      }]]
    },
    {
      'target_name': 'fontconfig',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags fontconfig)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other fontconfig)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l fontconfig)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gdk',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gdk-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gdk-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gdk-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gconf',
      'type': 'settings',
      'conditions': [
        ['use_gconf==1 and _toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gconf-2.0)',
            ],
            'defines': [
              'USE_GCONF',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gconf-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gconf-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gio',
      'type': 'settings',
      'conditions': [
        ['use_gio==1 and _toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gio-2.0)',
            ],
            'defines': [
              'USE_GIO',
            ],
            'conditions': [
              ['linux_link_gsettings==0', {
                'defines': ['DLOPEN_GSETTINGS'],
              }],
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gio-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gio-2.0)',
            ],
            'conditions': [
              ['linux_link_gsettings==0', {
                'libraries': [
                  '-ldl',
                ],
              }],
            ],
          },
      }]]
    },
    {
      'target_name': 'x11',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags x11)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other x11 xi)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l x11 xi)',
            ],
          },
      }],
      ],
    },
    {
      'target_name': 'xext',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags xext)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other xext)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l xext)',
            ],
          },
      }]]
    },
    {
      'target_name': 'libgcrypt',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target" and use_cups==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(libgcrypt-config --cflags)',
            ],
          },
          'link_settings': {
            'libraries': [
              '<!@(libgcrypt-config --libs)',
            ],
          },
      }]]
    },
    {
      'target_name': 'selinux',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'link_settings': {
            'libraries': [
              '-lselinux',
            ],
          },
      }]]
    },
    {
      'target_name': 'gnome_keyring',
      'type': 'settings',
      'conditions': [
        ['use_gnome_keyring==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gnome-keyring-1)',
            ],
            'defines': [
              'USE_GNOME_KEYRING',
            ],
            'conditions': [
              ['linux_link_gnome_keyring==0', {
                'defines': ['DLOPEN_GNOME_KEYRING'],
              }],
            ],
          },
          'conditions': [
            ['linux_link_gnome_keyring!=0', {
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other gnome-keyring-1)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l gnome-keyring-1)',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  '-ldl',
                ],
              },
            }],
          ],
        }],
      ],
    },
    {
      # The unit tests use a few convenience functions from the GNOME
      # Keyring library directly. We ignore linux_link_gnome_keyring and
      # link directly in this version of the target to allow this.
      # *** Do not use this target in the main binary! ***
      'target_name': 'gnome_keyring_direct',
      'type': 'settings',
      'conditions': [
        ['use_gnome_keyring==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gnome-keyring-1)',
            ],
            'defines': [
              'USE_GNOME_KEYRING',
            ],
            'conditions': [
              ['linux_link_gnome_keyring==0', {
                'defines': ['DLOPEN_GNOME_KEYRING'],
              }],
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gnome-keyring-1)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gnome-keyring-1)',
            ],
          },
        }],
      ],
    },
    {
      'target_name': 'dbus',
      'type': 'settings',
      'direct_dependent_settings': {
        'cflags': [
          '<!@(<(pkg-config) --cflags dbus-1)',
        ],
      },
      'link_settings': {
        'ldflags': [
          '<!@(<(pkg-config) --libs-only-L --libs-only-other dbus-1)',
        ],
        'libraries': [
          '<!@(<(pkg-config) --libs-only-l dbus-1)',
        ],
      },
    },
    {
      # TODO(satorux): Remove this once dbus-glib clients are gone.
      'target_name': 'dbus-glib',
      'type': 'settings',
      'direct_dependent_settings': {
        'cflags': [
          '<!@(<(pkg-config) --cflags dbus-glib-1)',
        ],
      },
      'link_settings': {
        'ldflags': [
          '<!@(<(pkg-config) --libs-only-L --libs-only-other dbus-glib-1)',
        ],
        'libraries': [
          '<!@(<(pkg-config) --libs-only-l dbus-glib-1)',
        ],
      },
    },
    {
      'target_name': 'libresolv',
      'type': 'settings',
      'link_settings': {
        'libraries': [
          '-lresolv',
        ],
      },
    },
    {
      'target_name': 'ibus',
      'type': 'settings',
      'conditions': [
        ['use_ibus==1', {
          'variables': {
            'ibus_min_version': '1.3.99.20110425',
          },
          'direct_dependent_settings': {
            'defines': ['HAVE_IBUS=1'],
            'cflags': [
              '<!@(<(pkg-config) --cflags "ibus-1.0 >= <(ibus_min_version)")',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other "ibus-1.0 >= <(ibus_min_version)")',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l "ibus-1.0 >= <(ibus_min_version)")',
            ],
          },
        }],
      ],
    },
    {
      'target_name': 'wayland',
      'type': 'settings',
      'conditions': [
        ['use_wayland == 1', {
          'cflags': [
            '<!@(<(pkg-config) --cflags cairo wayland-client wayland-egl xkbcommon)',
          ],
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags cairo wayland-client wayland-egl xkbcommon)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other wayland-client wayland-egl xkbcommon)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l wayland-client wayland-egl xkbcommon)',
            ],
          },
        }],
      ],
    },
  ],
}
<commit_msg>Add glib to system.gyp for cros static library (load.cc)<commit_after># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'conditions': [
    ['sysroot!=""', {
      'variables': {
        'pkg-config': './pkg-config-wrapper "<(sysroot)"',
      },
    }, {
      'variables': {
        'pkg-config': 'pkg-config'
      },
    }],
    [ 'os_posix==1 and OS!="mac"', {
      'variables': {
        # We use our own copy of libssl3, although we still need to link against
        # the rest of NSS.
        'use_system_ssl%': 0,
      },
    }, {
      'variables': {
        'use_system_ssl%': 1,
      },
    }],
  ],

  'targets': [
    {
      'target_name': 'gtk',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gtk+-2.0 gthread-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gtk+-2.0 gthread-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gtk+-2.0 gthread-2.0)',
            ],
          },
      }],
      [ 'chromeos==1', {
        'link_settings': {
          'libraries': [ '-lXtst' ]
        }
      }]]
    },
    {
      'target_name': 'gtkprint',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gtk+-unix-print-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gtk+-unix-print-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gtk+-unix-print-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'ssl',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'conditions': [
            ['use_openssl==1', {
              'dependencies': [
                '../../third_party/openssl/openssl.gyp:openssl',
              ],
            }],
            ['use_openssl==0 and use_system_ssl==0', {
              'dependencies': [
                '../../net/third_party/nss/ssl.gyp:ssl',
                '../../third_party/zlib/zlib.gyp:zlib',
              ],
              'direct_dependent_settings': {
                'include_dirs+': [
                  # We need for our local copies of the libssl3 headers to come
                  # before other includes, as we are shadowing system headers.
                  '<(DEPTH)/net/third_party/nss/ssl',
                ],
                'cflags': [
                  '<!@(<(pkg-config) --cflags nss)',
                ],
              },
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other nss)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l nss | sed -e "s/-lssl3//")',
                ],
              },
            }],
            ['use_openssl==0 and use_system_ssl==1', {
              'direct_dependent_settings': {
                'cflags': [
                  '<!@(<(pkg-config) --cflags nss)',
                ],
                'defines': [
                  'USE_SYSTEM_SSL',
                ],
              },
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other nss)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l nss)',
                ],
              },
            }],
          ]
        }],
      ],
    },
    {
      'target_name': 'freetype2',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags freetype2)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other freetype2)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l freetype2)',
            ],
          },
      }]]
    },
    {
      'target_name': 'fontconfig',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags fontconfig)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other fontconfig)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l fontconfig)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gdk',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gdk-2.0)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gdk-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gdk-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gconf',
      'type': 'settings',
      'conditions': [
        ['use_gconf==1 and _toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gconf-2.0)',
            ],
            'defines': [
              'USE_GCONF',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gconf-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gconf-2.0)',
            ],
          },
      }]]
    },
    {
      'target_name': 'gio',
      'type': 'settings',
      'conditions': [
        ['use_gio==1 and _toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gio-2.0)',
            ],
            'defines': [
              'USE_GIO',
            ],
            'conditions': [
              ['linux_link_gsettings==0', {
                'defines': ['DLOPEN_GSETTINGS'],
              }],
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gio-2.0)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gio-2.0)',
            ],
            'conditions': [
              ['linux_link_gsettings==0', {
                'libraries': [
                  '-ldl',
                ],
              }],
            ],
          },
      }]]
    },
    {
      'target_name': 'x11',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags x11)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other x11 xi)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l x11 xi)',
            ],
          },
      }],
      ],
    },
    {
      'target_name': 'xext',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags xext)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other xext)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l xext)',
            ],
          },
      }]]
    },
    {
      'target_name': 'libgcrypt',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target" and use_cups==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(libgcrypt-config --cflags)',
            ],
          },
          'link_settings': {
            'libraries': [
              '<!@(libgcrypt-config --libs)',
            ],
          },
      }]]
    },
    {
      'target_name': 'selinux',
      'type': 'settings',
      'conditions': [
        ['_toolset=="target"', {
          'link_settings': {
            'libraries': [
              '-lselinux',
            ],
          },
      }]]
    },
    {
      'target_name': 'gnome_keyring',
      'type': 'settings',
      'conditions': [
        ['use_gnome_keyring==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gnome-keyring-1)',
            ],
            'defines': [
              'USE_GNOME_KEYRING',
            ],
            'conditions': [
              ['linux_link_gnome_keyring==0', {
                'defines': ['DLOPEN_GNOME_KEYRING'],
              }],
            ],
          },
          'conditions': [
            ['linux_link_gnome_keyring!=0', {
              'link_settings': {
                'ldflags': [
                  '<!@(<(pkg-config) --libs-only-L --libs-only-other gnome-keyring-1)',
                ],
                'libraries': [
                  '<!@(<(pkg-config) --libs-only-l gnome-keyring-1)',
                ],
              },
            }, {
              'link_settings': {
                'libraries': [
                  '-ldl',
                ],
              },
            }],
          ],
        }],
      ],
    },
    {
      # The unit tests use a few convenience functions from the GNOME
      # Keyring library directly. We ignore linux_link_gnome_keyring and
      # link directly in this version of the target to allow this.
      # *** Do not use this target in the main binary! ***
      'target_name': 'gnome_keyring_direct',
      'type': 'settings',
      'conditions': [
        ['use_gnome_keyring==1', {
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags gnome-keyring-1)',
            ],
            'defines': [
              'USE_GNOME_KEYRING',
            ],
            'conditions': [
              ['linux_link_gnome_keyring==0', {
                'defines': ['DLOPEN_GNOME_KEYRING'],
              }],
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other gnome-keyring-1)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l gnome-keyring-1)',
            ],
          },
        }],
      ],
    },
    {
      'target_name': 'dbus',
      'type': 'settings',
      'direct_dependent_settings': {
        'cflags': [
          '<!@(<(pkg-config) --cflags dbus-1)',
        ],
      },
      'link_settings': {
        'ldflags': [
          '<!@(<(pkg-config) --libs-only-L --libs-only-other dbus-1)',
        ],
        'libraries': [
          '<!@(<(pkg-config) --libs-only-l dbus-1)',
        ],
      },
    },
    {
      # TODO(satorux): Remove this once dbus-glib clients are gone.
      'target_name': 'dbus-glib',
      'type': 'settings',
      'direct_dependent_settings': {
        'cflags': [
          '<!@(<(pkg-config) --cflags dbus-glib-1)',
        ],
      },
      'link_settings': {
        'ldflags': [
          '<!@(<(pkg-config) --libs-only-L --libs-only-other dbus-glib-1)',
        ],
        'libraries': [
          '<!@(<(pkg-config) --libs-only-l dbus-glib-1)',
        ],
      },
    },
    {
      'target_name': 'glib',
      'type': 'settings',
      'direct_dependent_settings': {
        'cflags': [
          '<!@(<(pkg-config) --cflags glib-2.0)',
        ],
      },
      'link_settings': {
        'ldflags': [
          '<!@(<(pkg-config) --libs-only-L --libs-only-other glib-2.0)',
        ],
        'libraries': [
          '<!@(<(pkg-config) --libs-only-l glib-2.0)',
        ],
      },
    },
    {
      'target_name': 'libresolv',
      'type': 'settings',
      'link_settings': {
        'libraries': [
          '-lresolv',
        ],
      },
    },
    {
      'target_name': 'ibus',
      'type': 'settings',
      'conditions': [
        ['use_ibus==1', {
          'variables': {
            'ibus_min_version': '1.3.99.20110425',
          },
          'direct_dependent_settings': {
            'defines': ['HAVE_IBUS=1'],
            'cflags': [
              '<!@(<(pkg-config) --cflags "ibus-1.0 >= <(ibus_min_version)")',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other "ibus-1.0 >= <(ibus_min_version)")',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l "ibus-1.0 >= <(ibus_min_version)")',
            ],
          },
        }],
      ],
    },
    {
      'target_name': 'wayland',
      'type': 'settings',
      'conditions': [
        ['use_wayland == 1', {
          'cflags': [
            '<!@(<(pkg-config) --cflags cairo wayland-client wayland-egl xkbcommon)',
          ],
          'direct_dependent_settings': {
            'cflags': [
              '<!@(<(pkg-config) --cflags cairo wayland-client wayland-egl xkbcommon)',
            ],
          },
          'link_settings': {
            'ldflags': [
              '<!@(<(pkg-config) --libs-only-L --libs-only-other wayland-client wayland-egl xkbcommon)',
            ],
            'libraries': [
              '<!@(<(pkg-config) --libs-only-l wayland-client wayland-egl xkbcommon)',
            ],
          },
        }],
      ],
    },
  ],
}
<|endoftext|>


<commit_before>"""
Tests the FeatureExporter class
"""

import os
import sys
import unittest

import pandas as pd

# Workaround to import featureEngineering module
PACKAGE_PARENT = '..'
SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from featureEngineering.FeatureExporter import FeatureExporter

class TestExporter(unittest.TestCase):
    @unittest.skip("No data downloaded")
    def test_output_to_file(self):
        """
        Tests if our FeatureExporter correctly outputs to file

        Returns: test case

        """
        train_df = pd.read_csv("data/training_set_test.txt", sep=" ", header=None)
        train_df.columns = ["source", "target", "label"]

        node_information_df = pd.read_csv("data/node_information_test.csv", sep=",", header=None)
        node_information_df.columns = ["ID", "year", "title", "authors", "journalName", "abstract"]
        node_information_df = node_information_df.reset_index().set_index("ID")
        node_information_df["authors"].fillna("", inplace=True)
        node_information_df["journalName"].fillna("", inplace=True)

        features = ["original"]
        feature = features[0]
        exporter = FeatureExporter()

        exporter.computeFeature(train_df, node_information_df, feature)
        exporter.exportTo('training_set_test.txt', feature)

        self.assertTrue(os.path.isfile("featureEngineering/originalFeatures/output/training_set_test.txt"))

    def test_absolute_truth_and_meaning(self):
        """
        Testing the tests

        Returns: test case

        """
        self.assertTrue(True)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Replaced data with ../data/<commit_after>"""
Tests the FeatureExporter class
"""

import os
import sys
import unittest

import pandas as pd

# Workaround to import featureEngineering module
PACKAGE_PARENT = '..'
SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from featureEngineering.FeatureExporter import FeatureExporter

class TestExporter(unittest.TestCase):
    @unittest.skip("Exporter.exportTo not working")
    def test_output_to_file(self):
        """
        Tests if our FeatureExporter correctly outputs to file

        Returns: test case

        """
        train_df = pd.read_csv("../data/training_set_test.txt", sep=" ", header=None)
        train_df.columns = ["source", "target", "label"]

        node_information_df = pd.read_csv("../data/node_information_test.csv", sep=",", header=None)
        node_information_df.columns = ["ID", "year", "title", "authors", "journalName", "abstract"]
        node_information_df = node_information_df.reset_index().set_index("ID")
        node_information_df["authors"].fillna("", inplace=True)
        node_information_df["journalName"].fillna("", inplace=True)

        features = ["original"]
        feature = features[0]
        exporter = FeatureExporter()

        exporter.computeFeature(train_df, node_information_df, feature)
        exporter.exportTo('training_set_test.txt', feature)

        self.assertTrue(os.path.isfile("featureEngineering/originalFeatures/output/training_set_test.txt"))

    def test_absolute_truth_and_meaning(self):
        """
        Testing the tests

        Returns: test case

        """
        self.assertTrue(True)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>