<commit_before>import re
from bs4 import BeautifulSoup
import requests
import datetime
import logging
import os
import json

from django.utils.timezone import utc
from pyparsing import commaSeparatedList
from .models import Station

STATION_LIST_URL = u'http://www.tel-o-fun.co.il/Default.aspx?TabID=64'

TEMPODB_KEY = os.environ['TEMPODB_KEY']
TEMPODB_SECRET = os.environ['TEMPODB_SECRET']
TEMPODB_BASEURL = 'https://api.tempo-db.com/v1'
TEMPO_JSONCT = {'content-type': 'application/json'}
tempo_auth = requests.auth.HTTPBasicAuth(TEMPODB_KEY, TEMPODB_SECRET)
tempo = requests.session(auth=tempo_auth)

FIELD_KEYS = ['longitude', 'latitude', 'id', 'name', 'address', 'poles', 'available']

def measure():
    logging.info("Measuring stations...")
    timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
    stations = parse_stations(scrape_stations())
    store_stations(stations)
    log_data(timestamp, stations)
    logging.info("Measured {} stations.".format(len(stations)))
    # TODO: periodically write more metadata about stations to the tempo series?

def scrape_stations():
    logging.debug("Scraping site content...")
    r = requests.get(STATION_LIST_URL)
    logging.debug("Parsing DOM...")
    soup = BeautifulSoup(r.content)
    raw_stations = soup.find_all(lambda t:
        t.name == 'script' and
        hasattr(t, 'text') and
        t.decode_contents().startswith('function loadMarkers'))[0].decode_contents()
    return raw_stations

def parse_stations(raw_stations):
    logging.debug("Extracting raw stations...")
    filtered = re.findall(r'setMarker\((.+?)\);(?=setMarker|\})', raw_stations)
    logging.debug("Parsing stations...")
    parsed = []
    for station in filtered:
        stripped = [s.strip("' ") for s in commaSeparatedList.parseString(station)[:7]]
        d = dict(zip(FIELD_KEYS, stripped))
        for floatkey in ['longitude', 'latitude']:
            d[floatkey] = float(d[floatkey])
        for intkey in ['id', 'poles', 'available']:
            d[intkey] = int(d[intkey])
        parsed.append(d)
    logging.debug("Parsed {} stations".format(len(parsed)))
    return parsed

def store_stations(stations):
    logging.debug("Create/update station metadata in Django DB...")
    for station in stations:
        fields = ['longitude', 'latitude', 'name', 'address']
        metadata = dict((f, station[f]) for f in fields)
        obj, created = Station.objects.get_or_create(
            id=station['id'], defaults=metadata)
        obj.poles = station['poles']
        obj.available = station['available']
        obj.save()

def station_poles_key(station):
    return "station:{id}.poles.{id}".format(**station)

def station_available_key(station):
    return "station:{id}.available.{id}".format(**station)

def log_data(timestamp, stations):
    logging.debug("Logging measurements to TempoDB...")
    payload = {}
    payload['t'] = timestamp.isoformat()
    data = []
    for station in stations:
        poles_key = station_poles_key(station)
        available_key = station_available_key(station)
        data.append({'key': poles_key, 'v': station['poles']})
        data.append({'key': available_key, 'v': station['available']})
    payload['data'] = data
    tempo.post(TEMPODB_BASEURL + '/data/',
               headers=TEMPO_JSONCT,
               data=json.dumps(payload))
<commit_msg>Fix missing defaults in store_stations() task<commit_after>import re
from bs4 import BeautifulSoup
import requests
import datetime
import logging
import os
import json

from django.utils.timezone import utc
from pyparsing import commaSeparatedList
from .models import Station

STATION_LIST_URL = u'http://www.tel-o-fun.co.il/Default.aspx?TabID=64'

TEMPODB_KEY = os.environ['TEMPODB_KEY']
TEMPODB_SECRET = os.environ['TEMPODB_SECRET']
TEMPODB_BASEURL = 'https://api.tempo-db.com/v1'
TEMPO_JSONCT = {'content-type': 'application/json'}
tempo_auth = requests.auth.HTTPBasicAuth(TEMPODB_KEY, TEMPODB_SECRET)
tempo = requests.session(auth=tempo_auth)

FIELD_KEYS = ['longitude', 'latitude', 'id', 'name', 'address', 'poles', 'available']

def measure():
    logging.info("Measuring stations...")
    timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
    stations = parse_stations(scrape_stations())
    store_stations(stations)
    log_data(timestamp, stations)
    logging.info("Measured {} stations.".format(len(stations)))
    # TODO: periodically write more metadata about stations to the tempo series?

def scrape_stations():
    logging.debug("Scraping site content...")
    r = requests.get(STATION_LIST_URL)
    logging.debug("Parsing DOM...")
    soup = BeautifulSoup(r.content)
    raw_stations = soup.find_all(lambda t:
        t.name == 'script' and
        hasattr(t, 'text') and
        t.decode_contents().startswith('function loadMarkers'))[0].decode_contents()
    return raw_stations

def parse_stations(raw_stations):
    logging.debug("Extracting raw stations...")
    filtered = re.findall(r'setMarker\((.+?)\);(?=setMarker|\})', raw_stations)
    logging.debug("Parsing stations...")
    parsed = []
    for station in filtered:
        stripped = [s.strip("' ") for s in commaSeparatedList.parseString(station)[:7]]
        d = dict(zip(FIELD_KEYS, stripped))
        for floatkey in ['longitude', 'latitude']:
            d[floatkey] = float(d[floatkey])
        for intkey in ['id', 'poles', 'available']:
            d[intkey] = int(d[intkey])
        parsed.append(d)
    logging.debug("Parsed {} stations".format(len(parsed)))
    return parsed

def store_stations(stations):
    logging.debug("Create/update station metadata in Django DB...")
    for station in stations:
        fields = ['longitude', 'latitude', 'name', 'address', 'poles', 'available']
        metadata = dict((f, station[f]) for f in fields)
        obj, created = Station.objects.get_or_create(
            id=station['id'], defaults=metadata)
        if not created:
            obj.poles = station['poles']
            obj.available = station['available']
            obj.save()

def station_poles_key(station):
    return "station:{id}.poles.{id}".format(**station)

def station_available_key(station):
    return "station:{id}.available.{id}".format(**station)

def log_data(timestamp, stations):
    logging.debug("Logging measurements to TempoDB...")
    payload = {}
    payload['t'] = timestamp.isoformat()
    data = []
    for station in stations:
        poles_key = station_poles_key(station)
        available_key = station_available_key(station)
        data.append({'key': poles_key, 'v': station['poles']})
        data.append({'key': available_key, 'v': station['available']})
    payload['data'] = data
    tempo.post(TEMPODB_BASEURL + '/data/',
               headers=TEMPO_JSONCT,
               data=json.dumps(payload))
<|endoftext|>


<commit_before>import argparse
import ipaddress
import functools
import http.client
import os
import pyudev
import signal
import sys
import threading
import wsgiref.simple_server

import prometheus_client
import prometheus_client.core as core

from . import exporter
from . import temper
from . import wsgiext

def main():
    '''
    You are here.
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('--bind-address', type=ipaddress.ip_address, default='::', help='IPv6 or IPv4 address to listen on')
    parser.add_argument('--bind-port', type=int, default=9204, help='Port to listen on')
    parser.add_argument('--bind-v6only', type=int, choices=[0, 1], help='If 1, prevent IPv6 sockets from accepting IPv4 connections; if 0, allow; if unspecified, use OS default')
    parser.add_argument('--thread-count', type=int, help='Number of request-handling threads to spawn')
    args = parser.parse_args()

    collector = exporter.Collector()
    core.REGISTRY.register(collector)

    server = wsgiext.Server((args.bind_address, args.bind_port), wsgiext.SilentRequestHandler, args.thread_count, args.bind_v6only)
    server.set_app(prometheus_client.make_wsgi_app())
    wsgi_thread = threading.Thread(target=functools.partial(server.serve_forever, poll_interval=86400), name='wsgi')

    ctx = pyudev.Context()
    mon = temper.monitor(ctx)
    observer_thread = pyudev.MonitorObserver(mon, name='monitor', callback=collector.handle_device_event)

    health_thread = Health(collector, server)

    def handle_sigterm(signum, frame):
        health_thread.send_stop()
        server.shutdown()
        observer_thread.send_stop()
    signal.signal(signal.SIGTERM, handle_sigterm)

    wsgi_thread.start()
    observer_thread.start()
    health_thread.start()

    collector.coldplug_scan(ctx)

    wsgi_thread.join()
    observer_thread.join()
    health_thread.join()

    server.server_close()

    sys.exit(health_thread.exit_status)

class Health(threading.Thread):
    def __init__(self, collector, server, interval=30, *args, **kwargs):
        super().__init__(name='health', *args, **kwargs)
        self.__collector = collector
        self.__server = server
        self.__interval = interval
        self.__event = threading.Event()
        self.exit_status = 0

    def send_stop(self):
        '''
        Cause the thread to exit.
        '''
        self.__event.set()

    def run(self):
        '''
        Monitor the health of the service.

        If something fails, sends SIGTERM to the process. The signal handler
        (which always runs in the main thread) will shut down the components
        and then the process will exit.

        We don't have to provide detailed error messages, since the component
        that failed should already have logged something useful.
        '''
        try:
            addr, port, *rest = self.__server.socket.getsockname()
            while not self.__event.wait(self.__interval):
                if self.__collector.exceptions._value.get() > 0:
                    raise Health.HealthException
                elif self.__collector.errors._value.get() > 0:
                    raise Health.HealthException

                c = http.client.HTTPConnection(addr, port, timeout=5)
                try:
                    c.request('GET', '/')
                except http.client.HTTPException:
                    raise Health.HealthException
                r = c.getresponse()
                if r.status != 200:
                    raise Health.HealthException
        except Exception as e:
            self.exit_status = 1
            os.kill(os.getpid(), signal.SIGTERM)
            if not isinstance(e, Health.HealthException):
                # Something went wrong in the health check itself... make sure it
                # hits the logs
                raise

    class HealthException(Exception):
        pass
<commit_msg>Health: don't use exceptions for control flow<commit_after>import argparse
import ipaddress
import functools
import http.client
import os
import pyudev
import signal
import sys
import threading
import wsgiref.simple_server

import prometheus_client
import prometheus_client.core as core

from . import exporter
from . import temper
from . import wsgiext

def main():
    '''
    You are here.
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('--bind-address', type=ipaddress.ip_address, default='::', help='IPv6 or IPv4 address to listen on')
    parser.add_argument('--bind-port', type=int, default=9204, help='Port to listen on')
    parser.add_argument('--bind-v6only', type=int, choices=[0, 1], help='If 1, prevent IPv6 sockets from accepting IPv4 connections; if 0, allow; if unspecified, use OS default')
    parser.add_argument('--thread-count', type=int, help='Number of request-handling threads to spawn')
    args = parser.parse_args()

    collector = exporter.Collector()
    core.REGISTRY.register(collector)

    server = wsgiext.Server((args.bind_address, args.bind_port), wsgiext.SilentRequestHandler, args.thread_count, args.bind_v6only)
    server.set_app(prometheus_client.make_wsgi_app())
    wsgi_thread = threading.Thread(target=functools.partial(server.serve_forever, poll_interval=86400), name='wsgi')

    ctx = pyudev.Context()
    mon = temper.monitor(ctx)
    observer_thread = pyudev.MonitorObserver(mon, name='monitor', callback=collector.handle_device_event)

    health_thread = Health(collector, server)

    def handle_sigterm(signum, frame):
        health_thread.send_stop()
        server.shutdown()
        observer_thread.send_stop()
    signal.signal(signal.SIGTERM, handle_sigterm)

    wsgi_thread.start()
    observer_thread.start()
    health_thread.start()

    collector.coldplug_scan(ctx)

    wsgi_thread.join()
    observer_thread.join()
    health_thread.join()

    server.server_close()

    sys.exit(health_thread.exit_status)

class Health(threading.Thread):
    def __init__(self, collector, server, interval=30, *args, **kwargs):
        super().__init__(name='health', *args, **kwargs)
        self.__collector = collector
        self.__server = server
        self.__interval = interval
        self.__event = threading.Event()
        self.exit_status = 0

    def send_stop(self):
        '''
        Cause the thread to exit.
        '''
        self.__event.set()

    def run(self):
        '''
        Monitor the health of the service.

        If something fails, sends SIGTERM to the process. The signal handler
        (which always runs in the main thread) will shut down the components
        and then the process will exit.

        We don't have to provide detailed error messages, since the component
        that failed should already have logged something useful.
        '''
        try:
            addr, port, *rest = self.__server.socket.getsockname()
            while not self.__event.wait(self.__interval):
                # Collector checks
                if self.__collector.exceptions._value.get() > 0:
                    self.exit_status = 1
                    break
                elif self.__collector.errors._value.get() > 0:
                    self.exit_status = 1
                    break
                # WSGIServer checks
                c = http.client.HTTPConnection(addr, port, timeout=5)
                try:
                    c.request('GET', '/')
                except http.client.HTTPException:
                    self.exit_status = 1
                    break
                r = c.getresponse()
                if r.status != 200:
                    self.exit_status = 1
                    break
        except:
            self.exit_status = 1
            raise
        finally:
            if self.exit_status != 0:
                os.kill(os.getpid(), signal.SIGTERM)
<|endoftext|>


<commit_before>import unittest
from pathlib import Path
import json
import subprocess

class TestCLI(unittest.TestCase):
    def assertRegex(self, string, pattern):
        assertRegexpMatches = getattr(self, 'assertRegexpMatches', None)
        if assertRegexpMatches:
            assertRegexpMatches(string, pattern)
            return

        super(TestCLI, self).assertRegex(string, pattern)

    def assertReturnedStdoutEqual(self, expected_stdout, cmd):
        got_stdout = '(no stdout)'

        try:
            got_stdout = subprocess.check_output(cmd, universal_newlines=True)
        except subprocess.CalledProcessError as err:
            print('Got stderr: `{err_message}'.format(err_message=err.output))
        finally:
            print('Got stdout: `{stdout}`'.format(stdout=got_stdout))

        self.assertEqual(expected_stdout, got_stdout)

    def test_exec_vint_with_valid_file_on_project_root(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid1.vim'))
        cmd = ['vint', valid_file]

        expected_output = ''

        self.assertReturnedStdoutEqual(expected_output, cmd)

    def test_exec_vint_with_valid_file_encoded_cp932_on_project_root(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid-cp932.vim'))
        cmd = ['vint', valid_file]

        expected_output = ''

        self.assertReturnedStdoutEqual(expected_output, cmd)

    def test_exec_vint_with_invalid_file_on_project_root(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd, universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '{file_path}:1:13:'.format(file_path=invalid_file)
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_no_args(self):
        cmd = ['vint']

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = r'^vint ERROR:'
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_unexistent_file(self):
        cmd = ['vint', '/path/to/unexistent']

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = r'^vint ERROR:'
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_stat_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--stat', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '{file_path}:1:13:'.format(file_path=invalid_file)
        expected_stat_pattern = 'Total'
        self.assertRegex(got_output, expected_output_pattern)
        self.assertRegex(got_output, expected_stat_pattern)

    def test_exec_vint_with_json_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--json', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            # We should not capture STRERR because coverage plugin use it.
            subprocess.check_output(cmd,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        print(got_output)
        self.assertIsInstance(json.loads(got_output), list)

    def test_exec_vint_with_verbose_flag(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid1.vim'))
        cmd = ['vint', '--verbose', valid_file]

        got_output = subprocess.check_output(cmd,
                                             universal_newlines=True,
                                             stderr=subprocess.STDOUT)

        expected_output_pattern = r'^vint INFO:'
        self.assertRegex(got_output, expected_output_pattern)

    @unittest.skip('Does drone.io not like ANSI color?')
    def test_exec_vint_with_color_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--color', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd, universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '\\033\['
        self.assertRegex(got_output, expected_output_pattern)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix an acceptance test<commit_after>import unittest
from pathlib import Path
import json
import subprocess

class TestCLI(unittest.TestCase):
    def assertRegex(self, string, pattern):
        assertRegexpMatches = getattr(self, 'assertRegexpMatches', None)
        if assertRegexpMatches:
            assertRegexpMatches(string, pattern)
            return

        super(TestCLI, self).assertRegex(string, pattern)

    def assertReturnedStdoutEqual(self, expected_stdout, cmd):
        got_stdout = '(no stdout)'

        try:
            got_stdout = subprocess.check_output(cmd, universal_newlines=True)
        except subprocess.CalledProcessError as err:
            print('Got stderr: `{err_message}'.format(err_message=err.output))
        finally:
            print('Got stdout: `{stdout}`'.format(stdout=got_stdout))

        self.assertEqual(expected_stdout, got_stdout)

    def test_exec_vint_with_valid_file_on_project_root(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid1.vim'))
        cmd = ['vint', valid_file]

        expected_output = ''

        self.assertReturnedStdoutEqual(expected_output, cmd)

    def test_exec_vint_with_valid_file_encoded_cp932_on_project_root(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid-cp932.vim'))
        cmd = ['vint', valid_file]

        expected_output = ''

        self.assertReturnedStdoutEqual(expected_output, cmd)

    def test_exec_vint_with_invalid_file_on_project_root(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd, universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '{file_path}:1:13:'.format(file_path=invalid_file)
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_no_args(self):
        cmd = ['vint']

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = r'^vint ERROR:'
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_unexistent_file(self):
        cmd = ['vint', '/path/to/unexistent']

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = r'^vint ERROR:'
        self.assertRegex(got_output, expected_output_pattern)

    def test_exec_vint_with_stat_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--stat', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '{file_path}:1:13:'.format(file_path=invalid_file)
        expected_stat_pattern = 'Total'
        self.assertRegex(got_output, expected_output_pattern)
        self.assertRegex(got_output, expected_stat_pattern)

    def test_exec_vint_with_json_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--json', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            # We should not capture STRERR because coverage plugin use it.
            subprocess.check_output(cmd,
                                    universal_newlines=True)

        got_output = context_manager.exception.output

        print(got_output)
        self.assertIsInstance(json.loads(got_output), list)

    def test_exec_vint_with_verbose_flag(self):
        valid_file = str(Path('test', 'fixture', 'cli', 'valid1.vim'))
        cmd = ['vint', '--verbose', valid_file]

        got_output = subprocess.check_output(cmd,
                                             universal_newlines=True,
                                             stderr=subprocess.STDOUT)

        expected_output_pattern = r'^vint DEBUG:'
        self.assertRegex(got_output, expected_output_pattern)

    @unittest.skip('Does drone.io not like ANSI color?')
    def test_exec_vint_with_color_flag(self):
        invalid_file = str(Path('test', 'fixture', 'cli', 'invalid1.vim'))
        cmd = ['vint', '--color', invalid_file]

        with self.assertRaises(subprocess.CalledProcessError) as context_manager:
            subprocess.check_output(cmd, universal_newlines=True)

        got_output = context_manager.exception.output

        expected_output_pattern = '\\033\['
        self.assertRegex(got_output, expected_output_pattern)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from threading import Event, Thread

import app.core.netutils as netutils
from app.core.messaging import discover_message_server
from app.services.discovery import DiscoveryService
from app.services.discovery.service import DiscoveryServer

class TestDiscoverMessageServer(object):
    @classmethod
    def setup_class(cls):
        DiscoveryServer.ACTIVE_POLL_TIME = 1

    def test_no_discovery_server(self):
        assert discover_message_server() is None

    def test_discovery_server_bad_json(self):
        service = DiscoveryService(None)
        service.server.process = lambda x, y: "sdf;lghkd;flkh".encode("UTF-8")
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        assert discover_message_server() is None
        service.on_service_stop()

    def test_discovery_server_unknown_json(self):
        service = DiscoveryService(None)
        service.server.process = lambda x, y: '{"vld": "json"}'.encode("UTF-8")
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        assert discover_message_server() is None
        service.on_service_stop()

    def test_valid_discovery_server(self):
        service = DiscoveryService(None)
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        ip_addresses = [x["addr"] for x in netutils.iter_ipv4_addresses()]
        try:
            assert discover_message_server()[0] in ip_addresses
        finally:
            service.on_service_stop()
<commit_msg>Add test cases for SnycMessenger.<commit_after>from threading import Event, Thread

import app.core.netutils as netutils
from app.core.messaging import discover_message_server, Sender, Receiver
from app.core.messaging import SyncMessenger
from app.services.discovery import DiscoveryService
from app.services.discovery.service import DiscoveryServer
from app.services.messaging import MessageService

CONFIG = {
    "redis_config": {
        "USE_FAKE_REDIS": True
    },
    "queues": {
        "custom_queues": [
            {
                "queue_name": "dummy",
                "request_schema": {"type": "object"}
            }
        ]
    }
}

class TestDiscoverMessageServer(object):
    @classmethod
    def setup_class(cls):
        DiscoveryServer.ACTIVE_POLL_TIME = 1

    def test_no_discovery_server(self):
        assert discover_message_server() is None

    def test_discovery_server_bad_json(self):
        service = DiscoveryService(None)
        service.server.process = lambda x, y: "sdf;lghkd;flkh".encode("UTF-8")
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        assert discover_message_server() is None
        service.on_service_stop()

    def test_discovery_server_unknown_json(self):
        service = DiscoveryService(None)
        service.server.process = lambda x, y: '{"vld": "json"}'.encode("UTF-8")
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        assert discover_message_server() is None
        service.on_service_stop()

    def test_valid_discovery_server(self):
        service = DiscoveryService(None)
        event = Event()
        service.notify_start = event.set
        Thread(target=service.on_service_start).start()
        event.wait()

        ip_addresses = [x["addr"] for x in netutils.iter_ipv4_addresses()]
        try:
            assert discover_message_server()[0] in ip_addresses
        finally:
            service.on_service_stop()

class TestSyncMessenger(object):
    @classmethod
    def setup_class(cls):
        event = Event()
        cls.service = MessageService(CONFIG)
        cls.service.notify_start = lambda: event.set()
        cls.service_thread = Thread(target=cls.service.on_service_start)
        cls.service_thread.start()
        event.wait()

        cls.start_echo_receiver("dummy")

    @classmethod
    def teardown_class(cls):
        cls.service.on_service_stop()
        cls.service_thread.join()

    @classmethod
    def start_echo_receiver(cls, queue):
        sender = Sender(queue)
        sender.start()

        def reply(msg):
            sender.send(msg)

        receiver = Receiver(queue)
        receiver.on_message = reply
        receiver.start()

    def test_send_sync(self):
        obj = {"test": "test-messsage", "arr": [1, 2, 3]}

        sync = SyncMessenger("dummy")
        sync.start()

        assert obj == sync.send(obj)

        sync.close()
<|endoftext|>


<commit_before>#! /usr/bin/python
# -.- coding: utf-8 -.-

# remote-test.py
#
# Copyright © 2009 Mikkel Kamstrup Erlandsen <mikkel.kamstrup@gmail.com>
# Copyright © 2010 Markus Korn <thekorn@gmx.de>
# Copyright © 2010 Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
# Copyright © 2011 Manish Sinha <manishsinha@ubuntu.com>
# Copyright © 2011 Collabora Ltd.
#             By Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Update python path to use local zeitgeist module
import sys
import os
import unittest
import gobject

sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from zeitgeist.client import ZeitgeistDBusInterface
from zeitgeist.datamodel import *
from testutils import RemoteTestCase

class BlacklistTest(RemoteTestCase):

	def __init__(self, methodName):
		super(BlacklistTest, self).__init__(methodName)
		self.blacklist = None

	def setUp(self):
		# lazy import to get a chance to use the private bus
		import dbus

		# We set up the connection lazily in order to wait for the
		# engine to come up
		super(BlacklistTest, self).setUp()
		obj = dbus.SessionBus().get_object("org.gnome.zeitgeist.Engine",
			"/org/gnome/zeitgeist/blacklist")
		self.blacklist = dbus.Interface(obj, "org.gnome.zeitgeist.Blacklist")

	def testClear(self):
		# Insert a blacklist template
		self.blacklist.AddTemplate("unicorns",
			Event.new_for_values(subject_uri="a"))
		self.assertTrue(len(self.blacklist.GetTemplates()) > 0)

		# Now remove all existing templates...
		allTemplates = self.blacklist.GetTemplates()
		[self.blacklist.RemoveTemplate(key) for key in allTemplates.iterkeys()]

		# And ensure that they are indeed gone.
		self.assertEquals(len(self.blacklist.GetTemplates()), 0)

	def testSetOne(self):
		orig = Event.new_for_values(interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")
		self.blacklist.AddTemplate("Foobar", orig)
		res = self.blacklist.GetTemplates()

		self.assertEquals(len(res), 1)
		self.assertEventsEqual(orig, Event(res["Foobar"]))

	def testApplyBlacklist(self):
		self.testSetOne()
		ev = Event.new_for_values(interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")
		ev.manifestation = Manifestation.USER_ACTIVITY
		ev.actor = "app.//foo.desktop"

		inserted_ids = self.insertEventsAndWait([ev])
		self.assertEquals(1, len(inserted_ids))
		self.assertEquals(0, int(inserted_ids[0]))

		# Now change the event to pass the blacklist
		ev.get_subjects()[0].uri = "htpp://totallyvaliduri.com"
		inserted_ids = self.insertEventsAndWait([ev])
		self.assertEquals(1, len(inserted_ids))
		self.assertTrue(0 != inserted_ids[0])

	def _get_blacklist_iface(self):
		"""
		Create a blacklist interface using the get_extension() method
		from client.py.
		"""
		del self.blacklist
		iface = ZeitgeistDBusInterface()
		blacklist = iface.get_extension("Blacklist", "blacklist")
		return blacklist

	def testBlacklistUsingClientDBusInterface(self):
		"""
		Ensure that get_extension() from client.py method works correctly.
		"""
		blacklist = self._get_blacklist_iface()
		allTemplates = blacklist.GetTemplates()
		[blacklist.RemoveTemplate(key) for key in allTemplates.iterkeys()]
		newAllTemplates = blacklist.GetTemplates()
		self.assertEquals(len(newAllTemplates), 0)

	def testBlacklistSignals(self):
		self.blacklist = self._get_blacklist_iface()
		mainloop = self.create_mainloop()

		template1 = Event.new_for_values(
			interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")

		global hit
		hit = 0

		def cb_added(template_id, event_template):
			global hit
			self.assertEquals(hit, 0)
			hit = 1
			self.assertEquals(template_id, "TestTemplate")
			self.assertEventsEqual(template1, event_template)

		def cb_removed(template_id, event_template):
			global hit
			self.assertEquals(hit, 1)
			hit = 2
			self.assertEquals(template_id, "TestTemplate")
			self.assertEventsEqual(template1, event_template)
			mainloop.quit()

		# Connect to signals
		self.blacklist.connect('TemplateAdded', cb_added)
		self.blacklist.connect('TemplateRemoved', cb_removed)

		def launch_tests():
			self.blacklist.AddTemplate("TestTemplate", template1)
			self.blacklist.RemoveTemplate("TestTemplate")
		gobject.idle_add(launch_tests)

		mainloop.run()

if __name__ == "__main__":
	unittest.main()
<commit_msg>set print for debugging the signal issue<commit_after>#! /usr/bin/python
# -.- coding: utf-8 -.-

# remote-test.py
#
# Copyright © 2009 Mikkel Kamstrup Erlandsen <mikkel.kamstrup@gmail.com>
# Copyright © 2010 Markus Korn <thekorn@gmx.de>
# Copyright © 2010 Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
# Copyright © 2011 Manish Sinha <manishsinha@ubuntu.com>
# Copyright © 2011 Collabora Ltd.
#             By Siegfried-Angel Gevatter Pujals <siegfried@gevatter.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Update python path to use local zeitgeist module
import sys
import os
import unittest
import gobject

sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from zeitgeist.client import ZeitgeistDBusInterface
from zeitgeist.datamodel import *
from testutils import RemoteTestCase

class BlacklistTest(RemoteTestCase):

	def __init__(self, methodName):
		super(BlacklistTest, self).__init__(methodName)
		self.blacklist = None

	def setUp(self):
		# lazy import to get a chance to use the private bus
		import dbus

		# We set up the connection lazily in order to wait for the
		# engine to come up
		super(BlacklistTest, self).setUp()
		obj = dbus.SessionBus().get_object("org.gnome.zeitgeist.Engine",
			"/org/gnome/zeitgeist/blacklist")
		self.blacklist = dbus.Interface(obj, "org.gnome.zeitgeist.Blacklist")

	def testClear(self):
		# Insert a blacklist template
		self.blacklist.AddTemplate("unicorns",
			Event.new_for_values(subject_uri="a"))
		self.assertTrue(len(self.blacklist.GetTemplates()) > 0)

		# Now remove all existing templates...
		allTemplates = self.blacklist.GetTemplates()
		[self.blacklist.RemoveTemplate(key) for key in allTemplates.iterkeys()]

		# And ensure that they are indeed gone.
		self.assertEquals(len(self.blacklist.GetTemplates()), 0)

	def testSetOne(self):
		orig = Event.new_for_values(interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")
		self.blacklist.AddTemplate("Foobar", orig)
		res = self.blacklist.GetTemplates()

		self.assertEquals(len(res), 1)
		self.assertEventsEqual(orig, Event(res["Foobar"]))

	def testApplyBlacklist(self):
		self.testSetOne()
		ev = Event.new_for_values(interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")
		ev.manifestation = Manifestation.USER_ACTIVITY
		ev.actor = "app.//foo.desktop"

		inserted_ids = self.insertEventsAndWait([ev])
		self.assertEquals(1, len(inserted_ids))
		self.assertEquals(0, int(inserted_ids[0]))

		# Now change the event to pass the blacklist
		ev.get_subjects()[0].uri = "htpp://totallyvaliduri.com"
		inserted_ids = self.insertEventsAndWait([ev])
		self.assertEquals(1, len(inserted_ids))
		self.assertTrue(0 != inserted_ids[0])

	def _get_blacklist_iface(self):
		"""
		Create a blacklist interface using the get_extension() method
		from client.py.
		"""
		del self.blacklist
		iface = ZeitgeistDBusInterface()
		blacklist = iface.get_extension("Blacklist", "blacklist")
		return blacklist

	def testBlacklistUsingClientDBusInterface(self):
		"""
		Ensure that get_extension() from client.py method works correctly.
		"""
		blacklist = self._get_blacklist_iface()
		allTemplates = blacklist.GetTemplates()
		[blacklist.RemoveTemplate(key) for key in allTemplates.iterkeys()]
		newAllTemplates = blacklist.GetTemplates()
		self.assertEquals(len(newAllTemplates), 0)

	def testBlacklistSignals(self):
		self.blacklist = self._get_blacklist_iface()
		mainloop = self.create_mainloop()

		template1 = Event.new_for_values(
			interpretation=Interpretation.ACCESS_EVENT,
			subject_uri="http://nothingtoseehere.gov")

		global hit
		hit = 0

		def cb_added(template_id, event_template):
			global hit
			print "*** cb_added with hit", hit
			self.assertEquals(hit, 0)
			hit = 1
			self.assertEquals(template_id, "TestTemplate")
			self.assertEventsEqual(template1, event_template)

		def cb_removed(template_id, event_template):
			global hit
			self.assertEquals(hit, 1)
			hit = 2
			self.assertEquals(template_id, "TestTemplate")
			self.assertEventsEqual(template1, event_template)
			mainloop.quit()

		# Connect to signals
		self.blacklist.connect('TemplateAdded', cb_added)
		self.blacklist.connect('TemplateRemoved', cb_removed)

		def launch_tests():
			self.blacklist.AddTemplate("TestTemplate", template1)
			self.blacklist.RemoveTemplate("TestTemplate")
		gobject.idle_add(launch_tests)

		mainloop.run()

if __name__ == "__main__":
	unittest.main()
<|endoftext|>


<commit_before>import fes_controller
import redis_data
import hbase_data
import test_utils
from random import randint

UPDATED_TEST_PAYLOAD = '{"a different list" : "some other stuff", "hark, a test datum" : ["this time with a", "two item list"]}'

def test_generate_hash():
    id_hashes = {}

    for i in range(100):
        id_ = test_utils.get_random_string()
        id_hash = fes_controller.generate_hash(id_)

        id_hashes[id_hash] = id_

    assert len(id_hashes) == 100

"""===============================================test add====================================================="""

def test_add_redis_only():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 900)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None

def test_add_hbase_only():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 900)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, expiration) == [id_hash]

def test_add_to_hbase_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(451, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_add_to_redis_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(450, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

def test_add_to_hbase_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 900)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_add_to_redis_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 900)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

"""==================================================test update expiration==============================================="""

def test_update_expiration_to_redis_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(451, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

def test_update_expiration_to_hbase_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(451, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_update_expiration_to_hbase_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 900)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_update_expiration_to_redis_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 900)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 900)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

"""==================================================test update event==============================================="""

def test_update_event_payload_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - 10

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_event_payload(id_, UPDATED_TEST_PAYLOAD)

    assert int(redis_data.get_expiration(id_hash)) == expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None

def test_update_event_payload_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + 10

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_event_payload(id_, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD

"""==================================================test delete==============================================="""
def test_delete_from_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - 10

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.delete(id_)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None

def test_delete_from_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + 10

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.delete(id_)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None<commit_msg>fixed a slight problem with my timestamp randomization in fes_controller_test module<commit_after>import fes_controller
import redis_data
import hbase_data
import test_utils
from random import randint

UPDATED_TEST_PAYLOAD = '{"a different list" : "some other stuff", "hark, a test datum" : ["this time with a", "two item list"]}'

def test_generate_hash():
    id_hashes = {}

    for i in range(100):
        id_ = test_utils.get_random_string()
        id_hash = fes_controller.generate_hash(id_)

        id_hashes[id_hash] = id_

    assert len(id_hashes) == 100

"""===============================================test add====================================================="""

def test_add_redis_only():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None

def test_add_hbase_only():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, expiration) == [id_hash]

def test_add_to_hbase_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(451, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_add_to_redis_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(450, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

def test_add_to_hbase_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_add_to_redis_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.add(id_, updated_expiration, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

"""==================================================test update expiration==============================================="""

def test_update_expiration_to_redis_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(451, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

def test_update_expiration_to_hbase_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 450)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(451, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_update_expiration_to_hbase_already_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(updated_expiration)
    assert future_event.payload == test_utils.TEST_PAYLOAD
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) == [id_hash]

def test_update_expiration_to_redis_already_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    original_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)
    updated_expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)

    fes_controller.add(id_, original_expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_expiration(id_, updated_expiration)

    assert redis_data.get_expiration(id_hash) == updated_expiration
    assert redis_data.get_event_payload(id_hash) == test_utils.TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, original_expiration) is None
    assert test_utils.scan_index(id_hash, updated_expiration) is None

"""==================================================test update event==============================================="""

def test_update_event_payload_in_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_event_payload(id_, UPDATED_TEST_PAYLOAD)

    assert int(redis_data.get_expiration(id_hash)) == expiration
    assert redis_data.get_event_payload(id_hash) == UPDATED_TEST_PAYLOAD
    assert hbase_data.read_event(id_hash) is None

def test_update_event_payload_in_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.update_event_payload(id_, UPDATED_TEST_PAYLOAD)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None

    future_event = hbase_data.read_event(id_hash)
    assert future_event is not None
    assert future_event.id_ is None
    assert future_event.expiration == str(expiration)
    assert future_event.payload == UPDATED_TEST_PAYLOAD

"""==================================================test delete==============================================="""
def test_delete_from_redis():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) - randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.delete(id_)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None

def test_delete_from_hbase():
    id_ = test_utils.get_random_string()
    id_hash = fes_controller.generate_hash(id_)
    expiration = test_utils.get_current_timestamp() + (fes_controller.STORAGE_CUTOFF_MINUTES * 60) + randint(1, 899)

    fes_controller.add(id_, expiration, test_utils.TEST_PAYLOAD)
    fes_controller.delete(id_)

    assert redis_data.get_expiration(id_hash) is None
    assert redis_data.get_event_payload(id_hash) is None
    assert hbase_data.read_event(id_hash) is None
    assert test_utils.scan_index(id_hash, expiration) is None<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import copy
import io
import os
import tarfile
import time
import uuid
import yaml

import docker
import pytest
import requests
import json

from openshift.client import models
from openshift.helper.ansible import AnsibleModuleHelper

if os.path.exists(os.path.join(os.getcwd(), 'KubeObjHelper.log')):
    os.remove(os.path.join(os.getcwd(), 'KubeObjHelper.log'))

@pytest.fixture(scope='session')
def openshift_container(request):
    client = docker.from_env()
    # TODO: bind to a random host port
    openshift_version = request.config.getoption('--openshift-version')
    if openshift_version is None:
        yield None
    else:
        image_name = 'openshift/origin:{}'.format(openshift_version)
        container = client.containers.run(image_name, 'start master', detach=True,
                                          ports={'8443/tcp': 8443})

        try:
            # Wait for the container to no longer be in the created state before
            # continuing
            while container.status == u'created':
                time.sleep(0.2)
                container = client.containers.get(container.id)

            # Wait for the api server to be ready before continuing
            for _ in range(10):
                try:
                    requests.head("https://127.0.0.1:8443/healthz/ready", verify=False)
                except requests.RequestException:
                    pass
                time.sleep(1)

            time.sleep(1)

            yield container
        finally:
            # Always remove the container
            container.remove(force=True)

@pytest.fixture(scope='session')
def kubeconfig(openshift_container, tmpdir_factory):
    # get_archive returns a stream of the tar archive containing the requested
    # files/directories, so we need use BytesIO as an intermediate step.
    if openshift_container is None:
        return None
    else:
        tar_stream, _ = openshift_container.get_archive(
            '/var/lib/origin/openshift.local.config/master/admin.kubeconfig')
        tar_obj = tarfile.open(fileobj=io.BytesIO(tar_stream.read()))
        kubeconfig_contents = tar_obj.extractfile('admin.kubeconfig').read()

        kubeconfig_file = tmpdir_factory.mktemp('kubeconfig').join('admin.kubeconfig')
        kubeconfig_file.write(kubeconfig_contents)
        return kubeconfig_file

@pytest.fixture(scope='module')
def ansible_helper(request, kubeconfig):
    _, api_version, resource = request.module.__name__.split('_', 2)
    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper(api_version, resource, debug=True, reset_logfile=False, auth=auth)
    helper.api_client.config.debug = True

    return helper

@pytest.fixture(scope='session')
def obj_compare():
    def compare_func(ansible_helper, k8s_obj, parameters):
        """ Assert that an object matches an expected object """
        requested = copy.deepcopy(k8s_obj)
        ansible_helper.object_from_params(parameters, obj=requested)

        ansible_helper.log('paramters:')
        ansible_helper.log(json.dumps(parameters, indent=4))
        ansible_helper.log('\n\n')

        ansible_helper.log('k8s_obj:')
        ansible_helper.log(json.dumps(k8s_obj.to_dict(), indent=4))
        ansible_helper.log('\n\n')

        ansible_helper.log('from params:')
        ansible_helper.log(json.dumps(requested.to_dict(), indent=4))
        ansible_helper.log('\n\n')

        match, diff = ansible_helper.objects_match(k8s_obj, requested)
        if not match:
            ansible_helper.log('\n\n')
            ansible_helper.log('Differences:')
            ansible_helper.log(list(diff))
            ansible_helper.log('\n\n')
        assert match

    return compare_func

@pytest.fixture(scope='module')
def namespace(kubeconfig):
    name = "test-{}".format(uuid.uuid4())

    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper('v1', 'namespace', debug=True, reset_logfile=False, **auth)

    k8s_obj = helper.create_object(models.V1Namespace(metadata=models.V1ObjectMeta(name=name)))
    assert k8s_obj is not None

    yield name

    helper.delete_object(name, None)

@pytest.fixture()
def object_name():
    name = 'test-{}'.format(uuid.uuid4())
    return name

@pytest.fixture(scope='module')
def project(kubeconfig):
    name = "test-{}".format(uuid.uuid4())
    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper('v1', 'project', debug=True, reset_logfile=False, **auth)

    k8s_obj = helper.create_project(metadata=models.V1ObjectMeta(name=name))
    assert k8s_obj is not None

    yield name

    helper.delete_object(name, None)

def _get_id(argvalue):
    op_type = ''
    if argvalue.get('create'):
        op_type = 'create'
    elif argvalue.get('patch'):
        op_type = 'patch'
    elif argvalue.get('remove'):
        op_type = 'remove'
    elif argvalue.get('replace'):
        op_type = 'replace'
    return op_type + '_' + argvalue[op_type]['name'] + '_' + "{:0>3}".format(argvalue['seq'])

def pytest_generate_tests(metafunc):
    _, api_version, resource = metafunc.module.__name__.split('_', 2)
    yaml_name = api_version + '_' + resource + '.yml'
    yaml_path = os.path.normpath(os.path.join(os.path.dirname(__file__),
                                              '../../openshift/ansiblegen/examples', yaml_name))
    if not os.path.exists(yaml_path):
        raise Exception("ansible_data: Unable to locate {}".format(yaml_path))
    with open(yaml_path, 'r') as f:
        data = yaml.load(f)
    seq = 0
    for task in data:
        seq += 1
        task['seq'] = seq

    if 'create_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('create')]
        metafunc.parametrize("create_tasks", tasks, False, _get_id)
    if 'patch_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('patch')]
        metafunc.parametrize("patch_tasks", tasks, False, _get_id)
    if 'remove_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('remove')]
        metafunc.parametrize("remove_tasks", tasks, False, _get_id)
    if 'replace_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('replace')]
        metafunc.parametrize("replace_tasks", tasks, False, _get_id)
    if 'namespaces' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('create') and x['create'].get('namespace')]
        unique_namespaces = dict()
        for task in tasks:
            unique_namespaces[task['create']['namespace']] = None
        metafunc.parametrize("namespaces", list(unique_namespaces.keys()))
<commit_msg>update name generation to work with v1.3 services<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import copy
import io
import os
import tarfile
import time
import uuid
import yaml

import docker
import pytest
import requests
import json

from openshift.client import models
from openshift.helper.ansible import AnsibleModuleHelper

if os.path.exists(os.path.join(os.getcwd(), 'KubeObjHelper.log')):
    os.remove(os.path.join(os.getcwd(), 'KubeObjHelper.log'))

@pytest.fixture(scope='session')
def openshift_container(request):
    client = docker.from_env()
    # TODO: bind to a random host port
    openshift_version = request.config.getoption('--openshift-version')
    if openshift_version is None:
        yield None
    else:
        image_name = 'openshift/origin:{}'.format(openshift_version)
        container = client.containers.run(image_name, 'start master', detach=True,
                                          ports={'8443/tcp': 8443})

        try:
            # Wait for the container to no longer be in the created state before
            # continuing
            while container.status == u'created':
                time.sleep(0.2)
                container = client.containers.get(container.id)

            # Wait for the api server to be ready before continuing
            for _ in range(10):
                try:
                    requests.head("https://127.0.0.1:8443/healthz/ready", verify=False)
                except requests.RequestException:
                    pass
                time.sleep(1)

            time.sleep(1)

            yield container
        finally:
            # Always remove the container
            container.remove(force=True)

@pytest.fixture(scope='session')
def kubeconfig(openshift_container, tmpdir_factory):
    # get_archive returns a stream of the tar archive containing the requested
    # files/directories, so we need use BytesIO as an intermediate step.
    if openshift_container is None:
        return None
    else:
        tar_stream, _ = openshift_container.get_archive(
            '/var/lib/origin/openshift.local.config/master/admin.kubeconfig')
        tar_obj = tarfile.open(fileobj=io.BytesIO(tar_stream.read()))
        kubeconfig_contents = tar_obj.extractfile('admin.kubeconfig').read()

        kubeconfig_file = tmpdir_factory.mktemp('kubeconfig').join('admin.kubeconfig')
        kubeconfig_file.write(kubeconfig_contents)
        return kubeconfig_file

@pytest.fixture(scope='module')
def ansible_helper(request, kubeconfig):
    _, api_version, resource = request.module.__name__.split('_', 2)
    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper(api_version, resource, debug=True, reset_logfile=False, auth=auth)
    helper.api_client.config.debug = True

    return helper

@pytest.fixture(scope='session')
def obj_compare():
    def compare_func(ansible_helper, k8s_obj, parameters):
        """ Assert that an object matches an expected object """
        requested = copy.deepcopy(k8s_obj)
        ansible_helper.object_from_params(parameters, obj=requested)

        ansible_helper.log('paramters:')
        ansible_helper.log(json.dumps(parameters, indent=4))
        ansible_helper.log('\n\n')

        ansible_helper.log('k8s_obj:')
        ansible_helper.log(json.dumps(k8s_obj.to_dict(), indent=4))
        ansible_helper.log('\n\n')

        ansible_helper.log('from params:')
        ansible_helper.log(json.dumps(requested.to_dict(), indent=4))
        ansible_helper.log('\n\n')

        match, diff = ansible_helper.objects_match(k8s_obj, requested)
        if not match:
            ansible_helper.log('\n\n')
            ansible_helper.log('Differences:')
            ansible_helper.log(list(diff))
            ansible_helper.log('\n\n')
        assert match

    return compare_func

@pytest.fixture(scope='module')
def namespace(kubeconfig):
    name = "test-{}".format(uuid.uuid4())

    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper('v1', 'namespace', debug=True, reset_logfile=False, **auth)

    k8s_obj = helper.create_object(models.V1Namespace(metadata=models.V1ObjectMeta(name=name)))
    assert k8s_obj is not None

    yield name

    helper.delete_object(name, None)

@pytest.fixture()
def object_name():
    # v1.3 services cannot be longer than 24 characters long
    # truncate at 23 to avoid a trailing '-'
    name = 'test-{}'.format(uuid.uuid4())[:23]
    return name

@pytest.fixture(scope='module')
def project(kubeconfig):
    name = "test-{}".format(uuid.uuid4())
    auth = None
    if kubeconfig is not None:
        auth = {
            'kubeconfig': str(kubeconfig),
            'host': 'https://localhost:8443',
            'verify_ssl': False
        }
    helper = AnsibleModuleHelper('v1', 'project', debug=True, reset_logfile=False, **auth)

    k8s_obj = helper.create_project(metadata=models.V1ObjectMeta(name=name))
    assert k8s_obj is not None

    yield name

    helper.delete_object(name, None)

def _get_id(argvalue):
    op_type = ''
    if argvalue.get('create'):
        op_type = 'create'
    elif argvalue.get('patch'):
        op_type = 'patch'
    elif argvalue.get('remove'):
        op_type = 'remove'
    elif argvalue.get('replace'):
        op_type = 'replace'
    return op_type + '_' + argvalue[op_type]['name'] + '_' + "{:0>3}".format(argvalue['seq'])

def pytest_generate_tests(metafunc):
    _, api_version, resource = metafunc.module.__name__.split('_', 2)
    yaml_name = api_version + '_' + resource + '.yml'
    yaml_path = os.path.normpath(os.path.join(os.path.dirname(__file__),
                                              '../../openshift/ansiblegen/examples', yaml_name))
    if not os.path.exists(yaml_path):
        raise Exception("ansible_data: Unable to locate {}".format(yaml_path))
    with open(yaml_path, 'r') as f:
        data = yaml.load(f)
    seq = 0
    for task in data:
        seq += 1
        task['seq'] = seq

    if 'create_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('create')]
        metafunc.parametrize("create_tasks", tasks, False, _get_id)
    if 'patch_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('patch')]
        metafunc.parametrize("patch_tasks", tasks, False, _get_id)
    if 'remove_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('remove')]
        metafunc.parametrize("remove_tasks", tasks, False, _get_id)
    if 'replace_tasks' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('replace')]
        metafunc.parametrize("replace_tasks", tasks, False, _get_id)
    if 'namespaces' in metafunc.fixturenames:
        tasks = [x for x in data if x.get('create') and x['create'].get('namespace')]
        unique_namespaces = dict()
        for task in tasks:
            unique_namespaces[task['create']['namespace']] = None
        metafunc.parametrize("namespaces", list(unique_namespaces.keys()))
<|endoftext|>


<commit_before>import socket
import sys

def create_client_socket():
    client_socket = socket.socket(
        socket.AF_INET,
        socket.SOCK_STREAM,
        socket.IPPROTO_IP)
    return client_socket

def comm(client_socket):
    client_socket.connect(('127.0.0.1', 50000))
    # Send argv from command line
    client_socket.sendall(', '.join(sys.argv[1:]))
    client_socket.shutdown(socket.SHUT_WR)

def receive(client_socket):
    reception = client_socket.recv(32)
    client_socket.close()
    return reception

if __name__ == '__main__':
    client_socket = create_client_socket()
    comm(client_socket)
    print receive(client_socket)
<commit_msg>Updated comm().<commit_after>import socket
import sys

def create_client_socket():
    client_socket = socket.socket(
        socket.AF_INET,
        socket.SOCK_STREAM,
        socket.IPPROTO_IP)
    return client_socket

def comm(client_socket, message):
    client_socket.connect(('127.0.0.1', 50000))
    # Send argv from command line
    client_socket.sendall(', '.join(message))
    client_socket.shutdown(socket.SHUT_WR)

def receive(client_socket):
    reception = client_socket.recv(32)
    client_socket.close()
    return reception

if __name__ == '__main__':
    client_socket = create_client_socket()
    comm(client_socket, sys.argv[1:])
    print receive(client_socket)
<|endoftext|>


<commit_before>import json
import os
import base64
import sys
import traceback
import glob

driver_path = '.'
driver_name = glob.glob("*.json")[0].replace('.json', '')
print("Building: %s" % driver_name)

if os.path.isdir(driver_path):

	driver_data = {}

	try:
		driver_json_file = open('%(driver_path)s/%(driver_name)s.json' % {'driver_path':driver_path, 'driver_name':driver_name}, 'r')
		driver_json = driver_json_file.read()
		driver_data = json.loads(driver_json)
		driver_json_file.close()

	except:
		print('No JSON descriptor for %s!' % driver_name)
		exit()

	try:
		setup_code_file = open('%(driver_path)s/setup.c' % {'driver_path':driver_path}, 'r')
		setup_code = unicode(setup_code_file.read(), errors='replace')
		setup_code_file.close()

		driver_data['code']['setup'] = setup_code

	except:
		pass

	try:
		global_code_file = open('%(driver_path)s/global.c' % {'driver_path':driver_path}, 'r')
		global_code = unicode(global_code_file.read(), errors='replace')
		global_code_file.close()

		driver_data['code']['global'] = global_code

	except:
		pass

	try:
		connected_code_file = open('%(driver_path)s/connected.c' % {'driver_path':driver_path}, 'r')
		connected_code = unicode(connected_code_file.read(), errors='replace')
		connected_code_file.close()

		driver_data['code']['connected'] = connected_code

	except:
		pass

	try:
		interval_code_file = open('%(driver_path)s/interval.c' % {'driver_path':driver_path}, 'r')
		interval_code = unicode(interval_code_file.read(), errors='replace')
		interval_code_file.close()

		driver_data['code']['interval'] = interval_code;

	except:
		pass

	for element_name in driver_data.get('elements', {}):

		try:
			try:
				element_code_file = open('%(driver_path)s/element_code/%(element_name)s.c' % {'driver_path':driver_path, 'element_name':element_name}, 'r')
				element_code = unicode(element_code_file.read(), errors='replace')

				driver_data['elements'][element_name]['code'] = element_code

				try:
					icon_file = open('%(driver_path)s/element_code/%(element_name)s.png' % {'driver_path':driver_path, 'element_name':element_name}, 'r')
					driver_data['elements'][element_name]['icon'] = 'data:image/png;base64,' + icon_file.read().encode('base64')
					icon_file.close()

				except:
					print >> sys.stderr, "Unexpected error:", sys.exc_info()[0]
					traceback.print_exc(file=sys.stderr)

				element_code_file.close()

			except:
				pass

		except:
			print >> sys.stderr, "Unexpected error:", sys.exc_info()[0]
			traceback.print_exc(file=sys.stderr)

	driver_data['files'] = {}

	for platform_name in os.listdir('%(driver_path)s/driver' % {'driver_path':driver_path}):
		platform_path = '%(driver_path)s/driver/%(platform_name)s' % {'driver_path':driver_path, 'platform_name':platform_name}

		if os.path.isdir(platform_path):
			driver_data['files'][platform_name] = {}

			folder_types = ['objects', 'headers', 'other']

			for type_name in folder_types:
				driver_data['files'][platform_name][type_name] = {}

				try:
					for object_filename in os.listdir('%(platform_path)s/%(type_name)s' % {'platform_path':platform_path, 'type_name':type_name}):
						object_location = '%(platform_path)s/%(type_name)s/%(object_filename)s' % {'platform_path':platform_path, 'object_filename':object_filename, 'type_name':type_name}

						if os.path.isfile(object_location):
							object_file = open(object_location, 'r')
							driver_data['files'][platform_name][type_name][object_filename] = unicode(object_file.read(), errors='replace')
							object_file.close()

				except:
					pass

	output_eel_file = open('%s.eel' % driver_name, 'w')
	output_eel_file.write(json.dumps(driver_data, sort_keys=True, indent=4, separators=(',', ': ')))
	output_eel_file.close()

				<commit_msg>updated eel builder<commit_after>import json
import os
import base64
import sys
import traceback
import glob

driver_path = '.'
driver_name = glob.glob("*.json")[0].replace('.json', '')
print("Building: %s" % driver_name)

if os.path.isdir(driver_path):

	driver_data = {}

	try:
		driver_json_file = open('%(driver_path)s/%(driver_name)s.json' % {'driver_path':driver_path, 'driver_name':driver_name}, 'r')
		driver_json = driver_json_file.read()
		driver_data = json.loads(driver_json)
		driver_json_file.close()

	except:
		print('No JSON descriptor for %s!' % driver_name)
		exit()

	try:
		setup_code_file = open('%(driver_path)s/setup.c' % {'driver_path':driver_path}, 'r')
		setup_code = unicode(setup_code_file.read(), errors='replace')
		setup_code_file.close()

		driver_data['code']['setup'] = setup_code

	except:
		pass

	try:
		global_code_file = open('%(driver_path)s/global.c' % {'driver_path':driver_path}, 'r')
		global_code = unicode(global_code_file.read(), errors='replace')
		global_code_file.close()

		driver_data['code']['global'] = global_code

	except:
		pass

	try:
		connected_code_file = open('%(driver_path)s/connected.c' % {'driver_path':driver_path}, 'r')
		connected_code = unicode(connected_code_file.read(), errors='replace')
		connected_code_file.close()

		driver_data['code']['connected'] = connected_code

	except:
		pass

	try:
		interval_code_file = open('%(driver_path)s/interval.c' % {'driver_path':driver_path}, 'r')
		interval_code = unicode(interval_code_file.read(), errors='replace')
		interval_code_file.close()

		driver_data['code']['interval'] = interval_code;

	except:
		pass

	for element_name in driver_data.get('elements', {}):

		try:
			try:
				element_code_file = open('%(driver_path)s/element_code/%(element_name)s.c' % {'driver_path':driver_path, 'element_name':element_name}, 'r')
				element_code = unicode(element_code_file.read(), errors='replace')

				driver_data['elements'][element_name]['code'] = element_code

				try:
					icon_file = open('%(driver_path)s/element_code/%(element_name)s.png' % {'driver_path':driver_path, 'element_name':element_name}, 'rb')
					driver_data['elements'][element_name]['icon'] = 'data:image/png;base64,' + icon_file.read().encode('base64')
					icon_file.close()

				except:
					print >> sys.stderr, "Unexpected error:", sys.exc_info()[0]
					traceback.print_exc(file=sys.stderr)

				element_code_file.close()

			except:
				pass

		except:
			print >> sys.stderr, "Unexpected error:", sys.exc_info()[0]
			traceback.print_exc(file=sys.stderr)

	driver_data['files'] = {}

	for platform_name in os.listdir('%(driver_path)s/driver' % {'driver_path':driver_path}):
		platform_path = '%(driver_path)s/driver/%(platform_name)s' % {'driver_path':driver_path, 'platform_name':platform_name}

		if os.path.isdir(platform_path):
			driver_data['files'][platform_name] = {}

			folder_types = ['objects', 'headers', 'other']

			for type_name in folder_types:
				driver_data['files'][platform_name][type_name] = {}

				try:
					for object_filename in os.listdir('%(platform_path)s/%(type_name)s' % {'platform_path':platform_path, 'type_name':type_name}):
						object_location = '%(platform_path)s/%(type_name)s/%(object_filename)s' % {'platform_path':platform_path, 'object_filename':object_filename, 'type_name':type_name}

						if os.path.isfile(object_location):
							object_file = open(object_location, 'r')
							driver_data['files'][platform_name][type_name][object_filename] = unicode(object_file.read(), errors='replace')
							object_file.close()

				except:
					pass

	output_eel_file = open('%s.eel' % driver_name, 'w')
	output_eel_file.write(json.dumps(driver_data, sort_keys=True, indent=4, separators=(',', ': ')))
	output_eel_file.close()

				<|endoftext|>


<commit_before>#!/usr/bin/env python3.4dm
# -*- coding: utf-8 -*-

"""
minimalistic browser levering off of Python, PyQt and Webkit

Original: https://code.google.com/p/foobrowser/
davydm@gmail.com

  Copyright (c) 2012, Davyd McColl; 2013, 2014 Jaime Soffer

   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

   Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.

   Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

   Neither the name of the involved organizations nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   THE POSSIBILITY OF SUCH DAMAGE.

"""

# to keep some support of python2
import sip
sip.setapi('QString', 2)

from PyQt4.Qt import QApplication
from PyQt4.QtNetwork import QNetworkProxy

# local
from MainWin import MainWin

from options import OPTIONS

from sys import argv
from os.path import expanduser

import tldextract

def all_urls(tab_widget):
    """
    Prints all the urls held on a tab widget
    """
    for i in range(tab_widget.count()):
        print(tab_widget.widget(i).webkit.url().toString())

def extract_options(url):
    """ Given a site, decide if cookies are allowed, if only some sites
    will not be blocked, etc.

    """

    options = OPTIONS['general']

    if url is None:
        return options

    host = tldextract.extract(url).domain

    if host not in OPTIONS.keys():
        print("GENERAL")
    else:
        options = OPTIONS[host]
        print("INSTANCE: %s" % options['prefix'])

    return options

APP = QApplication([])
APP.setApplicationName("Eilat")
APP.setApplicationVersion("1.3.002")

CLIPBOARD = APP.clipboard()

def main():
    """ Catch the url (if any); then choose adequate defaults and build
    a browser window.

    """

    if len(argv) == 2:
        site = argv[1]
    else:
        site = None

    options = extract_options(site)

    if options['cookie_file'] is not None:
        options['cookie_file'] = (
            expanduser("~/.eilat/cookies/") + options['cookie_file'])

    # Proxy
    if options['use_proxy']:
        proxy = QNetworkProxy()
        proxy.setType(QNetworkProxy.HttpProxy)
        proxy.setHostName('localhost')
        proxy.setPort(3128)
        QNetworkProxy.setApplicationProxy(proxy)

    mainwin = MainWin(clipboard=CLIPBOARD, options=options, parent=None)

    if site is not None:
        mainwin.add_tab(site)
    else:
        mainwin.add_tab()

    mainwin.show()

if __name__ == "__main__":
    main()
    APP.exec_()
<commit_msg>slighly cleaner<commit_after>#!/usr/bin/env python3.4dm
# -*- coding: utf-8 -*-

"""
minimalistic browser levering off of Python, PyQt and Webkit

Original: https://code.google.com/p/foobrowser/
davydm@gmail.com

  Copyright (c) 2012, Davyd McColl; 2013, 2014 Jaime Soffer

   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

   Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.

   Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

   Neither the name of the involved organizations nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   THE POSSIBILITY OF SUCH DAMAGE.

"""

# to keep some support of python2
import sip
sip.setapi('QString', 2)

from PyQt4.Qt import QApplication
from PyQt4.QtNetwork import QNetworkProxy

# local
from MainWin import MainWin

from options import OPTIONS

from sys import argv
from os.path import expanduser

import tldextract

def all_urls(tab_widget):
    """
    Prints all the urls held on a tab widget
    """
    for i in range(tab_widget.count()):
        print(tab_widget.widget(i).webkit.url().toString())

def extract_options(url):
    """ Given a site, decide if cookies are allowed, if only some sites
    will not be blocked, etc.

    """

    options = OPTIONS['general']

    host = tldextract.extract(url).domain

    if url is None or host not in OPTIONS.keys():
        print("GENERAL")
    else:
        options = OPTIONS[host]
        print("INSTANCE: %s" % options['prefix'])

    return options

APP = QApplication([])
APP.setApplicationName("Eilat")
APP.setApplicationVersion("1.3.002")

CLIPBOARD = APP.clipboard()

def main():
    """ Catch the url (if any); then choose adequate defaults and build
    a browser window.

    """

    if len(argv) == 2:
        site = argv[1]
    else:
        site = None

    options = extract_options(site)

    if options['cookie_file'] is not None:
        options['cookie_file'] = (
            expanduser("~/.eilat/cookies/") + options['cookie_file'])

    # Proxy
    if options['use_proxy']:
        proxy = QNetworkProxy()
        proxy.setType(QNetworkProxy.HttpProxy)
        proxy.setHostName('localhost')
        proxy.setPort(3128)
        QNetworkProxy.setApplicationProxy(proxy)

    mainwin = MainWin(clipboard=CLIPBOARD, options=options, parent=None)

    if site is not None:
        mainwin.add_tab(site)
    else:
        mainwin.add_tab()

    mainwin.show()

if __name__ == "__main__":
    main()
    APP.exec_()
<|endoftext|>