<commit_before>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2013, The SAGA Project"
__license__   = "MIT"

import saga
import saga.utils.test_config as sutc

# ------------------------------------------------------------------------------
#
def _silent_cancel(job_obj):
    # try to cancel job but silently ignore all errors
    try:
        print "silent cancel for  %s (%s)" % (job_obj.id, job_obj.state)
        job_obj.cancel()
        print "silent cancel done %s (%s)" % (job_obj.id, job_obj.state)
    except Exception:
        pass

# ------------------------------------------------------------------------------
#
def _silent_close_js(js_obj):
    # try to cancel job but silently ignore all errors
    try:
        js_obj.close()
        js_obj.close()
    except Exception:
        pass

# ------------------------------------------------------------------------------
#
def test_close():
    """ Test job service close()
    """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)
        js.close()
        js.get_url()
        assert False, "Subsequent calls should fail after close()"

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException:
        assert True

# ------------------------------------------------------------------------------
#
def test_open_close():
    """ Test job service create / close() in a big loop
    """
    try:
        tc = sutc.TestConfig()

        for i in range(0, 100):
            js = saga.job.Service(tc.js_url, tc.session)
            js.close()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_get_url():
    """ Test job service url/get_url()
    """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)
        assert str(js.get_url()) == str(tc.js_url)
        assert str(js.url) == str(tc.js_url)

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_list_jobs():
    """ Test if a submitted job shows up in Service.list() """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)

        # create job service and job
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)

        # run job - now it has an id, and js must know it
        j.run()
        all_jobs = js.list()
        assert j.id in all_jobs, \
            "%s not in %s" % (j.id, all_jobs)

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_cancel(j)
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_run_job():
    """ Test to submit a job via run_job, and retrieve id """
    try:
        print "test js.run_job 0"
        tc = sutc.TestConfig()
        print "test js.run_job 1"
        js = saga.job.Service(tc.js_url, tc.session)
        print "test js.run_job 2"

        # create job service and job
        j = js.run_job("/bin/sleep 10")
        print "test js.run_job 3 %s (%s)" % (j.id, j.state)
        assert j.id
        print "test js.run_job 4 %s (%s)" % (j.id, j.state)

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_get_job():
    """ Test to submit a job, and retrieve it by id """
    try:
        print "test get job 0"
        tc = sutc.TestConfig()
        print "test get job 1"
        js = saga.job.Service(tc.js_url, tc.session)

        # create job service and job
        print "test get job 2"
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        print "test get job 3 %s" % jd
        jd._attributes_dump ()
        j = js.create_job(jd)

        # run job - now it has an id, and js must be able to retrieve it by id
        print "test get job 3"
        j.run()
        print "test get job 4 %s" % j.id
        j_clone = js.get_job(j.id)
        print "test get job 5 %s" % j_clone.id
        assert j.id in j_clone.id

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_cancel(j)
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def helper_multiple_services(i):
    print "helper multiple services 0"
    tc = sutc.TestConfig()
    print "helper multiple services 1"
    js = saga.job.Service(tc.js_url, tc.session)
    print "helper multiple services 2"
    jd = saga.job.Description()
    print "helper multiple services 3"
    jd.executable = '/bin/sleep'
    print "helper multiple services 4"
    jd.arguments = ['10']
    print "helper multiple services 5"
    jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)
    print "helper multiple services 6"
    j = js.create_job(jd)
    print "helper multiple services 7"
    j.run()
    print "helper multiple services 8"
    assert (j.state in [saga.job.RUNNING, saga.job.PENDING]), "job submission failed"
    print "helper multiple services 9"
    _silent_cancel(j)
    print "helper multiple services 10"
    _silent_close_js(js)
    print "helper multiple services 11"

# ------------------------------------------------------------------------------
#
def test_multiple_services():
    """ Test to create multiple job service instances  (this test might take a while) """
    try:
        tc = sutc.TestConfig()
        for i in range(0, 20):
            print " ------------------------ %s" % i
            helper_multiple_services(i)
            print " ------------------------ %s ok" % i

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni

    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
<commit_msg>trigger tests<commit_after>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2013, The SAGA Project"
__license__   = "MIT"

import saga
import saga.utils.test_config as sutc

# ------------------------------------------------------------------------------
#
def _silent_cancel(job_obj):
    # try to cancel job but silently ignore all errors
    try:
        print "silent cancel for  %s (%s)" % (job_obj.id, job_obj.state)
        job_obj.cancel()
        print "silent cancel done %s (%s)" % (job_obj.id, job_obj.state)
    except Exception:
        pass

# ------------------------------------------------------------------------------
#
def _silent_close_js(js_obj):
    # try to cancel job but silently ignore all errors
    try:
        js_obj.close()
        js_obj.close()
    except Exception:
        pass

# ------------------------------------------------------------------------------
#
def test_close():
    """ Test job service close()
    """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)
        js.close()
        js.get_url()
        assert False, "Subsequent calls should fail after close()"

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException:
        assert True

# ------------------------------------------------------------------------------
#
def test_open_close():
    """ Test job service create / close() in a big loop
    """
    try:
        tc = sutc.TestConfig()

        for i in range(0, 100):
            js = saga.job.Service(tc.js_url, tc.session)
            js.close()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_get_url():
    """ Test job service url/get_url()
    """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)
        assert str(js.get_url()) == str(tc.js_url)
        assert str(js.url) == str(tc.js_url)

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_list_jobs():
    """ Test if a submitted job shows up in Service.list() """
    try:
        tc = sutc.TestConfig()
        js = saga.job.Service(tc.js_url, tc.session)

        # create job service and job
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)

        # run job - now it has an id, and js must know it
        j.run()
        all_jobs = js.list()
        assert j.id in all_jobs, \
            "%s not in %s" % (j.id, all_jobs)

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_cancel(j)
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_run_job():
    """ Test to submit a job via run_job, and retrieve id"""
    try:
        print "test js.run_job 0"
        tc = sutc.TestConfig()
        print "test js.run_job 1"
        js = saga.job.Service(tc.js_url, tc.session)
        print "test js.run_job 2"

        # create job service and job
        j = js.run_job("/bin/sleep 10")
        print "test js.run_job 3 %s (%s)" % (j.id, j.state)
        assert j.id
        print "test js.run_job 4 %s (%s)" % (j.id, j.state)

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def test_get_job():
    """ Test to submit a job, and retrieve it by id """
    try:
        print "test get job 0"
        tc = sutc.TestConfig()
        print "test get job 1"
        js = saga.job.Service(tc.js_url, tc.session)

        # create job service and job
        print "test get job 2"
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        print "test get job 3 %s" % jd
        jd._attributes_dump ()
        j = js.create_job(jd)

        # run job - now it has an id, and js must be able to retrieve it by id
        print "test get job 3"
        j.run()
        print "test get job 4 %s" % j.id
        j_clone = js.get_job(j.id)
        print "test get job 5 %s" % j_clone.id
        assert j.id in j_clone.id

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
    finally:
        _silent_cancel(j)
        _silent_close_js(js)

# ------------------------------------------------------------------------------
#
def helper_multiple_services(i):
    print "helper multiple services 0"
    tc = sutc.TestConfig()
    print "helper multiple services 1"
    js = saga.job.Service(tc.js_url, tc.session)
    print "helper multiple services 2"
    jd = saga.job.Description()
    print "helper multiple services 3"
    jd.executable = '/bin/sleep'
    print "helper multiple services 4"
    jd.arguments = ['10']
    print "helper multiple services 5"
    jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)
    print "helper multiple services 6"
    j = js.create_job(jd)
    print "helper multiple services 7"
    j.run()
    print "helper multiple services 8"
    assert (j.state in [saga.job.RUNNING, saga.job.PENDING]), "job submission failed"
    print "helper multiple services 9"
    _silent_cancel(j)
    print "helper multiple services 10"
    _silent_close_js(js)
    print "helper multiple services 11"

# ------------------------------------------------------------------------------
#
def test_multiple_services():
    """ Test to create multiple job service instances  (this test might take a while) """
    try:
        tc = sutc.TestConfig()
        for i in range(0, 20):
            print " ------------------------ %s" % i
            helper_multiple_services(i)
            print " ------------------------ %s ok" % i

    except saga.NotImplemented as ni:
        assert tc.notimpl_warn_only, "%s " % ni
        if tc.notimpl_warn_only:
            print "%s " % ni

    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Backscatter.io Module."""
import json
try:
    from backscatter import Backscatter
except ImportError:
    print("Backscatter.io library not installed.")

misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
moduleinfo = {'version': '1', 'author': 'brandon@backscatter.io',
              'description': 'Backscatter.io module to bring mass-scanning observations into MISP.',
              'module-type': ['expansion', 'hover']}
moduleconfig = ['api_key']
query_playbook = [
    {'inputs': ['ip-src', 'ip-dst'],
     'services': ['observations', 'enrichment'],
     'name': 'generic'}
]

def check_query(request):
    """Check the incoming request for a valid configuration."""
    output = {'success': False}
    config = request.get('config', None)
    if not config:
        misperrors['error'] = "Configuration is missing from the request."
        return output
    for item in moduleconfig:
        if config.get(item, None):
            continue
        misperrors['error'] = "Backscatter.io authentication is missing."
        return output
    if not request.get('ip-src') and request.get('ip-dst'):
        misperrors['error'] = "Unsupported attributes type."
        return output
    profile = {'success': True, 'config': config, 'playbook': 'generic'}
    if 'ip-src' in request:
        profile.update({'value': request.get('ip-src')})
    else:
        profile.update({'value': request.get('ip-dst')})
    return profile

def handler(q=False):
    """Handle gathering data."""
    if not q:
        return q
    request = json.loads(q)
    checks = check_query(request)
    if not checks['success']:
        return misperrors

    output = {'results': list()}

    try:
        bs = Backscatter(checks['config']['api_key'])
        response = bs.get_observations(query=output['value'], query_type='ip')
        if not response['success']:
            misperrors['error'] = '%s: %s' % (response['error'], response['message'])
            return misperrors
        r = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]}
    except Exception, e:
        misperrors['error'] = str(e)
        return misperrors

    return output

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

<commit_msg>Use the write var on return<commit_after># -*- coding: utf-8 -*-
"""Backscatter.io Module."""
import json
try:
    from backscatter import Backscatter
except ImportError:
    print("Backscatter.io library not installed.")

misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
moduleinfo = {'version': '1', 'author': 'brandon@backscatter.io',
              'description': 'Backscatter.io module to bring mass-scanning observations into MISP.',
              'module-type': ['expansion', 'hover']}
moduleconfig = ['api_key']
query_playbook = [
    {'inputs': ['ip-src', 'ip-dst'],
     'services': ['observations', 'enrichment'],
     'name': 'generic'}
]

def check_query(request):
    """Check the incoming request for a valid configuration."""
    output = {'success': False}
    config = request.get('config', None)
    if not config:
        misperrors['error'] = "Configuration is missing from the request."
        return output
    for item in moduleconfig:
        if config.get(item, None):
            continue
        misperrors['error'] = "Backscatter.io authentication is missing."
        return output
    if not request.get('ip-src') and request.get('ip-dst'):
        misperrors['error'] = "Unsupported attributes type."
        return output
    profile = {'success': True, 'config': config, 'playbook': 'generic'}
    if 'ip-src' in request:
        profile.update({'value': request.get('ip-src')})
    else:
        profile.update({'value': request.get('ip-dst')})
    return profile

def handler(q=False):
    """Handle gathering data."""
    if not q:
        return q
    request = json.loads(q)
    checks = check_query(request)
    if not checks['success']:
        return misperrors

    try:
        bs = Backscatter(checks['config']['api_key'])
        response = bs.get_observations(query=output['value'], query_type='ip')
        if not response['success']:
            misperrors['error'] = '%s: %s' % (response['error'], response['message'])
            return misperrors
        output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]}
    except Exception, e:
        misperrors['error'] = str(e)
        return misperrors

    return output

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

<|endoftext|>


<commit_before>__previous_settings_module = ''
__overriden_settings = {}

def establish_settings():
    '''Sets up the Django settings (in lieu of DJANGO_SETTINGS_MODULE)'''
    defaults = dict(
        INSTALLED_APPS = ['baseboard', ],
        CACHE_BACKEND = "dummy://",
        DATABASE_ENGINE = "sqlite3",
        DATABASE_NAME = ":memory:",
        TEST_DATABASE_CHARSET = 'utf8',
        TEST_DATABASE_COLLATION = 'utf8_unicode_ci',
    )

    import os
    __previous_settings_module = os.environ['DJANGO_SETTINGS_MODULE']
    os.environ['DJANGO_SETTINGS_MODULE'] = ''

    from django.conf import settings
    try:
        settings.configure(**defaults)
    except RuntimeError:
        for key, value in defaults.items():
            __overriden_settings[key] = value
            setattr(settings, key, value)

def restore_settings():
    import os
    os.environ['DJANGO_SETTINGS_MODULE'] =  __previous_settings_module

    from django.conf import settings
    for key, value in __overriden_settings.items():
        setattr(settings, key, value)

def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]):
    '''Sets up the environment and runs tests for django-baseboard.'''
    establish_settings()

    from django.test.simple import run_tests
    failures = run_tests(test_labels, verbosity=verbosity, interactive=interactive, extra_tests=extra_tests)
    if failures:
        sys.exit(failures)

    restore_settings()

if __name__ == "__main__":
    run_tests(["baseboard"], 0)
<commit_msg>Fixed missing import.<commit_after>import sys

__previous_settings_module = ''
__overriden_settings = {}

def establish_settings():
    '''Sets up the Django settings (in lieu of DJANGO_SETTINGS_MODULE)'''
    defaults = dict(
        INSTALLED_APPS = ['baseboard', ],
        CACHE_BACKEND = "dummy://",
        DATABASE_ENGINE = "sqlite3",
        DATABASE_NAME = ":memory:",
        TEST_DATABASE_CHARSET = 'utf8',
        TEST_DATABASE_COLLATION = 'utf8_unicode_ci',
    )

    import os
    __previous_settings_module = os.environ['DJANGO_SETTINGS_MODULE']
    os.environ['DJANGO_SETTINGS_MODULE'] = ''

    from django.conf import settings
    try:
        settings.configure(**defaults)
    except RuntimeError:
        for key, value in defaults.items():
            __overriden_settings[key] = value
            setattr(settings, key, value)

def restore_settings():
    import os
    os.environ['DJANGO_SETTINGS_MODULE'] =  __previous_settings_module

    from django.conf import settings
    for key, value in __overriden_settings.items():
        setattr(settings, key, value)

def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]):
    '''Sets up the environment and runs tests for django-baseboard.'''
    establish_settings()

    from django.test.simple import run_tests
    failures = run_tests(test_labels, verbosity=verbosity, interactive=interactive, extra_tests=extra_tests)
    if failures:
        sys.exit(failures)

    restore_settings()

if __name__ == "__main__":
    run_tests(["baseboard"], 0)
<|endoftext|>


<commit_before>
import os
import time
import signal
import saga
import saga.utils.pty_shell   as sups
import saga.utils.test_config as sutc

# ------------------------------------------------------------------------------
#
def test_ptyshell_ok () :
    """ Test pty_shell which runs command successfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)
    assert (out == txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_nok () :
    """ Test pty_shell which runs command unsuccessfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\" ; false" % txt)
    assert (ret == 1)
    assert (out == txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_async () :
    """ Test pty_shell which runs command successfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____\n"
    shell.run_async ("cat <<EOT")

    shell.send (txt)
    shell.send ('EOT\n')

    ret, out = shell.find_prompt ()

    assert (ret == 0)
    assert (out == "%s" % txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_prompt () :
    """ Test pty_shell with prompt change """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)
    assert (out == txt)

    shell.run_sync ('export PS1="HALLO-(\\$?)-PROMPT>"',
                     new_prompt='HALLO-\((\d)\)-PROMPT>')

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)
    assert (out == txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_file_stage () :
    """ Test pty_shell file staging """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    shell.write_to_file (txt, "/tmp/saga-test-staging")
    out = shell.read_from_file ("/tmp/saga-test-staging")

    assert (txt == out)

    ret, out, _ = shell.run_sync ("rm /tmp/saga-test-staging")
    assert (ret == 0)
    assert (out == "")

<commit_msg>add more verbosity to pty shell tests<commit_after>
import os
import time
import signal
import saga
import saga.utils.pty_shell   as sups
import saga.utils.test_config as sutc

# ------------------------------------------------------------------------------
#
def test_ptyshell_ok () :
    """ Test pty_shell which runs command successfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)    # "%s" % repr(ret)
    assert (out == txt)  # "%s == %s" % (repr(out), repr(txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_nok () :
    """ Test pty_shell which runs command unsuccessfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\" ; false" % txt)
    assert (ret == 1)    # "%s" % repr(ret)
    assert (out == txt)  # "%s == %s" % (repr(out), repr(txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_async () :
    """ Test pty_shell which runs command successfully """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____\n"
    shell.run_async ("cat <<EOT")

    shell.send (txt)
    shell.send ('EOT\n')

    ret, out = shell.find_prompt ()

    assert (ret == 0)   # "%s" % repr(ret)
    assert (out == txt) # "%s == %s" % (repr(out), repr(txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_prompt () :
    """ Test pty_shell with prompt change """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)    # "%s" % repr(ret)
    assert (out == txt)  # "%s == %s" % (repr(out), repr(txt)

    shell.run_sync ('export PS1="HALLO-(\\$?)-PROMPT>"',
                     new_prompt='HALLO-\((\d)\)-PROMPT>')

    txt = "______1______2_____3_____"
    ret, out, _ = shell.run_sync ("printf \"%s\"" % txt)
    assert (ret == 0)    # "%s" % repr(ret)
    assert (out == txt)  # "%s == %s" % (repr(out), repr(txt)

    assert (shell.alive ())
    shell.run_async ("exit")
    time.sleep (1)
    assert (not shell.alive ())

# ------------------------------------------------------------------------------
#
def test_ptyshell_file_stage () :
    """ Test pty_shell file staging """
    conf  = sutc.TestConfig()
    shell = sups.PTYShell (saga.Url(conf.js_url), conf.session)

    txt = "______1______2_____3_____"
    shell.write_to_file (txt, "/tmp/saga-test-staging")
    out = shell.read_from_file ("/tmp/saga-test-staging")

    assert (txt == out)# "%s == %s" % (repr(out), repr(txt)

    ret, out, _ = shell.run_sync ("rm /tmp/saga-test-staging")
    assert (ret == 0)    # "%s" % repr(ret)
    assert (out == "")   # "%s == ''" % (repr(out))

<|endoftext|>


<commit_before>import json
import logging
import sys
import time

from dnstrails import APIError
from dnstrails import DnsTrails

log = logging.getLogger('dnstrails')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['hostname', 'domain', 'ip-src', 'ip-dst'],
    'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email',
               'whois-registrant-email', 'whois-registrant-phone',
               'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'domain']
}

moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on securitytrails.com',
              'module-type': ['expansion', 'hover']}

# config fields that your code expects from the site admin
moduleconfig = ['apikey']

def handler(q=False):
    if q:

        request = json.loads(q)

        if not request.get('config') and not (request['config'].get('apikey')):
            misperrors['error'] = 'DNS authentication is missing'
            return misperrors

        api = DnsTrails(request['config'].get('apikey'))

        if not api:
            misperrors['error'] = 'Onyphe Error instance api'

        ip = ""
        dns_name = ""

        ip = ''
        if request.get('ip-src'):
            ip = request['ip-src']
            return handle_ip(api, ip, misperrors)
        elif request.get('ip-dst'):
            ip = request['ip-dst']
            return handle_ip(api, ip, misperrors)
        elif request.get('domain'):
            domain = request['domain']
            return handle_domain(api, domain, misperrors)
        elif request.get('hostname'):
            hostname = request['hostname']
            return handle_domain(api, hostname, misperrors)
        else:
            misperrors['error'] = "Unsupported attributes types"
            return misperrors
    else:
        return False

def handle_domain(api, domain, misperrors):
    result_filtered = {"results": []}

    r, status_ok = expand_domain_info(api, misperrors, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error DNS result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_subdomains(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error subdomains result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_whois(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error whois result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_history_ipv4_ipv6(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error history ipv4'
        return misperrors

    time.sleep(1)

    r, status_ok = expand_history_dns(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors[
                                  'error'] + ' Error in expand History DNS'
        return misperrors

    r, status_ok = expand_history_whois(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + \
                              ' Error in expand History Whois'
        return misperrors

    return result_filtered

def handle_ip(api, ip, misperrors):
    result_filtered = {"results": []}

    r, status_ok = expand_searching_domain(api, ip)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] += ' Error in expand searching domain'
        return misperrors

    return result_filtered

def expand_domain_info(api, misperror, domain):
    r = []
    status_ok = False
    ns_servers = []
    list_ipv4 = []
    list_ipv6 = []
    servers_mx = []
    soa_hostnames = []

    results = api.domain(domain)

    if results:
        status_ok = True
        if 'current_dns' in results:
            if 'values' in results['current_dns']['ns']:
                ns_servers = [ns_entry['nameserver'] for ns_entry in
                              results['current_dns']['ns']['values']
                              if 'nameserver' in ns_entry]
            if 'values' in results['current_dns']['a']:
                list_ipv4 = [a_entry['ip'] for a_entry in
                             results['current_dns']['a']['values'] if
                             'ip' in a_entry]

            if 'values' in results['current_dns']['aaaa']:
                list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in
                             results['current_dns']['aaaa']['values'] if
                             'ipv6' in ipv6_entry]

            if 'values' in results['current_dns']['mx']:
                servers_mx = [mx_entry['hostname'] for mx_entry in
                              results['current_dns']['mx']['values'] if
                              'hostname' in mx_entry]
            if 'values' in results['current_dns']['soa']:
                soa_hostnames = [soa_entry['email'] for soa_entry in
                                 results['current_dns']['soa']['values'] if
                                 'email' in soa_entry]

        if ns_servers:
            r.append({'types': ['domain'],
                      'values': ns_servers,
                      'categories': ['Network activity'],
                      'comment': 'List of name servers  of %s first seen %s ' %
                                 (domain,
                                  results['current_dns']['ns']['first_seen'])
                      })

        if list_ipv4:
            r.append({'types': ['domain|ip'],
                      'values': ['%s|%s' % (domain, ipv4) for ipv4 in
                                 list_ipv4],
                      'categories': ['Network activity'],

                      'comment': ' List ipv4 of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['a']['first_seen'])

                      })
        if list_ipv6:
            r.append({'types': ['domain|ip'],
                      'values': ['%s|%s' % (domain, ipv6) for ipv6 in
                                 list_ipv6],
                      'categories': ['Network activity'],
                      'comment': ' List ipv6 of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['aaaa']['first_seen'])

                      })

        if servers_mx:
            r.append({'types': ['domain'],
                      'values': servers_mx,
                      'categories': ['Network activity'],
                      'comment': ' List mx of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['mx']['first_seen'])

                      })
        if soa_hostnames:
            r.append({'types': ['domain'],
                      'values': soa_hostnames,
                      'categories': ['Network activity'],
                      'comment': ' List soa of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['soa']['first_seen'])
                      })

    return r, status_ok

def expand_subdomains(api, domain):
    r = []
    status_ok = False

    try:
        results = api.subdomains(domain)

        if results:
            status_ok = True
            if 'subdomains' in results:
                r.append({
                    'types': ['domain'],
                    'values': ['%s.%s' % (sub, domain)
                               for sub in results['subdomains']],
                    'categories': ['Network activity'],
                    'comment': 'subdomains of %s' % domain
                }

                )
    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_whois(api, domain):
    r = []
    status_ok = False

    try:
        results = api.whois(domain)

        if results:
            status_ok = True
            item_registrant = __select_registrant_item(results)
            if item_registrant:

                if 'email' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-email'],
                            'values': [item_registrant[0]['email']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'telephone' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-phone'],
                            'values': [item_registrant[0]['telephone']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'name' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-name'],
                            'values': [item_registrant[0]['name']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'registrarName' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrar'],
                            'values': [item_registrant[0]['registrarName']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'createdDate' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-creation-date'],
                            'values': [item_registrant[0]['createdDate']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_history_ipv4_ipv6(api, domain):
    r = []
    status_ok = False

    try:
        results = api.history_dns_ipv4(domain)

        if results:
            status_ok = True
            r.extend(__history_ip(results, domain))

        time.sleep(1)
        results = api.history_dns_aaaa(domain)

        if results:
            status_ok = True
            r.extend(__history_ip(results, domain, type_ip='ipv6'))

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_history_dns(api, domain):
    r = []
    status_ok = False

    try:

        results = api.history_dns_ns(domain)
        if results:
            r.extend(__history_dns(results, domain, 'nameserver', 'ns'))

        time.sleep(1)

        results = api.history_dns_soa(domain)

        if results:
            r.extend(__history_dns(results, domain, 'email', 'soa'))

        time.sleep(1)

        results = api.history_dns_mx(domain)

        if results:
            status_ok = True
            r.extend(__history_dns(results, domain, 'host', 'mx'))

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    status_ok = True

    return r, status_ok

def expand_history_whois(api, domain):
    r = []
    status_ok = False
    try:
        results = api.history_whois(domain)

        if results:

            if 'items' in results['result']:
                for item in results['result']['items']:
                    item_registrant = __select_registrant_item(item)

                    r.append(
                        {
                            'types': ['domain'],
                            'values': item['nameServers'],
                            'categories': ['Network activity'],
                            'comment': 'Whois history Name Servers of %s '
                                       'Status: %s ' % (
                                           domain, ' '.join(item['status']))

                        }
                    )
                    if item_registrant:

                        if 'email' in item_registrant[0]:
                            r.append(
                                {
                                    'types': ['whois-registrant-email'],
                                    'values': [item_registrant[0]['email']],
                                    'categories': ['Attribution'],
                                    'comment': 'Whois history registrant email of %s'
                                               'Status: %s' % (
                                                   domain,
                                                   ' '.join(item['status']))
                                }
                            )

                        if 'telephone' in item_registrant[0]:
                            r.append(
                                {
                                    'types': ['whois-registrant-phone'],
                                    'values': [item_registrant[0]['telephone']],
                                    'categories': ['Attribution'],
                                    'comment': 'Whois history registrant phone of %s'
                                               'Status: %s' % (
                                                   domain,
                                                   ' '.join(item['status']))
                                }
                            )

    except APIError as e:
        misperrors['error'] = e.value
        return [], False
    status_ok = True

    return r, status_ok

def __history_ip(results, domain, type_ip='ip'):
    r = []
    if 'records' in results:
        for record in results['records']:
            if 'values' in record:
                for item in record['values']:
                    r.append(
                        {'types': ['domain|ip'],
                         'values': ['%s|%s' % (domain, item[type_ip])],
                         'categories': ['Network activity'],
                         'comment': 'History IP on securitytrails %s '
                                    'last seen: %s first seen: %s' %
                                    (domain, record['last_seen'],
                                     record['first_seen'])
                         }
                    )

    return r

def __history_dns(results, domain, type_serv, service):
    r = []

    if 'records' in results:
        for record in results['records']:
            if 'values' in record:
                values = record['values']
                if type(values) is list:

                    for item in record['values']:
                        r.append(
                            {'types': ['domain|ip'],
                             'values': [item[type_serv]],
                             'categories': ['Network activity'],
                             'comment': 'history %s of %s last seen: %s first seen: %s' %
                                        (service, domain, record['last_seen'],
                                         record['first_seen'])
                             }
                        )
                else:
                    r.append(
                        {'types': ['domain|ip'],
                         'values': [values[type_serv]],
                         'categories': ['Network activity'],
                         'comment': 'history %s of %s last seen: %s first seen: %s' %
                                    (service, domain, record['last_seen'],
                                     record['first_seen'])
                         }
                    )
    return r

def expand_searching_domain(api, ip):
    r = []
    status_ok = False

    try:
        results = api.searching_domains(ipv4=ip)

        if results:
            if 'records' in results:
                res = [(r['host_provider'], r['hostname'], r['whois'])
                       for r in results['records']]

                for host_provider, hostname, whois in res:
                    comment = 'domain for %s by %s' % (ip, host_provider[0])
                    if whois['registrar']:
                        comment = comment + ' registrar %s' % whois['registrar']

                    r.append(
                        {
                            'types': ['domain'],
                            'category': ['Network activity'],
                            'values': [hostname],
                            'comment': comment

                        }
                    )
        status_ok = True
    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def __select_registrant_item(entry):
    res = None
    if 'contacts' in entry:
        res = list(filter(lambda x: x['type'] == 'registrant',
                          entry['contacts']))

    if 'contact' in entry:
        res = list(filter(lambda x: x['type'] == 'registrant',
                          entry['contact']))

    return res
<commit_msg>fix: Cleaned up not used variables<commit_after>import json
import logging
import sys
import time

from dnstrails import APIError
from dnstrails import DnsTrails

log = logging.getLogger('dnstrails')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

misperrors = {'error': 'Error'}
mispattributes = {
    'input': ['hostname', 'domain', 'ip-src', 'ip-dst'],
    'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email',
               'whois-registrant-email', 'whois-registrant-phone',
               'whois-registrant-name',
               'whois-registrar', 'whois-creation-date', 'domain']
}

moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on securitytrails.com',
              'module-type': ['expansion', 'hover']}

# config fields that your code expects from the site admin
moduleconfig = ['apikey']

def handler(q=False):
    if q:

        request = json.loads(q)

        if not request.get('config') and not (request['config'].get('apikey')):
            misperrors['error'] = 'DNS authentication is missing'
            return misperrors

        api = DnsTrails(request['config'].get('apikey'))

        if not api:
            misperrors['error'] = 'Onyphe Error instance api'
        if request.get('ip-src'):
            ip = request['ip-src']
            return handle_ip(api, ip, misperrors)
        elif request.get('ip-dst'):
            ip = request['ip-dst']
            return handle_ip(api, ip, misperrors)
        elif request.get('domain'):
            domain = request['domain']
            return handle_domain(api, domain, misperrors)
        elif request.get('hostname'):
            hostname = request['hostname']
            return handle_domain(api, hostname, misperrors)
        else:
            misperrors['error'] = "Unsupported attributes types"
            return misperrors
    else:
        return False

def handle_domain(api, domain, misperrors):
    result_filtered = {"results": []}

    r, status_ok = expand_domain_info(api, misperrors, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error DNS result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_subdomains(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error subdomains result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_whois(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error whois result'
        return misperrors

    time.sleep(1)
    r, status_ok = expand_history_ipv4_ipv6(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + ' Error history ipv4'
        return misperrors

    time.sleep(1)

    r, status_ok = expand_history_dns(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors[
                                  'error'] + ' Error in expand History DNS'
        return misperrors

    r, status_ok = expand_history_whois(api, domain)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] = misperrors['error'] + \
                              ' Error in expand History Whois'
        return misperrors

    return result_filtered

def handle_ip(api, ip, misperrors):
    result_filtered = {"results": []}

    r, status_ok = expand_searching_domain(api, ip)

    if status_ok:
        if r:
            result_filtered['results'].extend(r)
    else:
        misperrors['error'] += ' Error in expand searching domain'
        return misperrors

    return result_filtered

def expand_domain_info(api, misperror, domain):
    r = []
    status_ok = False
    ns_servers = []
    list_ipv4 = []
    list_ipv6 = []
    servers_mx = []
    soa_hostnames = []

    results = api.domain(domain)

    if results:
        status_ok = True
        if 'current_dns' in results:
            if 'values' in results['current_dns']['ns']:
                ns_servers = [ns_entry['nameserver'] for ns_entry in
                              results['current_dns']['ns']['values']
                              if 'nameserver' in ns_entry]
            if 'values' in results['current_dns']['a']:
                list_ipv4 = [a_entry['ip'] for a_entry in
                             results['current_dns']['a']['values'] if
                             'ip' in a_entry]

            if 'values' in results['current_dns']['aaaa']:
                list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in
                             results['current_dns']['aaaa']['values'] if
                             'ipv6' in ipv6_entry]

            if 'values' in results['current_dns']['mx']:
                servers_mx = [mx_entry['hostname'] for mx_entry in
                              results['current_dns']['mx']['values'] if
                              'hostname' in mx_entry]
            if 'values' in results['current_dns']['soa']:
                soa_hostnames = [soa_entry['email'] for soa_entry in
                                 results['current_dns']['soa']['values'] if
                                 'email' in soa_entry]

        if ns_servers:
            r.append({'types': ['domain'],
                      'values': ns_servers,
                      'categories': ['Network activity'],
                      'comment': 'List of name servers  of %s first seen %s ' %
                                 (domain,
                                  results['current_dns']['ns']['first_seen'])
                      })

        if list_ipv4:
            r.append({'types': ['domain|ip'],
                      'values': ['%s|%s' % (domain, ipv4) for ipv4 in
                                 list_ipv4],
                      'categories': ['Network activity'],

                      'comment': ' List ipv4 of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['a']['first_seen'])

                      })
        if list_ipv6:
            r.append({'types': ['domain|ip'],
                      'values': ['%s|%s' % (domain, ipv6) for ipv6 in
                                 list_ipv6],
                      'categories': ['Network activity'],
                      'comment': ' List ipv6 of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['aaaa']['first_seen'])

                      })

        if servers_mx:
            r.append({'types': ['domain'],
                      'values': servers_mx,
                      'categories': ['Network activity'],
                      'comment': ' List mx of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['mx']['first_seen'])

                      })
        if soa_hostnames:
            r.append({'types': ['domain'],
                      'values': soa_hostnames,
                      'categories': ['Network activity'],
                      'comment': ' List soa of %s first seen %s' %
                                 (domain,
                                  results['current_dns']['soa']['first_seen'])
                      })

    return r, status_ok

def expand_subdomains(api, domain):
    r = []
    status_ok = False

    try:
        results = api.subdomains(domain)

        if results:
            status_ok = True
            if 'subdomains' in results:
                r.append({
                    'types': ['domain'],
                    'values': ['%s.%s' % (sub, domain)
                               for sub in results['subdomains']],
                    'categories': ['Network activity'],
                    'comment': 'subdomains of %s' % domain
                }

                )
    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_whois(api, domain):
    r = []
    status_ok = False

    try:
        results = api.whois(domain)

        if results:
            status_ok = True
            item_registrant = __select_registrant_item(results)
            if item_registrant:

                if 'email' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-email'],
                            'values': [item_registrant[0]['email']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'telephone' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-phone'],
                            'values': [item_registrant[0]['telephone']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'name' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrant-name'],
                            'values': [item_registrant[0]['name']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'registrarName' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-registrar'],
                            'values': [item_registrant[0]['registrarName']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

                if 'createdDate' in item_registrant[0]:
                    r.append(
                        {
                            'types': ['whois-creation-date'],
                            'values': [item_registrant[0]['createdDate']],
                            'categories': ['Attribution'],
                            'comment': 'Whois information of %s by securitytrails'
                                       % domain
                        }
                    )

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_history_ipv4_ipv6(api, domain):
    r = []
    status_ok = False

    try:
        results = api.history_dns_ipv4(domain)

        if results:
            status_ok = True
            r.extend(__history_ip(results, domain))

        time.sleep(1)
        results = api.history_dns_aaaa(domain)

        if results:
            status_ok = True
            r.extend(__history_ip(results, domain, type_ip='ipv6'))

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def expand_history_dns(api, domain):
    r = []
    status_ok = False

    try:

        results = api.history_dns_ns(domain)
        if results:
            r.extend(__history_dns(results, domain, 'nameserver', 'ns'))

        time.sleep(1)

        results = api.history_dns_soa(domain)

        if results:
            r.extend(__history_dns(results, domain, 'email', 'soa'))

        time.sleep(1)

        results = api.history_dns_mx(domain)

        if results:
            status_ok = True
            r.extend(__history_dns(results, domain, 'host', 'mx'))

    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    status_ok = True

    return r, status_ok

def expand_history_whois(api, domain):
    r = []
    status_ok = False
    try:
        results = api.history_whois(domain)

        if results:

            if 'items' in results['result']:
                for item in results['result']['items']:
                    item_registrant = __select_registrant_item(item)

                    r.append(
                        {
                            'types': ['domain'],
                            'values': item['nameServers'],
                            'categories': ['Network activity'],
                            'comment': 'Whois history Name Servers of %s '
                                       'Status: %s ' % (
                                           domain, ' '.join(item['status']))

                        }
                    )
                    if item_registrant:

                        if 'email' in item_registrant[0]:
                            r.append(
                                {
                                    'types': ['whois-registrant-email'],
                                    'values': [item_registrant[0]['email']],
                                    'categories': ['Attribution'],
                                    'comment': 'Whois history registrant email of %s'
                                               'Status: %s' % (
                                                   domain,
                                                   ' '.join(item['status']))
                                }
                            )

                        if 'telephone' in item_registrant[0]:
                            r.append(
                                {
                                    'types': ['whois-registrant-phone'],
                                    'values': [item_registrant[0]['telephone']],
                                    'categories': ['Attribution'],
                                    'comment': 'Whois history registrant phone of %s'
                                               'Status: %s' % (
                                                   domain,
                                                   ' '.join(item['status']))
                                }
                            )

    except APIError as e:
        misperrors['error'] = e.value
        return [], False
    status_ok = True

    return r, status_ok

def __history_ip(results, domain, type_ip='ip'):
    r = []
    if 'records' in results:
        for record in results['records']:
            if 'values' in record:
                for item in record['values']:
                    r.append(
                        {'types': ['domain|ip'],
                         'values': ['%s|%s' % (domain, item[type_ip])],
                         'categories': ['Network activity'],
                         'comment': 'History IP on securitytrails %s '
                                    'last seen: %s first seen: %s' %
                                    (domain, record['last_seen'],
                                     record['first_seen'])
                         }
                    )

    return r

def __history_dns(results, domain, type_serv, service):
    r = []

    if 'records' in results:
        for record in results['records']:
            if 'values' in record:
                values = record['values']
                if type(values) is list:

                    for item in record['values']:
                        r.append(
                            {'types': ['domain|ip'],
                             'values': [item[type_serv]],
                             'categories': ['Network activity'],
                             'comment': 'history %s of %s last seen: %s first seen: %s' %
                                        (service, domain, record['last_seen'],
                                         record['first_seen'])
                             }
                        )
                else:
                    r.append(
                        {'types': ['domain|ip'],
                         'values': [values[type_serv]],
                         'categories': ['Network activity'],
                         'comment': 'history %s of %s last seen: %s first seen: %s' %
                                    (service, domain, record['last_seen'],
                                     record['first_seen'])
                         }
                    )
    return r

def expand_searching_domain(api, ip):
    r = []
    status_ok = False

    try:
        results = api.searching_domains(ipv4=ip)

        if results:
            if 'records' in results:
                res = [(r['host_provider'], r['hostname'], r['whois'])
                       for r in results['records']]

                for host_provider, hostname, whois in res:
                    comment = 'domain for %s by %s' % (ip, host_provider[0])
                    if whois['registrar']:
                        comment = comment + ' registrar %s' % whois['registrar']

                    r.append(
                        {
                            'types': ['domain'],
                            'category': ['Network activity'],
                            'values': [hostname],
                            'comment': comment

                        }
                    )
        status_ok = True
    except APIError as e:
        misperrors['error'] = e.value
        return [], False

    return r, status_ok

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def __select_registrant_item(entry):
    res = None
    if 'contacts' in entry:
        res = list(filter(lambda x: x['type'] == 'registrant',
                          entry['contacts']))

    if 'contact' in entry:
        res = list(filter(lambda x: x['type'] == 'registrant',
                          entry['contact']))

    return res
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os

from django.conf import settings

TESTS_DIR = os.path.join(os.path.dirname(__file__),
                         'src', 'django_markup', 'tests')

SETTINGS = {
    'DATABASES': {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'dev.db',
        },
    },
    'INSTALLED_APPS': [
        'django_markup',
    ],
    'TEMPLATE_DIRS': [
        os.path.join(TESTS_DIR, 'templates')
    ],
}
def runtests(*test_args):
    # Setup settings
    if not settings.configured:
        settings.configure(**SETTINGS)

    # New Django 1.7 app registry setup
    try:
        from django import setup
        setup()
    except ImportError:
        pass

    # New Django 1.8 test runner
    try:
        from django.test.runner import DiscoverRunner as TestRunner
    except ImportError:
        from django.test.simple import DjangoTestSuiteRunner as TestRunner

    test_runner = TestRunner(verbosity=1)
    failures = test_runner.run_tests(['django_markup'])
    if failures:
        sys.exit(failures)

if __name__ == '__main__':
    runtests(*sys.argv[1:])
<commit_msg>Cleanup<commit_after>#!/usr/bin/env python
import sys
import os

from django.conf import settings
from django import setup

TESTS_DIR = os.path.join(os.path.dirname(__file__),
                         'src', 'django_markup', 'tests')

SETTINGS = {
    'DATABASES': {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'dev.db',
        },
    },
    'INSTALLED_APPS': [
        'django_markup',
    ],
    'TEMPLATE_DIRS': [
        os.path.join(TESTS_DIR, 'templates')
    ],
}

def runtests(*test_args):
    # Setup settings
    if not settings.configured:
        settings.configure(**SETTINGS)

    setup()

    # New Django 1.8 test runner
    try:
        from django.test.runner import DiscoverRunner as TestRunner
    except ImportError:
        from django.test.simple import DjangoTestSuiteRunner as TestRunner

    test_runner = TestRunner(verbosity=1)
    failures = test_runner.run_tests(['django_markup'])
    if failures:
        sys.exit(failures)

if __name__ == '__main__':
    runtests(*sys.argv[1:])
<|endoftext|>


<commit_before># Copyright 2022 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX example_diff executor."""
import os
from typing import Any, Dict, List

from absl import logging
import apache_beam as beam
import tensorflow as tf
from tensorflow_data_validation.skew import feature_skew_detector
from tfx import types
from tfx.components.util import tfxio_utils
from tfx.dsl.components.base import base_beam_executor
from tfx.proto import example_diff_pb2
from tfx.types import artifact_utils
from tfx.types import standard_component_specs
from tfx.utils import io_utils
from tfx.utils import json_utils
from tfx_bsl.tfxio import record_based_tfxio

_STATS_FILE_NAME = 'skew_stats'
_SAMPLE_FILE_NAME = 'sample_pairs'

_TELEMETRY_DESCRIPTORS = ['ExampleDiff']

class _IncludedSplitPairs(object):
  """Checks includedness of split pairs."""

  def __init__(self, include_split_pairs: List[List[str]]):
    if include_split_pairs is None:
      self._include_split_pairs = None
    else:
      self._include_split_pairs = set(
          (test, base) for test, base in include_split_pairs)

  def included(self, test_split: str, base_split: str) -> bool:
    if self._include_split_pairs is None:
      return True
    return (test_split, base_split) in self._include_split_pairs

def _parse_example(serialized: bytes):
  # TODO(b/227361696): Validate that data are examples.
  ex = tf.train.Example()
  ex.ParseFromString(serialized)
  return ex

def _config_to_kwargs(config: example_diff_pb2.ExampleDiffConfig):
  """Convert ExampleDiffConfig to DetectFeatureSkewImpl kwargs."""
  kwargs = {}
  if not config.HasField('paired_example_skew'):
    raise ValueError('ExampleDiffConfig missing required paired_example_skew.')
  kwargs['identifier_features'] = list(
      config.paired_example_skew.identifier_features)
  kwargs['features_to_ignore'] = list(
      config.paired_example_skew.ignore_features)
  kwargs['sample_size'] = config.paired_example_skew.skew_sample_size
  kwargs['float_round_ndigits'] = config.paired_example_skew.float_round_ndigits
  kwargs[
      'allow_duplicate_identifiers'] = config.paired_example_skew.allow_duplicate_identifiers
  return kwargs

class Executor(base_beam_executor.BaseBeamExecutor):
  """Computes example level diffs. See TFDV feature_skew_detector.py."""

  def Do(self, input_dict: Dict[str, List[types.Artifact]],
         output_dict: Dict[str, List[types.Artifact]],
         exec_properties: Dict[str, Any]) -> None:
    """Computes example diffs for each split pair.

    Args:
      input_dict: Input dict from input key to a list of Artifacts.
      output_dict: Output dict from output key to a list of Artifacts.
      exec_properties: A dict of execution properties.

    Raises:
      ValueError: If examples are in a non- record-based format.

    Returns:
      None
    """
    self._log_startup(input_dict, output_dict, exec_properties)

    # Load and deserialize included split pairs from execution properties.
    included_split_pairs = _IncludedSplitPairs(
        json_utils.loads(
            exec_properties.get(
                standard_component_specs.INCLUDE_SPLIT_PAIRS_KEY, 'null')) or
        None)

    test_examples = artifact_utils.get_single_instance(
        input_dict[standard_component_specs.EXAMPLES_KEY])
    base_examples = artifact_utils.get_single_instance(
        input_dict[standard_component_specs.BASELINE_EXAMPLES_KEY])

    example_diff_artifact = artifact_utils.get_single_instance(
        output_dict[standard_component_specs.EXAMPLE_DIFF_RESULT_KEY])
    diff_config = exec_properties.get(
        standard_component_specs.EXAMPLE_DIFF_CONFIG_KEY)

    logging.info('Running examplediff with config %s', diff_config)

    test_tfxio_factory = tfxio_utils.get_tfxio_factory_from_artifact(
        examples=[test_examples], telemetry_descriptors=_TELEMETRY_DESCRIPTORS)
    base_tfxio_factory = tfxio_utils.get_tfxio_factory_from_artifact(
        examples=[base_examples], telemetry_descriptors=_TELEMETRY_DESCRIPTORS)

    # First set up the pairs we'll operate on.
    split_pairs = []
    for test_split in artifact_utils.decode_split_names(
        test_examples.split_names):
      for base_split in artifact_utils.decode_split_names(
          base_examples.split_names):
        if included_split_pairs.included(test_split, base_split):
          split_pairs.append((test_split, base_split))

    with self._make_beam_pipeline() as p:
      for test_split, base_split in split_pairs:
        test_uri = artifact_utils.get_split_uri([test_examples], test_split)
        base_uri = artifact_utils.get_split_uri([base_examples], base_split)
        test_tfxio = test_tfxio_factory(io_utils.all_files_pattern(test_uri))
        base_tfxio = base_tfxio_factory(io_utils.all_files_pattern(base_uri))
        if not isinstance(
            test_tfxio, record_based_tfxio.RecordBasedTFXIO) or not isinstance(
                base_tfxio, record_based_tfxio.RecordBasedTFXIO):
          # TODO(b/227361696): Support more general sources.
          raise ValueError('Only RecordBasedTFXIO supported, got %s, %s' %
                           (test_tfxio, base_tfxio))

        split_pair = '%s_%s' % (test_split, base_split)
        logging.info('Processing split pair %s', split_pair)
        # pylint: disable=cell-var-from-loop
        @beam.ptransform_fn
        def _iteration(p):
          base_examples = (
              p | 'TFXIORead[base]' >> test_tfxio.RawRecordBeamSource()
              | 'Parse[base]' >> beam.Map(_parse_example))
          test_examples = (
              p | 'TFXIORead[test]' >> base_tfxio.RawRecordBeamSource()
              | 'Parse[test]' >> beam.Map(_parse_example))
          skew_stats, samples = ((base_examples, test_examples)
                                 | feature_skew_detector.DetectFeatureSkewImpl(
                                     **_config_to_kwargs(diff_config)))
          output_uri = os.path.join(example_diff_artifact.uri,
                                    'SplitPair-%s' % split_pair)

          _ = (
              skew_stats
              | 'WriteStats' >> feature_skew_detector.skew_results_sink(
                  os.path.join(output_uri, _STATS_FILE_NAME)))
          _ = (
              samples | 'WriteSample' >> feature_skew_detector.skew_pair_sink(
                  os.path.join(output_uri, _SAMPLE_FILE_NAME)))
          # pylint: enable=cell-var-from-loop

        _ = p | 'ProcessSplits[%s]' % split_pair >> _iteration()  # pylint: disable=no-value-for-parameter
<commit_msg>nop<commit_after># Copyright 2022 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX example_diff executor."""
import os
from typing import Any, Dict, List

from absl import logging
import apache_beam as beam
import tensorflow as tf
from tensorflow_data_validation.skew import feature_skew_detector
from tfx import types
from tfx.components.util import tfxio_utils
from tfx.dsl.components.base import base_beam_executor
from tfx.proto import example_diff_pb2
from tfx.types import artifact_utils
from tfx.types import standard_component_specs
from tfx.utils import io_utils
from tfx.utils import json_utils
from tfx_bsl.tfxio import record_based_tfxio

STATS_FILE_NAME = 'skew_stats'
_SAMPLE_FILE_NAME = 'sample_pairs'

_TELEMETRY_DESCRIPTORS = ['ExampleDiff']

class _IncludedSplitPairs(object):
  """Checks includedness of split pairs."""

  def __init__(self, include_split_pairs: List[List[str]]):
    if include_split_pairs is None:
      self._include_split_pairs = None
    else:
      self._include_split_pairs = set(
          (test, base) for test, base in include_split_pairs)

  def included(self, test_split: str, base_split: str) -> bool:
    if self._include_split_pairs is None:
      return True
    return (test_split, base_split) in self._include_split_pairs

def _parse_example(serialized: bytes):
  # TODO(b/227361696): Validate that data are examples.
  ex = tf.train.Example()
  ex.ParseFromString(serialized)
  return ex

def _config_to_kwargs(config: example_diff_pb2.ExampleDiffConfig):
  """Convert ExampleDiffConfig to DetectFeatureSkewImpl kwargs."""
  kwargs = {}
  if not config.HasField('paired_example_skew'):
    raise ValueError('ExampleDiffConfig missing required paired_example_skew.')
  kwargs['identifier_features'] = list(
      config.paired_example_skew.identifier_features)
  kwargs['features_to_ignore'] = list(
      config.paired_example_skew.ignore_features)
  kwargs['sample_size'] = config.paired_example_skew.skew_sample_size
  kwargs['float_round_ndigits'] = config.paired_example_skew.float_round_ndigits
  kwargs[
      'allow_duplicate_identifiers'] = config.paired_example_skew.allow_duplicate_identifiers
  return kwargs

class Executor(base_beam_executor.BaseBeamExecutor):
  """Computes example level diffs. See TFDV feature_skew_detector.py."""

  def Do(self, input_dict: Dict[str, List[types.Artifact]],
         output_dict: Dict[str, List[types.Artifact]],
         exec_properties: Dict[str, Any]) -> None:
    """Computes example diffs for each split pair.

    Args:
      input_dict: Input dict from input key to a list of Artifacts.
      output_dict: Output dict from output key to a list of Artifacts.
      exec_properties: A dict of execution properties.

    Raises:
      ValueError: If examples are in a non- record-based format.

    Returns:
      None
    """
    self._log_startup(input_dict, output_dict, exec_properties)

    # Load and deserialize included split pairs from execution properties.
    included_split_pairs = _IncludedSplitPairs(
        json_utils.loads(
            exec_properties.get(
                standard_component_specs.INCLUDE_SPLIT_PAIRS_KEY, 'null')) or
        None)

    test_examples = artifact_utils.get_single_instance(
        input_dict[standard_component_specs.EXAMPLES_KEY])
    base_examples = artifact_utils.get_single_instance(
        input_dict[standard_component_specs.BASELINE_EXAMPLES_KEY])

    example_diff_artifact = artifact_utils.get_single_instance(
        output_dict[standard_component_specs.EXAMPLE_DIFF_RESULT_KEY])
    diff_config = exec_properties.get(
        standard_component_specs.EXAMPLE_DIFF_CONFIG_KEY)

    logging.info('Running examplediff with config %s', diff_config)

    test_tfxio_factory = tfxio_utils.get_tfxio_factory_from_artifact(
        examples=[test_examples], telemetry_descriptors=_TELEMETRY_DESCRIPTORS)
    base_tfxio_factory = tfxio_utils.get_tfxio_factory_from_artifact(
        examples=[base_examples], telemetry_descriptors=_TELEMETRY_DESCRIPTORS)

    # First set up the pairs we'll operate on.
    split_pairs = []
    for test_split in artifact_utils.decode_split_names(
        test_examples.split_names):
      for base_split in artifact_utils.decode_split_names(
          base_examples.split_names):
        if included_split_pairs.included(test_split, base_split):
          split_pairs.append((test_split, base_split))

    with self._make_beam_pipeline() as p:
      for test_split, base_split in split_pairs:
        test_uri = artifact_utils.get_split_uri([test_examples], test_split)
        base_uri = artifact_utils.get_split_uri([base_examples], base_split)
        test_tfxio = test_tfxio_factory(io_utils.all_files_pattern(test_uri))
        base_tfxio = base_tfxio_factory(io_utils.all_files_pattern(base_uri))
        if not isinstance(
            test_tfxio, record_based_tfxio.RecordBasedTFXIO) or not isinstance(
                base_tfxio, record_based_tfxio.RecordBasedTFXIO):
          # TODO(b/227361696): Support more general sources.
          raise ValueError('Only RecordBasedTFXIO supported, got %s, %s' %
                           (test_tfxio, base_tfxio))

        split_pair = '%s_%s' % (test_split, base_split)
        logging.info('Processing split pair %s', split_pair)
        # pylint: disable=cell-var-from-loop
        @beam.ptransform_fn
        def _iteration(p):
          base_examples = (
              p | 'TFXIORead[base]' >> test_tfxio.RawRecordBeamSource()
              | 'Parse[base]' >> beam.Map(_parse_example))
          test_examples = (
              p | 'TFXIORead[test]' >> base_tfxio.RawRecordBeamSource()
              | 'Parse[test]' >> beam.Map(_parse_example))
          skew_stats, samples = ((base_examples, test_examples)
                                 | feature_skew_detector.DetectFeatureSkewImpl(
                                     **_config_to_kwargs(diff_config)))
          output_uri = os.path.join(example_diff_artifact.uri,
                                    'SplitPair-%s' % split_pair)

          _ = (
              skew_stats
              | 'WriteStats' >> feature_skew_detector.skew_results_sink(
                  os.path.join(output_uri, STATS_FILE_NAME)))
          _ = (
              samples | 'WriteSample' >> feature_skew_detector.skew_pair_sink(
                  os.path.join(output_uri, _SAMPLE_FILE_NAME)))
          # pylint: enable=cell-var-from-loop

        _ = p | 'ProcessSplits[%s]' % split_pair >> _iteration()  # pylint: disable=no-value-for-parameter
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import django
from django.conf import settings, global_settings as default_settings
from django.core.management import execute_from_command_line
from os import path

# Give feedback on used versions
sys.stderr.write('Using Python version {0} from {1}\n'.format(sys.version[:5], sys.executable))
sys.stderr.write('Using Django version {0} from {1}\n'.format(
    django.get_version(),
    path.dirname(path.abspath(django.__file__)))
)

if not settings.configured:
    module_root = path.dirname(path.realpath(__file__))
    sys.path.insert(0, path.join(module_root, 'example'))

    settings.configure(
        DEBUG = False,  # will be False anyway by DjangoTestRunner.
        DATABASES = {
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
                'NAME': ':memory:'
            }
        },
        CACHES = {
            # By explicit since many tests also need the caching support
            'default': {
                'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
                'LOCATION': 'unique-snowflake',
            }
        },
        INSTALLED_APPS = (
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.sites',
            'django.contrib.admin',
            'django.contrib.sessions',
            'parler',
            'parler.tests.testapp',
            'article',
            'theme1',
        ),
        TEMPLATES = [
            {
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': (),
                'OPTIONS': {
                    'loaders': (
                        'django.template.loaders.filesystem.Loader',
                        'django.template.loaders.app_directories.Loader',
                    ),
                    'context_processors': (
                        'django.template.context_processors.debug',
                        'django.template.context_processors.i18n',
                        'django.template.context_processors.media',
                        'django.template.context_processors.request',
                        'django.template.context_processors.static',
                        'django.contrib.auth.context_processors.auth',
                    ),
                },
            },
        ],
        MIDDLEWARE = (
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.locale.LocaleMiddleware',  # / will be redirected to /<locale>/
        ),
        ROOT_URLCONF = 'example.urls',
        TEST_RUNNER = 'django.test.runner.DiscoverRunner',

        SITE_ID = 4,
        LANGUAGE_CODE = 'en',
        LANGUAGES = (
            ('nl', 'Dutch'),
            ('de', 'German'),
            ('en', 'English'),
            ('fr', 'French'),
        ),
        PARLER_LANGUAGES = {
            4: (
                {'code': 'nl'},
                {'code': 'de'},
                {'code': 'en'},
            ),
            'default': {
                'fallbacks': ['en'],
            },
        }
    )

DEFAULT_TEST_APPS = [
    'parler',
    'article',
]

def runtests():
    other_args = list(filter(lambda arg: arg.startswith('-'), sys.argv[1:]))
    test_apps = list(filter(lambda arg: not arg.startswith('-'), sys.argv[1:])) or DEFAULT_TEST_APPS
    argv = sys.argv[:1] + ['test', '--traceback'] + other_args + test_apps
    execute_from_command_line(argv)

if __name__ == '__main__':
    runtests()
<commit_msg>Fix runtest.py for Django 2.2<commit_after>#!/usr/bin/env python
import sys
import django
from django.conf import settings, global_settings as default_settings
from django.core.management import execute_from_command_line
from os import path

# Give feedback on used versions
sys.stderr.write('Using Python version {0} from {1}\n'.format(sys.version[:5], sys.executable))
sys.stderr.write('Using Django version {0} from {1}\n'.format(
    django.get_version(),
    path.dirname(path.abspath(django.__file__)))
)

if not settings.configured:
    module_root = path.dirname(path.realpath(__file__))
    sys.path.insert(0, path.join(module_root, 'example'))

    settings.configure(
        DEBUG = False,  # will be False anyway by DjangoTestRunner.
        DATABASES = {
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
                'NAME': ':memory:'
            }
        },
        CACHES = {
            # By explicit since many tests also need the caching support
            'default': {
                'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
                'LOCATION': 'unique-snowflake',
            }
        },
        INSTALLED_APPS = (
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.messages',
            'django.contrib.sessions',
            'django.contrib.sites',
            'django.contrib.admin',
            'parler',
            'parler.tests.testapp',
            'article',
            'theme1',
        ),
        TEMPLATES = [
            {
                'BACKEND': 'django.template.backends.django.DjangoTemplates',
                'DIRS': (),
                'OPTIONS': {
                    'loaders': (
                        'django.template.loaders.filesystem.Loader',
                        'django.template.loaders.app_directories.Loader',
                    ),
                    'context_processors': (
                        'django.template.context_processors.debug',
                        'django.template.context_processors.i18n',
                        'django.template.context_processors.media',
                        'django.template.context_processors.request',
                        'django.template.context_processors.static',
                        'django.contrib.messages.context_processors.messages',
                        'django.contrib.auth.context_processors.auth',
                    ),
                },
            },
        ],
        MIDDLEWARE = (
            'django.middleware.common.CommonMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.locale.LocaleMiddleware',  # / will be redirected to /<locale>/
        ),
        ROOT_URLCONF = 'example.urls',
        TEST_RUNNER = 'django.test.runner.DiscoverRunner',

        SITE_ID = 4,
        LANGUAGE_CODE = 'en',
        LANGUAGES = (
            ('nl', 'Dutch'),
            ('de', 'German'),
            ('en', 'English'),
            ('fr', 'French'),
        ),
        PARLER_LANGUAGES = {
            4: (
                {'code': 'nl'},
                {'code': 'de'},
                {'code': 'en'},
            ),
            'default': {
                'fallbacks': ['en'],
            },
        }
    )

DEFAULT_TEST_APPS = [
    'parler',
    'article',
]

def runtests():
    other_args = list(filter(lambda arg: arg.startswith('-'), sys.argv[1:]))
    test_apps = list(filter(lambda arg: not arg.startswith('-'), sys.argv[1:])) or DEFAULT_TEST_APPS
    argv = sys.argv[:1] + ['test', '--traceback'] + other_args + test_apps
    execute_from_command_line(argv)

if __name__ == '__main__':
    runtests()
<|endoftext|>