<commit_before>"""Pings utilities for Zinnia"""
import socket
import threading
from logging import getLogger
try:
    from urllib.request import urlopen
    from urllib.parse import urlsplit
    from xmlrpc.client import Error
    from xmlrpc.client import ServerProxy
except ImportError:  # Python 2
    from urllib2 import urlopen
    from urlparse import urlsplit
    from xmlrpclib import Error
    from xmlrpclib import ServerProxy

from bs4 import BeautifulSoup

from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse

from zinnia.flags import PINGBACK
from zinnia.settings import PROTOCOL

class URLRessources(object):
    """Object defining the ressources of the website"""

    def __init__(self):
        self.current_site = Site.objects.get_current()
        self.site_url = '%s://%s' % (PROTOCOL, self.current_site.domain)
        self.blog_url = '%s%s' % (self.site_url,
                                  reverse('zinnia_entry_archive_index'))
        self.blog_feed = '%s%s' % (self.site_url,
                                   reverse('zinnia_entry_latest_feed'))

class DirectoryPinger(threading.Thread):
    """Threaded Directory Pinger"""

    def __init__(self, server_name, entries, timeout=10, start_now=True):
        self.results = []
        self.timeout = timeout
        self.entries = entries
        self.server_name = server_name
        self.server = ServerProxy(self.server_name)
        self.ressources = URLRessources()

        threading.Thread.__init__(self)
        if start_now:
            self.start()

    def run(self):
        """Ping entries to a Directory in a Thread"""
        logger = getLogger('zinnia.ping.directory')
        socket.setdefaulttimeout(self.timeout)
        for entry in self.entries:
            reply = self.ping_entry(entry)
            self.results.append(reply)
            logger.info('%s : %s' % (self.server_name, reply['message']))
        socket.setdefaulttimeout(None)

    def ping_entry(self, entry):
        """Ping an entry to a Directory"""
        entry_url = '%s%s' % (self.ressources.site_url,
                              entry.get_absolute_url())
        categories = '|'.join([c.title for c in entry.categories.all()])

        try:
            reply = self.server.weblogUpdates.extendedPing(
                self.ressources.current_site.name,
                self.ressources.blog_url, entry_url,
                self.ressources.blog_feed, categories)
        except Exception:
            try:
                reply = self.server.weblogUpdates.ping(
                    self.ressources.current_site.name,
                    self.ressources.blog_url, entry_url,
                    categories)
            except Exception:
                reply = {'message': '%s is an invalid directory.' %
                         self.server_name,
                         'flerror': True}
        return reply

class ExternalUrlsPinger(threading.Thread):
    """Threaded ExternalUrls Pinger"""

    def __init__(self, entry, timeout=10, start_now=True):
        self.results = []
        self.entry = entry
        self.timeout = timeout
        self.ressources = URLRessources()
        self.entry_url = '%s%s' % (self.ressources.site_url,
                                   self.entry.get_absolute_url())

        threading.Thread.__init__(self)
        if start_now:
            self.start()

    def run(self):
        """Ping external URLS in a Thread"""
        logger = getLogger('zinnia.ping.external_urls')
        socket.setdefaulttimeout(self.timeout)

        external_urls = self.find_external_urls(self.entry)
        external_urls_pingable = self.find_pingback_urls(external_urls)

        for url, server_name in external_urls_pingable.items():
            reply = self.pingback_url(server_name, url)
            self.results.append(reply)
            logger.info('%s : %s' % (url, reply))

        socket.setdefaulttimeout(None)

    def is_external_url(self, url, site_url):
        """Check of the url in an external url"""
        url_splitted = urlsplit(url)
        if not url_splitted.netloc:
            return False
        return url_splitted.netloc != urlsplit(site_url).netloc

    def find_external_urls(self, entry):
        """Find external urls in an entry"""
        soup = BeautifulSoup(entry.html_content)
        external_urls = [a['href'] for a in soup.find_all('a')
                         if self.is_external_url(
                             a['href'], self.ressources.site_url)]
        return external_urls

    def find_pingback_href(self, content):
        """Try to find Link markup to pingback url"""
        soup = BeautifulSoup(content)
        for link in soup.find_all('link'):
            dict_attr = dict(link.attrs)
            if 'rel' in dict_attr and 'href' in dict_attr:
                for rel_type in dict_attr['rel']:
                    if rel_type.lower() == PINGBACK:
                        return dict_attr.get('href')

    def find_pingback_urls(self, urls):
        """Find the pingback urls of each urls"""
        pingback_urls = {}

        for url in urls:
            try:
                page = urlopen(url)
                headers = page.info()

                if 'text/' not in headers.get('Content-Type', '').lower():
                    continue

                server_url = headers.get('X-Pingback')
                if not server_url:
                    server_url = self.find_pingback_href(page.read())

                if server_url:
                    server_url_splitted = urlsplit(server_url)
                    if not server_url_splitted.netloc:
                        url_splitted = urlsplit(url)
                        server_url = '%s://%s%s' % (url_splitted.scheme,
                                                    url_splitted.netloc,
                                                    server_url)
                    pingback_urls[url] = server_url
            except IOError:
                pass
        return pingback_urls

    def pingback_url(self, server_name, target_url):
        """Do a pingback call for the target url"""
        try:
            server = ServerProxy(server_name)
            reply = server.pingback.ping(self.entry_url, target_url)
        except (Error, socket.error):
            reply = '%s cannot be pinged.' % target_url
        return reply
<commit_msg>Multilines docstrings for zinnia.ping<commit_after>"""Pings utilities for Zinnia"""
import socket
import threading
from logging import getLogger
try:
    from urllib.request import urlopen
    from urllib.parse import urlsplit
    from xmlrpc.client import Error
    from xmlrpc.client import ServerProxy
except ImportError:  # Python 2
    from urllib2 import urlopen
    from urlparse import urlsplit
    from xmlrpclib import Error
    from xmlrpclib import ServerProxy

from bs4 import BeautifulSoup

from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse

from zinnia.flags import PINGBACK
from zinnia.settings import PROTOCOL

class URLRessources(object):
    """
    Object defining the ressources of the Website.
    """

    def __init__(self):
        self.current_site = Site.objects.get_current()
        self.site_url = '%s://%s' % (PROTOCOL, self.current_site.domain)
        self.blog_url = '%s%s' % (self.site_url,
                                  reverse('zinnia_entry_archive_index'))
        self.blog_feed = '%s%s' % (self.site_url,
                                   reverse('zinnia_entry_latest_feed'))

class DirectoryPinger(threading.Thread):
    """
    Threaded web directory pinger.
    """

    def __init__(self, server_name, entries, timeout=10, start_now=True):
        self.results = []
        self.timeout = timeout
        self.entries = entries
        self.server_name = server_name
        self.server = ServerProxy(self.server_name)
        self.ressources = URLRessources()

        threading.Thread.__init__(self)
        if start_now:
            self.start()

    def run(self):
        """
        Ping entries to a directory in a thread.
        """
        logger = getLogger('zinnia.ping.directory')
        socket.setdefaulttimeout(self.timeout)
        for entry in self.entries:
            reply = self.ping_entry(entry)
            self.results.append(reply)
            logger.info('%s : %s' % (self.server_name, reply['message']))
        socket.setdefaulttimeout(None)

    def ping_entry(self, entry):
        """
        Ping an entry to a directory.
        """
        entry_url = '%s%s' % (self.ressources.site_url,
                              entry.get_absolute_url())
        categories = '|'.join([c.title for c in entry.categories.all()])

        try:
            reply = self.server.weblogUpdates.extendedPing(
                self.ressources.current_site.name,
                self.ressources.blog_url, entry_url,
                self.ressources.blog_feed, categories)
        except Exception:
            try:
                reply = self.server.weblogUpdates.ping(
                    self.ressources.current_site.name,
                    self.ressources.blog_url, entry_url,
                    categories)
            except Exception:
                reply = {'message': '%s is an invalid directory.' %
                         self.server_name,
                         'flerror': True}
        return reply

class ExternalUrlsPinger(threading.Thread):
    """
    Threaded external URLs pinger.
    """

    def __init__(self, entry, timeout=10, start_now=True):
        self.results = []
        self.entry = entry
        self.timeout = timeout
        self.ressources = URLRessources()
        self.entry_url = '%s%s' % (self.ressources.site_url,
                                   self.entry.get_absolute_url())

        threading.Thread.__init__(self)
        if start_now:
            self.start()

    def run(self):
        """
        Ping external URLs in a Thread.
        """
        logger = getLogger('zinnia.ping.external_urls')
        socket.setdefaulttimeout(self.timeout)

        external_urls = self.find_external_urls(self.entry)
        external_urls_pingable = self.find_pingback_urls(external_urls)

        for url, server_name in external_urls_pingable.items():
            reply = self.pingback_url(server_name, url)
            self.results.append(reply)
            logger.info('%s : %s' % (url, reply))

        socket.setdefaulttimeout(None)

    def is_external_url(self, url, site_url):
        """
        Check if the URL is an external URL.
        """
        url_splitted = urlsplit(url)
        if not url_splitted.netloc:
            return False
        return url_splitted.netloc != urlsplit(site_url).netloc

    def find_external_urls(self, entry):
        """
        Find external URLs in an entry.
        """
        soup = BeautifulSoup(entry.html_content)
        external_urls = [a['href'] for a in soup.find_all('a')
                         if self.is_external_url(
                             a['href'], self.ressources.site_url)]
        return external_urls

    def find_pingback_href(self, content):
        """
        Try to find LINK markups to pingback URL.
        """
        soup = BeautifulSoup(content)
        for link in soup.find_all('link'):
            dict_attr = dict(link.attrs)
            if 'rel' in dict_attr and 'href' in dict_attr:
                for rel_type in dict_attr['rel']:
                    if rel_type.lower() == PINGBACK:
                        return dict_attr.get('href')

    def find_pingback_urls(self, urls):
        """
        Find the pingback URL for each URLs.
        """
        pingback_urls = {}

        for url in urls:
            try:
                page = urlopen(url)
                headers = page.info()

                if 'text/' not in headers.get('Content-Type', '').lower():
                    continue

                server_url = headers.get('X-Pingback')
                if not server_url:
                    server_url = self.find_pingback_href(page.read())

                if server_url:
                    server_url_splitted = urlsplit(server_url)
                    if not server_url_splitted.netloc:
                        url_splitted = urlsplit(url)
                        server_url = '%s://%s%s' % (url_splitted.scheme,
                                                    url_splitted.netloc,
                                                    server_url)
                    pingback_urls[url] = server_url
            except IOError:
                pass
        return pingback_urls

    def pingback_url(self, server_name, target_url):
        """
        Do a pingback call for the target URL.
        """
        try:
            server = ServerProxy(server_name)
            reply = server.pingback.ping(self.entry_url, target_url)
        except (Error, socket.error):
            reply = '%s cannot be pinged.' % target_url
        return reply
<|endoftext|>


<commit_before>
try:
	import tempfile
	from lib.core.core import Core,InitDirFile
	from lib.core.config_parser import ConfigParser
except ImportError, err:
	from lib.core.core import Core
	Core.print_error(err)

class WebScan(InitDirFile):

	def __init__(self, args):

		self._scan_options = ConfigParser.get_screen_ports(args.config_file)
		core_options = "-n -Pn -T5 --open -p T:{0}".format(self._scan_options) if self._scan_options else "-n -Pn -T5 --open -p T:80,443"

		self._result_file = tempfile.NamedTemporaryFile(mode='w+t')
		InitDirFile.__init__(self, [Core._commands_path["phantomjs"], Core._commands_path["nmap"], args.rasterize], args, "screen")

		if args.destination:
			__destination =  " ".join([ip.strip() for ip in args.destination.split(",")])
			self._nmap_options = "{0} {1} -oG {2} {3}".format(core_options, Core.nmap_optimize, self._result_file.name, __destination) if args.nmap_optimize else  "{0} -oG {1} {2}".format(core_options, self._result_file.name, __destination)

<commit_msg>Update webscan.py<commit_after>
try:
	import tempfile
	from lib.core.core import Core,InitDirFile
	from lib.core.config_parser import ConfigParser
except ImportError, err:
	from lib.core.core import Core
	Core.print_error(err)

class WebScan(InitDirFile):

	def __init__(self, args):

		self._scan_options = ConfigParser.get_screen_ports(args.config_file)
		core_options = "-n -Pn -T5 --open -p T:{0}".format(self._scan_options) if self._scan_options else "-n -Pn -T5 --open -p T:80,443"

		self._result_file = tempfile.NamedTemporaryFile(mode='w+t')
		InitDirFile.__init__(self, [Core._commands_path["phantomjs"], Core._commands_path["nmap"], args.rasterize], args, "screen")

		if args.destination:
			__destination =  " ".join([ip.strip() for ip in args.destination.split(",")])
			self._nmap_options = "{0} {1} -oG {2} {3}".format(core_options, Core._nmap_optimize, self._result_file.name, __destination) if args.nmap_optimize else  "{0} -oG {1} {2}".format(core_options, self._result_file.name, __destination)

<|endoftext|>


<commit_before>"""
Test the autocomplete mode
"""
from pyqode.core.api import TextHelper
from pyqode.qt import QtCore, QtWidgets
from pyqode.qt.QtTest import QTest
from pyqode.python import modes as pymodes

def get_mode(editor):
    return editor.modes.get(pymodes.GoToAssignmentsMode)

def test_enabled(editor):
    mode = get_mode(editor)
    assert mode.enabled
    mode.enabled = False
    mode.enabled = True

def test_goto_variable(editor):
    editor.clear()
    code = "a = 15\nprint(a)"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(2, len('print(a)') - 2)
    mode.request_goto()
    QTest.qWait(1000)
    assert TextHelper(editor).current_line_nbr() == 0

out = False

def _on_out_of_doc(*args):
    global out
    out = True

def test_goto_out_of_doc(editor):
    global out
    out = False
    editor.clear()
    code = "a = 15\nprint(a)"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(1, len('print(a)') - 4)
    mode.out_of_doc.connect(_on_out_of_doc)
    assert out is False
    mode.request_goto()
    QTest.qWait(1000)
    assert out is True

flg_multi = False

def accept_dlg():
    global flg_multi
    flg_multi = True
    widgets = QtWidgets.QApplication.instance().topLevelWidgets()
    for w in widgets:
        if isinstance(w, QtWidgets.QDialog):
            QTest.keyPress(w, QtCore.Qt.Key_Tab)
            QTest.keyPress(w, QtCore.Qt.Key_Tab)
            QTest.keyPress(w, QtCore.Qt.Key_Return)

def test_multiple_results(editor):
    global flg_multi
    editor.clear()
    code = "import os\nos.path.abspath('..')"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(1, 9)
    QTest.qWait(1000)
    mode.request_goto()
    assert flg_multi is False
    QtCore.QTimer.singleShot(1000, accept_dlg)
    QTest.qWait(1000)
    assert flg_multi is True

no_results = False

def _on_no_results(*args):
    global no_results
    no_results = True

def test_no_results(editor):
    global no_results
    no_results = False
    editor.clear()
    code = "import foo"
    editor.setPlainText(code)
    mode = get_mode(editor)
    mode.no_results_found.connect(_on_no_results)
    TextHelper(editor).goto_line(0, len('import foo') -2)
    assert no_results is False
    mode.request_goto()
    QTest.qWait(1000)
    assert no_results is True

def test_make_unique(editor):
    seq = ['a', 'b', 'c', 'a']
    mode = get_mode(editor)
    new_seq = mode._unique(seq)
    assert len(new_seq) == len(seq) - 1
<commit_msg>Try the same fix as for the comments mode<commit_after>"""
Test the autocomplete mode
"""
from pyqode.core.api import TextHelper
from pyqode.qt import QtCore, QtWidgets
from pyqode.qt.QtTest import QTest
from pyqode.python import modes as pymodes
from test.helpers import preserve_editor_config, editor_open

def get_mode(editor):
    return editor.modes.get(pymodes.GoToAssignmentsMode)

@preserve_editor_config
@editor_open(__file__)
def test_enabled(editor):
    mode = get_mode(editor)
    assert mode.enabled
    mode.enabled = False
    mode.enabled = True

@preserve_editor_config
@editor_open(__file__)
def test_goto_variable(editor):
    editor.clear()
    code = "a = 15\nprint(a)"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(2, len('print(a)') - 2)
    mode.request_goto()
    QTest.qWait(1000)
    assert TextHelper(editor).current_line_nbr() == 0

out = False

def _on_out_of_doc(*args):
    global out
    out = True

@preserve_editor_config
@editor_open(__file__)
def test_goto_out_of_doc(editor):
    global out
    out = False
    editor.clear()
    code = "a = 15\nprint(a)"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(1, len('print(a)') - 4)
    mode.out_of_doc.connect(_on_out_of_doc)
    assert out is False
    mode.request_goto()
    QTest.qWait(1000)
    assert out is True

flg_multi = False

def accept_dlg():
    global flg_multi
    flg_multi = True
    widgets = QtWidgets.QApplication.instance().topLevelWidgets()
    for w in widgets:
        if isinstance(w, QtWidgets.QDialog):
            QTest.keyPress(w, QtCore.Qt.Key_Tab)
            QTest.keyPress(w, QtCore.Qt.Key_Tab)
            QTest.keyPress(w, QtCore.Qt.Key_Return)

@preserve_editor_config
@editor_open(__file__)
def test_multiple_results(editor):
    global flg_multi
    editor.clear()
    code = "import os\nos.path.abspath('..')"
    editor.setPlainText(code)
    mode = get_mode(editor)
    TextHelper(editor).goto_line(1, 9)
    QTest.qWait(1000)
    mode.request_goto()
    assert flg_multi is False
    QtCore.QTimer.singleShot(1000, accept_dlg)
    QTest.qWait(1000)
    assert flg_multi is True

no_results = False

def _on_no_results(*args):
    global no_results
    no_results = True

@preserve_editor_config
@editor_open(__file__)
def test_no_results(editor):
    global no_results
    no_results = False
    editor.clear()
    code = "import foo"
    editor.setPlainText(code)
    mode = get_mode(editor)
    mode.no_results_found.connect(_on_no_results)
    TextHelper(editor).goto_line(0, len('import foo') -2)
    assert no_results is False
    mode.request_goto()
    QTest.qWait(1000)
    assert no_results is True

@preserve_editor_config
@editor_open(__file__)
def test_make_unique(editor):
    seq = ['a', 'b', 'c', 'a']
    mode = get_mode(editor)
    new_seq = mode._unique(seq)
    assert len(new_seq) == len(seq) - 1
<|endoftext|>


<commit_before>from lexos.helpers.general_functions import *

def test_get_encoding():
    assert get_encoding(b"asdf") == "ascii"

def test_make_preview_from():
    one_char = "x"
    less_than_500_char = "modgaecq"
    str_250 = "gjzeqagitanbwnuwjkfbtpixhkcxltlcmvrbunoxovjzhyoiptckkxmdbrcnshy" \
              "efsrqexbdeczdbqjvprgiyjwwsacutlahuwhmscyuwkqxfnxqzxyozedtwmrztw" \
              "zzvoxrjnaypzbrkxfytpqeqmemxylvrvgtsthbalaibyzxnoxxbtofhnpdepatv" \
              "bihjoungenjidckhepgdlsmnrbqdgaalidwgccbardglcnedcqqxduuaauzyv"
    str_500 = str_250 + str_250
    more_than_500_char_even = str_250 + \
                              less_than_500_char + \
                              less_than_500_char + \
                              str_250
    more_than_500_char_odd = str_250 + \
                             less_than_500_char + \
                             one_char + \
                             less_than_500_char + \
                             str_250
    middle = '\u2026 ' + '/n/n' + '\u2026'
    assert make_preview_from(less_than_500_char) == less_than_500_char
    assert make_preview_from(str_500) == str_500
    assert make_preview_from(more_than_500_char_even) == str_250 + \
                                                         less_than_500_char + \
                                                         middle + \
                                                         str_250 + \
                                                         less_than_500_char
    assert make_preview_from(more_than_500_char_odd) == str_250 + \
                                                        less_than_500_char + \
                                                        middle + \
                                                        one_char + \
                                                        str_250 + \
                                                        less_than_500_char

def test_generate_d3_object():
    pass

def test_int_key():
    int_key(("a", "b"))

def test_natsort():
    assert natsort([10, 7, 1, 36, 92, 21, 9]) == [1, 7, 9, 10, 21, 36, 92]

def test_zip_dir():
    pass

def test_copy_dir():
    pass

def test_merge_list():
    assert merge_list([{1: "a", 2: "b"}, {3: "c", 4: "d"}]) == {1: 'a', 2: 'b', 3: 'c', 4: 'd'}

def test_load_stastic():
    assert load_stastic("this is a string string") == {"this": 1, "is": 1, "a": 1, "string": 2}

def test_matrix_to_dict():
    pass

def test_dict_to_matrix():
    pass

def test_xml_handling_options():
    pass

def test_html_escape():
    assert html_escape("&") == "&"
    assert html_escape('"') == """
    assert html_escape("'") == "'"
    assert html_escape(">") == ">"
    assert html_escape("<") == "<"

def test_apply_function_exclude_tags():
    pass

def test_decode_bytes():
    assert decode_bytes(u"asdf") == "asdf"
    assert decode_bytes( u'哈哈'.encode('gb2312')) == "¹þ¹þ"
<commit_msg>unitest<commit_after>from lexos.helpers.general_functions import *

def test_get_encoding():
    assert get_encoding(b"asdf") == "ascii"

def test_make_preview_from():
    one_char = "x"
    less_than_500_char = "modgaecq"
    str_250 = "gjzeqagitanbwnuwjkfbtpixhkcxltlcmvrbunoxovjzhyoiptckkxmdbrcnshy" \
              "efsrqexbdeczdbqjvprgiyjwwsacutlahuwhmscyuwkqxfnxqzxyozedtwmrztw" \
              "zzvoxrjnaypzbrkxfytpqeqmemxylvrvgtsthbalaibyzxnoxxbtofhnpdepatv" \
              "bihjoungenjidckhepgdlsmnrbqdgaalidwgccbardglcnedcqqxduuaauzyv"
    str_500 = str_250 + str_250
    more_than_500_char_even = str_250 + \
                              less_than_500_char + \
                              less_than_500_char + \
                              str_250
    more_than_500_char_odd = str_250 + \
                             less_than_500_char + \
                             one_char + \
                             less_than_500_char + \
                             str_250
    middle = '\u2026 ' + '/n/n' + '\u2026'
    assert make_preview_from(less_than_500_char) == less_than_500_char
    assert make_preview_from(str_500) == str_500
    assert make_preview_from(more_than_500_char_even) == str_250 + \
                                                         less_than_500_char + \
                                                         middle + \
                                                         str_250 + \
                                                         less_than_500_char
    assert make_preview_from(more_than_500_char_odd) == str_250 + \
                                                        less_than_500_char + \
                                                        middle + \
                                                        one_char + \
                                                        str_250 + \
                                                        less_than_500_char

def test_generate_d3_object():
    pass

def test_int_key():
    int_key(("a", "b"))

def test_natsort():
    assert natsort([10, 7, 1, 36, 92, 21, 9]) == [1, 7, 9, 10, 21, 36, 92]

def test_zip_dir():
    pass

def test_copy_dir():
    pass

def test_merge_list():
    assert merge_list([{1: "a", 2: "b"}, {3: "c", 4: "d"}]) == {1: 'a', 2: 'b', 3: 'c', 4: 'd'}

def test_load_stastic():
    assert load_stastic("this is a string string") == {"this": 1, "is": 1, "a": 1, "string": 2}

def test_matrix_to_dict():
    pass

def test_dict_to_matrix():
    pass

def test_xml_handling_options():
    pass

def test_html_escape():
    assert html_escape("&") == "&"
    assert html_escape('"') == """
    assert html_escape("'") == "'"
    assert html_escape(">") == ">"
    assert html_escape("<") == "<"

def test_apply_function_exclude_tags():
    pass

def test_decode_bytes():
    assert decode_bytes(u"asdf") == "asdf"
    assert decode_bytes(u'哈哈'.encode('gb2312')) == "¹þ¹þ"
<|endoftext|>


<commit_before>from os import path, listdir
import shutil
from .find_unmerged_branches_in_cwd import find_unmerged_branches_in_cwd
from .markdownify import markdownify
from .flatten import flatten
from .chdir import chdir
from .size import size
from .warn import warn
from .run import run

stogit = 'git@stogit.cs.stolaf.edu:sd-s16'
labnames = {
    'sound': ['lab2', 'lab3'],
    'images': ['lab4', 'lab5', 'lab6'],
}

def remove(student):
    shutil.rmtree(student)

def clone(student):
    if not path.exists(student):
        run(['git', 'clone', '--quiet', '{}/{}.git'.format(stogit, student)])

def has_changed_files():
    _, output = run(['git', 'status', '--porcelain'])
    return bool(output)

def stash(student, args):
    with chdir(student):
        if not args['no_update'] and has_changed_files():
            run(['git', 'stash', '-u'])
            run(['git', 'stash', 'clear'])

def pull(student, args):
    with chdir(student):
        if not args['no_update']:
            run(['git', 'pull', '--quiet', 'origin', 'master'])

def checkout_day(student, args):
    with chdir(student):
        if args['day']:
            rev_list = ['git', 'rev-list', '-n', '1', '--before="{} 18:00"'.format(day), 'master']
            _, rev = run(rev_list)
            run(['git', 'checkout', rev, '--force', '--quiet'])

def has_unmerged_branches(student, args):
    with chdir(student):
        if args['no_check']:
            return None
        else:
            return find_unmerged_branches_in_cwd()

def record(student, args):
    recordings = {}
    if not args['record']:
        return recordings

    with chdir(student):
        for to_record in args['record']:
            if path.exists(to_record):
                with chdir(to_record):
                    recording = markdownify(to_record, student, specs[to_record])
            else:
                recording = {
                    'spec': to_record,
                    'student': student,
                    'warnings': {'no submission': True},
                }

            recordings[to_record] = recording

    return recordings

def build_table_row(student, unmerged_branches):
    with chdir(student):
        all_folders = [folder
                       for folder in listdir('.')
                       if not folder.startswith('.') and path.isdir(folder)]
        filtered = [f for f in all_folders if size(f) > 100]

    FOLDERS = sorted([f.lower() for f in filtered])
    FOLDERS = list(flatten([(labnames[f] if f in labnames else f) for f in FOLDERS]))
    HWS = [f for f in FOLDERS if f.startswith('hw')]
    LABS = [f for f in FOLDERS if f.startswith('lab')]

    student_name = student + ' !' if unmerged_branches else student
    homework_list = ' '.join(HWS)
    lab_list = ' '.join(LABS)

    return "{}\t{}\t{}".format(student_name, homework_list, lab_list)

def reset(student, args):
    with chdir(student):
        if args['day']:
            run(['git', 'checkout', 'master', '--quiet', '--force'])

def single_student(student, args={}, specs={}):
    recordings = {}
    retval = ''

    if args['clean']:
        remove(student)

    clone(student)

    try:
        stash(student, args)
        pull(student, args)

        checkout_day(student, args)

        recordings = record(student, args)

        unmerged_branches = has_unmerged_branches(student, args)
        retval = build_table_row(student, unmerged_branches)

        reset(student, args)

    except Exception as err:
        retval = "{}: {}".format(student, err)

    return student, retval, recordings
<commit_msg>and another bug of forgetting to pass the specs<commit_after>from os import path, listdir
import shutil
from .find_unmerged_branches_in_cwd import find_unmerged_branches_in_cwd
from .markdownify import markdownify
from .flatten import flatten
from .chdir import chdir
from .size import size
from .warn import warn
from .run import run

stogit = 'git@stogit.cs.stolaf.edu:sd-s16'
labnames = {
    'sound': ['lab2', 'lab3'],
    'images': ['lab4', 'lab5', 'lab6'],
}

def remove(student):
    shutil.rmtree(student)

def clone(student):
    if not path.exists(student):
        run(['git', 'clone', '--quiet', '{}/{}.git'.format(stogit, student)])

def has_changed_files():
    _, output = run(['git', 'status', '--porcelain'])
    return bool(output)

def stash(student, args):
    with chdir(student):
        if not args['no_update'] and has_changed_files():
            run(['git', 'stash', '-u'])
            run(['git', 'stash', 'clear'])

def pull(student, args):
    with chdir(student):
        if not args['no_update']:
            run(['git', 'pull', '--quiet', 'origin', 'master'])

def checkout_day(student, args):
    with chdir(student):
        if args['day']:
            rev_list = ['git', 'rev-list', '-n', '1', '--before="{} 18:00"'.format(day), 'master']
            _, rev = run(rev_list)
            run(['git', 'checkout', rev, '--force', '--quiet'])

def has_unmerged_branches(student, args):
    with chdir(student):
        if args['no_check']:
            return None
        else:
            return find_unmerged_branches_in_cwd()

def record(student, specs, args):
    recordings = {}
    if not args['record']:
        return recordings

    with chdir(student):
        for to_record in args['record']:
            if path.exists(to_record):
                with chdir(to_record):
                    recording = markdownify(to_record, student, specs[to_record])
            else:
                recording = {
                    'spec': to_record,
                    'student': student,
                    'warnings': {'no submission': True},
                }

            recordings[to_record] = recording

    return recordings

def build_table_row(student, unmerged_branches):
    with chdir(student):
        all_folders = [folder
                       for folder in listdir('.')
                       if not folder.startswith('.') and path.isdir(folder)]
        filtered = [f for f in all_folders if size(f) > 100]

    FOLDERS = sorted([f.lower() for f in filtered])
    FOLDERS = list(flatten([(labnames[f] if f in labnames else f) for f in FOLDERS]))
    HWS = [f for f in FOLDERS if f.startswith('hw')]
    LABS = [f for f in FOLDERS if f.startswith('lab')]

    student_name = student + ' !' if unmerged_branches else student
    homework_list = ' '.join(HWS)
    lab_list = ' '.join(LABS)

    return "{}\t{}\t{}".format(student_name, homework_list, lab_list)

def reset(student, args):
    with chdir(student):
        if args['day']:
            run(['git', 'checkout', 'master', '--quiet', '--force'])

def single_student(student, args={}, specs={}):
    recordings = {}
    retval = ''

    if args['clean']:
        remove(student)

    clone(student)

    try:
        stash(student, args)
        pull(student, args)

        checkout_day(student, args)

        recordings = record(student, specs, args)

        unmerged_branches = has_unmerged_branches(student, args)
        retval = build_table_row(student, unmerged_branches)

        reset(student, args)

    except Exception as err:
        retval = "{}: {}".format(student, err)

    return student, retval, recordings
<|endoftext|>


<commit_before>#! /usr/bin/env python3

import sys
import os
os.chdir(os.path.dirname(sys.argv[0]))

from mule_local.JobMule import *
from itertools import product
from mule.exec_program import *

exec_program('mule.benchmark.cleanup_all', catch_output=False)

jg = JobGeneration()
jg.compile.unit_test = "test_sphere_coordinates"
jg.compile.sphere_spectral_space = "enable"
jg.runtime.verbosity = 5

jg.gen_jobscript_directory()

exitcode = exec_program('mule.benchmark.jobs_run_directly', catch_output=False)
if exitcode != 0:
    sys.exit(exitcode)

print("Benchmarks successfully finished")

exec_program('mule.benchmark.cleanup_all', catch_output=False)
<commit_msg>updated test to run in debug mode<commit_after>#! /usr/bin/env python3

import sys
import os
os.chdir(os.path.dirname(sys.argv[0]))

from mule_local.JobMule import *
from itertools import product
from mule.exec_program import *

exec_program('mule.benchmark.cleanup_all', catch_output=False)

jg = JobGeneration()
jg.compile.unit_test = "test_sphere_coordinates"
jg.compile.sphere_spectral_space = "enable"
jg.runtime.verbosity = 5
jg.compile.mode = 'debug'

jg.gen_jobscript_directory()

exitcode = exec_program('mule.benchmark.jobs_run_directly', catch_output=False)
if exitcode != 0:
    sys.exit(exitcode)

print("Benchmarks successfully finished")

exec_program('mule.benchmark.cleanup_all', catch_output=False)
<|endoftext|>


<commit_before>import os
import re
import traceback
from distutils.version import LooseVersion
from importlib import import_module

import yaml
import yaml.representer

from dmoj import judgeenv
from dmoj.executors import executors
from dmoj.testsuite import Tester
from dmoj.utils.ansi import ansi_style

def find_directory(parent, expr):
    regex = re.compile(expr)
    dirs = [dir for dir in os.listdir(parent) if regex.match(dir)]
    if not dirs:
        return
    return os.path.join(parent, max(dirs, key=LooseVersion))

def make_override(home, parent, expr):
    directory = find_directory(parent, expr)
    if not directory:
        return
    return {home: directory}

TEST_ON_TRAVIS = ['ADA', 'AWK', 'BF', 'C', 'CBL', 'D', 'DART', 'CPP0X', 'CPP03', 'CPP11', 'CLANG', 'CLANGX',
                  'F95', 'GO', 'GROOVY', 'HASK', 'JAVA7', 'JAVA8', 'JAVA9',
                  'PAS', 'PRO', 'GAS32', 'GAS64', 'LUA', 'NASM', 'NASM64',
                  'PERL', 'PHP', 'PY2', 'PY3', 'PYPY', 'PYPY3',
                  'RUBY2', 'RUST', 'SCM', 'SED', 'SWIFT', 'TCL', 'TEXT']
RVM_DIR = os.path.expanduser('~/.rvm/rubies/')
PYENV_DIR = '/opt/python/'
JVM_DIR = '/usr/lib/jvm/'

OVERRIDES = {
    'PY2':    make_override('py2_home',    PYENV_DIR, r'2\.'),
    'RUBY2':  make_override('ruby2_home',  RVM_DIR,   r'ruby-2\.'),
    'PYPY':  {'pypy_home':  os.path.abspath('pypy2')},
    'PYPY3': {'pypy3_home': os.path.abspath('pypy3')},
}

def get_dirs(dir):
    try:
        return [item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item))]
    except OSError:
        return []

def main():
    result = {}

    judgeenv.env['runtime'] = {}
    judgeenv.env['extra_fs'] = {
        'PHP': ['/etc/php5/', '/etc/terminfo/', '/etc/protocols$'],
        'SWIFT': [os.path.abspath(os.path.join(os.path.dirname(__file__), 'swift'))],
    }

    failed = False

    print 'Available JVMs:'
    for jvm in get_dirs(JVM_DIR):
        print '  -', jvm
    print

    print 'Available Pythons:'
    for python in get_dirs(PYENV_DIR):
        print '  -', python
    print

    print 'Available Rubies:'
    for ruby in get_dirs(RVM_DIR):
        print '  -', ruby
    print

    print 'Using extra allowed filesystems:'
    for lang, fs in judgeenv.env['extra_fs'].iteritems():
        print '%-6s: %s' % (lang, '|'.join(fs))
    print

    print 'Testing executors...'

    for name in TEST_ON_TRAVIS:
        executor = import_module('dmoj.executors.' + name)

        print ansi_style('%-34s%s' % ('Testing #ansi[%s](|underline):' % name, '')),

        if not hasattr(executor, 'Executor'):
            failed = True
            print ansi_style('#ansi[Does not export](red|bold) #ansi[Executor](red|underline)')
            continue

        if not hasattr(executor.Executor, 'autoconfig'):
            print ansi_style('#ansi[Could not autoconfig](magenta|bold)')
            continue

        try:
            if name in OVERRIDES:
                if not OVERRIDES[name]:
                    print ansi_style('#ansi[Environment not found on Travis](red)')
                    continue
                print ansi_style('#ansi[(manual config)](yellow)'),
                data = executor.Executor.autoconfig_run_test(OVERRIDES[name])
            else:
                data = executor.Executor.autoconfig()
            config = data[0]
            success = data[1]
            feedback = data[2]
            errors = '' if len(data) < 4 else data[3]
        except Exception:
            failed = True
            print ansi_style('#ansi[Autoconfig broken](red|bold)')
            traceback.print_exc()
        else:
            print ansi_style(['#ansi[%s](red|bold)', '#ansi[%s](green|bold)'][success] %
                             (feedback or ['Failed', 'Success'][success]))

            if success:
                result.update(config)
                executor.Executor.runtime_dict = config
                executors[name] = executor
                for runtime, ver in executor.Executor.get_runtime_versions():
                    print ansi_style('  #ansi[%s](cyan): %s' % (runtime, '.'.join(map(str, ver)) if ver else 'unknown'))
            else:
                if feedback == 'Could not find JVM':
                    continue

                if config:
                    print '  Attempted:'
                    print '   ', yaml.dump(config, default_flow_style=False).rstrip().replace('\n', '\n' + ' ' * 4)

                if errors:
                    print '  Errors:'
                    print '   ', errors.replace('\n', '\n' + ' ' * 4)
                failed = True

    print
    print ansi_style('#ansi[Configuration result](green|bold|underline):')
    print yaml.dump({'runtime': result}, default_flow_style=False).rstrip()
    print
    if failed:
        print ansi_style('#ansi[Executor configuration failed.](red|bold).')
    else:
        print ansi_style('#ansi[Executor configuration succeeded.](green|bold).')
    print
    print
    print 'Running test cases...'
    judgeenv.problem_dirs = [os.path.join(os.path.dirname(__file__), 'testsuite')]
    tester = Tester()
    fails = tester.test_all()
    print
    print 'Test complete'
    if fails:
        print ansi_style('#ansi[A total of %d case(s) failed](red|bold).') % fails
    else:
        print ansi_style('#ansi[All cases passed.](green|bold)')
    failed |= fails != 0
    raise SystemExit(int(failed))

if __name__ == '__main__':
    main()
<commit_msg>Allow Travis ruby to access .gem<commit_after>import os
import re
import traceback
from distutils.version import LooseVersion
from importlib import import_module

import yaml
import yaml.representer

from dmoj import judgeenv
from dmoj.executors import executors
from dmoj.testsuite import Tester
from dmoj.utils.ansi import ansi_style

def find_directory(parent, expr):
    regex = re.compile(expr)
    dirs = [dir for dir in os.listdir(parent) if regex.match(dir)]
    if not dirs:
        return
    return os.path.join(parent, max(dirs, key=LooseVersion))

def make_override(home, parent, expr):
    directory = find_directory(parent, expr)
    if not directory:
        return
    return {home: directory}

TEST_ON_TRAVIS = ['ADA', 'AWK', 'BF', 'C', 'CBL', 'D', 'DART', 'CPP0X', 'CPP03', 'CPP11', 'CLANG', 'CLANGX',
                  'F95', 'GO', 'GROOVY', 'HASK', 'JAVA7', 'JAVA8', 'JAVA9',
                  'PAS', 'PRO', 'GAS32', 'GAS64', 'LUA', 'NASM', 'NASM64',
                  'PERL', 'PHP', 'PY2', 'PY3', 'PYPY', 'PYPY3',
                  'RUBY2', 'RUST', 'SCM', 'SED', 'SWIFT', 'TCL', 'TEXT']
RVM_DIR = os.path.expanduser('~/.rvm/rubies/')
PYENV_DIR = '/opt/python/'
JVM_DIR = '/usr/lib/jvm/'

OVERRIDES = {
    'PY2':    make_override('py2_home',    PYENV_DIR, r'2\.'),
    'RUBY2':  make_override('ruby2_home',  RVM_DIR,   r'ruby-2\.'),
    'PYPY':  {'pypy_home':  os.path.abspath('pypy2')},
    'PYPY3': {'pypy3_home': os.path.abspath('pypy3')},
}

def get_dirs(dir):
    try:
        return [item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item))]
    except OSError:
        return []

def main():
    result = {}

    judgeenv.env['runtime'] = {}
    judgeenv.env['extra_fs'] = {
        'PHP': ['/etc/php5/', '/etc/terminfo/', '/etc/protocols$'],
        'SWIFT': [os.path.abspath(os.path.join(os.path.dirname(__file__), 'swift'))],
        'RUBY2': ['/home/travis/.gem/'],
    }

    failed = False

    print 'Available JVMs:'
    for jvm in get_dirs(JVM_DIR):
        print '  -', jvm
    print

    print 'Available Pythons:'
    for python in get_dirs(PYENV_DIR):
        print '  -', python
    print

    print 'Available Rubies:'
    for ruby in get_dirs(RVM_DIR):
        print '  -', ruby
    print

    print 'Using extra allowed filesystems:'
    for lang, fs in judgeenv.env['extra_fs'].iteritems():
        print '%-6s: %s' % (lang, '|'.join(fs))
    print

    print 'Testing executors...'

    for name in TEST_ON_TRAVIS:
        executor = import_module('dmoj.executors.' + name)

        print ansi_style('%-34s%s' % ('Testing #ansi[%s](|underline):' % name, '')),

        if not hasattr(executor, 'Executor'):
            failed = True
            print ansi_style('#ansi[Does not export](red|bold) #ansi[Executor](red|underline)')
            continue

        if not hasattr(executor.Executor, 'autoconfig'):
            print ansi_style('#ansi[Could not autoconfig](magenta|bold)')
            continue

        try:
            if name in OVERRIDES:
                if not OVERRIDES[name]:
                    print ansi_style('#ansi[Environment not found on Travis](red)')
                    continue
                print ansi_style('#ansi[(manual config)](yellow)'),
                data = executor.Executor.autoconfig_run_test(OVERRIDES[name])
            else:
                data = executor.Executor.autoconfig()
            config = data[0]
            success = data[1]
            feedback = data[2]
            errors = '' if len(data) < 4 else data[3]
        except Exception:
            failed = True
            print ansi_style('#ansi[Autoconfig broken](red|bold)')
            traceback.print_exc()
        else:
            print ansi_style(['#ansi[%s](red|bold)', '#ansi[%s](green|bold)'][success] %
                             (feedback or ['Failed', 'Success'][success]))

            if success:
                result.update(config)
                executor.Executor.runtime_dict = config
                executors[name] = executor
                for runtime, ver in executor.Executor.get_runtime_versions():
                    print ansi_style('  #ansi[%s](cyan): %s' % (runtime, '.'.join(map(str, ver)) if ver else 'unknown'))
            else:
                if feedback == 'Could not find JVM':
                    continue

                if config:
                    print '  Attempted:'
                    print '   ', yaml.dump(config, default_flow_style=False).rstrip().replace('\n', '\n' + ' ' * 4)

                if errors:
                    print '  Errors:'
                    print '   ', errors.replace('\n', '\n' + ' ' * 4)
                failed = True

    print
    print ansi_style('#ansi[Configuration result](green|bold|underline):')
    print yaml.dump({'runtime': result}, default_flow_style=False).rstrip()
    print
    if failed:
        print ansi_style('#ansi[Executor configuration failed.](red|bold).')
    else:
        print ansi_style('#ansi[Executor configuration succeeded.](green|bold).')
    print
    print
    print 'Running test cases...'
    judgeenv.problem_dirs = [os.path.join(os.path.dirname(__file__), 'testsuite')]
    tester = Tester()
    fails = tester.test_all()
    print
    print 'Test complete'
    if fails:
        print ansi_style('#ansi[A total of %d case(s) failed](red|bold).') % fails
    else:
        print ansi_style('#ansi[All cases passed.](green|bold)')
    failed |= fails != 0
    raise SystemExit(int(failed))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import traceback
from distutils.spawn import find_executable
from importlib import import_module

import yaml
import yaml.representer

from dmoj import judgeenv
from dmoj.executors import executors
from dmoj.testsuite import Tester
from dmoj.utils.ansi import ansi_style

TEST_ON_TRAVIS = ['ADA', 'AWK', 'BF', 'C', 'CPP03', 'CPP11', 'CPP14', 'CLANG', 'CLANGX',
                  'COFFEE', 'GO', 'HASK', 'JAVA7', 'JAVA8', 'MONOCS',
                  'GAS32', 'GAS64', 'NASM', 'NASM64',
                  'PERL', 'PHP', 'PY2', 'PY3', 'PYPY', 'PYPY3', 'RUBY19', 'SCALA', 'TEXT']
OVERRIDES = {
    'PY2': {'python': '/usr/bin/python'},
    'RUBY19': {'ruby19': find_executable('ruby')},
    'PYPY': {'pypy_home': '/opt/python/pypy-2.5.0'},
    'PYPY3': {'pypy3_home': '/opt/python/pypy3-2.4.0'},
}

def get_dirs(dir):
    try:
        return [item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item))]
    except OSError:
        return []

def main():
    result = {}

    judgeenv.env['runtime'] = {}
    judgeenv.env['extra_fs'] = {
        'PHP': ['/etc/php5/', '/etc/terminfo/', '/etc/protocols$'],
        'RUBY19': ['/home/travis/.rvm/rubies/'],
    }

    failed = False

    print 'Available JVMs:'
    for jvm in get_dirs('/usr/lib/jvm/'):
        print '  -', jvm
    print

    print 'Available Pythons:'
    for python in get_dirs('/opt/python'):
        print '  -', python
    print

    print 'Available Rubies:'
    for ruby in get_dirs(os.path.expanduser('~/.rvm/rubies')):
        print '  -', ruby
    print

    print 'Testing executors...'

    for name in TEST_ON_TRAVIS:
        executor = import_module('dmoj.executors.' + name)

        print ansi_style('%-34s%s' % ('Testing #ansi[%s](|underline):' % name, '')),

        if not hasattr(executor, 'Executor'):
            failed = True
            print ansi_style('#ansi[Does not export](red|bold) #ansi[Executor](red|underline)')
            continue

        if not hasattr(executor.Executor, 'autoconfig'):
            print ansi_style('#ansi[Could not autoconfig](magenta|bold)')
            continue

        try:
            if name in OVERRIDES:
                print ansi_style('#ansi[(manual config)](yellow)'),
                data = executor.Executor.autoconfig_run_test(OVERRIDES[name])
            else:
                data = executor.Executor.autoconfig()
            config = data[0]
            success = data[1]
            feedback = data[2]
            errors = '' if len(data) < 4 else data[3]
        except Exception:
            failed = True
            print ansi_style('#ansi[Autoconfig broken](red|bold)')
            traceback.print_exc()
        else:
            print ansi_style(['#ansi[%s](red|bold)', '#ansi[%s](green|bold)'][success] %
                             (feedback or ['Failed', 'Success'][success]))

            if success:
                result.update(config)
                executor.Executor.runtime_dict = config
                executors[name] = executor
            else:
                if config:
                    print '  Attempted:'
                    print '   ', yaml.dump(config, default_flow_style=False).rstrip().replace('\n', '\n' + ' ' * 4)

                if errors:
                    print '  Errors:'
                    print '   ', errors.replace('\n', '\n' + ' ' * 4)
                failed = True

    print
    print ansi_style('#ansi[Configuration result](green|bold|underline):')
    print yaml.dump({'runtime': result}, default_flow_style=False).rstrip()
    print
    print
    print 'Running test cases...'
    judgeenv.problem_dirs = [os.path.join(os.path.dirname(__file__), 'testsuite')]
    tester = Tester()
    fails = tester.test_all()
    print
    print 'Test complete'
    if fails:
        print ansi_style('#ansi[A total of %d case(s) failed](red|bold).') % fails
    else:
        print ansi_style('#ansi[All cases passed.](green|bold)')
    failed |= fails != 0
    raise SystemExit(int(failed))

if __name__ == '__main__':
    main()
<commit_msg>Give runtime versions a testing too<commit_after>import os
import traceback
from distutils.spawn import find_executable
from importlib import import_module

import yaml
import yaml.representer

from dmoj import judgeenv
from dmoj.executors import executors
from dmoj.testsuite import Tester
from dmoj.utils.ansi import ansi_style

TEST_ON_TRAVIS = ['ADA', 'AWK', 'BF', 'C', 'CPP03', 'CPP11', 'CPP14', 'CLANG', 'CLANGX',
                  'COFFEE', 'GO', 'HASK', 'JAVA7', 'JAVA8', 'MONOCS',
                  'GAS32', 'GAS64', 'NASM', 'NASM64',
                  'PERL', 'PHP', 'PY2', 'PY3', 'PYPY', 'PYPY3', 'RUBY19', 'SCALA', 'TEXT']
OVERRIDES = {
    'PY2': {'python': '/usr/bin/python'},
    'RUBY19': {'ruby19': find_executable('ruby')},
    'PYPY': {'pypy_home': '/opt/python/pypy-2.5.0'},
    'PYPY3': {'pypy3_home': '/opt/python/pypy3-2.4.0'},
}

def get_dirs(dir):
    try:
        return [item for item in os.listdir(dir) if os.path.isdir(os.path.join(dir, item))]
    except OSError:
        return []

def main():
    result = {}

    judgeenv.env['runtime'] = {}
    judgeenv.env['extra_fs'] = {
        'PHP': ['/etc/php5/', '/etc/terminfo/', '/etc/protocols$'],
        'RUBY19': ['/home/travis/.rvm/rubies/'],
    }

    failed = False

    print 'Available JVMs:'
    for jvm in get_dirs('/usr/lib/jvm/'):
        print '  -', jvm
    print

    print 'Available Pythons:'
    for python in get_dirs('/opt/python'):
        print '  -', python
    print

    print 'Available Rubies:'
    for ruby in get_dirs(os.path.expanduser('~/.rvm/rubies')):
        print '  -', ruby
    print

    print 'Testing executors...'

    for name in TEST_ON_TRAVIS:
        executor = import_module('dmoj.executors.' + name)

        print ansi_style('%-34s%s' % ('Testing #ansi[%s](|underline):' % name, '')),

        if not hasattr(executor, 'Executor'):
            failed = True
            print ansi_style('#ansi[Does not export](red|bold) #ansi[Executor](red|underline)')
            continue

        if not hasattr(executor.Executor, 'autoconfig'):
            print ansi_style('#ansi[Could not autoconfig](magenta|bold)')
            continue

        try:
            if name in OVERRIDES:
                print ansi_style('#ansi[(manual config)](yellow)'),
                data = executor.Executor.autoconfig_run_test(OVERRIDES[name])
            else:
                data = executor.Executor.autoconfig()
            config = data[0]
            success = data[1]
            feedback = data[2]
            errors = '' if len(data) < 4 else data[3]
        except Exception:
            failed = True
            print ansi_style('#ansi[Autoconfig broken](red|bold)')
            traceback.print_exc()
        else:
            print ansi_style(['#ansi[%s](red|bold)', '#ansi[%s](green|bold)'][success] %
                             (feedback or ['Failed', 'Success'][success]))

            if success:
                result.update(config)
                executor.Executor.runtime_dict = config
                executors[name] = executor
                for name, version in executor.get_runtime_versions():
                    print ansi_style('  #ansi[%s](cyan): %s' % (name, '.'.join(map(str, version))))
            else:
                if config:
                    print '  Attempted:'
                    print '   ', yaml.dump(config, default_flow_style=False).rstrip().replace('\n', '\n' + ' ' * 4)

                if errors:
                    print '  Errors:'
                    print '   ', errors.replace('\n', '\n' + ' ' * 4)
                failed = True

    print
    print ansi_style('#ansi[Configuration result](green|bold|underline):')
    print yaml.dump({'runtime': result}, default_flow_style=False).rstrip()
    print
    print
    print 'Running test cases...'
    judgeenv.problem_dirs = [os.path.join(os.path.dirname(__file__), 'testsuite')]
    tester = Tester()
    fails = tester.test_all()
    print
    print 'Test complete'
    if fails:
        print ansi_style('#ansi[A total of %d case(s) failed](red|bold).') % fails
    else:
        print ansi_style('#ansi[All cases passed.](green|bold)')
    failed |= fails != 0
    raise SystemExit(int(failed))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
import boto

import lib.logger as logger
import lib.utils as utils
import settings

class AWSS3(logger.Logger):

    FIRST_DAY_OF_MONTH = '01'

    log_file = settings.BACKUP_LOG_FILE
    ref_class = 'aws-s3'
    conn = None
    bucket = None
    bucket_name = None
    access_key = None
    secret_access_key = None

    def __init__(self, bucket_name, access_key, secret_access_key):
        logger.Logger.__init__(self)
        self.bucket_name = bucket_name
        self.access_key = access_key
        self.secret_access_key = secret_access_key

    def connect(self):
        self.log.debug('connecting to AWS S3...')
        self.conn = boto.connect_s3(self.access_key, self.secret_access_key)
        self.bucket = self.conn.get_bucket(self.bucket_name)
        self.log.debug('connected, bucket set to <%s>' % self.bucket_name)

    def disconnect(self):
        self.conn.close()
        self.log.debug('connection to AWS S3 closed')

    def upload(self, filename):
        filepath = '%s/%s' % (settings.AWS_DIR, filename)
        k = boto.s3.key.Key(self.bucket)
        k.key = filepath
        self.log.debug('uploading...')
        k.set_contents_from_filename(filename)
        self.log.debug('upload successful')

    def sync(self, filename):
        # TODO(nandersson):
        # * add some logic so we only get backups from this server here,
        #   e.g. match against server_name, suffix of 'tar.bz2' etc
        key_list = self.bucket.list(prefix=settings.AWS_DIR)
        for k in key_list:
            if self._get_backup_filename(k.key) == filename:  continue
            remove_file = True
            k_date = self._get_backup_date(k.key)
            k_day = k_date[6:8]  # fmt e.g.: 20130812
            remove_file = not (settings.BACKUP_SAVE_EVERY_MONTH and
                               k_day == self.FIRST_DAY_OF_MONTH)
            if remove_file:
                for i in range(settings.BACKUP_DAYS_TO_STORE):
                    if k_date == utils.get_timestamp('%Y%m%d', i):
                        remove_file = False
                        break
            if remove_file:
                k.delete()
                self.log.debug('removed outdated backup <%s>' % k.key)
        self.log.debug('everything in sync')

    @staticmethod
    def _get_backup_filename(path):
        path_tree = path.split('/')
        return path_tree[len(path_tree)-1]

    @staticmethod
    def _get_backup_date(path):
        file_day = AWSS3._get_backup_filename(path).split('-')
        if len(file_day) == 0:  return None
        return file_day[len(file_day)-2]<commit_msg>fixed support for uploading files in chunks, 1GB each as default<commit_after>
import boto
import glob
import os

import lib.logger as logger
import lib.utils as utils
import settings

class AWSS3(logger.Logger):

    FIRST_DAY_OF_MONTH = '01'
    MB_IN_BYTES = 1049000
    MULTIPART_UPLOAD_CHUNK_SIZE = 1000 * MB_IN_BYTES
    SPLIT_FILES_CMD = 'split -b %d %s %s'
    FILES_ALL_PREFIX = '%s*'

    log_file = settings.BACKUP_LOG_FILE
    ref_class = 'aws-s3'
    conn = None
    bucket = None
    bucket_name = None
    access_key = None
    secret_access_key = None

    def __init__(self, bucket_name, access_key, secret_access_key):
        logger.Logger.__init__(self)
        self.bucket_name = bucket_name
        self.access_key = access_key
        self.secret_access_key = secret_access_key

    def connect(self):
        self.log.debug('connecting to AWS S3...')
        self.conn = boto.connect_s3(self.access_key, self.secret_access_key)
        self.bucket = self.conn.get_bucket(self.bucket_name)
        self.log.debug('connected, bucket set to <%s>' % self.bucket_name)

    def disconnect(self):
        self.conn.close()
        self.log.debug('connection to AWS S3 closed')

    def upload(self, src):
        dest = '%s/%s' % (settings.AWS_DIR, src)
        src_file_size = os.path.getsize(src)
        k = boto.s3.key.Key(self.bucket)
        k.key = dest
        self.log.debug('created S3 key %r' % k)
        self.log.debug('source filesize is <%d> bytes' % src_file_size)
        if src_file_size < self.MULTIPART_UPLOAD_CHUNK_SIZE:
            self._standard_transfer(k, src, dest)
        else:
            self._multipart_transfer(k, src, src_file_size, dest)

    def sync(self, filename):
        # TODO(nandersson):
        # * add some logic so we only get backups from this server here,
        #   e.g. match against server_name, suffix of 'tar.bz2' etc
        key_list = self.bucket.list(prefix=settings.AWS_DIR)
        for k in key_list:
            if self._get_backup_filename(k.key) == filename:  continue
            remove_file = True
            k_date = self._get_backup_date(k.key)
            k_day = k_date[6:8]  # fmt e.g.: 20130812
            remove_file = not (settings.BACKUP_SAVE_EVERY_MONTH and
                               k_day == self.FIRST_DAY_OF_MONTH)
            if remove_file:
                for i in range(settings.BACKUP_DAYS_TO_STORE):
                    if k_date == utils.get_timestamp('%Y%m%d', i):
                        remove_file = False
                        break
            if remove_file:
                k.delete()
                self.log.debug('removed outdated backup <%s>' % k.key)
        self.log.debug('everything in sync')

    def _standard_transfer(self, key, src, dest):
        self.log.debug('uploading...')
        key.set_contents_from_filename(src)
        self.log.debug('upload successful')

    def _multipart_transfer(self, key, src, src_file_size, dest):
        # TODO(nandersson):
        # * should be able to parallelize this, one for each core for example,
        #   to increase throughput (upload parts at the same time)
        # NOTE(nandersson):
        # * max filesize for a PUT request to AWS S3 API is 5GB, so need to
        #   split it up for files larger than that.. uploading in chunks of size
        #   set in self.MULTIPART_UPLOAD_CHUNK_SIZE
        self.log.debug('uploading using multipart...')
        chunks = (src_file_size / self.MULTIPART_UPLOAD_CHUNK_SIZE) + 1
        self.log.debug('splitting <%d> bytes in <%d> chunks of max <%d> bytes'
                       % (src_file_size, chunks,
                          self.MULTIPART_UPLOAD_CHUNK_SIZE))
        files = self._split_file(src, src_file_size,
                                 self.MULTIPART_UPLOAD_CHUNK_SIZE)
        mp = self.bucket.initiate_multipart_upload(key)
        for i, filename in enumerate(files):
            f = open(filename, 'r')
            self.log.debug('uploading chunk <%d>, part file <%s>..'
                           % (i+1, filename))
            mp.upload_part_from_file(f, i+1)
            f.close()
            os.remove(filename)
        mp.complete_upload()
        self.log.debug('upload successful')

    @staticmethod
    def _split_file(path, chunks, chunk_bytes):
        # TODO(nandersson):
        # * might want to move this to a more general file class
        os.system(AWSS3.SPLIT_FILES_CMD % (chunk_bytes, path, path))
        splitted_files = glob.glob(AWSS3.FILES_ALL_PREFIX % path)
        splitted_files.remove(path)
        return sorted(splitted_files)

    @staticmethod
    def _get_backup_filename(path):
        path_tree = path.split('/')
        return path_tree[len(path_tree)-1]

    @staticmethod
    def _get_backup_date(path):
        file_day = AWSS3._get_backup_filename(path).split('-')
        if len(file_day) == 0:  return None
        return file_day[len(file_day)-2]
<|endoftext|>


<commit_before>import os
import shutil
from unittest.mock import patch

from django.http import HttpRequest
from django.test import TestCase

from conversion_job.models import Extent, ConversionJob, GISFormat
from conversion_job.serializers import ConversionJobSerializer, GISFormatStatusSerializer
from converters import converter_options
from shared import ConversionProgress

class RQJobMock:
    id = ''.join(['1' for _ in range(36)])

class GISFormatListSerializerTest(TestCase):
    def setUp(self):
        self.extent = Extent.objects.create(west=0, south=0, east=0, north=0)
        self.conversion_job = ConversionJob.objects.create(extent=self.extent)
        self.gis_format_1 = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[0]
        )
        self.gis_format_2 = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[3]
        )

    @patch('conversion_job.serializers.ConversionJobSerializer._enqueue_rq_job', return_value=RQJobMock)
    def test_create(self, mock):
        self.assertEqual(GISFormat.objects.count(), 2)
        self.assertEqual(Extent.objects.count(), 1)
        self.assertEqual(ConversionJob.objects.count(), 1)
        data = {
            "gis_formats": converter_options.get_output_formats(),
            "callback_url": "http://example.com",
            "gis_options": {
                "coordinate_reference_system": "WGS_84",
                "detail_level": 1,
            },
            "extent": {
                "west": 29.525547623634335,
                "south": 40.77546776498174,
                "east": 29.528980851173397,
                "north": 40.77739734768811,
                "polyfile": None,
            }
        }
        conversion_job_serializer = ConversionJobSerializer(data=data)
        conversion_job_serializer.is_valid()
        conversion_job_serializer.save()

        self.assertEqual(Extent.objects.count(), 2)
        self.assertEqual(GISFormat.objects.count(), 2 + len(converter_options.get_output_formats()))
        self.assertEqual(ConversionJob.objects.count(), 2)

        args, kwargs = mock.call_args
        self.assertCountEqual(kwargs['format_options'].output_formats, converter_options.get_output_formats())

        self.assertNotEqual(self.conversion_job, ConversionJob.objects.last())

class GISFormatStatusSerializerTest(TestCase):
    def setUp(self):
        extent = Extent.objects.create(west=0, south=0, east=0, north=0)
        self.conversion_job = ConversionJob.objects.create(extent=extent)
        self.gis_format = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[0]
        )
        request = HttpRequest()
        request.META['HTTP_HOST'] = 'some-host'
        self.format_status_serializer = GISFormatStatusSerializer(self.gis_format, context={'request': request})

    def tearDown(self):
        shutil.rmtree(self.conversion_job.output_directory)
        super().tearDown()

    def _create_valid_file(self):
        matching_file_names = ['{}.zip'.format(f) for f in converter_options.get_output_formats()]
        for matching_file_name in matching_file_names:
            open(os.path.join(self.conversion_job.output_directory, matching_file_name), 'x').close()

    def test_get_download_url_is_none_when_file_is_not_available(self):
        self.gis_format.progress = ConversionProgress.SUCCESSFUL.value
        self.gis_format.save()
        self.assertIsNone(self.format_status_serializer.data.get('result_url'))

    def test_get_download_url_is_avilable_if_file_is_avaiable_even_if_progress_is_not_success(self):
        self._create_valid_file()
        os.path.join(self.conversion_job.output_directory)
        self.assertEqual(
            'http://some-host/gis_format_status/1/download_result/',
            self.format_status_serializer.data.get('result_url')
        )

    def test_get_download_url_is_defined_when_status_is_success_and_file_available(self):
        self._create_valid_file()
        self.gis_format.progress = ConversionProgress.SUCCESSFUL.value
        self.gis_format.save()
        self.assertIsNotNone(self.format_status_serializer.data.get('result_url'))

    def test_get_download_url_is_defined_when_status_raises_error_when_deleted(self):
        self.gis_format.delete()
        with self.assertRaises(GISFormat.DoesNotExist):
            # self.format_status_serializer.data already raises, but .get does raise as well.
            self.format_status_serializer.data.get('result_url')
<commit_msg>follow test naming a bit better<commit_after>import os
import shutil
from unittest.mock import patch

from django.http import HttpRequest
from django.test import TestCase

from conversion_job.models import Extent, ConversionJob, GISFormat
from conversion_job.serializers import ConversionJobSerializer, GISFormatStatusSerializer
from converters import converter_options
from shared import ConversionProgress

class RQJobMock:
    id = ''.join(['1' for _ in range(36)])

class GISFormatListSerializerTest(TestCase):
    def setUp(self):
        self.extent = Extent.objects.create(west=0, south=0, east=0, north=0)
        self.conversion_job = ConversionJob.objects.create(extent=self.extent)
        self.gis_format_1 = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[0]
        )
        self.gis_format_2 = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[3]
        )

    @patch('conversion_job.serializers.ConversionJobSerializer._enqueue_rq_job', return_value=RQJobMock)
    def test_create_succeeds(self, mock):
        self.assertEqual(GISFormat.objects.count(), 2)
        self.assertEqual(Extent.objects.count(), 1)
        self.assertEqual(ConversionJob.objects.count(), 1)
        data = {
            "gis_formats": converter_options.get_output_formats(),
            "callback_url": "http://example.com",
            "gis_options": {
                "coordinate_reference_system": "WGS_84",
                "detail_level": 1,
            },
            "extent": {
                "west": 29.525547623634335,
                "south": 40.77546776498174,
                "east": 29.528980851173397,
                "north": 40.77739734768811,
                "polyfile": None,
            }
        }
        conversion_job_serializer = ConversionJobSerializer(data=data)
        conversion_job_serializer.is_valid()
        conversion_job_serializer.save()

        self.assertEqual(Extent.objects.count(), 2)
        self.assertEqual(GISFormat.objects.count(), 2 + len(converter_options.get_output_formats()))
        self.assertEqual(ConversionJob.objects.count(), 2)

        args, kwargs = mock.call_args
        self.assertCountEqual(kwargs['format_options'].output_formats, converter_options.get_output_formats())

        self.assertNotEqual(self.conversion_job, ConversionJob.objects.last())

class GISFormatStatusSerializerTest(TestCase):
    def setUp(self):
        extent = Extent.objects.create(west=0, south=0, east=0, north=0)
        self.conversion_job = ConversionJob.objects.create(extent=extent)
        self.gis_format = GISFormat.objects.create(
            conversion_job=self.conversion_job,
            format=converter_options.get_output_formats()[0]
        )
        request = HttpRequest()
        request.META['HTTP_HOST'] = 'some-host'
        self.format_status_serializer = GISFormatStatusSerializer(self.gis_format, context={'request': request})

    def tearDown(self):
        shutil.rmtree(self.conversion_job.output_directory)
        super().tearDown()

    def _create_valid_files(self):
        matching_file_names = ['{}.zip'.format(f) for f in converter_options.get_output_formats()]
        for matching_file_name in matching_file_names:
            open(os.path.join(self.conversion_job.output_directory, matching_file_name), 'x').close()

    def test_get_download_url_when_file_is_not_available_is_none(self):
        self.gis_format.progress = ConversionProgress.SUCCESSFUL.value
        self.gis_format.save()
        self.assertIsNone(self.format_status_serializer.data.get('result_url'))

    def test_get_download_url_when_status_is_success_and_file_available_is_defined(self):
        self._create_valid_files()
        self.gis_format.progress = ConversionProgress.SUCCESSFUL.value
        self.gis_format.save()
        self.assertIsNotNone(self.format_status_serializer.data.get('result_url'))

    def test_get_download_url_when_status_raises_error_when_deleted_is_defined(self):
        self.gis_format.delete()
        with self.assertRaises(GISFormat.DoesNotExist):
            # self.format_status_serializer.data already raises, but .get does raise as well.
            self.format_status_serializer.data.get('result_url')
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
from django.test import TestCase

from django_mysql_tests.models import Author

class SoundexTests(TestCase):

    def test_sounds_like_lookup(self):
        principles = ["principle", "principal", "princpl"]
        created = {Author.objects.create(name=name) for name in principles}

        for name in principles:
            sounding = Author.objects.filter(name__sounds_like=name)
            self.assertEqual(set(sounding), created)

        sounding = Author.objects.filter(name__sounds_like='')
        self.assertEqual(set(sounding), set())

        sounding = Author.objects.filter(name__sounds_like='nothing')
        self.assertEqual(set(sounding), set())

    def test_soundex_strings(self):
        author = Author.objects.create(name='hi')
        self.assertEqual(Author.objects.get(name__soundex='H000'), author)
<commit_msg>More plausible author name in test<commit_after># -*- coding:utf-8 -*-
from django.test import TestCase

from django_mysql_tests.models import Author

class SoundexTests(TestCase):

    def test_sounds_like_lookup(self):
        principles = ["principle", "principal", "princpl"]
        created = {Author.objects.create(name=name) for name in principles}

        for name in principles:
            sounding = Author.objects.filter(name__sounds_like=name)
            self.assertEqual(set(sounding), created)

        sounding = Author.objects.filter(name__sounds_like='')
        self.assertEqual(set(sounding), set())

        sounding = Author.objects.filter(name__sounds_like='nothing')
        self.assertEqual(set(sounding), set())

    def test_soundex_strings(self):
        author = Author.objects.create(name='Robert')
        self.assertEqual(Author.objects.get(name__soundex='R163'), author)
<|endoftext|>


<commit_before># -*- coding: utf-8
from __future__ import unicode_literals

import wrappers
from behave import when, then

@when('we build {image_name} from Dockerfile')
def step_build_dockerfile(context, image_name):
    """
    Send "build -t test-image ./tests/features/fixture_data/".
    """
    context.cli.sendline('build -t {0} {1}'.format(image_name, context.data_dir))

@then('we see image built')
def step_see_image_built(context):
    """
    Expect to see image built.
    """
    wrappers.expect_exact(context, 'Successfully built')

@when('we pull {image_name} image')
def step_pull_image(context, image_name):
    """
    Send "pull {image_name}".
    """
    context.cli.sendline('pull ' + image_name)

@then('we see {image_name} pulled')
def step_see_image_pulled(context, image_name):
    """
    Expect to see image pulled.
    """
    wrappers.expect_exact(context, [
        'Downloaded newer image for ' + image_name,
        'Image is up to date for ' + image_name
    ])

@when('we log in as {user}, {email} with {password}')
def step_log_in(context, user, email, password):
    """
    Send "login" command.
    """
    context.cli.sendline('login -u {0} -e {1} -p {2}'.format(user, email, password))

@then('we see login success')
def step_see_log_in_success(context):
    """
    Expect to see login succeeded.
    """
    wrappers.expect_exact(context, 'Login Succeeded', 30)

@when('we tag {image_name} into {repo_name}')
def step_tag_image(context, image_name, repo_name):
    """
    Send "tag" command.
    """
    context.cli.sendline('tag {0} {1}'.format(image_name, repo_name))

@then('we see {image_name} tagged into {repo_name}')
def step_see_image_tagged(context, image_name, repo_name):
    """
    Expect to see image tagged.
    """
    wrappers.expect_exact(context, 'Tagged {0} into {1}'.format(image_name, repo_name))

@when('we remove image {image_name}')
def step_remove_image(context, image_name):
    """
    Send "rmi" command.
    """
    context.cli.sendline('rmi {0}'.format(image_name))

@then('we see image {image_name} removed')
def step_see_image_removed(context, image_name):
    """
    Expect to see image removed.
    """
    wrappers.expect_exact(context, image_name)

@when('we list images')
def step_list_images(context):
    """
    Send "images" command.
    """
    context.cli.sendline('images')

@then('we see image {image_name} listed')
def step_see_image_listed(context, image_name):
    """
    Expect to see image listed.
    """
    wrappers.expect_exact(context, 'hello-world')

@when('we inspect image {name}')
def step_inspect_image(context, name):
    """
    Send "inspect" command.
    """
    context.cli.sendline('inspect {0}'.format(name))

@when('we search for {name}')
def step_search(context, name):
    """
    Send "search" command.
    """
    context.cli.sendline('search {0}'.format(name))
<commit_msg>Increase pull timeout to 180.<commit_after># -*- coding: utf-8
from __future__ import unicode_literals

import wrappers
from behave import when, then

@when('we build {image_name} from Dockerfile')
def step_build_dockerfile(context, image_name):
    """
    Send "build -t test-image ./tests/features/fixture_data/".
    """
    context.cli.sendline('build -t {0} {1}'.format(image_name, context.data_dir))

@then('we see image built')
def step_see_image_built(context):
    """
    Expect to see image built.
    """
    wrappers.expect_exact(context, 'Successfully built')

@when('we pull {image_name} image')
def step_pull_image(context, image_name):
    """
    Send "pull {image_name}".
    """
    context.cli.sendline('pull ' + image_name)

@then('we see {image_name} pulled')
def step_see_image_pulled(context, image_name):
    """
    Expect to see image pulled.
    """
    wrappers.expect_exact(context, [
        'Downloaded newer image for ' + image_name,
        'Image is up to date for ' + image_name
    ],
    timeout=180)

@when('we log in as {user}, {email} with {password}')
def step_log_in(context, user, email, password):
    """
    Send "login" command.
    """
    context.cli.sendline('login -u {0} -e {1} -p {2}'.format(user, email, password))

@then('we see login success')
def step_see_log_in_success(context):
    """
    Expect to see login succeeded.
    """
    wrappers.expect_exact(context, 'Login Succeeded', 30)

@when('we tag {image_name} into {repo_name}')
def step_tag_image(context, image_name, repo_name):
    """
    Send "tag" command.
    """
    context.cli.sendline('tag {0} {1}'.format(image_name, repo_name))

@then('we see {image_name} tagged into {repo_name}')
def step_see_image_tagged(context, image_name, repo_name):
    """
    Expect to see image tagged.
    """
    wrappers.expect_exact(context, 'Tagged {0} into {1}'.format(image_name, repo_name))

@when('we remove image {image_name}')
def step_remove_image(context, image_name):
    """
    Send "rmi" command.
    """
    context.cli.sendline('rmi {0}'.format(image_name))

@then('we see image {image_name} removed')
def step_see_image_removed(context, image_name):
    """
    Expect to see image removed.
    """
    wrappers.expect_exact(context, image_name)

@when('we list images')
def step_list_images(context):
    """
    Send "images" command.
    """
    context.cli.sendline('images')

@then('we see image {image_name} listed')
def step_see_image_listed(context, image_name):
    """
    Expect to see image listed.
    """
    wrappers.expect_exact(context, 'hello-world')

@when('we inspect image {name}')
def step_inspect_image(context, name):
    """
    Send "inspect" command.
    """
    context.cli.sendline('inspect {0}'.format(name))

@when('we search for {name}')
def step_search(context, name):
    """
    Send "search" command.
    """
    context.cli.sendline('search {0}'.format(name))
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import pytest
import re
import time

from tests.common.environ import specific_build_type_timeout
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal

WAIT_TIME_MS = specific_build_type_timeout(60000, slow_build_timeout=100000)

# Some of the queries in runtime_filters consume a lot of memory, leading to
# significant memory reservations in parallel tests.
@pytest.mark.execute_serially
@SkipIfLocal.multiple_impalad
class TestRuntimeFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestRuntimeFilters, cls).add_test_dimensions()
    # Runtime filters are disabled on HBase
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format not in ['hbase'])

  def test_basic_filters(self, vector):
    self.run_test_case('QueryTest/runtime_filters', vector,
        test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS' : str(WAIT_TIME_MS)})

  def test_wait_time(self, vector):
    """Test that a query that has global filters does not wait for them if run in LOCAL
    mode"""
    now = time.time()
    self.run_test_case('QueryTest/runtime_filters_wait', vector)
    duration = time.time() - now
    assert duration < WAIT_TIME_MS, \
        "Query took too long (%ss, possibly waiting for missing filters?)" % str(duration)

  def test_file_filtering(self, vector):
    if 'kudu' in str(vector.get_value('table_format')):
      return
    self.change_database(self.client, vector.get_value('table_format'))
    self.execute_query("SET RUNTIME_FILTER_MODE=GLOBAL")
    self.execute_query("SET RUNTIME_FILTER_WAIT_TIME_MS=10000")
    result = self.execute_query("""select STRAIGHT_JOIN * from alltypes inner join
                                (select * from alltypessmall where smallint_col=-1) v
                                on v.year = alltypes.year""")
    assert re.search("Files rejected: 8 \(8\)", result.runtime_profile) is not None
    assert re.search("Splits rejected: [^0] \([^0]\)", result.runtime_profile) is None

@SkipIfLocal.multiple_impalad
class TestBloomFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestBloomFilters, cls).add_test_dimensions()
    # Bloom filters are disabled on HBase, Kudu
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format not in ['hbase', 'kudu'])

  def test_bloom_filters(self, vector):
    self.run_test_case('QueryTest/bloom_filters', vector)

  def test_bloom_wait_time(self, vector):
    """Test that a query that has global filters does not wait for them if run in LOCAL
    mode"""
    now = time.time()
    self.run_test_case('QueryTest/bloom_filters_wait', vector)
    duration = time.time() - now
    assert duration < 60, \
        "Query took too long (%ss, possibly waiting for missing filters?)" % str(duration)

@SkipIfLocal.multiple_impalad
class TestMinMaxFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestMinMaxFilters, cls).add_test_dimensions()
    # Min-max filters are only implemented for Kudu.
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format in ['kudu'])

  def test_min_max_filters(self, vector):
    self.run_test_case('QueryTest/min_max_filters', vector)

  def test_large_strings(self, cursor, unique_database):
    """Tests that truncation of large strings by min-max filters still gives correct
    results"""
    table1 = "%s.min_max_filter_large_strings1" % unique_database
    cursor.execute(
        "create table %s (string_col string primary key) stored as kudu" % table1)
    # Min-max bounds are truncated at 1024 characters, so construct some strings that are
    # longer than that, as well as some that are very close to the min/max bounds.
    matching_vals =\
        ('b' * 1100, 'b' * 1099 + 'c', 'd' * 1100, 'f'* 1099 + 'e', 'f' * 1100)
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s'), ('%s'), ('%s')"
        % ((table1,) + matching_vals))
    non_matching_vals = ('b' * 1099 + 'a', 'c', 'f' * 1099 + 'g')
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s')"
        % ((table1,) + non_matching_vals))

    table2 = "%s.min_max_filter_large_strings2" % unique_database
    cursor.execute(
        "create table %s (string_col string primary key) stored as kudu" % table2)
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s'), ('%s'), ('%s')"
        % ((table2,) + matching_vals))

    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals),)]

    # Insert a string that will have the max char (255) trailing after truncation, to
    # test the path where adding 1 to the max bound after trunc overflows.
    max_trail_str = "concat(repeat('h', 1000), repeat(chr(255), 50))"
    cursor.execute("insert into %s values (%s)" % (table1, max_trail_str))
    cursor.execute("insert into %s values (%s)" % (table2, max_trail_str))
    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals) + 1,)]

    # Insert a string that is entirely the max char to test the path where the max can't
    # have 1 added to it after truncation and the filter is disabled.
    all_max_str = "repeat(chr(255), 1030)"
    cursor.execute("insert into %s values (%s)" % (table1, all_max_str))
    cursor.execute("insert into %s values (%s)" % (table2, all_max_str))
    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals) + 2,)]

@SkipIfLocal.multiple_impalad
class TestRuntimeRowFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestRuntimeRowFilters, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.add_constraint(lambda v:
        v.get_value('table_format').file_format in ['parquet'])

  def test_row_filters(self, vector):
    self.run_test_case('QueryTest/runtime_row_filters', vector,
                       test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS' : str(WAIT_TIME_MS)})
<commit_msg>IMPALA-6998: test_bloom_wait_time fails due to late arrival of filters on Isilon<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import pytest
import re
import time

from tests.common.environ import specific_build_type_timeout
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal, SkipIfIsilon

WAIT_TIME_MS = specific_build_type_timeout(60000, slow_build_timeout=100000)

# Some of the queries in runtime_filters consume a lot of memory, leading to
# significant memory reservations in parallel tests.
# Skipping Isilon due to IMPALA-6998. TODO: Remove when there's a holistic revamp of
# what tests to run for non-HDFS platforms
@pytest.mark.execute_serially
@SkipIfLocal.multiple_impalad
@SkipIfIsilon.jira(reason="IMPALA-6998")
class TestRuntimeFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestRuntimeFilters, cls).add_test_dimensions()
    # Runtime filters are disabled on HBase
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format not in ['hbase'])

  def test_basic_filters(self, vector):
    self.run_test_case('QueryTest/runtime_filters', vector,
        test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS' : str(WAIT_TIME_MS)})

  def test_wait_time(self, vector):
    """Test that a query that has global filters does not wait for them if run in LOCAL
    mode"""
    now = time.time()
    self.run_test_case('QueryTest/runtime_filters_wait', vector)
    duration = time.time() - now
    assert duration < WAIT_TIME_MS, \
        "Query took too long (%ss, possibly waiting for missing filters?)" % str(duration)

  def test_file_filtering(self, vector):
    if 'kudu' in str(vector.get_value('table_format')):
      return
    self.change_database(self.client, vector.get_value('table_format'))
    self.execute_query("SET RUNTIME_FILTER_MODE=GLOBAL")
    self.execute_query("SET RUNTIME_FILTER_WAIT_TIME_MS=10000")
    result = self.execute_query("""select STRAIGHT_JOIN * from alltypes inner join
                                (select * from alltypessmall where smallint_col=-1) v
                                on v.year = alltypes.year""")
    assert re.search("Files rejected: 8 \(8\)", result.runtime_profile) is not None
    assert re.search("Splits rejected: [^0] \([^0]\)", result.runtime_profile) is None

@SkipIfLocal.multiple_impalad
class TestBloomFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestBloomFilters, cls).add_test_dimensions()
    # Bloom filters are disabled on HBase, Kudu
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format not in ['hbase', 'kudu'])

  def test_bloom_filters(self, vector):
    self.run_test_case('QueryTest/bloom_filters', vector)

  def test_bloom_wait_time(self, vector):
    """Test that a query that has global filters does not wait for them if run in LOCAL
    mode"""
    now = time.time()
    self.run_test_case('QueryTest/bloom_filters_wait', vector)
    duration = time.time() - now
    assert duration < 60, \
        "Query took too long (%ss, possibly waiting for missing filters?)" % str(duration)

@SkipIfLocal.multiple_impalad
class TestMinMaxFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestMinMaxFilters, cls).add_test_dimensions()
    # Min-max filters are only implemented for Kudu.
    cls.ImpalaTestMatrix.add_constraint(
        lambda v: v.get_value('table_format').file_format in ['kudu'])

  def test_min_max_filters(self, vector):
    self.run_test_case('QueryTest/min_max_filters', vector)

  def test_large_strings(self, cursor, unique_database):
    """Tests that truncation of large strings by min-max filters still gives correct
    results"""
    table1 = "%s.min_max_filter_large_strings1" % unique_database
    cursor.execute(
        "create table %s (string_col string primary key) stored as kudu" % table1)
    # Min-max bounds are truncated at 1024 characters, so construct some strings that are
    # longer than that, as well as some that are very close to the min/max bounds.
    matching_vals =\
        ('b' * 1100, 'b' * 1099 + 'c', 'd' * 1100, 'f'* 1099 + 'e', 'f' * 1100)
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s'), ('%s'), ('%s')"
        % ((table1,) + matching_vals))
    non_matching_vals = ('b' * 1099 + 'a', 'c', 'f' * 1099 + 'g')
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s')"
        % ((table1,) + non_matching_vals))

    table2 = "%s.min_max_filter_large_strings2" % unique_database
    cursor.execute(
        "create table %s (string_col string primary key) stored as kudu" % table2)
    cursor.execute("insert into %s values ('%s'), ('%s'), ('%s'), ('%s'), ('%s')"
        % ((table2,) + matching_vals))

    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals),)]

    # Insert a string that will have the max char (255) trailing after truncation, to
    # test the path where adding 1 to the max bound after trunc overflows.
    max_trail_str = "concat(repeat('h', 1000), repeat(chr(255), 50))"
    cursor.execute("insert into %s values (%s)" % (table1, max_trail_str))
    cursor.execute("insert into %s values (%s)" % (table2, max_trail_str))
    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals) + 1,)]

    # Insert a string that is entirely the max char to test the path where the max can't
    # have 1 added to it after truncation and the filter is disabled.
    all_max_str = "repeat(chr(255), 1030)"
    cursor.execute("insert into %s values (%s)" % (table1, all_max_str))
    cursor.execute("insert into %s values (%s)" % (table2, all_max_str))
    cursor.execute("select count(*) from %s a, %s b where a.string_col = b.string_col"
        % (table1, table2))
    assert cursor.fetchall() == [(len(matching_vals) + 2,)]

@SkipIfLocal.multiple_impalad
class TestRuntimeRowFilters(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestRuntimeRowFilters, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.add_constraint(lambda v:
        v.get_value('table_format').file_format in ['parquet'])

  def test_row_filters(self, vector):
    self.run_test_case('QueryTest/runtime_row_filters', vector,
                       test_file_vars={'$RUNTIME_FILTER_WAIT_TIME_MS' : str(WAIT_TIME_MS)})
<|endoftext|>