<commit_before># This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2020 ScyllaDB

import logging
import os
import uuid
import random
import json
import time

from sdcm.sct_events import Severity
from sdcm.utils.common import FileFollowerThread
from sdcm.sct_events.loaders import GeminiStressEvent, GeminiStressLogEvent
from sdcm.stress_thread import DockerBasedStressThread
from sdcm.utils.docker_remote import RemoteDocker

LOGGER = logging.getLogger(__name__)

class NotGeminiErrorResult:  # pylint: disable=too-few-public-methods
    def __init__(self, error):
        self.exited = 1
        self.stdout = "n/a"
        self.stderr = str(error)

class GeminiEventsPublisher(FileFollowerThread):
    def __init__(self, node, gemini_log_filename, verbose=False, event_id=None):
        super().__init__()
        self.gemini_log_filename = gemini_log_filename
        self.node = str(node)
        self.verbose = verbose
        self.event_id = event_id

    def run(self):
        while not self.stopped():
            if not os.path.isfile(self.gemini_log_filename):
                time.sleep(0.5)
                continue
            for line_number, line in enumerate(self.follow_file(self.gemini_log_filename), start=1):
                gemini_event = GeminiStressLogEvent.GeminiEvent(verbose=self.verbose)
                gemini_event.add_info(node=self.node, line=line, line_number=line_number)
                gemini_event.event_id = self.event_id
                gemini_event.publish(warn_not_ready=False)

                if self.stopped():
                    break

class GeminiStressThread(DockerBasedStressThread):  # pylint: disable=too-many-instance-attributes

    def __init__(self, test_cluster, oracle_cluster, loaders, stress_cmd, timeout=None, params=None):  # pylint: disable=too-many-arguments
        super().__init__(loader_set=loaders, stress_cmd=stress_cmd, timeout=timeout, params=params)
        self.test_cluster = test_cluster
        self.oracle_cluster = oracle_cluster
        self._gemini_result_file = None
        self.gemini_commands = []
        self.gemini_request_timeout = 180
        self.gemini_connect_timeout = 120

    @property
    def gemini_result_file(self):
        if not self._gemini_result_file:
            self._gemini_result_file = os.path.join("/gemini", "gemini_result_{}.log".format(uuid.uuid4()))
        return self._gemini_result_file

    def _generate_gemini_command(self):
        seed = self.params.get('gemini_seed')
        table_options = self.params.get('gemini_table_options')
        if not seed:
            seed = random.randint(1, 100)
        test_nodes = ",".join(self.test_cluster.get_node_cql_ips())
        oracle_nodes = ",".join(self.oracle_cluster.get_node_cql_ips()) if self.oracle_cluster else None

        cmd = "{} --test-cluster={} --outfile {} --seed {} --request-timeout {}s --connect-timeout {}s".format(
            self.stress_cmd.strip(),
            test_nodes,
            self.gemini_result_file,
            seed,
            self.gemini_request_timeout,
            self.gemini_connect_timeout)
        if oracle_nodes:
            cmd += "--oracle-cluster={} ".format(oracle_nodes)
        if table_options:
            cmd += " ".join([f"--table-options \"{table_opt}\"" for table_opt in table_options])
        self.gemini_commands.append(cmd)
        return cmd

    def _run_stress(self, loader, loader_idx, cpu_idx):

        cpu_options = ""
        if self.stress_num > 1:
            cpu_options = f'--cpuset-cpus="{cpu_idx}"'

        docker = RemoteDocker(loader, self.params.get("stress_image.gemini"),
                              extra_docker_opts=f'{cpu_options} --label shell_marker={self.shell_marker} --network=host')

        if not os.path.exists(loader.logdir):
            os.makedirs(loader.logdir, exist_ok=True)
        log_file_name = os.path.join(loader.logdir, 'gemini-l%s-c%s-%s.log' %
                                     (loader_idx, cpu_idx, uuid.uuid4()))
        LOGGER.debug('gemini local log: %s', log_file_name)

        gemini_cmd = self._generate_gemini_command()
        with GeminiEventsPublisher(node=loader, gemini_log_filename=log_file_name) as publisher, \
                GeminiStressEvent(node=loader, cmd=gemini_cmd, log_file_name=log_file_name) as gemini_stress_event:
            try:
                publisher.event_id = gemini_stress_event.event_id
                gemini_stress_event.log_file_name = log_file_name
                result = docker.run(cmd=gemini_cmd,
                                    timeout=self.timeout,
                                    ignore_status=False,
                                    log_file=log_file_name)
                # sleep to gather all latest log messages
                time.sleep(5)
            except Exception as details:  # pylint: disable=broad-except
                LOGGER.error(details)
                result = getattr(details, "result", NotGeminiErrorResult(details))

            if result.exited:
                gemini_stress_event.add_result(result=result)
                gemini_stress_event.severity = Severity.ERROR
            else:
                if result.stderr:
                    gemini_stress_event.add_result(result=result)
                    gemini_stress_event.severity = Severity.WARNING

        return docker, result, self.gemini_result_file

    def get_gemini_results(self):
        parsed_results = []
        raw_results = self.get_results()
        for docker, _, result_file in raw_results:

            local_gemini_result_file = os.path.join(docker.node.logdir, os.path.basename(result_file))
            results_copied = docker.receive_files(src=result_file, dst=local_gemini_result_file)
            assert results_copied, "gemini results aren't available, did gemini even run ?"
            with open(local_gemini_result_file, encoding="utf-8") as local_file:
                content = local_file.read()
                res = self._parse_gemini_summary_json(content)
                if res:
                    parsed_results.append(res)

        return parsed_results

    @staticmethod
    def verify_gemini_results(results):

        stats = {'results': [], 'errors': {}}
        if not results:
            LOGGER.error('Gemini results are not found')
            stats['status'] = 'FAILED'
        else:
            for res in results:
                stats['results'].append(res)
                for err_type in ['write_errors', 'read_errors', 'errors']:
                    if res.get(err_type, None):
                        LOGGER.error("Gemini {} errors: {}".format(err_type, res[err_type]))
                        stats['status'] = 'FAILED'
                        stats['errors'][err_type] = res[err_type]
        if not stats.get('status'):
            stats['status'] = "PASSED"

        return stats

    @staticmethod
    def _parse_gemini_summary_json(json_str):
        results = {'result': {}}
        try:
            results = json.loads(json_str)

        except Exception as details:  # pylint: disable=broad-except
            LOGGER.error("Invalid json document {}".format(details))

        return results.get('result')

    @staticmethod
    def _parse_gemini_summary(lines):
        results = {}
        enable_parse = False

        for line in lines:
            line.strip()
            if 'Results:' in line:
                enable_parse = True
                continue
            if "run completed" in line:
                enable_parse = False
                continue
            if not enable_parse:
                continue

            split_idx = line.index(':')
            key = line[:split_idx].strip()
            value = line[split_idx + 1:].split()[0]
            results[key] = int(value)
        return results
<commit_msg>fix(gemini_thread): correctly build gemini comand with oracle<commit_after># This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright (c) 2020 ScyllaDB

import logging
import os
import uuid
import random
import json
import time

from sdcm.sct_events import Severity
from sdcm.utils.common import FileFollowerThread
from sdcm.sct_events.loaders import GeminiStressEvent, GeminiStressLogEvent
from sdcm.stress_thread import DockerBasedStressThread
from sdcm.utils.docker_remote import RemoteDocker

LOGGER = logging.getLogger(__name__)

class NotGeminiErrorResult:  # pylint: disable=too-few-public-methods
    def __init__(self, error):
        self.exited = 1
        self.stdout = "n/a"
        self.stderr = str(error)

class GeminiEventsPublisher(FileFollowerThread):
    def __init__(self, node, gemini_log_filename, verbose=False, event_id=None):
        super().__init__()
        self.gemini_log_filename = gemini_log_filename
        self.node = str(node)
        self.verbose = verbose
        self.event_id = event_id

    def run(self):
        while not self.stopped():
            if not os.path.isfile(self.gemini_log_filename):
                time.sleep(0.5)
                continue
            for line_number, line in enumerate(self.follow_file(self.gemini_log_filename), start=1):
                gemini_event = GeminiStressLogEvent.GeminiEvent(verbose=self.verbose)
                gemini_event.add_info(node=self.node, line=line, line_number=line_number)
                gemini_event.event_id = self.event_id
                gemini_event.publish(warn_not_ready=False)

                if self.stopped():
                    break

class GeminiStressThread(DockerBasedStressThread):  # pylint: disable=too-many-instance-attributes

    def __init__(self, test_cluster, oracle_cluster, loaders, stress_cmd, timeout=None, params=None):  # pylint: disable=too-many-arguments
        super().__init__(loader_set=loaders, stress_cmd=stress_cmd, timeout=timeout, params=params)
        self.test_cluster = test_cluster
        self.oracle_cluster = oracle_cluster
        self._gemini_result_file = None
        self.gemini_commands = []
        self.gemini_request_timeout = 180
        self.gemini_connect_timeout = 120

    @property
    def gemini_result_file(self):
        if not self._gemini_result_file:
            self._gemini_result_file = os.path.join("/gemini", "gemini_result_{}.log".format(uuid.uuid4()))
        return self._gemini_result_file

    def _generate_gemini_command(self):
        seed = self.params.get('gemini_seed')
        table_options = self.params.get('gemini_table_options')
        if not seed:
            seed = random.randint(1, 100)
        test_nodes = ",".join(self.test_cluster.get_node_cql_ips())
        oracle_nodes = ",".join(self.oracle_cluster.get_node_cql_ips()) if self.oracle_cluster else None

        cmd = "{} --test-cluster={} --outfile {} --seed {} --request-timeout {}s --connect-timeout {}s ".format(
            self.stress_cmd.strip(),
            test_nodes,
            self.gemini_result_file,
            seed,
            self.gemini_request_timeout,
            self.gemini_connect_timeout)
        if oracle_nodes:
            cmd += "--oracle-cluster={} ".format(oracle_nodes)
        if table_options:
            cmd += " ".join([f"--table-options \"{table_opt}\"" for table_opt in table_options])
        self.gemini_commands.append(cmd)
        return cmd

    def _run_stress(self, loader, loader_idx, cpu_idx):

        cpu_options = ""
        if self.stress_num > 1:
            cpu_options = f'--cpuset-cpus="{cpu_idx}"'

        docker = RemoteDocker(loader, self.params.get("stress_image.gemini"),
                              extra_docker_opts=f'{cpu_options} --label shell_marker={self.shell_marker} --network=host')

        if not os.path.exists(loader.logdir):
            os.makedirs(loader.logdir, exist_ok=True)
        log_file_name = os.path.join(loader.logdir, 'gemini-l%s-c%s-%s.log' %
                                     (loader_idx, cpu_idx, uuid.uuid4()))
        LOGGER.debug('gemini local log: %s', log_file_name)

        gemini_cmd = self._generate_gemini_command()
        with GeminiEventsPublisher(node=loader, gemini_log_filename=log_file_name) as publisher, \
                GeminiStressEvent(node=loader, cmd=gemini_cmd, log_file_name=log_file_name) as gemini_stress_event:
            try:
                publisher.event_id = gemini_stress_event.event_id
                gemini_stress_event.log_file_name = log_file_name
                result = docker.run(cmd=gemini_cmd,
                                    timeout=self.timeout,
                                    ignore_status=False,
                                    log_file=log_file_name)
                # sleep to gather all latest log messages
                time.sleep(5)
            except Exception as details:  # pylint: disable=broad-except
                LOGGER.error(details)
                result = getattr(details, "result", NotGeminiErrorResult(details))

            if result.exited:
                gemini_stress_event.add_result(result=result)
                gemini_stress_event.severity = Severity.ERROR
            else:
                if result.stderr:
                    gemini_stress_event.add_result(result=result)
                    gemini_stress_event.severity = Severity.WARNING

        return docker, result, self.gemini_result_file

    def get_gemini_results(self):
        parsed_results = []
        raw_results = self.get_results()
        for docker, _, result_file in raw_results:

            local_gemini_result_file = os.path.join(docker.node.logdir, os.path.basename(result_file))
            results_copied = docker.receive_files(src=result_file, dst=local_gemini_result_file)
            assert results_copied, "gemini results aren't available, did gemini even run ?"
            with open(local_gemini_result_file, encoding="utf-8") as local_file:
                content = local_file.read()
                res = self._parse_gemini_summary_json(content)
                if res:
                    parsed_results.append(res)

        return parsed_results

    @staticmethod
    def verify_gemini_results(results):

        stats = {'results': [], 'errors': {}}
        if not results:
            LOGGER.error('Gemini results are not found')
            stats['status'] = 'FAILED'
        else:
            for res in results:
                stats['results'].append(res)
                for err_type in ['write_errors', 'read_errors', 'errors']:
                    if res.get(err_type, None):
                        LOGGER.error("Gemini {} errors: {}".format(err_type, res[err_type]))
                        stats['status'] = 'FAILED'
                        stats['errors'][err_type] = res[err_type]
        if not stats.get('status'):
            stats['status'] = "PASSED"

        return stats

    @staticmethod
    def _parse_gemini_summary_json(json_str):
        results = {'result': {}}
        try:
            results = json.loads(json_str)

        except Exception as details:  # pylint: disable=broad-except
            LOGGER.error("Invalid json document {}".format(details))

        return results.get('result')

    @staticmethod
    def _parse_gemini_summary(lines):
        results = {}
        enable_parse = False

        for line in lines:
            line.strip()
            if 'Results:' in line:
                enable_parse = True
                continue
            if "run completed" in line:
                enable_parse = False
                continue
            if not enable_parse:
                continue

            split_idx = line.index(':')
            key = line[:split_idx].strip()
            value = line[split_idx + 1:].split()[0]
            results[key] = int(value)
        return results
<|endoftext|>


<commit_before>"""
seaworthy
~~~~~~~~~

.. todo::

   Write some API reference docs for :mod:`seaworthy`.
"""

from .helpers import DockerHelper
from .logs import output_lines, wait_for_logs_matching

__all__ = ['DockerHelper', 'output_lines', 'wait_for_logs_matching']

__version__ = '0.2.2.dev0'
<commit_msg>Version 0.3.0<commit_after>"""
seaworthy
~~~~~~~~~

.. todo::

   Write some API reference docs for :mod:`seaworthy`.
"""

from .helpers import DockerHelper
from .logs import output_lines, wait_for_logs_matching

__all__ = ['DockerHelper', 'output_lines', 'wait_for_logs_matching']

__version__ = '0.3.0'
<|endoftext|>


<commit_before>"""
Plugin to create a quick panel lookup that lets you jump between comment
titles
"""

import os
import imp
import time
import sys
import sublime
import sublime_plugin
import re

#
# > Plugin command
#
class table_of_comments_command(sublime_plugin.TextCommand):

    def run(self, edit, move=None, fold=None, unfold=None):
        if move is not None:
            self.traverse_comments(edit, move)
        elif fold is not None or unfold is not None:
            self.fold_comments(edit, fold, unfold)
        else:
            self.show_quick_panel(edit)

    # >> Quick panel
    def show_quick_panel(self, edit):
        view = self.view
        toc = TableOfComments(view, edit)
        toc._debug_start('Show quick panel')
        toc.create_toc()
        # Store position for returning to
        return_to = []
        for each in view.sel():
            return_to.append(each)
        toc.return_to = return_to
        # Pop up the panel
        titles = toc.get_comment_titles('string')
        self.window = sublime.active_window()
        if sys.version_info < (3, 0):
            self.window.show_quick_panel(titles, toc.on_list_selected_done)
        else:
            self.window.show_quick_panel(  # Pass on_highlighted callback
                titles, toc.on_list_selected_done, False, 0,
                toc.on_list_selected_done)
        toc._debug_stop('Show quick panel')

    # >> Up down
    # Allows moving up and down through comments
    def traverse_comments(self, edit, move):
        view = self.view
        toc = TableOfComments(view, edit)
        titles = toc.get_comment_titles()
        sel = view.sel()
        if len(sel) == 1:
            current_line_no, col_no = view.rowcol(sel[0].b)
            for x in range(len(titles)):
                item = titles[x]
                if move == 'up':  # moving up
                    if item['line'] < current_line_no:
                        if x+1 < len(titles):
                            if titles[x+1]['line'] >= current_line_no:
                                return toc.on_list_selected_done(x)
                        else:
                            return self.on_list_selected_done(x)
                else:  # moving down
                    if item['line'] > current_line_no:
                        return toc.on_list_selected_done(x)

    # >> Fold comments
    def fold_comments(self, edit, fold, unfold):
        toc = TableOfComments(self.view, edit)
        comments = self.view.find_by_selector('comment')
        titles = toc.get_comment_titles()
        is_all = fold == 'all' or unfold == 'all'
        # Current selection
        sels = self.view.sel()
        for each in sels:
            sel = each
        # Only get comment blocks with titles within them
        regions = []
        for i in range(len(comments)):
            region = comments[i]
            for title in titles:
                if region.contains(title['region']):
                    regions.append(region)
        # Get the fold regions (content blocks)
        fold_regions = []
        for i in range(len(regions)):
            region = regions[i]
            for title in titles:
                if region.contains(title['region']):
                    fold_start = region.b + 1
                    fold_end = self.view.size()
                    if i < len(regions)-1:
                        fold_end = regions[i+1].a - 1
                    region_content = sublime.Region(fold_start, fold_end)
                    if region.contains(sel) or is_all:
                        fold_regions.append(region_content)
                    elif region_content.contains(sel):
                        fold_regions.append(region_content)

        # Fold, unfold or toggle
        if fold is not None:
            self.view.fold(fold_regions)
        elif unfold is not None:
            self.view.unfold(fold_regions)
        elif self.view.fold(fold_regions) is False:
            self.view.unfold(fold_regions)

#
# > Plugin class
#
class TableOfComments:

    def __init__(self, view, edit):
        self.view = view
        self.edit = edit

#
# Debug timing functions
#
#
    timers = {}

    def _debug_start(self, ref):
        self.timers[ref] = time.time()

    def _debug_stop(self, ref):
        start_time = self.timers[ref]
        duration = time.time() - start_time
        self.timers[ref] = duration

#
# Table TOC tag
#

    def get_toc_region(self, view):
        title = get_setting('toc_title', str)
        pattern = r'\/\*(\s|\*)*'+title+r'[^\/]*\/'
        matches = view.find_all(pattern)
        for region in (matches):
            if self.is_scope_or_comment(view, region):
                return region
        return None

    def is_in_toc_region(self, view, region):
        toc_region = self.get_toc_region(view)
        if toc_region:
            if region.a > toc_region.a and region.a < toc_region.b:
                return True
        return False

    def create_toc(self):
        view = self.view
        edit = self.edit
        region = self.get_toc_region(view)
        if region:
            toc = self.compile_toc(view)
            existing = view.substr(region)
            if existing != toc:
                view.replace(edit, region, toc)

    def compile_toc(self, view):
        self._debug_start('compile-toc')
        titles = self.get_comment_titles('string')
        title = get_setting('toc_title', str)
        start = get_setting('toc_start', str)
        line = get_setting('toc_line', str)
        end = get_setting('toc_end', str)
        front = "\n" + line
        output = start + front + title + front.rstrip()
        for title in titles:
            comment_level = title.count('-') + 1
            try:
                level = int(get_setting('toc_level', int))
                if level >= comment_level:
                    output += front + title
            except TypeError:
                output += front + title
        output += "\n"+end
        self._debug_stop('compile-toc')
        return output

#
# >> Quick panel
#

    # Jump list quick menu selected
    def on_list_selected_done(self, picked):
        if picked == -1:
            self.view.sel().clear()
            for each in self.return_to:
                self.view.sel().add(each)
            self.view.show(self.view.sel())
        else:
            titles = self.get_comment_titles()
            title = titles[picked]
            row = title['line']
            point = self.view.text_point(row, 0)
            line_region = self.view.line(point)
            text = title['text']
            text = re.escape(text)
            text = text.replace('\>', '>')  # ">" does not work when escaped
            text_region = self.view.find(
                text, line_region.a)
            self.view.sel().clear()
            self.view.sel().add(text_region)
            self.view.show_at_center(text_region.b)

#
# >> Parse
#

    # Core parse function (returned as dict or list)
    def get_comment_titles(self, format='dict', test=None):
        self._debug_start('get-comment-titles')
        view = self.view
        level_char = get_setting('level_char', str)
        comment_chars = get_setting('comment_chars', str)
        escaped_chars = re.escape(comment_chars)
        comment = list(comment_chars)
        comment = 'DIV'.join(comment_chars)
        start = r'\s|'+re.escape(comment).replace('DIV', '|')
        # build the pattern to match the comment
        pattern = r'^('+start+')*?('+format_pattern(level_char)+'+)\s*' + \
            r'([^'+escaped_chars+']+)('+start+')*?$'

        matches = view.find_all(pattern)
        results = []
        toc_title = get_setting('toc_title', str)

        for match in matches:
            bits = view.lines(match)  # go through each line
            for region in bits:
                # Ensure it's comment or source
                if not self.is_scope_or_comment(view, region):
                    continue
                # Ensure not in toc region already
                if self.is_in_toc_region(view, region):
                    continue

                line = view.substr(region)
                line_match = re.match(pattern, line)

                if not line_match:
                    continue

                if level_char in line:
                    # Add the level chars
                    label = line_match.group(2)

                    # Replace level char with toc char
                    label = self.replace_level_chars(label)
                    if label != '':
                        label += ' '

                    # append the heading text
                    text = line_match.group(3).strip()
                    label += text

                    # Get the position
                    if line != '' and line != toc_title:
                        line_no, col_no = view.rowcol(region.b)
                        if format == 'dict':
                            results.append(
                                {'label': label, 'text': text,
                                    'region': region, 'line': line_no})
                        else:
                            results.append(label)
        self._debug_stop('get-comment-titles')
        return results

    # Only find titles within genuine comments
    # This will no doubt need to be improved over time for various syntaxes
    # ('string.quoted' makes python """ comments """ not trigger)
    def is_scope_or_comment(self, view, region):
        line = view.substr(region)
        # Trim to scope
        # If line starts with whitespace, the syntax returned is "source" not
        # "comment" for the initial char
        trimmed = line.lstrip()
        diff = len(line) - len(trimmed)
        scope = view.scope_name(region.a + diff)
        # Check out scope
        comments_scope = ['comment']
        disallow = ['string.quoted']
        for each in comments_scope:
            if scope.find(each) < 0:
                return False
        for each in disallow:
            if scope.find(each) > 0:
                return False
        return True

    def replace_level_chars(self, line):
        level_char = get_setting('level_char', str)
        toc_char = get_setting('toc_char', str)
        # remove the last char so level one has no indent
        line = line[:-1].replace(level_char, toc_char)
        return line

#
# Helpers
#

def format_pattern(pattern):
    pattern = re.escape(pattern)
    pattern = pattern.replace('\>', '>')
    return pattern

def get_setting(name, typeof=str):
    settings = sublime.load_settings('tableofcomments.sublime-settings')
    setting = settings.get(name)
    if setting:
        if typeof == str:
            return setting
        if typeof == bool:
            return setting is True
        elif typeof == int:
            return int(settings.get(name, 500))
    else:
        if typeof == str:
            return ''
        else:
            return None

#
# Testing infrastructure
#

if sys.version_info < (3, 0):
    import tests
else:
    from . import tests

class table_of_comments_run_tests_command(sublime_plugin.TextCommand):
    def run(self, edit):
        reload_test_bootstrap()
        tests.run(self.view, edit)

# For developing, reload tests.* which in turn reloads it's sub packages
basedir = os.getcwd()

def reload_test_bootstrap():
    os.chdir(basedir)
    path = 'tests'
    if sys.version_info < (3, 0):
        __import__(path)
        sys.modules[path] = reload(sys.modules[path])
    else:
        imp.reload(eval(path))
<commit_msg>Made quick jump open at current location within comment titles Fixed bug stopping 'move up' from bottom of document Added plugin sections which returns all elements of the plugin output for titles and content (used for folding and quick panel jump now)<commit_after>"""
Plugin to create a quick panel lookup that lets you jump between comment
titles
"""

import os
import imp
import time
import sys
import sublime
import sublime_plugin
import re

#
# > Plugin command
#
class table_of_comments_command(sublime_plugin.TextCommand):

    def run(self, edit, move=None, fold=None, unfold=None):
        toc = TableOfComments(self.view, edit)
        if move is not None:
            self.traverse_comments(toc, move)
        elif fold is not None or unfold is not None:
            self.fold_comments(toc, fold, unfold)
        else:
            self.show_quick_panel(toc)

    # >> Quick panel
    def show_quick_panel(self, toc):
        view = self.view
        toc._debug_start('Show quick panel')
        toc.create_toc()
        # Get current section from cursor
        show_index = 0
        current_section = toc.get_section_from_cursor()
        if current_section:
            show_index = current_section['index']
        # Store position for returning to
        return_to = []
        for each in view.sel():
            return_to.append(each)
        toc.return_to = return_to
        # Pop up the panel
        titles = toc.get_comment_titles('string')
        self.window = sublime.active_window()
        if sys.version_info < (3, 0):
            self.window.show_quick_panel(titles, toc.on_list_selected_done)
        else:
            self.window.show_quick_panel(  # Pass on_highlighted callback
                titles, toc.on_list_selected_done, False, show_index,
                toc.on_list_selected_done)
        toc._debug_stop('Show quick panel')

    # >> Up down
    # Allows moving up and down through comments
    def traverse_comments(self, toc, move):
        view = self.view
        titles = toc.get_comment_titles()
        sel = view.sel()
        if len(sel) == 1:
            current_line_no, col_no = view.rowcol(sel[0].b)
            for x in range(len(titles)):
                item = titles[x]
                if move == 'up':  # moving up
                    if item['line'] < current_line_no:
                        if x+1 < len(titles):
                            if titles[x+1]['line'] >= current_line_no:
                                return toc.on_list_selected_done(x)
                        else:
                            return toc.on_list_selected_done(x)
                else:  # moving down
                    if item['line'] > current_line_no:
                        return toc.on_list_selected_done(x)

    # >> Fold comments
    def fold_comments(self, toc, fold, unfold):
        comments = self.view.find_by_selector('comment')
        titles = toc.get_comment_titles()
        is_all = fold == 'all' or unfold == 'all'
        # Current selection
        sels = self.view.sel()
        for each in sels:
            sel = each
        # Get content regions to fold
        fold_regions = []
        sections = toc.get_plugin_sections()
        for each in sections:
            title_region = each['title_region']
            content_region = each['content_region']
            if title_region.contains(sel) or is_all:
                fold_regions.append(content_region)
            elif content_region.contains(sel):
                fold_regions.append(content_region)
        # Fold, unfold or toggle
        if fold is not None:
            self.view.fold(fold_regions)
        elif unfold is not None:
            self.view.unfold(fold_regions)
        elif self.view.fold(fold_regions) is False:
            self.view.unfold(fold_regions)

#
# > Plugin class
#
class TableOfComments:

    def __init__(self, view, edit):
        self.view = view
        self.edit = edit

#
# Debug timing functions
#
#
    timers = {}

    def _debug_start(self, ref):
        self.timers[ref] = time.time()

    def _debug_stop(self, ref):
        start_time = self.timers[ref]
        duration = time.time() - start_time
        self.timers[ref] = duration

#
# Table TOC tag
#

    def get_toc_region(self, view):
        title = get_setting('toc_title', str)
        pattern = r'\/\*(\s|\*)*'+title+r'[^\/]*\/'
        matches = view.find_all(pattern)
        for region in (matches):
            if self.is_scope_or_comment(view, region):
                return region
        return None

    def is_in_toc_region(self, view, region):
        toc_region = self.get_toc_region(view)
        if toc_region:
            if region.a > toc_region.a and region.a < toc_region.b:
                return True
        return False

    def create_toc(self):
        view = self.view
        edit = self.edit
        region = self.get_toc_region(view)
        if region:
            toc = self.compile_toc(view)
            existing = view.substr(region)
            if existing != toc:
                view.replace(edit, region, toc)

    def compile_toc(self, view):
        self._debug_start('compile-toc')
        titles = self.get_comment_titles('string')
        title = get_setting('toc_title', str)
        start = get_setting('toc_start', str)
        line = get_setting('toc_line', str)
        end = get_setting('toc_end', str)
        front = "\n" + line
        output = start + front + title + front.rstrip()
        for title in titles:
            comment_level = title.count('-') + 1
            try:
                level = int(get_setting('toc_level', int))
                if level >= comment_level:
                    output += front + title
            except TypeError:
                output += front + title
        output += "\n"+end
        self._debug_stop('compile-toc')
        return output

#
# >> Quick panel
#

    # Jump list quick menu selected
    def on_list_selected_done(self, picked):
        if picked == -1:
            self.view.sel().clear()
            for each in self.return_to:
                self.view.sel().add(each)
            self.view.show(self.view.sel())
        else:
            titles = self.get_comment_titles()
            title = titles[picked]
            row = title['line']
            point = self.view.text_point(row, 0)
            line_region = self.view.line(point)
            text = title['text']
            text = re.escape(text)
            text = text.replace('\>', '>')  # ">" does not work when escaped
            text_region = self.view.find(
                text, line_region.a)
            self.view.sel().clear()
            self.view.sel().add(text_region)
            self.view.show_at_center(text_region.b)

#
# >> Parse
#

    # Core parse function (returned as dict or list)
    def get_comment_titles(self, format='dict', test=None):
        self._debug_start('get-comment-titles')
        view = self.view
        level_char = get_setting('level_char', str)
        comment_chars = get_setting('comment_chars', str)
        escaped_chars = re.escape(comment_chars)
        comment = list(comment_chars)
        comment = 'DIV'.join(comment_chars)
        start = r'\s|'+re.escape(comment).replace('DIV', '|')
        # build the pattern to match the comment
        pattern = r'^('+start+')*?('+format_pattern(level_char)+'+)\s*' + \
            r'([^'+escaped_chars+']+)('+start+')*?$'

        matches = view.find_all(pattern)
        results = []
        toc_title = get_setting('toc_title', str)

        for match in matches:
            bits = view.lines(match)  # go through each line
            for region in bits:
                # Ensure it's comment or source
                if not self.is_scope_or_comment(view, region):
                    continue
                # Ensure not in toc region already
                if self.is_in_toc_region(view, region):
                    continue

                line = view.substr(region)
                line_match = re.match(pattern, line)

                if not line_match:
                    continue

                if level_char in line:
                    # Add the level chars
                    label = line_match.group(2)

                    # Replace level char with toc char
                    label = self.replace_level_chars(label)
                    if label != '':
                        label += ' '

                    # append the heading text
                    text = line_match.group(3).strip()
                    label += text

                    # Get the position
                    if line != '' and line != toc_title:
                        line_no, col_no = view.rowcol(region.b)
                        if format == 'dict':
                            results.append(
                                {'label': label, 'text': text,
                                    'region': region, 'line': line_no})
                        else:
                            results.append(label)
        self._debug_stop('get-comment-titles')
        return results

#
# >> Plugin sections (regions)
#

    # Returns list of sections dicts with all related values
    def get_plugin_sections(self):
        comments = self.view.find_by_selector('comment')
        titles = self.get_comment_titles()
        # Only get comment blocks with titles within them
        sections = []
        for i in range(len(comments)):
            comment = comments[i]
            for title in titles:
                if comment.contains(title['region']):
                    title['title_region'] = comment
                    sections.append(title)
        # Get the fold regions (content blocks)
        output = []
        for i in range(len(sections)):
            section = sections[i]
            section['index'] = i
            region = section['title_region']
            for title in titles:
                if region.contains(title['region']):
                    fold_start = region.b + 1
                    fold_end = self.view.size()
                    if i < len(sections)-1:
                        fold_end = sections[i+1]['region'].a - 1
                    content_region = sublime.Region(fold_start, fold_end)
                    section['content_region'] = content_region
            output.append(section)
        return output

    # Returns the title and content region from cursor
    def get_section_from_cursor(self):
        # Current selection
        sels = self.view.sel()
        for each in sels:
            sel = each
        # Find within sections
        sections = self.get_plugin_sections()
        for section in sections:
            if section['title_region'].contains(sel) \
                    or section['content_region'].contains(sel):
                return section
        return False

    # Only find titles within genuine comments
    # This will no doubt need to be improved over time for various syntaxes
    # ('string.quoted' makes python """ comments """ not trigger)
    def is_scope_or_comment(self, view, region):
        line = view.substr(region)
        # Trim to scope
        # If line starts with whitespace, the syntax returned is "source" not
        # "comment" for the initial char
        trimmed = line.lstrip()
        diff = len(line) - len(trimmed)
        scope = view.scope_name(region.a + diff)
        # Check out scope
        comments_scope = ['comment']
        disallow = ['string.quoted']
        for each in comments_scope:
            if scope.find(each) < 0:
                return False
        for each in disallow:
            if scope.find(each) > 0:
                return False
        return True

    def replace_level_chars(self, line):
        level_char = get_setting('level_char', str)
        toc_char = get_setting('toc_char', str)
        # remove the last char so level one has no indent
        line = line[:-1].replace(level_char, toc_char)
        return line

#
# Helpers
#

def format_pattern(pattern):
    pattern = re.escape(pattern)
    pattern = pattern.replace('\>', '>')
    return pattern

def get_setting(name, typeof=str):
    settings = sublime.load_settings('tableofcomments.sublime-settings')
    setting = settings.get(name)
    if setting:
        if typeof == str:
            return setting
        if typeof == bool:
            return setting is True
        elif typeof == int:
            return int(settings.get(name, 500))
    else:
        if typeof == str:
            return ''
        else:
            return None

#
# Testing infrastructure
#

if sys.version_info < (3, 0):
    import tests
else:
    from . import tests

class table_of_comments_run_tests_command(sublime_plugin.TextCommand):
    def run(self, edit):
        reload_test_bootstrap()
        tests.run(self.view, edit)

# For developing, reload tests.* which in turn reloads it's sub packages
basedir = os.getcwd()

def reload_test_bootstrap():
    os.chdir(basedir)
    path = 'tests'
    if sys.version_info < (3, 0):
        __import__(path)
        sys.modules[path] = reload(sys.modules[path])
    else:
        imp.reload(eval(path))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple
from .iterator import Iterator

class Table(object):
    """Table representation.

    Args:
        loader (tabulator.loaders.API): table loader
        parser (tabulator.parsers.API): table parser

    """

    # Public

    def __init__(self, loader, parser):
        self.__loader = loader
        self.__parser = parser
        self.__processors = []
        self.__bytes = None
        self.__items = None
        self.__iterator = None

    def __enter__(self):
        """Enter context manager by opening table.
        """
        self.open()
        return self

    def __exit__(self, type, value, traceback):
        """Exit context manager by closing table.
        """
        self.close()

    def add_processor(self, processor):
        """Add processor to pipeline.
        """
        self.__processors.append(processor)

    def open(self):
        """Open table by opening source stream.
        """
        if self.closed:
            self.__bytes = self.__loader.load()
            self.__items = self.__parser.parse(self.__bytes)
            self.__iterator = Iterator(
                    self.__bytes, self.__items, self.__processors)

    def close(self):
        """Close table by closing source stream.
        """
        if not self.closed:
            self.__bytes.close()

    @property
    def closed(self):
        """Return true if table is closed.
        """
        return not self.__bytes or self.__bytes.closed

    def read(self, with_headers=False, limit=None):
        """Return full table.
        """
        rows = []
        rows_iter = self.readrow(with_headers=with_headers)
        for row_index, row in enumerate(rows_iter):
            if row_index > limit:
                break
            rows.append(row)
        return rows

    def readrow(self, with_headers=False):
        """Return next row from the source stream.
        """
        self.__require_not_closed()
        for iterator in self.__iterator:
            row = iterator.values
            if with_headers:
                if iterator.headers is None:
                    raise RuntimeError('No headers are available.')
                Row = namedtuple('Row', iterator.headers)
                row = Row(*iterator.values)
            yield row

    @property
    def headers(self):
        """Return table headers.
        """
        self.__require_not_closed()
        if self.__iterator.headers is None:
            if self.__iterator.input_index == 0:
                for iterator in self.__iterator:
                    if iterator.headers is not None:
                        break
                self.__iterator.reset()
        return self.__iterator.headers

    def reset(self):
        """Reset pointer to the first row.
        """
        self.__require_not_closed()
        self.__iterator.reset()

    # Private

    def __require_not_closed(self):
        if self.closed:
            message = (
               'Table have to be opened by `table.open()` before '
               'iteration interface will be available.')
            raise RuntimeError(message)
<commit_msg>refactored readrow, read<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple
from .iterator import Iterator

class Table(object):
    """Table representation.

    Args:
        loader (tabulator.loaders.API): table loader
        parser (tabulator.parsers.API): table parser

    """

    # Public

    def __init__(self, loader, parser):
        self.__loader = loader
        self.__parser = parser
        self.__processors = []
        self.__bytes = None
        self.__items = None
        self.__iterator = None

    def __enter__(self):
        """Enter context manager by opening table.
        """
        self.open()
        return self

    def __exit__(self, type, value, traceback):
        """Exit context manager by closing table.
        """
        self.close()

    def add_processor(self, processor):
        """Add processor to pipeline.
        """
        self.__processors.append(processor)

    def open(self):
        """Open table by opening source stream.
        """
        if self.closed:
            self.__bytes = self.__loader.load()
            self.__items = self.__parser.parse(self.__bytes)
            self.__iterator = Iterator(
                    self.__bytes, self.__items, self.__processors)

    def close(self):
        """Close table by closing source stream.
        """
        if not self.closed:
            self.__bytes.close()

    @property
    def closed(self):
        """Return true if table is closed.
        """
        return not self.__bytes or self.__bytes.closed

    def read(self, with_headers=False, limit=None):
        """Return full table with row limit.
        """
        return list(self.readrow(with_headers=with_headers, limit=limit))

    @property
    def headers(self):
        """Return table headers.
        """
        self.__require_not_closed()
        if self.__iterator.headers is None:
            if self.__iterator.input_index == 1:
                for iterator in self.__iterator:
                    if iterator.headers is not None:
                        break
                if self.__iterator.input_index > 1:
                    self.__iterator.reset()
        return self.__iterator.headers

    def readrow(self, with_headers=False, limit=None):
        """Return next row from the source stream.
        """
        self.__require_not_closed()
        for iterator in self.__iterator:
            if limit is not None:
                if iterator.output_index > limit:
                    break
            row = iterator.values
            if with_headers:
                if iterator.headers is None:
                    raise RuntimeError('No headers are available.')
                Row = namedtuple('Row', iterator.headers)
                row = Row(*iterator.values)
            yield row

    def reset(self):
        """Reset pointer to the first row.
        """
        self.__require_not_closed()
        self.__iterator.reset()

    # Private

    def __require_not_closed(self):
        if self.closed:
            message = (
               'Table have to be opened by `table.open()` before '
               'iteration interface will be available.')
            raise RuntimeError(message)
<|endoftext|>


<commit_before>import copy
import functools
import json
import logging
import datetime
import traceback

from sen.exceptions import TerminateApplication

import docker
import humanize

logger = logging.getLogger(__name__)

class ImageNameStruct(object):
    """
    stolen from atomic-reactor; thanks @mmilata!
    """
    def __init__(self, registry=None, namespace=None, repo=None, tag=None):
        self.registry = registry
        self.namespace = namespace
        self.repo = repo
        self.tag = tag

    @classmethod
    def parse(cls, image_name):
        result = cls()

        # registry.org/namespace/repo:tag
        s = image_name.split('/', 2)

        if len(s) == 2:
            if '.' in s[0] or ':' in s[0]:
                result.registry = s[0]
            else:
                result.namespace = s[0]
        elif len(s) == 3:
            result.registry = s[0]
            result.namespace = s[1]
        if result.namespace == 'library':
            # https://github.com/projectatomic/atomic-reactor/issues/45
            logger.debug("namespace 'library' -> ''")
            result.namespace = None
        result.repo = s[-1]

        try:
            result.repo, result.tag = result.repo.rsplit(':', 1)
        except ValueError:
            pass

        return result

    def to_str(self, registry=True, tag=True, explicit_tag=False,
               explicit_namespace=False):
        if self.repo is None:
            raise RuntimeError('No image repository specified')

        result = self.repo

        if tag and self.tag:
            result = '{0}:{1}'.format(result, self.tag)
        elif tag and explicit_tag:
            result = '{0}:{1}'.format(result, 'latest')

        if self.namespace:
            result = '{0}/{1}'.format(self.namespace, result)
        elif explicit_namespace:
            result = '{0}/{1}'.format('library', result)

        if registry and self.registry:
            result = '{0}/{1}'.format(self.registry, result)

        return result

    def __str__(self):
        return self.to_str(registry=True, tag=True)

    def __repr__(self):
        return "ImageName(image=%s)" % repr(self.to_str())

    def __eq__(self, other):
        return type(self) == type(other) and self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self.to_str())

    def copy(self):
        return ImageNameStruct(
            registry=self.registry,
            namespace=self.namespace,
            repo=self.repo,
            tag=self.tag)

def operation(fmt_str):
    def wrap(func):
        @functools.wraps(func)
        def wrapper(self, *args, **kwargs):
            command_took = None
            pretty_message = ""
            # cached queries do not access backend -- we don't care about that
            if kwargs.get("cached", False) is True:
                response = func(self, *args, **kwargs)
            else:
                before = datetime.datetime.now()
                response = func(self, *args, **kwargs)
                after = datetime.datetime.now()

                # we want milliseconds, not seconds
                command_took = (after - before).total_seconds() * 1000
                logger.debug("%s(%s, %s) %s -> [%f ms]", func.__name__, args, kwargs, self,
                             command_took)
            if fmt_str:
                pretty_message = fmt_str.format(object_type=getattr(self, "pretty_object_type", ""),
                                                object_short_name=getattr(self, "short_name", ""))
            return Operation(response, pretty_message=pretty_message, took=command_took)
        return wrapper
    return wrap

class Operation:
    """
    class for describing performed operation
    """

    def __init__(self, response, pretty_message="", took=None):
        self.response = response
        self.pretty_message = pretty_message
        self.took = took

class DockerObject:
    """
    Common base for images and containers
    """
    def __init__(self, data, docker_backend, object_id=None):
        self._id = object_id
        self._short_id = None
        self.data = data  # `client.containers` or `client.images`
        self.docker_backend = docker_backend
        self._created = None
        self._inspect = None
        self._names = None

    @property
    def d(self):
        """
        shortcut for instance of Docker client
        """
        return self.docker_backend.client

    @property
    def created(self):
        if self._created is None:
            self._created = datetime.datetime.fromtimestamp(self.data["Created"])
        return self._created

    def set_id(self):
        if self._id is None:
            try:
                self._id = self.data["Id"]
            except KeyError:
                raise RuntimeError("initial data not specified")

    @property
    def short_id(self):
        if self._short_id is None:
            self.set_id()
            self._short_id = self._id[:12]
        return self._short_id

    def display_time_created(self):
        return humanize.naturaltime(self.created)

    def inspect(self):
        raise NotImplementedError()

    def display_inspect(self):
        return json.dumps(self.inspect().response, indent=2)

def graceful_chain_get(d, *args):
    if not d:
        return None
    t = copy.deepcopy(d)
    for arg in args:
        try:
            t = t[arg]
        except (AttributeError, KeyError, TypeError):
            return None
    return t

class DockerImage(DockerObject):
    @property
    def image_id(self):
        if self._id is None:
            self.set_id()
        return self._id

    @property
    def parent_id(self):
        if self.data:
            return self.data.get("ParentId", None)
        else:
            inspect_operation = self.inspect(cached=True)
            return inspect_operation.response.get("Parent", None)

    @property
    def pretty_object_type(self):
        return "Image"

    @property
    def parent_image(self):
        try:
            parent_id = self.parent_id
        except Exception as ex:
            logger.error("error while getting parent ID of image %s: %r", self, ex)
            logger.info(traceback.format_exc())
            raise
        if parent_id:
            return DockerImage(None, self.docker_backend, object_id=parent_id)

    @property
    def command(self):
        cmd = graceful_chain_get(self.inspect(cached=True), "Config", "Cmd")
        if cmd:
            return " ".join(cmd)
        return ""

    @property
    def names(self):
        if self._names is None:
            self._names = []
            if self.data is None:
                return self._names
            for t in self.data["RepoTags"]:
                self._names.append(ImageNameStruct.parse(t))
            # sort by name length
            self._names.sort(key=lambda x: len(x.to_str()))
        return self._names

    @property
    def short_name(self):
        try:
            ins = self.names[0]
        except IndexError:
            return self.short_id
        if ins.repo == "<none>":
            return self.short_id
        return ins.to_str()

    def base_image(self):
        child_image = self
        while True:
            try:
                parent_image = self.docker_backend.get_image_by_id(child_image.parent_id)
            except Exception as ex:
                logger.warning("error while getting image by ID: %r", ex)
                parent_image = None
            if parent_image is None:
                try:
                    child_image = child_image.parent_image
                except Exception as ex:
                    logger.error("error while getting parent image for image %s: %r", self, ex)
                    return None
                if child_image is None:
                    return None
            else:
                return parent_image

    @operation("Inspect image {object_short_name}.")
    def inspect(self, cached=False):
        if self._inspect is None or cached is False:
            self._inspect = self.d.inspect_image(self.image_id)
        return self._inspect

    @operation("{object_type} {object_short_name} removed!")
    def remove(self):
        return self.d.remove_image(self.image_id)

    def matches_search(self, s):
        return s in self.image_id or \
            s in self.short_name

    def __str__(self):
        return "{} ({})".format(self.image_id, self.names)

class DockerContainer(DockerObject):
    """
    Container related logic
    """

    def __str__(self):
        return "{} ({})".format(self.container_id, self.short_name)

    # properties

    @property
    def container_id(self):
        if self._id is None:
            self.set_id()
        return self._id

    @property
    def names(self):
        if self._names is None:
            self._names = []
            for t in self.data["Names"] or []:
                self._names.append(t)
            # sort by name length
            self._names.sort(key=lambda x: len(x))
        return self._names

    @property
    def command(self):
        return self.data["Command"]

    @property
    def status(self):
        return self.data["Status"]

    @property
    def running(self):
        return self.status.startswith("Up")

    @property
    def short_name(self):
        try:
            return self.names[0]
        except IndexError:
            return self.short_id

    @property
    def pretty_object_type(self):
        return "Container"

    # methods

    def image_name(self):
        image_id = self.data["Image"]
        image = self.docker_backend.get_image_by_id(image_id)
        if image is not None:
            return image.short_name
        else:
            return image_id[:12]

    def matches_search(self, s):
        return s in self.container_id or \
               s in self.short_name
    # api calls

    @operation("Inspect container {object_short_name}.")
    def inspect(self, cached=False):
        if cached is False or self._inspect is None:
            self._inspect = self.d.inspect_container(self.container_id)
        return self._inspect

    @operation("Logs of container {object_short_name} received.")
    def logs(self, follow=False):
        # when tail is set to all, it takes ages to populate widget
        logs_data = self.d.logs(self.container_id, stream=follow, tail=16)
        return logs_data

    @operation("{object_type} {object_short_name} removed!")
    def remove(self):
        self.d.remove_container(self.container_id)

    @operation("{object_type} {object_short_name} started.")
    def start(self):
        self.d.start(self.container_id)

    @operation("{object_type} {object_short_name} stopped.")
    def stop(self):
        self.d.stop(self.container_id)

    @operation("{object_type} {object_short_name} restarted.")
    def restart(self):
        self.d.restart(self.container_id)

    @operation("{object_type} {object_short_name} killed.")
    def kill(self):
        self.d.kill(self.container_id)

    @operation("{object_type} {object_short_name} paused.")
    def pause(self):
        self.d.pause(self.container_id)

    @operation("{object_type} {object_short_name} unpaused.")
    def unpause(self):
        self.d.unpause(self.container_id)

class DockerBackend:
    """
    backend for docker
    """

    def __init__(self):
        self._containers = None
        self._images = None
        kwargs = docker.utils.kwargs_from_env(assert_hostname=False)
        try:
            self.client = docker.AutoVersionClient(**kwargs)
        except docker.errors.DockerException as ex:
            raise TerminateApplication("can't establish connection to docker daemon: {0}".format(str(ex)))

    # backend queries

    @operation("Get list of images.")
    def get_images(self, cached=True):
        if cached is False or self._images is None:
            logger.debug("doing images() query")
            self._images = {}
            for i in self.client.images():
                img = DockerImage(i, self)
                self._images[img.image_id] = img
        return list(self._images.values())

    @operation("Get list of containers.")
    def get_containers(self, cached=True, stopped=True):
        if cached is False or self._containers is None:
            logger.debug("doing containers() query")
            self._containers = {}
            for c in self.client.containers(all=stopped):
                container = DockerContainer(c, self)
                self._containers[container.container_id] = container
        if not stopped:
            return [x for x in list(self._containers.values()) if x.running]
        return list(self._containers.values())

    def realtime_updates(self):
        for event in self.client.events(decode=True):
            logger.debug("RT event: %s", event)

            # event["from'] means it's a container
            if "from" in event:
                # inspect doesn't contain info about status and you can't query just one
                # container with containers()
                # let's do full-blown containers() query; it's not that expensive
                self.get_containers(cached=False)
            else:
                # similar as ^
                # images() doesn't support query by ID
                # inspect doesn't contain info about repositories
                self.get_images(cached=False)
            content, _, _ = self.filter(containers=True, images=True, stopped=True,
                                        cached=True, sort_by_created=True)
            yield content

    # service methods

    def get_image_by_id(self, image_id):
        return self._images.get(image_id)

    def get_container_by_id(self, container_id):
        return self._containers.get(container_id)

    def filter(self, containers=True, images=True, stopped=True, cached=False, sort_by_created=True):
        """
        since django is so awesome, let's use their ORM API

        :return:
        """
        content = []
        containers_o = None
        images_o = None
        # return containers when containers=False and running=True
        if containers or not stopped:
            containers_o = self.get_containers(cached=cached, stopped=stopped)
            content += containers_o.response
        if images:
            images_o = self.get_images(cached=cached)
            content += images_o.response
        if sort_by_created:
            content.sort(key=lambda x: x.created, reverse=True)
        return content, containers_o, images_o
<commit_msg>backend: get containers for image<commit_after>import copy
import functools
import json
import logging
import datetime
import traceback

from sen.exceptions import TerminateApplication

import docker
import humanize

logger = logging.getLogger(__name__)

class ImageNameStruct(object):
    """
    stolen from atomic-reactor; thanks @mmilata!
    """
    def __init__(self, registry=None, namespace=None, repo=None, tag=None):
        self.registry = registry
        self.namespace = namespace
        self.repo = repo
        self.tag = tag

    @classmethod
    def parse(cls, image_name):
        result = cls()

        # registry.org/namespace/repo:tag
        s = image_name.split('/', 2)

        if len(s) == 2:
            if '.' in s[0] or ':' in s[0]:
                result.registry = s[0]
            else:
                result.namespace = s[0]
        elif len(s) == 3:
            result.registry = s[0]
            result.namespace = s[1]
        if result.namespace == 'library':
            # https://github.com/projectatomic/atomic-reactor/issues/45
            logger.debug("namespace 'library' -> ''")
            result.namespace = None
        result.repo = s[-1]

        try:
            result.repo, result.tag = result.repo.rsplit(':', 1)
        except ValueError:
            pass

        return result

    def to_str(self, registry=True, tag=True, explicit_tag=False,
               explicit_namespace=False):
        if self.repo is None:
            raise RuntimeError('No image repository specified')

        result = self.repo

        if tag and self.tag:
            result = '{0}:{1}'.format(result, self.tag)
        elif tag and explicit_tag:
            result = '{0}:{1}'.format(result, 'latest')

        if self.namespace:
            result = '{0}/{1}'.format(self.namespace, result)
        elif explicit_namespace:
            result = '{0}/{1}'.format('library', result)

        if registry and self.registry:
            result = '{0}/{1}'.format(self.registry, result)

        return result

    def __str__(self):
        return self.to_str(registry=True, tag=True)

    def __repr__(self):
        return "ImageName(image=%s)" % repr(self.to_str())

    def __eq__(self, other):
        return type(self) == type(other) and self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self.to_str())

    def copy(self):
        return ImageNameStruct(
            registry=self.registry,
            namespace=self.namespace,
            repo=self.repo,
            tag=self.tag)

def operation(fmt_str):
    def wrap(func):
        @functools.wraps(func)
        def wrapper(self, *args, **kwargs):
            command_took = None
            pretty_message = ""
            # cached queries do not access backend -- we don't care about that
            if kwargs.get("cached", False) is True:
                response = func(self, *args, **kwargs)
            else:
                before = datetime.datetime.now()
                response = func(self, *args, **kwargs)
                after = datetime.datetime.now()

                # we want milliseconds, not seconds
                command_took = (after - before).total_seconds() * 1000
                logger.debug("%s(%s, %s) %s -> [%f ms]", func.__name__, args, kwargs, self,
                             command_took)
            if fmt_str:
                pretty_message = fmt_str.format(object_type=getattr(self, "pretty_object_type", ""),
                                                object_short_name=getattr(self, "short_name", ""))
            return Operation(response, pretty_message=pretty_message, took=command_took)
        return wrapper
    return wrap

class Operation:
    """
    class for describing performed operation
    """

    def __init__(self, response, pretty_message="", took=None):
        self.response = response
        self.pretty_message = pretty_message
        self.took = took

class DockerObject:
    """
    Common base for images and containers
    """
    def __init__(self, data, docker_backend, object_id=None):
        self._id = object_id
        self._short_id = None
        self.data = data  # `client.containers` or `client.images`
        self.docker_backend = docker_backend
        self._created = None
        self._inspect = None
        self._names = None

    @property
    def d(self):
        """
        shortcut for instance of Docker client
        """
        return self.docker_backend.client

    @property
    def created(self):
        if self._created is None:
            self._created = datetime.datetime.fromtimestamp(self.data["Created"])
        return self._created

    def set_id(self):
        if self._id is None:
            try:
                self._id = self.data["Id"]
            except KeyError:
                raise RuntimeError("initial data not specified")

    @property
    def short_id(self):
        if self._short_id is None:
            self.set_id()
            self._short_id = self._id[:12]
        return self._short_id

    def display_time_created(self):
        return humanize.naturaltime(self.created)

    def inspect(self):
        raise NotImplementedError()

    def display_inspect(self):
        return json.dumps(self.inspect().response, indent=2)

def graceful_chain_get(d, *args):
    if not d:
        return None
    t = copy.deepcopy(d)
    for arg in args:
        try:
            t = t[arg]
        except (AttributeError, KeyError, TypeError):
            return None
    return t

class DockerImage(DockerObject):
    @property
    def image_id(self):
        if self._id is None:
            self.set_id()
        return self._id

    @property
    def parent_id(self):
        if self.data:
            return self.data.get("ParentId", None)
        else:
            inspect_operation = self.inspect(cached=True)
            return inspect_operation.response.get("Parent", None)

    @property
    def pretty_object_type(self):
        return "Image"

    @property
    def parent_image(self):
        try:
            parent_id = self.parent_id
        except Exception as ex:
            logger.error("error while getting parent ID of image %s: %r", self, ex)
            logger.info(traceback.format_exc())
            raise
        if parent_id:
            return DockerImage(None, self.docker_backend, object_id=parent_id)

    @property
    def command(self):
        cmd = graceful_chain_get(self.inspect(cached=True), "Config", "Cmd")
        if cmd:
            return " ".join(cmd)
        return ""

    @property
    def names(self):
        if self._names is None:
            self._names = []
            if self.data is None:
                return self._names
            for t in self.data["RepoTags"]:
                self._names.append(ImageNameStruct.parse(t))
            # sort by name length
            self._names.sort(key=lambda x: len(x.to_str()))
        return self._names

    @property
    def short_name(self):
        try:
            ins = self.names[0]
        except IndexError:
            return self.short_id
        if ins.repo == "<none>":
            return self.short_id
        return ins.to_str()

    def base_image(self):
        child_image = self
        while True:
            try:
                parent_image = self.docker_backend.get_image_by_id(child_image.parent_id)
            except Exception as ex:
                logger.warning("error while getting image by ID: %r", ex)
                parent_image = None
            if parent_image is None:
                try:
                    child_image = child_image.parent_image
                except Exception as ex:
                    logger.error("error while getting parent image for image %s: %r", self, ex)
                    return None
                if child_image is None:
                    return None
            else:
                return parent_image

    @operation("Inspect image {object_short_name}.")
    def inspect(self, cached=False):
        if self._inspect is None or cached is False:
            self._inspect = self.d.inspect_image(self.image_id)
        return self._inspect

    @operation("{object_type} {object_short_name} removed!")
    def remove(self):
        return self.d.remove_image(self.image_id)

    def matches_search(self, s):
        return s in self.image_id or \
            s in self.short_name

    def __str__(self):
        return "{} ({})".format(self.image_id, self.names)

    def containers(self):
        return self.docker_backend.get_containers_for_image(self.image_id)

class DockerContainer(DockerObject):
    """
    Container related logic
    """

    def __str__(self):
        return "{} ({})".format(self.container_id, self.short_name)

    # properties

    @property
    def container_id(self):
        if self._id is None:
            self.set_id()
        return self._id

    @property
    def names(self):
        if self._names is None:
            self._names = []
            for t in self.data["Names"] or []:
                self._names.append(t)
            # sort by name length
            self._names.sort(key=lambda x: len(x))
        return self._names

    @property
    def command(self):
        return self.data["Command"]

    @property
    def status(self):
        return self.data["Status"]

    @property
    def running(self):
        return self.status.startswith("Up")

    @property
    def short_name(self):
        try:
            return self.names[0]
        except IndexError:
            return self.short_id

    @property
    def pretty_object_type(self):
        return "Container"

    # methods

    def image_name(self):
        image_id = self.data["Image"]
        image = self.docker_backend.get_image_by_id(image_id)
        if image is not None:
            return image.short_name
        else:
            return image_id[:12]

    def matches_search(self, s):
        return s in self.container_id or \
               s in self.short_name
    # api calls

    @operation("Inspect container {object_short_name}.")
    def inspect(self, cached=False):
        if cached is False or self._inspect is None:
            self._inspect = self.d.inspect_container(self.container_id)
        return self._inspect

    @operation("Logs of container {object_short_name} received.")
    def logs(self, follow=False):
        # when tail is set to all, it takes ages to populate widget
        logs_data = self.d.logs(self.container_id, stream=follow, tail=16)
        return logs_data

    @operation("{object_type} {object_short_name} removed!")
    def remove(self):
        self.d.remove_container(self.container_id)

    @operation("{object_type} {object_short_name} started.")
    def start(self):
        self.d.start(self.container_id)

    @operation("{object_type} {object_short_name} stopped.")
    def stop(self):
        self.d.stop(self.container_id)

    @operation("{object_type} {object_short_name} restarted.")
    def restart(self):
        self.d.restart(self.container_id)

    @operation("{object_type} {object_short_name} killed.")
    def kill(self):
        self.d.kill(self.container_id)

    @operation("{object_type} {object_short_name} paused.")
    def pause(self):
        self.d.pause(self.container_id)

    @operation("{object_type} {object_short_name} unpaused.")
    def unpause(self):
        self.d.unpause(self.container_id)

class DockerBackend:
    """
    backend for docker
    """

    def __init__(self):
        self._containers = None
        self._images = None
        kwargs = docker.utils.kwargs_from_env(assert_hostname=False)
        try:
            self.client = docker.AutoVersionClient(**kwargs)
        except docker.errors.DockerException as ex:
            raise TerminateApplication("can't establish connection to docker daemon: {0}".format(str(ex)))

    # backend queries

    @operation("Get list of images.")
    def get_images(self, cached=True):
        if cached is False or self._images is None:
            logger.debug("doing images() query")
            self._images = {}
            for i in self.client.images():
                img = DockerImage(i, self)
                self._images[img.image_id] = img
        return list(self._images.values())

    @operation("Get list of containers.")
    def get_containers(self, cached=True, stopped=True):
        if cached is False or self._containers is None:
            logger.debug("doing containers() query")
            self._containers = {}
            for c in self.client.containers(all=stopped):
                container = DockerContainer(c, self)
                self._containers[container.container_id] = container
        if not stopped:
            return [x for x in list(self._containers.values()) if x.running]
        return list(self._containers.values())

    def realtime_updates(self):
        for event in self.client.events(decode=True):
            logger.debug("RT event: %s", event)

            # event["from'] means it's a container
            if "from" in event:
                # inspect doesn't contain info about status and you can't query just one
                # container with containers()
                # let's do full-blown containers() query; it's not that expensive
                self.get_containers(cached=False)
            else:
                # similar as ^
                # images() doesn't support query by ID
                # inspect doesn't contain info about repositories
                self.get_images(cached=False)
            content, _, _ = self.filter(containers=True, images=True, stopped=True,
                                        cached=True, sort_by_created=True)
            yield content

    # service methods

    def get_image_by_id(self, image_id):
        return self._images.get(image_id)

    def get_container_by_id(self, container_id):
        return self._containers.get(container_id)

    def get_containers_for_image(self, image_id):
        return [container for container in self._containers.values() if container.image_id == image_id]

    def filter(self, containers=True, images=True, stopped=True, cached=False, sort_by_created=True):
        """
        since django is so awesome, let's use their ORM API

        :return:
        """
        content = []
        containers_o = None
        images_o = None
        # return containers when containers=False and running=True
        if containers or not stopped:
            containers_o = self.get_containers(cached=cached, stopped=stopped)
            content += containers_o.response
        if images:
            images_o = self.get_images(cached=cached)
            content += images_o.response
        if sort_by_created:
            content.sort(key=lambda x: x.created, reverse=True)
        return content, containers_o, images_o
<|endoftext|>


<commit_before>import subprocess
import re
import os
import time
import urllib2
import platform

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def handleDarwin():

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    print err

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    commandLine = 'ffmpeg -f qtkit -i %s -f mpeg1video -b 400k -r 30 -s 320x240 http://runmyrobot.com:8082/hello/320/240/' % deviceAnswer

    while(True):
        os.system(commandLine)
        print "Press Ctrl-C to quit"
        time.sleep(3)
        print "Retrying"

    print commandLine

def handleWindows():

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    cameraIDAnswer = raw_input("Enter the Camera ID for your robot, you can get it from the runmyrobot.com website: ")

    url = 'http://runmyrobot.com:3100/init_video_port/%s' % cameraIDAnswer
    print "GET", url

    response = urllib2.urlopen(url).read()
    print response

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -s 320x240 -f dshow -i video="%s" -f mpeg1video -b 400k -r 20 http://runmyrobot.com:8082/hello/320/240/' % device

    while(True):
        os.system(commandLine)
        print "Press Ctrl-C to quit"
        time.sleep(3)
        print "Retrying"

    print commandLine

if platform.system() == 'Windows':
    handleWindows()
elif platform.system() == 'Darwin':
    handleDarwin()
else:
    print "unknown platform", platform.system()

<commit_msg>commit<commit_after>import subprocess
import re
import os
import time
import urllib2
import platform

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def handleDarwin():

    response = getVideoPort()
    print "video port:", response

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    print err

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    commandLine = 'ffmpeg -f qtkit -i %s -f mpeg1video -b 400k -r 30 -s 320x240 http://runmyrobot.com:8082/hello/320/240/' % deviceAnswer

    while(True):
        os.system(commandLine)
        print "Press Ctrl-C to quit"
        time.sleep(3)
        print "Retrying"

    print commandLine

def getVideoPort():

    cameraIDAnswer = raw_input("Enter the Camera ID for your robot, you can get it from the runmyrobot.com website: ")
    url = 'http://runmyrobot.com:3100/init_video_port/%s' % cameraIDAnswer
    print "GET", url
    response = urllib2.urlopen(url).read()

    return response

def handleWindows():

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    response = getVideoPort()
    print "video port:", response

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -s 320x240 -f dshow -i video="%s" -f mpeg1video -b 400k -r 20 http://runmyrobot.com:8082/hello/320/240/' % device

    while(True):
        os.system(commandLine)
        print "Press Ctrl-C to quit"
        time.sleep(3)
        print "Retrying"

    print commandLine

if platform.system() == 'Darwin':
    handleDarwin()
elif platform.system() == 'Windows':
    handleWindows()
else:
    print "unknown platform", platform.system()

<|endoftext|>


<commit_before>import textract
import sys

###################################
# Extracts text from a pdf file and
# selects one sentence, which it
# then prints.
#
# Created by Fredrik Omstedt.
###################################

text = textract.process(sys.argv[1])
print(text)
<commit_msg>Finish the sentence extractor.<commit_after>import textract
import sys
import os
import re
import random

###################################
# Extracts text from a pdf file and
# selects one sentence, which it
# then prints.
#
# Created by Fredrik Omstedt.
###################################

# Extracts texts from pdf files. If given a directory, the
# program will return texts from all pdf files in that directory.
def extractTexts():
    file = sys.argv[1]
    texts = []
    if os.path.isdir(file):
        for f in os.listdir(file):
            if re.match(r'^.*\.pdf$', f):
                texts.append(textract.process(file + "/" + f))
    else:
        texts.append(textract.process(file))
    return texts

# Chooses one sentence randomly from each of the given texts.
def selectSentences(texts):
    chosen_sentences = []
    for text in texts:
        sentence_structure = re.compile(r'([A-Z][^\.!?]*[\.!?])', re.M)
        sentences = sentence_structure.findall(text)
        chosen_sentences.append(
            sentences[random.randint(0, len(sentences)-1)].replace("\n", " ")
        )
    return chosen_sentences

def main():
    texts = extractTexts()
    sentences = selectSentences(texts)
    for sentence in sentences:
        print(sentence)
        print("\n")

if __name__ == '__main__':
    main()
<|endoftext|>