<commit_before>import sqlite3
import sys

# queue.Empty seems to have moved
if sys.version_info[0] >= 3:
    import queue
else:
    from multiprocessing import queues as queue

import time
import unittest
from multiprocessing import Process, Queue
from optparse import make_option

from billiard import cpu_count
from django.conf import settings
from django.db import connections
from django.db.models import get_apps
from django.test.runner import DiscoverRunner
from termcolor import colored

class DiscoverRoadRunner(DiscoverRunner):
    option_list = DiscoverRunner.option_list + (
        make_option(
            '-c', '--concurrency',
            action='store', dest='concurrency', default=None,
            help='Number of additional parallel processes to run. '
                 '--concurrency=0 is thus special - it means run in the '
                 'same Python process.'),
    )

    def __init__(self, concurrency, *args, **kwargs):
        if concurrency is None or int(concurrency) < 0:
            concurrency = cpu_count()
        self.concurrency = int(concurrency)
        self.stream = sys.stderr
        self.original_stream = self.stream
        super(DiscoverRoadRunner, self).__init__(*args, **kwargs)

    @staticmethod
    def get_apps_after_exclusions():
        """
        Lists all apps in the active Django project, but excluding any
        listed in ``TEST_RUNNER_EXCLUDE_APPS``.
        """
        exclude_apps = getattr(settings, 'TEST_RUNNER_EXCLUDE_APPS', ())
        excluded_names = ['%s.models' % app for app in exclude_apps]
        is_not_excluded = lambda app: app.__name__ not in excluded_names
        return filter(is_not_excluded, get_apps())

    def run_suite(self, suite, **kwargs):

        class HijackTextTestResult(unittest.TextTestResult):

            @staticmethod
            def repro(test):
                """
                String designed to be copied and pasted
                directly after `manage.py test`.
                """
                fast_repro = '.'.join((
                    test.__module__,
                    test.__class__.__name__,
                    test._testMethodName,
                ))
                return colored(fast_repro, 'red')

            def addError(self, test, err):
                super(HijackTextTestResult, self).addError(test, err)
                write = self._original_stderr.write
                error_str = self._exc_info_to_string(err, test)
                write('\n%s %s' % ('ERROR:', self.repro(test)))
                write('\n%s' % error_str)

            def addFailure(self, test, err):
                super(HijackTextTestResult, self).addFailure(test, err)
                write = self._original_stderr.write
                error_str = self._exc_info_to_string(err, test)
                write('\n%s %s' % ('FAIL:', self.repro(test)))
                write('\n%s' % error_str)

        class HijackMoreOutputTestRunner(unittest.TextTestRunner):
            resultclass = HijackTextTestResult

        class HijackUnitTestOutput(object):
            """
            Modify unit test output so only the valuable actionable information
            is printed to the stream, as soon as it is known,
            to boost developer productivity.
            """

            def __init__(self, original_stream):
                self.original_stream = original_stream

            def flush(self):
                pass

            def write(self, string):
                pass

            def writeln(self, string):
                pass

        if kwargs.get('stream', None):
            self.stream = HijackUnitTestOutput(self.original_stream)

        # Need to pass the stream in to silence the output,
        # then use the result to print relevant output fast
        return HijackMoreOutputTestRunner(
            stream=self.stream,
            verbosity=self.verbosity,
            failfast=self.failfast,
        ).run(suite)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        start = time.time()
        if not test_labels:
            # If no test labels were provided, provide them
            # and remove our custom exclusions
            test_labels = [
                # Don't double-discover tests?
                app.__name__.replace('.models', '')
                for app in self.get_apps_after_exclusions()
            ]
            # Hide most of the test output so we can focus on failures,
            # unless the user wanted to see the full output per app.
            if self.verbosity == 1:
                self.verbosity = 0

        processes = []
        source_queue = Queue(maxsize=len(test_labels))
        for item in test_labels:
            source_queue.put(item)

        # Run slow migrations first, then copy resulting DB later
        # to get most of the performance boost in downstream projects,
        # should get even more with --keepdb
        old_config = self.setup_databases()
        queries = [
            ''.join(line for line in database_wrapper.connection.iterdump())
            for database_wrapper in connections.all()
        ]
        print(queries)

        result_queue = Queue(maxsize=len(test_labels))
        process_args = (self, source_queue, result_queue, extra_tests, queries)
        for _ in range(min(self.concurrency, len(test_labels))):
            p = Process(target=multi_proc_run_tests, args=process_args)
            p.start()
            processes.append(p)
        else:
            # Concurrency == 0 - run in same process
            multi_proc_run_tests(*process_args)

        for p in processes:
            p.join()

        results = []
        while not result_queue.empty():
            results.append(result_queue.get())

        mars = [
            r['test_label']
            for r in results
            if r['fail_count'] or r['error_count']
        ]
        skippy = [
            r['test_label']
            for r in results
            if r['skip_count'] and r['test_label'] not in mars
        ]
        if mars or skippy:
            line = ''.join((
                '---Copy/Paste-after-manage-py-test---',
                colored('Skipped', 'yellow'),
                '-or-',
                colored('MARS', 'red'),
                '-' * 28,
            ))
            print(line)
            if skippy:
                print(colored(' '.join(skippy), 'yellow'))
            if mars:
                print(colored(' '.join(mars), 'red'))

        merged = {
            'test_label': 'OVERALL',
            'run': sum([r['run'] for r in results]),
            'fail_count': sum([r['fail_count'] for r in results]),
            'error_count': sum([r['error_count'] for r in results]),
            'skip_count': sum([r['skip_count'] for r in results]),
        }
        merged['short_summary'] = build_short_summary(merged)
        end = time.time()
        merged['took'] = end - start

        final_result = ''.join((
            '_  \~ ', '-meep-meep', '-' * 64, '\n',
            ' `=/  ', build_message(merged), '\n',
            '~` `~ ', '-' * 74,
        ))
        msg = colored(final_result, color=get_colour(merged), attrs=['bold'])
        print(msg)
        self.teardown_databases(old_config)

def build_short_summary(extra_msg_dict):
    short_summary = []
    failed = extra_msg_dict['fail_count']
    errored = extra_msg_dict['error_count']
    skipped = extra_msg_dict['skip_count']
    if failed:
        short_summary.append('failures=%s' % failed)
    if errored:
        short_summary.append('errors=%s' % errored)
    if skipped:
        short_summary.append('skipped=%s' % skipped)
    if short_summary:
        # Add extra , into printed message formatting
        short_summary.append('')
    return ', '.join(short_summary)

def extra_msg_dict_from(test_label, result):
    # Add some often useful details into the message
    extra_msg_dict = {
        'test_label': test_label,
        'run': result.testsRun,
        'fail_count': len(result.failures),
        'error_count': len(result.errors),
        'skip_count': len(result.skipped),
    }
    extra_msg_dict['short_summary'] = build_short_summary(extra_msg_dict)
    return extra_msg_dict

def get_colour(extra_msg_dict):
    if extra_msg_dict['error_count'] or extra_msg_dict['fail_count']:
        colour = 'red'
    elif extra_msg_dict['skip_count']:
        colour = 'yellow'
    else:
        colour = 'green'
    return colour

def build_message(extra_msg_dict):
    msg = (
        '{test_label} '
        '(run={run}, '
        '{short_summary}'
        'took={took:.3f}s)'
    ).format(**extra_msg_dict)
    return msg

def multi_proc_run_tests(pickled_self, source_queue, result_queue, extra_tests, queries):
    """
    This is a version of `DiscoverRunner.run_tests` that is written to be
    run as a single thread, but run in parallel with other test processes.
    It has also been augmented to provide informative breakdowns for each of
    the test_label(s) placed into the source_queue.
    """

    # Get any test apps / labels in the source_queue until it is empty
    while True:
        try:
            test_label = source_queue.get(block=False)
        except queue.Empty:
            return

        # Set up and run the suite, capturing most of the stream output
        # Printing here can't be made atomic cleanly at verbosity >= 2,
        # i.e. without hacks I don't want to do and it's off the default flows
        start = time.time()
        pickled_self.setup_test_environment()
        suite = pickled_self.build_suite([test_label], extra_tests)

        # Can't safely setup_databases until after suite has been built
        create_cloned_sqlite_db(queries)
        # old_config = pickled_self.setup_databases()

        stream = getattr(pickled_self, 'stream', sys.stderr)
        result = pickled_self.run_suite(suite, stream=stream)

        # Build the final message
        extra_msg_dict = extra_msg_dict_from(test_label, result)
        end = time.time()
        extra_msg_dict['took'] = end - start
        msg = build_message(extra_msg_dict)
        msg_coloured = colored(msg, color=get_colour(extra_msg_dict))
        full_msg = msg_coloured

        # Only one print statement so it is atomic
        # i.e. no annoying interleaving of test output should be possible
        print(full_msg)

        # Copy the behaviour of the old run_tests method
        # pickled_self.teardown_databases(old_config)
        pickled_self.teardown_test_environment()
        result_queue.put(extra_msg_dict)

def create_cloned_sqlite_db(queries):
    """
    Magic. Inspired by:
    http://stackoverflow.com/questions/8045602/how-can-i-copy-an-in-memory-sqlite-database-to-another-in-memory-sqlite-database
    """
    for query, database_wrapper in zip(queries, connections.all()):
        new_connection = sqlite3.connect(':memory:')
        new_connection.executescript(query)
        database_wrapper.connection = new_connection
<commit_msg>Fix creating SQL at least under Django 1.7... still lots to figure out but this finally gets a lot closer! Rethink concurrency model so we build the suites first, then setup_databases(), which is closer to what DiscoverRunner does.<commit_after>import sys

# queue.Empty seems to have moved
if sys.version_info[0] >= 3:
    import queue
else:
    from multiprocessing import queues as queue

import time
import unittest
from multiprocessing import Process, Queue
from optparse import make_option

from billiard import cpu_count
from django.conf import settings
from django.db import connections
from django.db.backends import BaseDatabaseWrapper
from django.db.models import get_apps
from django.test.runner import DiscoverRunner
from termcolor import colored

class DiscoverRoadRunner(DiscoverRunner):
    option_list = DiscoverRunner.option_list + (
        make_option(
            '-c', '--concurrency',
            action='store', dest='concurrency', default=None,
            help='Number of additional parallel processes to run. '
                 '--concurrency=0 is thus special - it means run in the '
                 'same Python process.'),
    )

    def __init__(self, concurrency, *args, **kwargs):
        if concurrency is None or int(concurrency) < 0:
            concurrency = cpu_count()
        self.concurrency = int(concurrency)
        self.stream = sys.stderr
        self.original_stream = self.stream
        super(DiscoverRoadRunner, self).__init__(*args, **kwargs)

    @staticmethod
    def get_apps_after_exclusions():
        """
        Lists all apps in the active Django project, but excluding any
        listed in ``TEST_RUNNER_EXCLUDE_APPS``.
        """
        exclude_apps = getattr(settings, 'TEST_RUNNER_EXCLUDE_APPS', ())
        excluded_names = ['%s.models' % app for app in exclude_apps]
        is_not_excluded = lambda app: app.__name__ not in excluded_names
        return filter(is_not_excluded, get_apps())

    def run_suite(self, suite, **kwargs):

        class HijackTextTestResult(unittest.TextTestResult):

            @staticmethod
            def repro(test):
                """
                String designed to be copied and pasted
                directly after `manage.py test`.
                """
                fast_repro = '.'.join((
                    test.__module__,
                    test.__class__.__name__,
                    test._testMethodName,
                ))
                return colored(fast_repro, 'red')

            def addError(self, test, err):
                super(HijackTextTestResult, self).addError(test, err)
                write = self._original_stderr.write
                error_str = self._exc_info_to_string(err, test)
                write('\n%s %s' % ('ERROR:', self.repro(test)))
                write('\n%s' % error_str)

            def addFailure(self, test, err):
                super(HijackTextTestResult, self).addFailure(test, err)
                write = self._original_stderr.write
                error_str = self._exc_info_to_string(err, test)
                write('\n%s %s' % ('FAIL:', self.repro(test)))
                write('\n%s' % error_str)

        class HijackMoreOutputTestRunner(unittest.TextTestRunner):
            resultclass = HijackTextTestResult

        class HijackUnitTestOutput(object):
            """
            Modify unit test output so only the valuable actionable information
            is printed to the stream, as soon as it is known,
            to boost developer productivity.
            """

            def __init__(self, original_stream):
                self.original_stream = original_stream

            def flush(self):
                pass

            def write(self, string):
                pass

            def writeln(self, string):
                pass

        if kwargs.get('stream', None):
            self.stream = HijackUnitTestOutput(self.original_stream)

        # Need to pass the stream in to silence the output,
        # then use the result to print relevant output fast
        return HijackMoreOutputTestRunner(
            stream=self.stream,
            verbosity=self.verbosity,
            failfast=self.failfast,
        ).run(suite)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        start = time.time()
        if not test_labels:
            # If no test labels were provided, provide them
            # and remove our custom exclusions
            test_labels = [
                # Don't double-discover tests?
                app.__name__.replace('.models', '')
                for app in self.get_apps_after_exclusions()
            ]
            # Hide most of the test output so we can focus on failures,
            # unless the user wanted to see the full output per app.
            if self.verbosity == 1:
                self.verbosity = 0

        self.setup_test_environment()

        # Prepare (often many) test suites to be run across multiple processes
        # suite = self.build_suite(test_labels, extra_tests)
        processes = []
        source_queue = Queue(maxsize=len(test_labels) + 1)

        for label in test_labels:
            suite = self.build_suite([label])
            source_queue.put((label, suite))
        if extra_tests:
            source_queue.put(
                ('extra_tests', self.build_suite(None, extra_tests))
            )

        # Run slow migrations first, then copy resulting DB later
        # to get most of the performance boost in downstream projects
        old_config = self.setup_databases()
        queries = []
        for database_wrapper in connections.all():
            queries.append(
                ''.join(line
                        for line in database_wrapper.connection.iterdump())
            )
            # Work around :memory: in django/db/backends/sqlite3/base.py
            BaseDatabaseWrapper.close(database_wrapper)

        result_queue = Queue(maxsize=len(test_labels) + 1)
        process_args = (self, source_queue, result_queue, extra_tests, queries)
        for _ in range(min(self.concurrency, len(test_labels))):
            p = Process(target=multi_proc_run_tests, args=process_args)
            p.start()
            processes.append(p)
        else:
            # Concurrency == 0 - run in same process
            multi_proc_run_tests(*process_args)

        for p in processes:
            p.join()

        results = []
        while not result_queue.empty():
            results.append(result_queue.get())

        mars = [
            r['test_label']
            for r in results
            if r['fail_count'] or r['error_count']
        ]
        skippy = [
            r['test_label']
            for r in results
            if r['skip_count'] and r['test_label'] not in mars
        ]
        if mars or skippy:
            line = ''.join((
                '---Copy/Paste-after-manage-py-test---',
                colored('Skipped', 'yellow'),
                '-or-',
                colored('MARS', 'red'),
                '-' * 28,
            ))
            print(line)
            if skippy:
                print(colored(' '.join(skippy), 'yellow'))
            if mars:
                print(colored(' '.join(mars), 'red'))

        merged = {
            'test_label': 'OVERALL',
            'run': sum([r['run'] for r in results]),
            'fail_count': sum([r['fail_count'] for r in results]),
            'error_count': sum([r['error_count'] for r in results]),
            'skip_count': sum([r['skip_count'] for r in results]),
        }
        merged['short_summary'] = build_short_summary(merged)
        end = time.time()
        merged['took'] = end - start

        final_result = ''.join((
            '_  \~ ', '-meep-meep', '-' * 64, '\n',
            ' `=/  ', build_message(merged), '\n',
            '~` `~ ', '-' * 74,
        ))
        msg = colored(final_result, color=get_colour(merged), attrs=['bold'])
        print(msg)
        self.teardown_databases(old_config)
        self.teardown_test_environment()

def build_short_summary(extra_msg_dict):
    short_summary = []
    failed = extra_msg_dict['fail_count']
    errored = extra_msg_dict['error_count']
    skipped = extra_msg_dict['skip_count']
    if failed:
        short_summary.append('failures=%s' % failed)
    if errored:
        short_summary.append('errors=%s' % errored)
    if skipped:
        short_summary.append('skipped=%s' % skipped)
    if short_summary:
        # Add extra , into printed message formatting
        short_summary.append('')
    return ', '.join(short_summary)

def extra_msg_dict_from(test_label, result):
    # Add some often useful details into the message
    extra_msg_dict = {
        'test_label': test_label,
        'run': result.testsRun,
        'fail_count': len(result.failures),
        'error_count': len(result.errors),
        'skip_count': len(result.skipped),
    }
    extra_msg_dict['short_summary'] = build_short_summary(extra_msg_dict)
    return extra_msg_dict

def get_colour(extra_msg_dict):
    if extra_msg_dict['error_count'] or extra_msg_dict['fail_count']:
        colour = 'red'
    elif extra_msg_dict['skip_count']:
        colour = 'yellow'
    else:
        colour = 'green'
    return colour

def build_message(extra_msg_dict):
    msg = (
        '{test_label} '
        '(run={run}, '
        '{short_summary}'
        'took={took:.3f}s)'
    ).format(**extra_msg_dict)
    return msg

def multi_proc_run_tests(pickled_self, source_queue, result_queue, extra_tests, queries):
    """
    This is a version of `DiscoverRunner.run_tests` that is written to be
    run as a single thread, but run in parallel with other test processes.
    It has also been augmented to provide informative breakdowns for each of
    the test_label(s) placed into the source_queue.
    """
    # Can't safely setup_databases until after suite has been built
    create_cloned_sqlite_db(queries)
    # old_config = pickled_self.setup_databases()

    # Get any test apps / labels in the source_queue until it is empty
    while True:
        try:
            test_label, suite = source_queue.get(block=False)
        except queue.Empty:
            return

        # Set up and run the suite, capturing most of the stream output
        # Printing here can't be made atomic cleanly at verbosity >= 2,
        # i.e. without hacks I don't want to do and it's off the default flows
        start = time.time()
        stream = getattr(pickled_self, 'stream', sys.stderr)
        result = pickled_self.run_suite(suite, stream=stream)

        # Build the final message
        extra_msg_dict = extra_msg_dict_from(test_label, result)
        end = time.time()
        extra_msg_dict['took'] = end - start
        msg = build_message(extra_msg_dict)
        msg_coloured = colored(msg, color=get_colour(extra_msg_dict))
        full_msg = msg_coloured

        # Only one print statement so it is atomic
        # i.e. no annoying interleaving of test output should be possible
        print(full_msg)

        result_queue.put(extra_msg_dict)

def create_cloned_sqlite_db(queries):
    """
    Magic. Inspired by:
    http://stackoverflow.com/questions/8045602/how-can-i-copy-an-in-memory-sqlite-database-to-another-in-memory-sqlite-database
    """
    for query_list, database_wrapper in zip(queries, connections.all()):
        cursor = database_wrapper.cursor()
        for sql in query_list.split(';'):
            sql += ';'
            cursor.execute(sql)
        database_wrapper.connection.commit()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
from __future__ import with_statement

import unittest
from cStringIO import StringIO
import xml.etree.cElementTree as ET

import elementflow

class XML(unittest.TestCase):
    def test_xml(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            with xml.container(u'container', {u'key': u'"значение"'}):
                xml.text(u'<Текст> контейнера')
                xml.element(u'item')
            xml.element(u'item', text=u'Текст')
        buffer.seek(0)
        tree = ET.parse(buffer)
        buffer = StringIO()
        tree.write(buffer, encoding='utf-8')
        self.assertEqual(
          buffer.getvalue(),
          '<root>'
            '<container key=""значение"">'
              '<Текст> контейнера'
              '<item />'
            '</container>'
            '<item>Текст</item>'
          '</root>'
        )

    def test_non_well_formed_on_exception(self):
        buffer = StringIO()
        try:
            with elementflow.xml(buffer, u'root') as xml:
                xml.text(u'Text')
                raise Exception()
        except:
            pass
        buffer.seek(0)
        # Parsing this buffer should cause a parsing error due to unclosed
        # root element
        self.assertRaises(SyntaxError, lambda: ET.parse(buffer))

    def test_comment(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.comment(u'comment')
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
            '<?xml version="1.0" encoding="utf-8"?><root></root>'
        )

    def test_comment_with_double_hyphen(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.comment(u'--comm-->ent--')
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
            '<?xml version="1.0" encoding="utf-8"?><root></root>'
        )

    def test_namespaces(self):
        buffer = StringIO()
        with elementflow.xml(buffer, 'root', namespaces={'': 'urn:n', 'n1': 'urn:n1'}) as xml:
            xml.element('item')
            with xml.container('n2:item', namespaces={'n2': 'urn:n2'}):
                xml.element('item')
                xml.element('n1:item')
        buffer.seek(0)
        tree = ET.parse(buffer)
        root = tree.getroot()
        self.assertEquals(root.tag, '{urn:n}root')
        self.assertNotEqual(root.find('{urn:n}item'), None)
        self.assertNotEqual(root.find('{urn:n2}item/{urn:n}item'), None)
        self.assertNotEqual(root.find('{urn:n2}item/{urn:n1}item'), None)

    def test_bad_namespace(self):
        buffer = StringIO()
        def g():
            with elementflow.xml(buffer, 'n1:root', namespaces={'n': 'urn:n'}) as xml:
                pass
        self.assertRaises(ValueError, g)
        def g():
            with elementflow.xml(buffer, 'n:root', attrs={'n1:k': 'v'}, namespaces={'n': 'urn:n'}) as xml:
                pass
        self.assertRaises(ValueError, g)

    def test_map(self):
        data = [(1, u'One'), (2, u'Two'), (3, u'Three')]
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.map(lambda (k, v): (
                'item',
                {'key': unicode(k)},
                v,
            ), data)
            xml.map(lambda (k, v): (v,), data)
        buffer.seek(0)
        tree = ET.parse(buffer)
        buffer = StringIO()
        tree.write(buffer, encoding='utf-8')
        self.assertEqual(
          buffer.getvalue(),
          '<root>'
            '<item key="1">One</item>'
            '<item key="2">Two</item>'
            '<item key="3">Three</item>'
            '<One /><Two /><Three />'
          '</root>'
        )

    def test_indent(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root', indent = True) as xml:
            with xml.container(u'a'):
                xml.element(u'b', text = ''.join(['blah '] * 20))
                xml.comment(' '.join(['comment'] * 20))
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
"""<?xml version="1.0" encoding="utf-8"?>
<root>

      blah blah blah blah blah blah blah blah blah blah blah
      blah blah blah blah blah blah blah blah blah

</root>
""")

    def test_indent_nowrap(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root', indent = True, text_wrap = False) as xml:
            with xml.container(u'a'):
                xml.element(u'b', text = ''.join(['blah '] * 20))
                xml.comment(' '.join(['comment'] * 20))
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
"""<?xml version="1.0" encoding="utf-8"?>
<root>

    blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah

</root>
""")

if __name__ == '__main__':
    unittest.main()
<commit_msg>unicode strings on tests.<commit_after># -*- coding:utf-8 -*-
from __future__ import with_statement

import unittest
from cStringIO import StringIO
import xml.etree.cElementTree as ET

import elementflow

class XML(unittest.TestCase):
    def test_xml(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            with xml.container(u'container', {u'key': u'"значение"'}):
                xml.text(u'<Текст> контейнера')
                xml.element(u'item')
            xml.element(u'item', text=u'Текст')
        buffer.seek(0)
        tree = ET.parse(buffer)
        buffer = StringIO()
        tree.write(buffer, encoding='utf-8')
        self.assertEqual(
          buffer.getvalue(),
          '<root>'
            '<container key=""значение"">'
              '<Текст> контейнера'
              '<item />'
            '</container>'
            '<item>Текст</item>'
          '</root>'
        )

    def test_non_well_formed_on_exception(self):
        buffer = StringIO()
        try:
            with elementflow.xml(buffer, u'root') as xml:
                xml.text(u'Text')
                raise Exception()
        except:
            pass
        buffer.seek(0)
        # Parsing this buffer should cause a parsing error due to unclosed
        # root element
        self.assertRaises(SyntaxError, lambda: ET.parse(buffer))

    def test_comment(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.comment(u'comment')
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
            '<?xml version="1.0" encoding="utf-8"?><root></root>'
        )

    def test_comment_with_double_hyphen(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.comment(u'--comm-->ent--')
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
            '<?xml version="1.0" encoding="utf-8"?><root></root>'
        )

    def test_namespaces(self):
        buffer = StringIO()
        with elementflow.xml(buffer, 'root', namespaces={'': 'urn:n', 'n1': 'urn:n1'}) as xml:
            xml.element('item')
            with xml.container('n2:item', namespaces={'n2': 'urn:n2'}):
                xml.element('item')
                xml.element('n1:item')
        buffer.seek(0)
        tree = ET.parse(buffer)
        root = tree.getroot()
        self.assertEquals(root.tag, '{urn:n}root')
        self.assertNotEqual(root.find('{urn:n}item'), None)
        self.assertNotEqual(root.find('{urn:n2}item/{urn:n}item'), None)
        self.assertNotEqual(root.find('{urn:n2}item/{urn:n1}item'), None)

    def test_bad_namespace(self):
        buffer = StringIO()
        def g():
            with elementflow.xml(buffer, 'n1:root', namespaces={'n': 'urn:n'}) as xml:
                pass
        self.assertRaises(ValueError, g)
        def g():
            with elementflow.xml(buffer, 'n:root', attrs={'n1:k': 'v'}, namespaces={'n': 'urn:n'}) as xml:
                pass
        self.assertRaises(ValueError, g)

    def test_map(self):
        data = [(1, u'One'), (2, u'Two'), (3, u'Three')]
        buffer = StringIO()
        with elementflow.xml(buffer, u'root') as xml:
            xml.map(lambda (k, v): (
                'item',
                {'key': unicode(k)},
                v,
            ), data)
            xml.map(lambda (k, v): (v,), data)
        buffer.seek(0)
        tree = ET.parse(buffer)
        buffer = StringIO()
        tree.write(buffer, encoding='utf-8')
        self.assertEqual(
          buffer.getvalue(),
          '<root>'
            '<item key="1">One</item>'
            '<item key="2">Two</item>'
            '<item key="3">Three</item>'
            '<One /><Two /><Three />'
          '</root>'
        )

    def test_indent(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root', indent = True) as xml:
            with xml.container(u'a'):
                xml.element(u'b', text = ''.join(['blah '] * 20))
                xml.comment(u' '.join(['comment'] * 20))
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
"""<?xml version="1.0" encoding="utf-8"?>
<root>

      blah blah blah blah blah blah blah blah blah blah blah
      blah blah blah blah blah blah blah blah blah

</root>
""")

    def test_indent_nowrap(self):
        buffer = StringIO()
        with elementflow.xml(buffer, u'root', indent = True, text_wrap = False) as xml:
            with xml.container(u'a'):
                xml.element(u'b', text = ''.join(['blah '] * 20))
                xml.comment(u' '.join(['comment'] * 20))
        buffer.seek(0)
        self.assertEqual(
            buffer.getvalue(),
"""<?xml version="1.0" encoding="utf-8"?>
<root>

    blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah

</root>
""")

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from optparse import OptionParser
import requests

def get_arg_parser():
    parser = OptionParser()
    parser.add_option('-o', '--host', dest='host', help='Hostname of graphite server', metavar='HOST')
    parser.add_option('-p', '--prefix', dest='prefix', help='Prefix of metric for which to dump the tree', metavar='PREFIX')
    parser.add_option('-s', '--start', dest='start', help='Start date for query', metavar='DATE')
    parser.add_option('-e', '--end', dest='end', help='End date for query', metavar='DATE')

    return parser

def get_children(host, prefix, min, max):
    url = 'http://%s/metrics/expand' % host
    leaves = []
    for i in range(min, max + 1):
        params = {'query': '%s%s' % (prefix, '.*' * i), 'leavesOnly': '1'}
        json_url = requests.get(url, params=params)
        json_results = json_url.json()
        leaves.extend(json_results['results'])
        print len(leaves)

    return leaves

def get_bounds(host, prefix):
    params = {'query': '%s.*' % prefix, 'leavesOnly': '1'}
    url = 'http://%s/metrics/expand' % host
    json_url = requests.get(url, params=params)
    json_results = json_url.json()
    bounds = [int(bound.replace(prefix + '.', '')) for bound in json_results['results']]

    return (min(bounds), max(bounds))

def get_max_metric(host, metric, start, end):
    params = {'target': 'keepLastValue(%s)' % metric, 'format': 'json', 'from': start, 'until': end}
    url = 'http://%s/render' % host
    json_url = requests.get(url, params=params)
    json_results = json_url.json()
    return max([point[0] for point in json_results[0]['datapoints']])

def get_tree(host, prefix, start, end):
    nodes_to_process = [prefix]
    (min, max) = get_bounds(host, prefix)
    leaves = get_children(host, prefix, min, max)

    results = {}
    for leaf in leaves:
        results[leaf] = get_max_metric(host, leaf, start, end)

    return results

def format_metric(metric, prefix):
    no_prefix = metric.replace(prefix + '.', '')
    frames = no_prefix.split('.')
    frames.reverse()
    return ';'.join(frames).replace('-', '.')

def format_output(prefix, results):
    for metric, value in results.iteritems():
        print '%s %d' % (format_metric(metric, prefix), value)

if __name__ == '__main__':
    parser = get_arg_parser()
    args, _ = parser.parse_args()
    if not(args.host and args.prefix and args.start and args.end):
        parser.print_help()
        sys.exit(255)

    results = get_tree(args.host, args.prefix, args.start, args.end)
    format_output(args.prefix, results)
<commit_msg>Fixes #3: Support custom port in graphite_dump.py<commit_after>#!/usr/bin/env python
from optparse import OptionParser
import requests

def get_arg_parser():
    parser = OptionParser()
    parser.add_option('-o', '--host', dest='host', help='Hostname of graphite server', metavar='HOST')
    parser.add_option('-r', '--port', dest='port', help='Port for graphite server', metavar='PORT')
    parser.add_option('-p', '--prefix', dest='prefix', help='Prefix of metric for which to dump the tree', metavar='PREFIX')
    parser.add_option('-s', '--start', dest='start', help='Start date for query', metavar='DATE')
    parser.add_option('-e', '--end', dest='end', help='End date for query', metavar='DATE')

    return parser

def get_children(host, prefix, min, max):
    url = 'http://%s/metrics/expand' % host
    leaves = []
    for i in range(min, max + 1):
        params = {'query': '%s%s' % (prefix, '.*' * i), 'leavesOnly': '1'}
        json_url = requests.get(url, params=params)
        json_results = json_url.json()
        leaves.extend(json_results['results'])

    return leaves

def get_bounds(host, prefix):
    params = {'query': '%s.*' % prefix, 'leavesOnly': '1'}
    url = 'http://%s/metrics/expand' % host
    json_url = requests.get(url, params=params)
    json_results = json_url.json()
    bounds = [int(bound.replace(prefix + '.', '')) for bound in json_results['results']]

    return (min(bounds), max(bounds))

def get_max_metric(host, metric, start, end):
    params = {'target': 'keepLastValue(%s)' % metric, 'format': 'json', 'from': start, 'until': end}
    url = 'http://%s/render' % host
    json_url = requests.get(url, params=params)
    json_results = json_url.json()
    return max([point[0] for point in json_results[0]['datapoints']])

def get_tree(host, prefix, start, end):
    nodes_to_process = [prefix]
    (min, max) = get_bounds(host, prefix)
    leaves = get_children(host, prefix, min, max)

    results = {}
    for leaf in leaves:
        results[leaf] = get_max_metric(host, leaf, start, end)

    return results

def format_metric(metric, prefix):
    no_prefix = metric.replace(prefix + '.', '')
    frames = no_prefix.split('.')
    frames.reverse()
    return ';'.join(frames).replace('-', '.')

def format_output(prefix, results):
    for metric, value in results.iteritems():
        print '%s %d' % (format_metric(metric, prefix), value)

if __name__ == '__main__':
    parser = get_arg_parser()
    args, _ = parser.parse_args()
    if not(args.host and args.prefix and args.start and args.end):
        parser.print_help()
        sys.exit(255)

    host = args.host
    if args.port is not None:
        host = '%s:%s' % (host, args.port)

    results = get_tree(host, args.prefix, args.start, args.end)
    format_output(args.prefix, results)
<|endoftext|>


<commit_before>import queue
import sys
from multiprocessing import Process, Queue
import time
import unittest

from billiard import cpu_count
from django.conf import settings
from django.db.models import get_apps
from django.test.runner import DiscoverRunner
from termcolor import colored

def get_apps_after_exclusions():
    """
    Lists all apps in the active Django project, but excluding any
    listed in ``TEST_RUNNER_EXCLUDE_APPS``.
    """
    exclude_apps = getattr(settings, 'TEST_RUNNER_EXCLUDE_APPS', ())
    excluded_names = ['%s.models' % app for app in exclude_apps]
    is_not_excluded = lambda app: app.__name__ not in excluded_names
    return filter(is_not_excluded, get_apps())

class HijackTextTestResult(unittest.TextTestResult):
    def addError(self, test, err):
        super(HijackTextTestResult, self).addError(test, err)
        write = self._original_stderr.write
        fast_repro = '.'.join((
            test.__module__,
            test.__class__.__name__,
            test._testMethodName,
        ))
        write("\n%s: %s" % ('ERROR', self.getDescription(test)))
        error_str = self._exc_info_to_string(err, test)
        write("\n%s" % error_str)

    def addFailure(self, test, err):
        super(HijackTextTestResult, self).addFailure(test, err)
        write = self._original_stderr.write
        write("\n%s: %s" % ('FAIL', self.getDescription(test)))
        error_str = self._exc_info_to_string(err, test)
        write("\n%s" % error_str)

class HijackMoreOutputTestRunner(unittest.TextTestRunner):
    resultclass = HijackTextTestResult

class DiscoverRoadRunner(DiscoverRunner):

    def __init__(self, *args, **kwargs):
        self.stream = sys.stderr
        self.original_stream = self.stream
        super(DiscoverRoadRunner, self).__init__(*args, **kwargs)

    def run_suite(self, suite, **kwargs):
        if kwargs.get('stream', None):
            self.stream = HijackUnitTestOutput(self.original_stream)

        # Need to pass the stream in to silence the output,
        # then use the result to print relevant output fast
        return HijackMoreOutputTestRunner(
            stream=self.stream,
            verbosity=self.verbosity,
            failfast=self.failfast
        ).run(suite)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        start = time.time()
        if not test_labels:
            # If no test labels were provided, provide them
            # and remove our custom exclusions
            test_labels = [
                # Don't double-discover tests?
                app.__name__.replace('.models', '')
                for app in get_apps_after_exclusions()
            ]
            # Hide most of the test output so we can focus on failures,
            # unless the user wanted to see the full output per app.
            if self.verbosity == 1:
                self.verbosity = 0

        processes = []
        source_queue = Queue(maxsize=len(test_labels))
        for item in test_labels:
            source_queue.put(item)

        result_queue = Queue(maxsize=len(test_labels))
        for _ in range(min(cpu_count(), len(test_labels))):
            # Limit number of processes to cpu_count
            # so performance tests run more reliably
            p = Process(
                target=multi_proc_run_tests,
                args=[self, source_queue, result_queue, extra_tests]
            )
            p.start()
            processes.append(p)
        for p in processes:
            p.join()

        results = []
        while not result_queue.empty():
            results.append(result_queue.get())

        unpacked = [item
                    for sub_list in results
                    for item in sub_list]
        merged = {
            'test_label': 'OVERALL',
            'run': sum([r['run'] for r in unpacked]),
            'fail_count': sum([r['fail_count'] for r in unpacked]),
            'error_count': sum([r['error_count'] for r in unpacked]),
            'skip_count': sum([r['skip_count'] for r in unpacked]),
        }
        merged['short_summary'] = build_short_summary(merged)
        end = time.time()
        merged['took'] = end - start

        final_result = '\n'.join((
            '-' * 80,
            build_message(merged),
            '-' * 80,
        ))
        msg = colored(final_result, color=get_colour(merged), attrs=['bold'])
        print(msg)

class HijackUnitTestOutput(object):
    """
    Modify unit test output so only the valuable actionable information is
    printed to the stream, as soon as it is known,
    to boost developer productivity.
    """

    def __init__(self, original_stream):
        self.original_stream = original_stream

    def flush(self):
        pass

    def write(self, string):
        if len(string) == 1 and string not in '\r\n':
            # Keep the progress skipped or dots that give feedback that
            # a test is taking a long time to run
            self.original_stream.write(string)

    def writeln(self, string):
        pass

def build_short_summary(extra_msg_dict):
    short_summary = []
    failed = extra_msg_dict['fail_count']
    errored = extra_msg_dict['error_count']
    skipped = extra_msg_dict['skip_count']
    if failed:
        short_summary.append('failures=%s' % failed)
    if errored:
        short_summary.append('errors=%s' % errored)
    if skipped:
        short_summary.append('skipped=%s' % skipped)
    if short_summary:
        # Add extra , into printed message formatting
        short_summary.append('')
    return ', '.join(short_summary)

def extra_msg_dict_from(test_label, result):
    # Add some often useful details into the message
    extra_msg_dict = {
        'test_label': test_label,
        'run': result.testsRun,
        'fail_count': len(result.failures),
        'error_count': len(result.errors),
        'skip_count': len(result.skipped),
    }
    extra_msg_dict['short_summary'] = build_short_summary(extra_msg_dict)
    return extra_msg_dict

def get_colour(extra_msg_dict):
    if extra_msg_dict['error_count'] or extra_msg_dict['fail_count']:
        colour = 'red'
    elif extra_msg_dict['skip_count']:
        colour = 'yellow'
    else:
        colour = 'green'
    return colour

def build_message(extra_msg_dict):
    msg = (
        '{test_label} '
        '(run={run}, '
        '{short_summary}'
        'took={took:.3f}s)'
    ).format(**extra_msg_dict)
    return msg

def multi_proc_run_tests(pickled_self, source_queue, result_queue, extra_tests):
    """
    This is a version of `DiscoverRunner.run_tests` that is written to be
    run as a single thread, but run in parallel with other test processes.
    It has also been augmented to provide informative breakdowns for each of
    the test_label(s) placed into the source_queue.
    """

    # Get any test apps / labels in the source_queue until it is empty
    while True:
        try:
            test_label = source_queue.get(block=False)
        except queue.Empty:
            return

        # Set up and run the suite, capturing most of the stream output
        # Printing here can't be made atomic cleanly at verbosity >= 2,
        # i.e. without hacks I don't want to do and it's off the default flows
        start = time.time()
        pickled_self.setup_test_environment()
        suite = pickled_self.build_suite([test_label], extra_tests)
        old_config = pickled_self.setup_databases()
        stream = getattr(pickled_self, 'stream', sys.stderr)
        result = pickled_self.run_suite(suite, stream=stream)

        # Build the final message
        extra_msg_dict = extra_msg_dict_from(test_label, result)
        end = time.time()
        extra_msg_dict['took'] = end - start
        msg = build_message(extra_msg_dict)
        msg_coloured = colored(msg, color=get_colour(extra_msg_dict))
        full_msg = msg_coloured

        # Only one print statement so it is atomic
        # i.e. no annoying interleaving of test output should be possible
        print(full_msg)

        # Copy the behaviour of the old run_tests method
        pickled_self.teardown_databases(old_config)
        pickled_self.teardown_test_environment()
        result_queue.put([extra_msg_dict])
<commit_msg>Immutable tuples and trailing comma usually preferred.<commit_after>import queue
import sys
from multiprocessing import Process, Queue
import time
import unittest

from billiard import cpu_count
from django.conf import settings
from django.db.models import get_apps
from django.test.runner import DiscoverRunner
from termcolor import colored

def get_apps_after_exclusions():
    """
    Lists all apps in the active Django project, but excluding any
    listed in ``TEST_RUNNER_EXCLUDE_APPS``.
    """
    exclude_apps = getattr(settings, 'TEST_RUNNER_EXCLUDE_APPS', ())
    excluded_names = ['%s.models' % app for app in exclude_apps]
    is_not_excluded = lambda app: app.__name__ not in excluded_names
    return filter(is_not_excluded, get_apps())

class HijackTextTestResult(unittest.TextTestResult):
    def addError(self, test, err):
        super(HijackTextTestResult, self).addError(test, err)
        write = self._original_stderr.write
        fast_repro = '.'.join((
            test.__module__,
            test.__class__.__name__,
            test._testMethodName,
        ))
        write("\n%s: %s" % ('ERROR', self.getDescription(test)))
        error_str = self._exc_info_to_string(err, test)
        write("\n%s" % error_str)

    def addFailure(self, test, err):
        super(HijackTextTestResult, self).addFailure(test, err)
        write = self._original_stderr.write
        write("\n%s: %s" % ('FAIL', self.getDescription(test)))
        error_str = self._exc_info_to_string(err, test)
        write("\n%s" % error_str)

class HijackMoreOutputTestRunner(unittest.TextTestRunner):
    resultclass = HijackTextTestResult

class DiscoverRoadRunner(DiscoverRunner):

    def __init__(self, *args, **kwargs):
        self.stream = sys.stderr
        self.original_stream = self.stream
        super(DiscoverRoadRunner, self).__init__(*args, **kwargs)

    def run_suite(self, suite, **kwargs):
        if kwargs.get('stream', None):
            self.stream = HijackUnitTestOutput(self.original_stream)

        # Need to pass the stream in to silence the output,
        # then use the result to print relevant output fast
        return HijackMoreOutputTestRunner(
            stream=self.stream,
            verbosity=self.verbosity,
            failfast=self.failfast
        ).run(suite)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        start = time.time()
        if not test_labels:
            # If no test labels were provided, provide them
            # and remove our custom exclusions
            test_labels = [
                # Don't double-discover tests?
                app.__name__.replace('.models', '')
                for app in get_apps_after_exclusions()
            ]
            # Hide most of the test output so we can focus on failures,
            # unless the user wanted to see the full output per app.
            if self.verbosity == 1:
                self.verbosity = 0

        processes = []
        source_queue = Queue(maxsize=len(test_labels))
        for item in test_labels:
            source_queue.put(item)

        result_queue = Queue(maxsize=len(test_labels))
        for _ in range(min(cpu_count(), len(test_labels))):
            # Limit number of processes to cpu_count
            # so performance tests run more reliably
            p = Process(
                target=multi_proc_run_tests,
                args=(self, source_queue, result_queue, extra_tests),
            )
            p.start()
            processes.append(p)
        for p in processes:
            p.join()

        results = []
        while not result_queue.empty():
            results.append(result_queue.get())

        unpacked = [item
                    for sub_list in results
                    for item in sub_list]
        merged = {
            'test_label': 'OVERALL',
            'run': sum([r['run'] for r in unpacked]),
            'fail_count': sum([r['fail_count'] for r in unpacked]),
            'error_count': sum([r['error_count'] for r in unpacked]),
            'skip_count': sum([r['skip_count'] for r in unpacked]),
        }
        merged['short_summary'] = build_short_summary(merged)
        end = time.time()
        merged['took'] = end - start

        final_result = '\n'.join((
            '-' * 80,
            build_message(merged),
            '-' * 80,
        ))
        msg = colored(final_result, color=get_colour(merged), attrs=['bold'])
        print(msg)

class HijackUnitTestOutput(object):
    """
    Modify unit test output so only the valuable actionable information is
    printed to the stream, as soon as it is known,
    to boost developer productivity.
    """

    def __init__(self, original_stream):
        self.original_stream = original_stream

    def flush(self):
        pass

    def write(self, string):
        if len(string) == 1 and string not in '\r\n':
            # Keep the progress skipped or dots that give feedback that
            # a test is taking a long time to run
            self.original_stream.write(string)

    def writeln(self, string):
        pass

def build_short_summary(extra_msg_dict):
    short_summary = []
    failed = extra_msg_dict['fail_count']
    errored = extra_msg_dict['error_count']
    skipped = extra_msg_dict['skip_count']
    if failed:
        short_summary.append('failures=%s' % failed)
    if errored:
        short_summary.append('errors=%s' % errored)
    if skipped:
        short_summary.append('skipped=%s' % skipped)
    if short_summary:
        # Add extra , into printed message formatting
        short_summary.append('')
    return ', '.join(short_summary)

def extra_msg_dict_from(test_label, result):
    # Add some often useful details into the message
    extra_msg_dict = {
        'test_label': test_label,
        'run': result.testsRun,
        'fail_count': len(result.failures),
        'error_count': len(result.errors),
        'skip_count': len(result.skipped),
    }
    extra_msg_dict['short_summary'] = build_short_summary(extra_msg_dict)
    return extra_msg_dict

def get_colour(extra_msg_dict):
    if extra_msg_dict['error_count'] or extra_msg_dict['fail_count']:
        colour = 'red'
    elif extra_msg_dict['skip_count']:
        colour = 'yellow'
    else:
        colour = 'green'
    return colour

def build_message(extra_msg_dict):
    msg = (
        '{test_label} '
        '(run={run}, '
        '{short_summary}'
        'took={took:.3f}s)'
    ).format(**extra_msg_dict)
    return msg

def multi_proc_run_tests(pickled_self, source_queue, result_queue, extra_tests):
    """
    This is a version of `DiscoverRunner.run_tests` that is written to be
    run as a single thread, but run in parallel with other test processes.
    It has also been augmented to provide informative breakdowns for each of
    the test_label(s) placed into the source_queue.
    """

    # Get any test apps / labels in the source_queue until it is empty
    while True:
        try:
            test_label = source_queue.get(block=False)
        except queue.Empty:
            return

        # Set up and run the suite, capturing most of the stream output
        # Printing here can't be made atomic cleanly at verbosity >= 2,
        # i.e. without hacks I don't want to do and it's off the default flows
        start = time.time()
        pickled_self.setup_test_environment()
        suite = pickled_self.build_suite([test_label], extra_tests)
        old_config = pickled_self.setup_databases()
        stream = getattr(pickled_self, 'stream', sys.stderr)
        result = pickled_self.run_suite(suite, stream=stream)

        # Build the final message
        extra_msg_dict = extra_msg_dict_from(test_label, result)
        end = time.time()
        extra_msg_dict['took'] = end - start
        msg = build_message(extra_msg_dict)
        msg_coloured = colored(msg, color=get_colour(extra_msg_dict))
        full_msg = msg_coloured

        # Only one print statement so it is atomic
        # i.e. no annoying interleaving of test output should be possible
        print(full_msg)

        # Copy the behaviour of the old run_tests method
        pickled_self.teardown_databases(old_config)
        pickled_self.teardown_test_environment()
        result_queue.put([extra_msg_dict])
<|endoftext|>


<commit_before>"""
GBDX Catalog Image Interface.

Contact: chris.helm@digitalglobe.com
"""
from __future__ import print_function
import json

from gbdxtools import IdahoImage
from gbdxtools.ipe.util import calc_toa_gain_offset
from gbdxtools.images.ipe_image import IpeImage
from gbdxtools.vectors import Vectors
from gbdxtools.ipe.interface import Ipe
from gbdxtools.ipe.error import MissingMetadata, MissingIdahoImages

from shapely import wkt
from shapely.geometry import box

import requests

ipe = Ipe()

band_types = {
    'MS': 'WORLDVIEW_8_BAND',
    'Panchromatic': 'PAN',
    'Pan': 'PAN',
    'pan': 'PAN'
}

class WVImage(IpeImage):
    _parts = None

    def __new__(cls, cat_id, **kwargs):
        options = {
            "band_type": kwargs.get("band_type", "MS"),
            "product": kwargs.get("product", "toa_reflectance"),
            "proj": kwargs.get("proj", "EPSG:4326"),
            "pansharpen": kwargs.get("pansharpen", False),
            "gsd": kwargs.get("gsd", None)
        }

        if options["pansharpen"]:
            options["band_type"] = "MS"
            options["product"] = "pansharpened"

        standard_products = cls._build_standard_products(cat_id, options["band_type"], options["proj"], gsd=options["gsd"])
        pan_products = cls._build_standard_products(cat_id, "pan", options["proj"], options["gsd"])
        pan = ipe.Format(ipe.MultiplyConst(pan_products['toa_reflectance'], constants=json.dumps([1000])), dataType="1")
        ms = ipe.Format(ipe.MultiplyConst(standard_products['toa_reflectance'],
                                          constants=json.dumps([1000]*8)), dataType="1")
        standard_products["pansharpened"] = ipe.LocallyProjectivePanSharpen(ms, pan)

        try:
            self = super(WVImage, cls).__new__(cls, standard_products[options["product"]])
        except KeyError as e:
            print(e)
            print("Specified product not implemented: {}".format(options["product"]))
            raise
        self = self.aoi(**kwargs)
        self.cat_id = cat_id
        self._products = standard_products
        self.options = options
        return self

    @property
    def parts(self):
        if self._parts is None:
            self._parts = [IdahoImage(rec['properties']['attributes']['idahoImageId'],
                                      product=self.options["product"], proj=self.options["proj"], gsd=self.options["gsd"])
                           for rec in self._find_parts(self.cat_id, self.options["band_type"])]
        return self._parts

    def get_product(self, product):
        return self.__class__(self.cat_id, proj=self.proj, product=product)

    @staticmethod
    def _find_parts(cat_id, band_type):
        def vendor_id(rec):
          _id = rec['properties']['attributes']['vendorDatasetIdentifier']
          return _id.split(':')[1].split('_')[0]

        vectors = Vectors()
        aoi = wkt.dumps(box(-180, -90, 180, 90))
        query = "item_type:IDAHOImage AND attributes.catalogID:{} " \
                "AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type])
        _parts = sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])
        if not len(_parts):
            raise MissingIdahoImages('Unable to find IDAHO imagery in the catalog: {}'.format(query))
        _id = vendor_id(_parts[0])
        return [p for p in _parts if vendor_id(p) == _id]

    @classmethod
    def _build_standard_products(cls, cat_id, band_type, proj, gsd=None):
        # TODO: Switch to direct metadata access (ie remove this block)
        _parts = cls._find_parts(cat_id, band_type)
        _id = _parts[0]['properties']['attributes']['idahoImageId']
        try:
            idaho_md = requests.get('http://idaho.timbr.io/{}.json'.format(_id)).json()
        except ValueError:
            raise MissingMetadata('Metadata not found in S3 for image: {}'.format(_id))
        meta = idaho_md["properties"]
        gains_offsets = calc_toa_gain_offset(meta)
        radiance_scales, reflectance_scales, radiance_offsets = zip(*gains_offsets)
        # ---

        dn_ops = [ipe.IdahoRead(bucketName=p['properties']['attributes']['bucketName'], imageId=p['properties']['attributes']['idahoImageId'],
                                objectStore="S3") for p in _parts]
        mosaic_params = {"Dest SRS Code": proj}
        if gsd is not None:
            mosaic_params["Requested GSD"] = str(gsd)
        ortho_op = ipe.GeospatialMosaic(*dn_ops, **mosaic_params)

        radiance_op = ipe.AddConst(
            ipe.MultiplyConst(
                ipe.Format(ortho_op, dataType="4"),
                constants=radiance_scales),
            constants=radiance_offsets)

        toa_reflectance_op = ipe.MultiplyConst(radiance_op,
                                               constants=reflectance_scales)

        return {"ortho": ortho_op, "toa_reflectance": toa_reflectance_op, "radiance": radiance_op}

class WV03_SWIR(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV03_SWIR, cls).__new__(cls, cat_id, **kwargs)

    @staticmethod
    def _find_parts(cat_id, band_type):
        vectors = Vectors()
        aoi = wkt.dumps(box(-180, -90, 180, 90))
        query = "item_type:IDAHOImage AND attributes.catalogID:{}".format(cat_id)
        return sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])

class WV03_VNIR(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV03_VNIR, cls).__new__(cls, cat_id, **kwargs)

class WV02(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV02, cls).__new__(cls, cat_id, **kwargs)

class WV01(WVImage):
    def __new__(cls, cat_id, **kwargs):
        options = {
            "band_type": "pan",
            "product": kwargs.get("product", "ortho"),
            "proj": kwargs.get("proj", "EPSG:4326"),
            "gsd": kwargs.get("gsd", None)
        }

        standard_products = cls._build_standard_products(cat_id, options["band_type"], options["proj"], gsd=options["gsd"])
        try:
            self = super(WVImage, cls).__new__(cls, standard_products[options["product"]])
        except KeyError as e:
            print(e)
            print("Specified product not implemented: {}".format(options["product"]))
            raise
        self = self.aoi(**kwargs)
        self.cat_id = cat_id
        self._products = standard_products
        self.options = options
        return self

    @classmethod
    def _build_standard_products(cls, cat_id, band_type, proj, gsd=None):
        _parts = cls._find_parts(cat_id, band_type)
        _id = _parts[0]['properties']['attributes']['idahoImageId']
        dn_ops = [ipe.IdahoRead(bucketName=p['properties']['attributes']['bucketName'], imageId=p['properties']['attributes']['idahoImageId'],
                                objectStore="S3") for p in _parts]
        mosaic_params = {"Dest SRS Code": proj}
        if gsd is not None:
            mosaic_params["Requested GSD"] = str(gsd)
        ortho_op = ipe.GeospatialMosaic(*dn_ops, **mosaic_params)

        return {"ortho": ortho_op}
<commit_msg>adding bucket name to idaho image inits<commit_after>"""
GBDX Catalog Image Interface.

Contact: chris.helm@digitalglobe.com
"""
from __future__ import print_function
import json

from gbdxtools import IdahoImage
from gbdxtools.ipe.util import calc_toa_gain_offset
from gbdxtools.images.ipe_image import IpeImage
from gbdxtools.vectors import Vectors
from gbdxtools.ipe.interface import Ipe
from gbdxtools.ipe.error import MissingMetadata, MissingIdahoImages

from shapely import wkt
from shapely.geometry import box

import requests

ipe = Ipe()

band_types = {
    'MS': 'WORLDVIEW_8_BAND',
    'Panchromatic': 'PAN',
    'Pan': 'PAN',
    'pan': 'PAN'
}

class WVImage(IpeImage):
    _parts = None

    def __new__(cls, cat_id, **kwargs):
        options = {
            "band_type": kwargs.get("band_type", "MS"),
            "product": kwargs.get("product", "toa_reflectance"),
            "proj": kwargs.get("proj", "EPSG:4326"),
            "pansharpen": kwargs.get("pansharpen", False),
            "gsd": kwargs.get("gsd", None)
        }

        if options["pansharpen"]:
            options["band_type"] = "MS"
            options["product"] = "pansharpened"

        standard_products = cls._build_standard_products(cat_id, options["band_type"], options["proj"], gsd=options["gsd"])
        pan_products = cls._build_standard_products(cat_id, "pan", options["proj"], options["gsd"])
        pan = ipe.Format(ipe.MultiplyConst(pan_products['toa_reflectance'], constants=json.dumps([1000])), dataType="1")
        ms = ipe.Format(ipe.MultiplyConst(standard_products['toa_reflectance'],
                                          constants=json.dumps([1000]*8)), dataType="1")
        standard_products["pansharpened"] = ipe.LocallyProjectivePanSharpen(ms, pan)

        try:
            self = super(WVImage, cls).__new__(cls, standard_products[options["product"]])
        except KeyError as e:
            print(e)
            print("Specified product not implemented: {}".format(options["product"]))
            raise
        self = self.aoi(**kwargs)
        self.cat_id = cat_id
        self._products = standard_products
        self.options = options
        return self

    @property
    def parts(self):
        if self._parts is None:
            self._parts = [IdahoImage(rec['properties']['attributes']['idahoImageId'],
                                      product=self.options["product"],
                                      proj=self.options["proj"],
                                      bucket=rec['properties']['attributes']['bucketName']
                                      gsd=self.options["gsd"])
                           for rec in self._find_parts(self.cat_id, self.options["band_type"])]
        return self._parts

    def get_product(self, product):
        return self.__class__(self.cat_id, proj=self.proj, product=product)

    @staticmethod
    def _find_parts(cat_id, band_type):
        def vendor_id(rec):
          _id = rec['properties']['attributes']['vendorDatasetIdentifier']
          return _id.split(':')[1].split('_')[0]

        vectors = Vectors()
        aoi = wkt.dumps(box(-180, -90, 180, 90))
        query = "item_type:IDAHOImage AND attributes.catalogID:{} " \
                "AND attributes.colorInterpretation:{}".format(cat_id, band_types[band_type])
        _parts = sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])
        if not len(_parts):
            raise MissingIdahoImages('Unable to find IDAHO imagery in the catalog: {}'.format(query))
        _id = vendor_id(_parts[0])
        return [p for p in _parts if vendor_id(p) == _id]

    @classmethod
    def _build_standard_products(cls, cat_id, band_type, proj, gsd=None):
        # TODO: Switch to direct metadata access (ie remove this block)
        _parts = cls._find_parts(cat_id, band_type)
        _id = _parts[0]['properties']['attributes']['idahoImageId']
        try:
            idaho_md = requests.get('http://idaho.timbr.io/{}.json'.format(_id)).json()
        except ValueError:
            raise MissingMetadata('Metadata not found in S3 for image: {}'.format(_id))
        meta = idaho_md["properties"]
        gains_offsets = calc_toa_gain_offset(meta)
        radiance_scales, reflectance_scales, radiance_offsets = zip(*gains_offsets)
        # ---

        dn_ops = [ipe.IdahoRead(bucketName=p['properties']['attributes']['bucketName'], imageId=p['properties']['attributes']['idahoImageId'],
                                objectStore="S3") for p in _parts]
        mosaic_params = {"Dest SRS Code": proj}
        if gsd is not None:
            mosaic_params["Requested GSD"] = str(gsd)
        ortho_op = ipe.GeospatialMosaic(*dn_ops, **mosaic_params)

        radiance_op = ipe.AddConst(
            ipe.MultiplyConst(
                ipe.Format(ortho_op, dataType="4"),
                constants=radiance_scales),
            constants=radiance_offsets)

        toa_reflectance_op = ipe.MultiplyConst(radiance_op,
                                               constants=reflectance_scales)

        return {"ortho": ortho_op, "toa_reflectance": toa_reflectance_op, "radiance": radiance_op}

class WV03_SWIR(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV03_SWIR, cls).__new__(cls, cat_id, **kwargs)

    @staticmethod
    def _find_parts(cat_id, band_type):
        vectors = Vectors()
        aoi = wkt.dumps(box(-180, -90, 180, 90))
        query = "item_type:IDAHOImage AND attributes.catalogID:{}".format(cat_id)
        return sorted(vectors.query(aoi, query=query), key=lambda x: x['properties']['id'])

class WV03_VNIR(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV03_VNIR, cls).__new__(cls, cat_id, **kwargs)

class WV02(WVImage):
    def __new__(cls, cat_id, **kwargs):
        return super(WV02, cls).__new__(cls, cat_id, **kwargs)

class WV01(WVImage):
    def __new__(cls, cat_id, **kwargs):
        options = {
            "band_type": "pan",
            "product": kwargs.get("product", "ortho"),
            "proj": kwargs.get("proj", "EPSG:4326"),
            "gsd": kwargs.get("gsd", None)
        }

        standard_products = cls._build_standard_products(cat_id, options["band_type"], options["proj"], gsd=options["gsd"])
        try:
            self = super(WVImage, cls).__new__(cls, standard_products[options["product"]])
        except KeyError as e:
            print(e)
            print("Specified product not implemented: {}".format(options["product"]))
            raise
        self = self.aoi(**kwargs)
        self.cat_id = cat_id
        self._products = standard_products
        self.options = options
        return self

    @classmethod
    def _build_standard_products(cls, cat_id, band_type, proj, gsd=None):
        _parts = cls._find_parts(cat_id, band_type)
        _id = _parts[0]['properties']['attributes']['idahoImageId']
        dn_ops = [ipe.IdahoRead(bucketName=p['properties']['attributes']['bucketName'], imageId=p['properties']['attributes']['idahoImageId'],
                                objectStore="S3") for p in _parts]
        mosaic_params = {"Dest SRS Code": proj}
        if gsd is not None:
            mosaic_params["Requested GSD"] = str(gsd)
        ortho_op = ipe.GeospatialMosaic(*dn_ops, **mosaic_params)

        return {"ortho": ortho_op}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
"""

import wave
import pyaudio as pa

class AudioRecorder:
    """
    """

    def __init__(self, fmt=pa.paInt16, channels=2, rate=44100, frames_per_buffer=1024):
        self.__format = fmt
        self.__channels = channels
        self.__rate = rate
        self.__frames_per_buffer = frames_per_buffer

        self.__audio = pa.PyAudio()
        self.__stream = None
        self.__status = None
        self.__frames = b''

    def __del__(self):
        self.stop_recording()
        self.__audio.terminate()

    @property
    def frames(self):
        return self.__frames

    @property
    def sample_size(self):
        return self.__audio.get_sample_size(self.__format)

    @property
    def is_recording(self):
        if self.__stream is None:
            return False
        return self.__stream.is_active()

    def start_recording(self, input_device_index=None):
        if self.__stream is not None:
            return

        self.__status = pa.paContinue

        self.__stream = self.__audio.open(
            input=True,
            output=False,
            input_device_index=input_device_index,
            format=self.__format,
            channels=self.__channels,
            rate=self.__rate,
            frames_per_buffer=self.__frames_per_buffer,
            stream_callback=self.__record_callback,
        )

    def stop_recording(self):
        if self.__stream is None:
            return

        self.__status = pa.paComplete

        self.__stream.stop_stream()
        self.__stream.close()
        self.__stream = None

    def clear_recorded_frames(self):
        self.__frames = b''

    def write_wav(self, filename):
        with wave.open(filename, 'wb') as wf:
            wf.setnchannels(self.__channels)
            wf.setsampwidth(self.sample_size)
            wf.setframerate(self.__rate)
            wf.writeframes(self.__frames)

    def __record_callback(self, data, frame_count, time_info, status):
        self.__frames += data
        return data, self.__status
<commit_msg>Refactoring API<commit_after># -*- coding: utf-8 -*-

"""
"""

import wave
import pyaudio as pa

class AudioRecorder:
    """
    """

    def __init__(self, fmt=pa.paInt16, channels=2, rate=44100, frames_per_buffer=1024):
        self.__format = fmt
        self.__channels = channels
        self.__rate = rate
        self.__frames_per_buffer = frames_per_buffer

        self.__audio = pa.PyAudio()
        self.__stream = None
        self.__status = None
        self.__frames = b''

    def __del__(self):
        self.stop()
        self.__audio.terminate()

    @property
    def frames(self):
        return self.__frames

    @property
    def sample_size(self):
        return self.__audio.get_sample_size(self.__format)

    @property
    def is_recording(self):
        if self.__stream is None:
            return False
        return self.__stream.is_active()

    def record(self, input_device_index=None):
        if self.__stream is not None:
            return

        self.__status = pa.paContinue

        self.__stream = self.__audio.open(
            input=True,
            output=False,
            input_device_index=input_device_index,
            format=self.__format,
            channels=self.__channels,
            rate=self.__rate,
            frames_per_buffer=self.__frames_per_buffer,
            stream_callback=self.__record,
        )

    def stop(self):
        if self.__stream is None:
            return

        self.__status = pa.paComplete

        self.__stream.stop_stream()
        self.__stream.close()
        self.__stream = None

    def clear(self):
        self.__frames = b''

    def write_wav(self, filename):
        with wave.open(filename, 'wb') as wf:
            wf.setnchannels(self.__channels)
            wf.setsampwidth(self.sample_size)
            wf.setframerate(self.__rate)
            wf.writeframes(self.__frames)

    def __record(self, data, frame_count, time_info, status):
        self.__frames += data
        return data, self.__status
<|endoftext|>


<commit_before>#!/usr/bin/env python3

## Note
# Ho inserito alcune annotazioni all'interno del programma.
# Riassumendo il discorso multi-sensore, forse e` bene realizzare
# qualcosa di differente ? Ma cosa ? E come ?

import time
import os
import json

# Lettura file sonda temperatura
# ATTENZIONE: il file temperatura e` differente per ogni sensore !!!!
# Qua e` da pensare una possibile variante o cosa/come
# Ci ho pensato su, sarebbe coumnque una cosa da verificare,
# Perche` non e` detto che la sonda 1 sia in bagno, piuttosto che in
# camera, la 2 potrebbe essere nel salotto ..
# Quindi non so se e quanto ne vale la pena. Oggi!
FileSondaT = open('/sys/bus/w1/devices/10-0008029444b6/w1_slave','r')

# Devo aggiungere il file dei setpoin temperature
#FileSetPoint = open('setpointtemperature','r')

# Anche questo file, la riga di intestazione sara` "in dipendenza di",
# quindi un'altra cosa da valutare.
# Apertura file temperature.csv in "append mode"
FileTempsCSV = open('temperature.csv','a')

# Serve il file dei set point e dei giorni
if os.path.exists("setpointarray.json"):
	with open("setpointarray.json") as JsonFileSetPoints:
		SetPoints = json.load(JsonFileSetPoints)
else:
	print ("Errore, manca il file \"setpointarray.json\"")
# Questo e` il file di setpoint giornaliero
# Se il file esiste lo apro, se no, genero entrambi SetPoints e file
if os.path.exists("dayssetpointarray.json"):
	with open("dayssetpointarray.json") as JsonFileDays:
		DaysSetPoints = json.load(JsonFileDays)
else:
	print ("Errore, manca il file \"daysetpointarray.json\"")

## Devo scrivere il file CSV con i parametri separati da virgole
## La struttura che ho pensato di utilizzare e`:
## Data,Temperatura di Setpoint, Temperatura 1
# Quindi per primo, scrivo la data+ora nel file
FileTempsCSV.write(time.strftime("%Y/%m/%d %H:%M:%S", time.localtime()))
# Aggiungo la virgola di separazione
FileTempsCSV.write(',')

### Prima devo "calcolare" il setpoint
Giorno=time.strftime("%w", time.localtime())
Ora=time.strftime("%H", time.localtime())
# 0 Domenica .. 6
if Giorno == "0":
	CalcoloGiorno="6"
else:
	CalcoloGiorno=int(Giorno)-1

# Puntatore giorno, ore, ora, temperatura
# nel file json
#print (DaysSetPoints[int(CalcoloGiorno)]["hours"][int(Ora)]["temperature"])
# Ricerca della temperatura nei files json
for i in range(len(SetPoints)):
	if (SetPoints[i]["name"]) == (DaysSetPoints[int(CalcoloGiorno)]["hours"][int(Ora)]["temperature"]):
		SetPoint = (SetPoints[i]["temperature"])

# Poi scrivo il setpoit richiesto
FileTempsCSV.write(SetPoint)
# Aggiungo la virgola di separazione
FileTempsCSV.write(',')

## Qua si dovrebbe aggiungere un ciclo "for", per ogni sonda di
## temperatura da inserie nel grafico, per adesso e` una, quindi
## semplifichiamo.
# Controlla se gli ultimi caratteri della stringa letta sono uguali a
if (FileSondaT.readline()[-4:-1]) == "YES":
    # Effettuo il calcolo della temperatura indicando che il valore e` intero
    # prima del calcolo e che e` una stringa per passarla in scrittura
    FileTempsCSV.write(str(int(FileSondaT.readline()[-6:-1])/1000))
else:
    FileTempsCSV.write('err')	# err, se errore sonda, non so ancora cosa fara` il grafico perche` non e` successo.

# Aggiungo il ritorno a capo di fine riga per i valori successivi
FileTempsCSV.write('\n')

FileSondaT.close()
FileTempsCSV.close()
<commit_msg>Rimosso files obsoleti e non piu` usati<commit_after><|endoftext|>


<commit_before>import sublime

from . import SublimeHelper as SH
from . import OsShell

class ShellCommandCommand(SH.TextCommand):

    def __init__(self, plugin, default_prompt=None, **kwargs):

        SH.TextCommand.__init__(self, plugin, **kwargs)
        if default_prompt is None:
            self.default_prompt = 'Shell Command'
        else:
            self.default_prompt = default_prompt
        self.data_key = 'ShellCommand'
        self.output_written = False

    def run(self, edit, command=None, command_prefix=None, prompt=None, region=None, arg_required=None, panel=None, title=None, syntax=None, refresh=None):

        if region is None:
            region is False

        if arg_required is None:
            arg_required = False

        if panel is None:
            panel = False

        if refresh is None:
            refresh = False

        arg = None

        # If regions should be used then work them out, and append
        # them to the command:
        #
        if region is True:
            arg = self.get_region().strip()

            if arg == '':
                if arg_required is True:
                    sublime.message_dialog('This command requires a parameter.')
                    return

        # Setup a closure to run the command:
        #
        def _C(command):

            if command_prefix is not None:
                command = command_prefix + ' ' + command

            if arg is not None:
                command = command + ' ' + arg

            self.run_shell_command(command, panel=panel, title=title, syntax=syntax, refresh=refresh)

        # If no command is specified then we prompt for one, otherwise
        # we can just execute the command:
        #
        if command is None:
            if prompt is None:
                prompt = self.default_prompt
            self.view.window().show_input_panel(prompt, '', _C, None, None)
        else:
            _C(command)

    def run_shell_command(self, command=None, panel=False, title=None, syntax=None, refresh=False, console=None, working_dir=None):

        view = self.view
        window = view.window()

        if command is None:
            sublime.message_dialog('No command provided.')
            return

        if working_dir is None:
            working_dir = self.get_working_dir()

        # Run the command and write any output to the buffer:
        #
        output_target = SH.OutputTarget(window, self.data_key, command, working_dir, title=title, syntax=syntax, panel=panel, console=console)

        def _C(output):

            if output is not None:
                output_target.append_text(output)
                self.output_written = True
            else:
                # If there has been no output:
                #
                if self.output_written is False:
                    settings = sublime.load_settings('ShellCommand.sublime-settings')
                    show_message = settings.get('show_success_but_no_output_message')
                    if show_message:
                        output = settings.get('success_but_no_output_message')

                # Check whether the initiating view needs refreshing:
                #
                if refresh is True:
                    view.run_command('shell_command_refresh')

        OsShell.process(command, _C, working_dir=working_dir)

class ShellCommandOnRegionCommand(ShellCommandCommand):

    def run(self, edit, command=None, command_prefix=None, prompt=None, arg_required=None, panel=None, title=None, syntax=None, refresh=None):

        ShellCommandCommand.run(self, edit, command=command, command_prefix=command_prefix, prompt=prompt, region=True, arg_required=True, panel=panel, title=title, syntax=syntax, refresh=refresh)

# Refreshing a shell command simply involves re-running the original command:
#
class ShellCommandRefreshCommand(ShellCommandCommand):

    def run(self, edit, callback=None):

        view = self.view

        settings = view.settings()
        if settings.has(self.data_key):
            data = settings.get(self.data_key + '_data', None)
            if data is not None:

                # Create a local function that will re-write the buffer contents:
                #
                def _C(output, **kwargs):

                    console = view

                    console.set_read_only(False)
                    console.run_command('sublime_helper_clear_buffer')
                    console.run_command('sublime_helper_insert_text', {'pos': 0, 'msg': output})
                    console.set_read_only(True)

                    if callback is not None:
                        callback()

                OsShell.process(data['command'], _C, working_dir=data['working_dir'])
<commit_msg>Factor the refresh command to use the main 'run shell command' command.<commit_after>import sublime

from . import SublimeHelper as SH
from . import OsShell

class ShellCommandCommand(SH.TextCommand):

    def __init__(self, plugin, default_prompt=None, **kwargs):

        SH.TextCommand.__init__(self, plugin, **kwargs)
        if default_prompt is None:
            self.default_prompt = 'Shell Command'
        else:
            self.default_prompt = default_prompt
        self.data_key = 'ShellCommand'
        self.output_written = False

    def run(self, edit, command=None, command_prefix=None, prompt=None, region=None, arg_required=None, panel=None, title=None, syntax=None, refresh=None):

        if region is None:
            region is False

        if arg_required is None:
            arg_required = False

        if panel is None:
            panel = False

        if refresh is None:
            refresh = False

        arg = None

        # If regions should be used then work them out, and append
        # them to the command:
        #
        if region is True:
            arg = self.get_region().strip()

            if arg == '':
                if arg_required is True:
                    sublime.message_dialog('This command requires a parameter.')
                    return

        # Setup a closure to run the command:
        #
        def _C(command):

            if command_prefix is not None:
                command = command_prefix + ' ' + command

            if arg is not None:
                command = command + ' ' + arg

            self.run_shell_command(command, panel=panel, title=title, syntax=syntax, refresh=refresh)

        # If no command is specified then we prompt for one, otherwise
        # we can just execute the command:
        #
        if command is None:
            if prompt is None:
                prompt = self.default_prompt
            self.view.window().show_input_panel(prompt, '', _C, None, None)
        else:
            _C(command)

    def run_shell_command(self, command=None, panel=False, title=None, syntax=None, refresh=False, console=None, working_dir=None):

        view = self.view
        window = view.window()

        if command is None:
            sublime.message_dialog('No command provided.')
            return

        if working_dir is None:
            working_dir = self.get_working_dir()

        # Run the command and write any output to the buffer:
        #
        output_target = SH.OutputTarget(window, self.data_key, command, working_dir, title=title, syntax=syntax, panel=panel, console=console)

        def _C(output):

            if output is not None:
                output_target.append_text(output)
                self.output_written = True
            else:
                # If there has been no output:
                #
                if self.output_written is False:
                    settings = sublime.load_settings('ShellCommand.sublime-settings')
                    show_message = settings.get('show_success_but_no_output_message')
                    if show_message:
                        output = settings.get('success_but_no_output_message')

                # Check whether the initiating view needs refreshing:
                #
                if refresh is True:
                    view.run_command('shell_command_refresh')

        OsShell.process(command, _C, working_dir=working_dir)

class ShellCommandOnRegionCommand(ShellCommandCommand):

    def run(self, edit, command=None, command_prefix=None, prompt=None, arg_required=None, panel=None, title=None, syntax=None, refresh=None):

        ShellCommandCommand.run(self, edit, command=command, command_prefix=command_prefix, prompt=prompt, region=True, arg_required=True, panel=panel, title=title, syntax=syntax, refresh=refresh)

# Refreshing a shell command simply involves re-running the original command:
#
class ShellCommandRefreshCommand(ShellCommandCommand):

    def run(self, edit, callback=None):

        console = self.view

        settings = console.settings()
        if settings.has(self.data_key):
            data = settings.get(self.data_key + '_data', None)
            if data is not None:

                console.set_read_only(False)
                console.run_command('sublime_helper_clear_buffer')
                console.set_read_only(True)

                self.run_shell_command(command=data['command'], console=console, working_dir=data['working_dir'])

<|endoftext|>