<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

import py.test
from mock import Mock
from docutils import nodes
from sphinx.addnodes import pending_xref

from sphinxcontrib import issuetracker

def pytest_funcarg__issue_info(request):
    return dict(uri='http://example.com/issues/10', closed=False)

def pytest_funcarg__get_issue_information(request):
    get_issue_information = Mock()
    info = request.getfuncargvalue('issue_info')
    get_issue_information.return_value = info
    return get_issue_information

def pytest_funcarg__resolver(request):
    return issuetracker.make_issue_reference_resolver(
        request.getfuncargvalue('get_issue_information'))

def pytest_funcarg__config(request):
    config = Mock()
    config.project = 'issuetracker'
    config.issuetracker = 'spamtracker'
    config.issuetracker_user = 'foobar'
    config.issuetracker_project = None
    config.issuetracker_issue_pattern = re.compile(r'#(\d+)')
    return config

def pytest_funcarg__app(request):
    app = Mock()
    app.config = request.getfuncargvalue('config')
    return app

def pytest_funcarg__env(request):
    env = Mock()
    env.config = request.getfuncargvalue('config')
    return env

def pytest_funcarg__node(request):
    node = pending_xref()
    node['reftype'] = 'issue'
    node['reftarget'] = '10'
    node.append(nodes.Text('#10'))
    return node

def pytest_funcarg__doc(request):
    doc = nodes.paragraph()
    doc.append(nodes.Text('foo #1 bar'))
    em = nodes.emphasis()
    em.append(nodes.Text('see #2 if you are #abc brave'))
    doc.append(em)
    doc.settings = Mock()
    doc.settings.language_code = ''
    doc.settings.env = request.getfuncargvalue('env')
    return doc

def test_get_github_issue_information(env):
    info = issuetracker.get_github_issue_information(
        'pyudev', 'lunaryorn', '2', env)
    assert info == {'closed': True,
                    'uri': 'https://github.com/lunaryorn/pyudev/issues/2'}

def test_get_bitbucket_issue_information_resolved(env):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '22', env)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/22/'}

def test_get_bitbucket_issue_information_invalid(env):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '36', env)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/36/'}

def test_get_bitbucket_issue_information_duplicate(env):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '42', env)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/42/'}

def test_get_google_code_issue_information_fixed(env):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '2', env)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=2'}

def test_get_google_code_issue_information_invalid(env):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '5', env)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=5'}

def test_get_google_code_issue_information_wontfix(env):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '6', env)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=6'}

def test_get_debian_issue_information_fixed(env):
    info = issuetracker.get_debian_issue_information(
        'ldb-tools', None, '584227', env)
    assert info == {
        'closed': True,
        'uri': 'http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=584227'}

def test_get_debian_issue_information_open(env):
    info = issuetracker.get_debian_issue_information(
        'xul-ext-sync', None, '600890', env)
    assert info == {
        'closed': False,
        'uri': 'http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=600890'}

def test_get_debian_issue_information_invalid(env):
    info = issuetracker.get_debian_issue_information(
        'release.debian.org', None, '1', env)
    assert info == None

def test_make_isssue_reference_resolver_invalid_reftype(
    app, env, resolver, node):
    node['reftype'] = 'spam'
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_no_issue(
    app, env, resolver, node, get_issue_information):
    get_issue_information.return_value = None
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_no_uri(
    app, env, resolver, node, issue_info):
    del issue_info['uri']
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_open_issue(
    app, env, resolver, node, issue_info):
    contnode = node[0]
    node = resolver(app, env, node, node[0])
    assert isinstance(node, nodes.reference)
    assert node[0] is contnode
    assert node['refuri'] == issue_info['uri']
    assert 'reference-issue' in node['classes']
    assert 'issue-closed' not in node['classes']

def test_make_issue_reference_resolver_closed_issue(
    app, env, resolver, node, issue_info):
    issue_info['closed'] = True
    contnode = node[0]
    node = resolver(app, env, node, node[0])
    assert isinstance(node, nodes.reference)
    assert node[0] is contnode
    assert node['refuri'] == issue_info['uri']
    assert 'reference-issue' in node['classes']
    assert 'issue-closed' in node['classes']

def test_get_issue_information_called(
    app, env, resolver, node, get_issue_information):
    resolver(app, env, node, node[0])
    get_issue_information.assert_called_with(
        'issuetracker', 'foobar', '10', app)
    app.config.issuetracker_project = 'spam with eggs'
    resolver(app, env, node, node[0])
    get_issue_information.assert_called_with(
        'spam with eggs', 'foobar', '10', app)

def test_builtin_issue_trackers():
    for tracker in ('github', 'bitbucket',
                    'launchpad', 'google code', 'debian'):
        assert tracker in issuetracker.BUILTIN_ISSUE_TRACKERS

def _assert_text(node, text):
    assert isinstance(node, nodes.Text)
    assert node.astext() == text

def _assert_xref(node, target):
    assert isinstance(node, pending_xref)
    _assert_text(node[0], '#%s' % target)
    assert node['reftype'] == 'issue'
    assert node['reftarget'] == target

def test_issues_references(doc):
    transformer = issuetracker.IssuesReferences(doc)
    transformer.apply()
    assert isinstance(doc, nodes.paragraph)
    assert doc.astext() == 'foo #1 barsee #2 if you are #abc brave'
    _assert_text(doc[0], 'foo ')
    _assert_xref(doc[1], '1')
    _assert_text(doc[2], ' bar')
    em = doc[3]
    assert isinstance(em, nodes.emphasis)
    _assert_text(em[0], 'see ')
    _assert_xref(em[1], '2')
    _assert_text(em[2], ' if you are #abc brave')

def test_issues_references_too_many_groups(doc, config):
    config.issuetracker_issue_pattern = r'(#)(\d+)'
    transformer = issuetracker.IssuesReferences(doc)
    with py.test.raises(ValueError) as exc_info:
        transformer.apply()
    error = exc_info.value
    assert str(error) == ('issuetracker_issue_pattern must have '
                          'exactly one group: %r' % ((u'#', u'1'),))

def test_auto_connect_builtin_issue_resolvers_known_tracker(app):
    app.config.issuetracker = 'bitbucket'
    issuetracker.auto_connect_builtin_issue_resolvers(app)
    assert app.connect.called

def test_auto_connect_builtin_issue_resolvers_unknown_tracker(app):
    app.config.issuetracker = 'spamtracker'
    with py.test.raises(KeyError):
        issuetracker.auto_connect_builtin_issue_resolvers(app)

def test_auto_connect_builtin_issue_resolvers_no_tracker(app):
    app.config.issuetracker = None
    issuetracker.auto_connect_builtin_issue_resolvers(app)
    assert not app.connect.called

def test_add_stylesheet(app):
    issuetracker.add_stylesheet(app)
    app.add_stylesheet.assert_called_with('issuetracker.css')

def test_setup(app):
    issuetracker.setup(app)
    app.require_sphinx.assert_called_with('1.0')
    app.add_transform.assert_called_with(issuetracker.IssuesReferences)
    app.connect.call_args_list = [
        (('builder-inited',
          issuetracker.auto_connect_builtin_issue_resolvers), {}),
        (('builder-inited', issuetracker.add_stylesheet), {}),
        (('build-finished', issuetracker.copy_stylesheet), {})]
    app.add_config_value.call_args_list = [
        (('issuetracker_issue_pattern', re.compile(r'#(\d+)'), 'env'), {}),
        (('issuetracker_user', None, 'env'), {}),
        (('issuetracker_project', None, 'env'), {}),
        (('issuetracker', None, 'env'), {})]

def main():
    import py
    py.cmdline.pytest()

if __name__ == '__main__':
    main()
<commit_msg>Fixed tests: reference resolvers get the application and not the environment as argument<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

import py.test
from mock import Mock
from docutils import nodes
from sphinx.addnodes import pending_xref

from sphinxcontrib import issuetracker

def pytest_funcarg__issue_info(request):
    return dict(uri='http://example.com/issues/10', closed=False)

def pytest_funcarg__get_issue_information(request):
    get_issue_information = Mock()
    info = request.getfuncargvalue('issue_info')
    get_issue_information.return_value = info
    return get_issue_information

def pytest_funcarg__resolver(request):
    return issuetracker.make_issue_reference_resolver(
        request.getfuncargvalue('get_issue_information'))

def pytest_funcarg__config(request):
    config = Mock()
    config.project = 'issuetracker'
    config.issuetracker = 'spamtracker'
    config.issuetracker_user = 'foobar'
    config.issuetracker_project = None
    config.issuetracker_issue_pattern = re.compile(r'#(\d+)')
    return config

def pytest_funcarg__app(request):
    app = Mock()
    app.config = request.getfuncargvalue('config')
    return app

def pytest_funcarg__env(request):
    env = Mock()
    env.config = request.getfuncargvalue('config')
    return env

def pytest_funcarg__node(request):
    node = pending_xref()
    node['reftype'] = 'issue'
    node['reftarget'] = '10'
    node.append(nodes.Text('#10'))
    return node

def pytest_funcarg__doc(request):
    doc = nodes.paragraph()
    doc.append(nodes.Text('foo #1 bar'))
    em = nodes.emphasis()
    em.append(nodes.Text('see #2 if you are #abc brave'))
    doc.append(em)
    doc.settings = Mock()
    doc.settings.language_code = ''
    doc.settings.env = request.getfuncargvalue('env')
    return doc

def test_get_github_issue_information(app):
    info = issuetracker.get_github_issue_information(
        'pyudev', 'lunaryorn', '2', app)
    assert info == {'closed': True,
                    'uri': 'https://github.com/lunaryorn/pyudev/issues/2'}

def test_get_bitbucket_issue_information_resolved(app):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '22', app)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/22/'}

def test_get_bitbucket_issue_information_invalid(app):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '36', app)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/36/'}

def test_get_bitbucket_issue_information_duplicate(app):
    info = issuetracker.get_bitbucket_issue_information(
        'synaptiks', 'lunar', '42', app)
    assert info == {'closed': True,
                    'uri': 'http://bitbucket.org/lunar/synaptiks/issue/42/'}

def test_get_google_code_issue_information_fixed(app):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '2', app)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=2'}

def test_get_google_code_issue_information_invalid(app):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '5', app)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=5'}

def test_get_google_code_issue_information_wontfix(app):
    info = issuetracker.get_google_code_issue_information(
        'pytox', None, '6', app)
    assert info == {
        'closed': True,
        'uri': 'http://code.google.com/p/pytox/issues/detail?id=6'}

def test_get_debian_issue_information_fixed(app):
    info = issuetracker.get_debian_issue_information(
        'ldb-tools', None, '584227', app)
    assert info == {
        'closed': True,
        'uri': 'http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=584227'}

def test_get_debian_issue_information_open(app):
    info = issuetracker.get_debian_issue_information(
        'xul-ext-sync', None, '600890', app)
    assert info == {
        'closed': False,
        'uri': 'http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=600890'}

def test_get_debian_issue_information_invalid(app):
    info = issuetracker.get_debian_issue_information(
        'release.debian.org', None, '1', app)
    assert info == None

def test_make_isssue_reference_resolver_invalid_reftype(
    app, env, resolver, node):
    node['reftype'] = 'spam'
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_no_issue(
    app, env, resolver, node, get_issue_information):
    get_issue_information.return_value = None
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_no_uri(
    app, env, resolver, node, issue_info):
    del issue_info['uri']
    assert resolver(app, env, node, node[0]) is None

def test_make_issue_reference_resolver_open_issue(
    app, env, resolver, node, issue_info):
    contnode = node[0]
    node = resolver(app, env, node, node[0])
    assert isinstance(node, nodes.reference)
    assert node[0] is contnode
    assert node['refuri'] == issue_info['uri']
    assert 'reference-issue' in node['classes']
    assert 'issue-closed' not in node['classes']

def test_make_issue_reference_resolver_closed_issue(
    app, env, resolver, node, issue_info):
    issue_info['closed'] = True
    contnode = node[0]
    node = resolver(app, env, node, node[0])
    assert isinstance(node, nodes.reference)
    assert node[0] is contnode
    assert node['refuri'] == issue_info['uri']
    assert 'reference-issue' in node['classes']
    assert 'issue-closed' in node['classes']

def test_get_issue_information_called(
    app, env, resolver, node, get_issue_information):
    resolver(app, env, node, node[0])
    get_issue_information.assert_called_with(
        'issuetracker', 'foobar', '10', app)
    app.config.issuetracker_project = 'spam with eggs'
    resolver(app, env, node, node[0])
    get_issue_information.assert_called_with(
        'spam with eggs', 'foobar', '10', app)

def test_builtin_issue_trackers():
    for tracker in ('github', 'bitbucket',
                    'launchpad', 'google code', 'debian'):
        assert tracker in issuetracker.BUILTIN_ISSUE_TRACKERS

def _assert_text(node, text):
    assert isinstance(node, nodes.Text)
    assert node.astext() == text

def _assert_xref(node, target):
    assert isinstance(node, pending_xref)
    _assert_text(node[0], '#%s' % target)
    assert node['reftype'] == 'issue'
    assert node['reftarget'] == target

def test_issues_references(doc):
    transformer = issuetracker.IssuesReferences(doc)
    transformer.apply()
    assert isinstance(doc, nodes.paragraph)
    assert doc.astext() == 'foo #1 barsee #2 if you are #abc brave'
    _assert_text(doc[0], 'foo ')
    _assert_xref(doc[1], '1')
    _assert_text(doc[2], ' bar')
    em = doc[3]
    assert isinstance(em, nodes.emphasis)
    _assert_text(em[0], 'see ')
    _assert_xref(em[1], '2')
    _assert_text(em[2], ' if you are #abc brave')

def test_issues_references_too_many_groups(doc, config):
    config.issuetracker_issue_pattern = r'(#)(\d+)'
    transformer = issuetracker.IssuesReferences(doc)
    with py.test.raises(ValueError) as exc_info:
        transformer.apply()
    error = exc_info.value
    assert str(error) == ('issuetracker_issue_pattern must have '
                          'exactly one group: %r' % ((u'#', u'1'),))

def test_auto_connect_builtin_issue_resolvers_known_tracker(app):
    app.config.issuetracker = 'bitbucket'
    issuetracker.auto_connect_builtin_issue_resolvers(app)
    assert app.connect.called

def test_auto_connect_builtin_issue_resolvers_unknown_tracker(app):
    app.config.issuetracker = 'spamtracker'
    with py.test.raises(KeyError):
        issuetracker.auto_connect_builtin_issue_resolvers(app)

def test_auto_connect_builtin_issue_resolvers_no_tracker(app):
    app.config.issuetracker = None
    issuetracker.auto_connect_builtin_issue_resolvers(app)
    assert not app.connect.called

def test_add_stylesheet(app):
    issuetracker.add_stylesheet(app)
    app.add_stylesheet.assert_called_with('issuetracker.css')

def test_setup(app):
    issuetracker.setup(app)
    app.require_sphinx.assert_called_with('1.0')
    app.add_transform.assert_called_with(issuetracker.IssuesReferences)
    app.connect.call_args_list = [
        (('builder-inited',
          issuetracker.auto_connect_builtin_issue_resolvers), {}),
        (('builder-inited', issuetracker.add_stylesheet), {}),
        (('build-finished', issuetracker.copy_stylesheet), {})]
    app.add_config_value.call_args_list = [
        (('issuetracker_issue_pattern', re.compile(r'#(\d+)'), 'env'), {}),
        (('issuetracker_user', None, 'env'), {}),
        (('issuetracker_project', None, 'env'), {}),
        (('issuetracker', None, 'env'), {})]

def main():
    import py
    py.cmdline.pytest()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from pychecker2.Check import Check
from pychecker2.Options import Opt, BoolOpt
from pychecker2.Warning import Warning
from pychecker2.util import flatten
from pychecker2 import symbols

import compiler

def _is_method(scope):
    return scope.__class__ is symbols.FunctionScope and \
           scope.parent.__class__ is symbols.ClassScope

def _is_self(scope, name):
    return _is_method(scope) and name in scope.node.argnames[:1]

def is_arg_and_defaulted_to_same_name(name, scope):
    if isinstance(scope, symbols.FunctionScope):
        if name in scope.node.argnames and scope.node.defaults:
            # compute default args
            args = scope.node.argnames[:]
            # knock off varags
            if scope.node.varargs:
                args = args[:-1]
            # knock off kwargs
            if scope.node.kwargs:
                args = args[:-1]
            # take the last args as the defaults
            args = args[-len(scope.node.defaults):]
            try:
                # get the corresponding default arg value
                default = scope.node.defaults[args.index(name)]
                # must be a Name node of the same name
                if isinstance(default, compiler.ast.Name) and \
                   default.name == name:
                    return 1
            except ValueError:
                pass
    return None

class Parents:
    def __init__(self, scope):
        self.scope = scope
    def __call__(self):
        retval = self.scope.parent
        self.scope = retval
        return retval

def parents(scope):
    return iter(Parents(scope), None)

class ShadowCheck(Check):
    """Use symbol information to check that no scope defines a name
    already known to a parent scope"""

    shadowBuiltins = Warning('Report names that shadow builtins',
                            'Identifier (%s) shadows builtin', 0)
    shadowIdentifier = Warning('Report names already defined in outer scopes',
                               'Identifier (%s) shadows definition in scope %s')
    def check(self, file):
        # warn if any name defined in a scope is defined in a parent scope
        # or even the builtins
        for scope in file.scopes.values():
            # skip methods of classes
            if isinstance(scope.node, compiler.ast.Class):
                continue
            for name in scope.defs:
                if name in scope.globals:
                    continue
                if is_arg_and_defaulted_to_same_name(name, scope):
                    continue
                if _is_self(scope, name):
                    continue
                if __builtins__.has_key(name):
                    file.warning(scope.defs[name], self.shadowBuiltins, name)
                for p in parents(scope):
                    if p.defs.has_key(name) and not isinstance(p, symbols.ClassScope):
                        file.warning(scope.defs[name], self.shadowIdentifier, name, `p`)

class UnusedCheck(Check):
    """Use symbol information to check that no scope defines a name
    not used in this or any child scope"""

    unused = Warning('Report names not used', 'Identifier (%s) not used')

    def get_options(self, options):
        desc = 'Ignore unused identifiers that start with these values'
        default = ['unused', 'empty', 'dummy',
                   '__pychecker__', '__all__', '__version__', 'ignored']
        options.add(Opt(self, 'unusedPrefixes', desc, default))

        desc = 'Ignore unused method "self" parameter'
        options.add(BoolOpt(self, 'reportUnusedSelf', desc))

    def check(self, file):
        if type(self.unusedPrefixes) == type(''):
            self.unusedPrefixes = eval(self.unusedPrefixes)

        def used(name, parent_scope):
            # don't report global classes and functions that
            # don't start with "_"
            if parent_scope in file.root_scope.get_children() and \
               parent_scope.__class__ in (symbols.ClassScope, \
                                          symbols.FunctionScope) and \
               name == parent_scope.node.name and \
               not name.startswith('_'):
                    return 1

            if parent_scope.uses.has_key(name):
                return 1
            for c in parent_scope.get_children():
                if used(name, c):
                    return 1
            return 0

        for nodes, scope in file.scopes.items():
            if isinstance(nodes, compiler.ast.Function):
                code = nodes.code.nodes
                # functions which only raise exceptions are unlikely to use
                # their local variables
                if isinstance(code[0], compiler.ast.Raise):
                    continue
                # functions which do nothing are unlikely to use their
                # local variables
                if len(code) == 1 and isinstance(code[0], compiler.ast.Pass):
                    continue
                # functions which only assert falsehood are unlikely to use
                # their local variables
                if isinstance(code[0], compiler.ast.Assert) and \
                   isinstance(code[0].test, compiler.ast.Const) and \
                   not code[0].test.value:
                    continue

            # ignore names defined in a class scope
            if isinstance(scope, symbols.ClassScope):
                continue

            # ensure that every defined variable is used in some scope
            for var in scope.defs:
                # check for method self
                if not self.reportUnusedSelf and _is_self(scope, var):
                    continue

                # ignore names in the root scope which are not imported:
                # class defs, function defs, variables, etc.
                if scope == file.root_scope:
                    if not scope.imports.has_key(var):
                        continue

                # ignore variables global to this scope
                if scope.globals.has_key(var):
                    continue

                for prefix in self.unusedPrefixes:
                    if var.startswith(prefix):
                        break
                else:
                    if not used(var, scope):
                        file.warning(scope.defs[var], self.unused, var)

def _importedName(scope, name):
    if scope.imports.has_key(name):
        return 1
    if scope.parent:
        return _importedName(scope.parent, name)
    return None

class UnknownCheck(Check):
    """Use symbol information to check that no scope uses a name
    not defined in a parent scope"""

    unknown = Warning('Report names that are not defined',
                      'Unknown identifier: %s')

    builtins = {}
    builtins.update(__builtins__)
    builtins['__builtins__'] = __builtins__
    builtins['WindowsError'] = getattr(__builtins__, 'WindowsError', None)

    def check(self, file):

        # if a name used is not found in the defined variables, complain
        for scope in file.scopes.values():
            for var in scope.uses:
                if not scope.defs.has_key(var):
                    for p in parents(scope):
                        if p.defs.has_key(var):
                            break
                    else:
                        if not _importedName(scope, var):
                            if not UnknownCheck.builtins.has_key(var):
                                file.warning(scope.uses[var], self.unknown, var)

class SelfCheck(Check):
    'Report any methods whose first argument is not self'

    selfName = Warning(__doc__,
                       'First argument to method %s (%s) is not in %s')

    def get_options(self, options):
        desc = 'Name of self parameter'
        default = ["self", "this", "s"]
        options.add(Opt(self, 'selfNames', desc, default))

    def check(self, file):
        if type(self.selfNames) == type(''):
            self.selfNames = eval(self.selfNames)

        for scope in file.scopes.values():
            for var in scope.defs:
                if _is_self(scope, var) and var not in self.selfNames:
                    file.warning(scope.defs[var], self.selfName,
                                 scope.node.name, var, `self.selfNames`)

class UnpackCheck(Check):
    'Mark all unpacked variables as used'

    def get_options(self, options):
        desc = 'Do not treat variables used in tuple assignment as used'
        options.add(BoolOpt(self, 'unpackedUsed', desc, 1))

    def check(self, file):
        if not self.unpackedUsed:
            return

        class Visitor:
            def visitAssTuple(self, node):
                for c in node.getChildNodes():
                    n = node
                    while n:
                        try:
                            file.scopes[n].uses[c.name] = node.lineno
                            break
                        except AttributeError:
                            pass
                        except KeyError:
                            pass
                        n = n.parent
            visitAssList = visitAssTuple

        # local args unpacked on the `def' line are used, too
        for scope in file.scopes.values():
            if isinstance(scope.node, compiler.ast.Function):
                for arg in scope.node.argnames:
                    if isinstance(arg, tuple):
                        for unpacked in flatten(arg):
                            scope.uses[unpacked] = scope.uses.get(unpacked,
                                                                  scope.node)

        if file.root_scope:
            compiler.walk(file.root_scope.node, Visitor())
<commit_msg>clean up exceptions remove kwargs first<commit_after>from pychecker2.Check import Check
from pychecker2.Options import Opt, BoolOpt
from pychecker2.Warning import Warning
from pychecker2.util import flatten
from pychecker2 import symbols

import compiler

def _is_method(scope):
    return scope.__class__ is symbols.FunctionScope and \
           scope.parent.__class__ is symbols.ClassScope

def _is_self(scope, name):
    return _is_method(scope) and name in scope.node.argnames[:1]

def is_arg_and_defaulted_to_same_name(name, scope):
    if isinstance(scope, symbols.FunctionScope):
        if name in scope.node.argnames and scope.node.defaults:
            # compute default args
            args = scope.node.argnames[:]
            # knock off kwargs
            if scope.node.kwargs:
                args = args[:-1]
            # knock off varags
            if scope.node.varargs:
                args = args[:-1]
            # take the last args as the defaults
            args = args[-len(scope.node.defaults):]
            try:
                # get the corresponding default arg value
                default = scope.node.defaults[args.index(name)]
                # must be a Name node of the same name
                if isinstance(default, compiler.ast.Name) and \
                   default.name == name:
                    return 1
            except ValueError:
                pass
    return None

class Parents:
    def __init__(self, scope):
        self.scope = scope
    def __call__(self):
        retval = self.scope.parent
        self.scope = retval
        return retval

def parents(scope):
    return iter(Parents(scope), None)

class ShadowCheck(Check):
    """Use symbol information to check that no scope defines a name
    already known to a parent scope"""

    shadowBuiltins = Warning('Report names that shadow builtins',
                            'Identifier (%s) shadows builtin', 0)
    shadowIdentifier = Warning('Report names already defined in outer scopes',
                               'Identifier (%s) shadows definition in scope %s')
    def check(self, file):
        # warn if any name defined in a scope is defined in a parent scope
        # or even the builtins
        for scope in file.scopes.values():
            # skip methods of classes
            if isinstance(scope.node, compiler.ast.Class):
                continue
            for name in scope.defs:
                if name in scope.globals:
                    continue
                if is_arg_and_defaulted_to_same_name(name, scope):
                    continue
                if _is_self(scope, name):
                    continue
                if __builtins__.has_key(name):
                    file.warning(scope.defs[name], self.shadowBuiltins, name)
                for p in parents(scope):
                    if p.defs.has_key(name) and not isinstance(p, symbols.ClassScope):
                        file.warning(scope.defs[name], self.shadowIdentifier, name, `p`)

class UnusedCheck(Check):
    """Use symbol information to check that no scope defines a name
    not used in this or any child scope"""

    unused = Warning('Report names not used', 'Identifier (%s) not used')

    def get_options(self, options):
        desc = 'Ignore unused identifiers that start with these values'
        default = ['unused', 'empty', 'dummy',
                   '__pychecker__', '__all__', '__version__', 'ignored']
        options.add(Opt(self, 'unusedPrefixes', desc, default))

        desc = 'Ignore unused method "self" parameter'
        options.add(BoolOpt(self, 'reportUnusedSelf', desc))

    def check(self, file):
        if type(self.unusedPrefixes) == type(''):
            self.unusedPrefixes = eval(self.unusedPrefixes)

        def used(name, parent_scope):
            # don't report global classes and functions that
            # don't start with "_"
            if parent_scope in file.root_scope.get_children() and \
               parent_scope.__class__ in (symbols.ClassScope, \
                                          symbols.FunctionScope) and \
               name == parent_scope.node.name and \
               not name.startswith('_'):
                    return 1

            if parent_scope.uses.has_key(name):
                return 1
            for c in parent_scope.get_children():
                if used(name, c):
                    return 1
            return 0

        for nodes, scope in file.scopes.items():
            if isinstance(nodes, compiler.ast.Function):
                code = nodes.code.nodes
                # functions which only raise exceptions are unlikely to use
                # their local variables
                if isinstance(code[0], compiler.ast.Raise):
                    continue
                # functions which do nothing are unlikely to use their
                # local variables
                if len(code) == 1 and isinstance(code[0], compiler.ast.Pass):
                    continue
                # functions which only assert falsehood are unlikely to use
                # their local variables
                if isinstance(code[0], compiler.ast.Assert) and \
                   isinstance(code[0].test, compiler.ast.Const) and \
                   not code[0].test.value:
                    continue

            # ignore names defined in a class scope
            if isinstance(scope, symbols.ClassScope):
                continue

            # ensure that every defined variable is used in some scope
            for var in scope.defs:
                # check for method self
                if not self.reportUnusedSelf and _is_self(scope, var):
                    continue

                # ignore names in the root scope which are not imported:
                # class defs, function defs, variables, etc.
                if scope == file.root_scope:
                    if not scope.imports.has_key(var):
                        continue

                # ignore variables global to this scope
                if scope.globals.has_key(var):
                    continue

                for prefix in self.unusedPrefixes:
                    if var.startswith(prefix):
                        break
                else:
                    if not used(var, scope):
                        file.warning(scope.defs[var], self.unused, var)

def _importedName(scope, name):
    if scope.imports.has_key(name):
        return 1
    if scope.parent:
        return _importedName(scope.parent, name)
    return None

class UnknownCheck(Check):
    """Use symbol information to check that no scope uses a name
    not defined in a parent scope"""

    unknown = Warning('Report names that are not defined',
                      'Unknown identifier: %s')

    builtins = {}
    builtins.update(__builtins__)
    builtins['__builtins__'] = __builtins__
    builtins['WindowsError'] = getattr(__builtins__, 'WindowsError', None)

    def check(self, file):

        # if a name used is not found in the defined variables, complain
        for scope in file.scopes.values():
            for var in scope.uses:
                if not scope.defs.has_key(var):
                    for p in parents(scope):
                        if p.defs.has_key(var):
                            break
                    else:
                        if not _importedName(scope, var):
                            if not UnknownCheck.builtins.has_key(var):
                                file.warning(scope.uses[var], self.unknown, var)

class SelfCheck(Check):
    'Report any methods whose first argument is not self'

    selfName = Warning(__doc__,
                       'First argument to method %s (%s) is not in %s')

    def get_options(self, options):
        desc = 'Name of self parameter'
        default = ["self", "this", "s"]
        options.add(Opt(self, 'selfNames', desc, default))

    def check(self, file):
        if type(self.selfNames) == type(''):
            self.selfNames = eval(self.selfNames)

        for scope in file.scopes.values():
            for var in scope.defs:
                if _is_self(scope, var) and var not in self.selfNames:
                    file.warning(scope.defs[var], self.selfName,
                                 scope.node.name, var, `self.selfNames`)

class UnpackCheck(Check):
    'Mark all unpacked variables as used'

    def get_options(self, options):
        desc = 'Do not treat variables used in tuple assignment as used'
        options.add(BoolOpt(self, 'unpackedUsed', desc, 1))

    def check(self, file):
        if not self.unpackedUsed:
            return

        class Visitor:
            def visitAssTuple(self, node):
                for c in node.getChildNodes():
                    n = node
                    while n:
                        try:
                            file.scopes[n].uses[c.name] = node.lineno
                            break
                        except (AttributeError, KeyError):
                            pass
                        n = n.parent
            visitAssList = visitAssTuple

        # local args unpacked on the `def' line are used, too
        for scope in file.scopes.values():
            if isinstance(scope.node, compiler.ast.Function):
                for arg in scope.node.argnames:
                    if isinstance(arg, tuple):
                        for unpacked in flatten(arg):
                            scope.uses[unpacked] = scope.uses.get(unpacked,
                                                                  scope.node)

        if file.root_scope:
            compiler.walk(file.root_scope.node, Visitor())
<|endoftext|>


<commit_before>
# -*- coding: utf-8 -*-
r"""
Initialization code of pyopencl.
"""
__author__ = "Konstantin Klementiev, Roman Chernikov"
__date__ = "19 Mar 2017"
import numpy as np
import os
from .. import raycing
try:
    import pyopencl as cl
    os.environ['PYOPENCL_COMPILER_OUTPUT'] = '1'
    isOpenCL = True
except ImportError:
    isOpenCL = False

__dir__ = os.path.dirname(__file__)
_DEBUG = 20

class XRT_CL(object):
    def __init__(self, filename=None, targetOpenCL=raycing.targetOpenCL,
                 precisionOpenCL=raycing.precisionOpenCL, kernelsource=None):
        self.kernelsource = kernelsource
        self.cl_filename = filename
        self.lastTargetOpenCL = None
        self.lastPrecisionOpenCL = None
        self.set_cl(targetOpenCL, precisionOpenCL)
        self.cl_is_blocking = True

    def set_cl(self, targetOpenCL=raycing.targetOpenCL,
               precisionOpenCL=raycing.precisionOpenCL):
        if (targetOpenCL == self.lastTargetOpenCL) and\
                (precisionOpenCL == self.lastPrecisionOpenCL):
            return
        self.lastTargetOpenCL = targetOpenCL
        self.lastPrecisionOpenCL = precisionOpenCL
        if not isOpenCL:
            raise EnvironmentError("pyopencl is not available!")
        else:
            if isinstance(targetOpenCL, (tuple, list)):
                iDevice = []
                targetOpenCL = list(targetOpenCL)
                if isinstance(targetOpenCL[0], int):
                    nPlatform, nDevice = targetOpenCL
                    platform = cl.get_platforms()[nPlatform]
                    dev = platform.get_devices()[nDevice]
                    iDevice.extend([dev])
                else:
                    for target in targetOpenCL:
                        if isinstance(target, (tuple, list)):
                            target = list(target)
                            if len(target) > 1:
                                nPlatform, nDevice = target
                                platform = cl.get_platforms()[nPlatform]
                                iDevice.extend(
                                    [platform.get_devices()[nDevice]])
                            else:
                                nPlatform = target[0]
                                platform = cl.get_platforms()[nPlatform]
                                iDevice.extend(platform.get_devices())
            elif isinstance(targetOpenCL, int):
                nPlatform = targetOpenCL
                platform = cl.get_platforms()[nPlatform]
                iDevice = platform.get_devices()
            elif isinstance(targetOpenCL, str):
                iDeviceCPU = []
                iDeviceGPU = []
                iDeviceAcc = []
                iDevice = []
                for platform in cl.get_platforms():
                    if 'mesa' in platform.vendor.lower():
                        continue  # for new Linuxes Mesa provides OpenCL 1.1
                    CPUdevices = []
                    GPUdevices = []
                    AccDevices = []
                    try:  # at old pyopencl versions:
                        CPUdevices =\
                            platform.get_devices(
                                device_type=cl.device_type.CPU)
                        GPUdevices =\
                            platform.get_devices(
                                device_type=cl.device_type.GPU)
                        AccDevices =\
                            platform.get_devices(
                                device_type=cl.device_type.ACCELERATOR)
                    except cl.RuntimeError:
                        pass

                    if len(CPUdevices) > 0:
                        if len(iDeviceCPU) > 0:
                            if CPUdevices[0].vendor == \
                                    CPUdevices[0].platform.vendor:
                                try:
                                    tmpctx = cl.Context(devices=CPUdevices)
                                    iDeviceCPU = CPUdevices
                                except:
                                    pass
                        else:
                            try:
                                tmpctx = cl.Context(devices=CPUdevices)
                                iDeviceCPU.extend(CPUdevices)
                            except:
                                pass
                    for GPUDevice in GPUdevices:
                        try:
                            tmpctx = cl.Context(devices=[GPUDevice])
                            if GPUDevice.double_fp_config > 0:
                                iDeviceGPU.extend([GPUDevice])
                        except:
                            pass
                    iDeviceAcc.extend(AccDevices)
                if _DEBUG > 10:
                    print("OpenCL: bulding {0} ...".format(self.cl_filename))
                    print("OpenCL: found {0} CPU{1}".format(
                          len(iDeviceCPU) if len(iDeviceCPU) > 0 else 'none',
                          's' if len(iDeviceCPU) > 1 else ''))
                    print("OpenCL: found {0} GPU{1}".format(
                          len(iDeviceGPU) if len(iDeviceGPU) > 0 else 'none',
                          's' if len(iDeviceGPU) > 1 else ''))
                    print("OpenCL: found {0} other accelerator{1}".format(
                          len(iDeviceAcc) if len(iDeviceAcc) > 0 else 'none',
                          's' if len(iDeviceAcc) > 1 else ''))

                if targetOpenCL.upper().startswith('GPU'):
                    iDevice.extend(iDeviceGPU)
                elif targetOpenCL.upper().startswith('CPU'):
                    iDevice.extend(iDeviceCPU)
                elif targetOpenCL.upper().startswith('ALL'):
                    iDevice.extend(iDeviceGPU)
                    iDevice.extend(iDeviceCPU)
                    iDevice.extend(iDeviceAcc)
                else:  # auto
                    if len(iDeviceGPU) > 0:
                        iDevice = iDeviceGPU
                    elif len(iDeviceAcc) > 0:
                        iDevice = iDeviceAcc
                    else:
                        iDevice = iDeviceCPU
                if len(iDevice) == 0:
                    targetOpenCL = None
                    self.lastTargetOpenCL = targetOpenCL
            else:  # None
                targetOpenCL = None
                self.lastTargetOpenCL = targetOpenCL
        if targetOpenCL is not None:
            if _DEBUG > 10:
                autoStr = "Autos" if isinstance(targetOpenCL, str) else "S"
                for idn, idv in enumerate(iDevice):
                    print("OpenCL for {0}: {1}elected device {2}: {3}".format(
                        self.cl_filename, autoStr, idn, idv.name))

            if self.kernelsource is None:
                cl_file = os.path.join(os.path.dirname(__file__),
                                       self.cl_filename)
                with open(cl_file, 'r') as f:
                    kernelsource = f.read()
            else:
                kernelsource = self.kernelsource
            if precisionOpenCL == 'auto':
                try:
                    for device in iDevice:
                        if device.double_fp_config == 63:
                            precisionOpenCL = 'float64'
                        else:
                            raise AttributeError
                except AttributeError:
                    precisionOpenCL = 'float32'
            if _DEBUG > 10:
                print('precisionOpenCL = {0}'.format(precisionOpenCL))
            if precisionOpenCL == 'float64':
                self.cl_precisionF = np.float64
                self.cl_precisionC = np.complex128
                kernelsource = kernelsource.replace('float', 'double')
            elif precisionOpenCL == 'float32':
                self.cl_precisionF = np.float32
                self.cl_precisionC = np.complex64
            else:
                raise ValueError("Unknown precision")
            self.cl_queue = []
            self.cl_ctx = []
            self.cl_program = []
            for device in iDevice:
                cl_ctx = cl.Context(devices=[device])
                self.cl_queue.extend([cl.CommandQueue(cl_ctx, device)])
                self.cl_program.extend(
                    [cl.Program(cl_ctx, kernelsource).build(
                        options=['-I', __dir__])])
                self.cl_ctx.extend([cl_ctx])

            self.cl_mf = cl.mem_flags

    def run_parallel(self, kernelName='', scalarArgs=None,
                     slicedROArgs=None, nonSlicedROArgs=None,
                     slicedRWArgs=None, nonSlicedRWArgs=None, dimension=0):

        ka_offset = len(scalarArgs) if scalarArgs is not None else 0
        ro_offset = len(slicedROArgs) if slicedROArgs is not None else 0
        ns_offset = len(nonSlicedROArgs) if nonSlicedROArgs is not None else 0
        rw_offset = len(slicedRWArgs) if slicedRWArgs is not None else 0
        rw_pos = ka_offset + ro_offset + ns_offset
        nsrw_pos = ka_offset + ro_offset + ns_offset + rw_offset

        nctx = len(self.cl_ctx)
        kernel_bufs = []
        global_size = []
        ev_h2d = []
        ev_run = []
        nCU = []
        ndstart = []
        ndslice = []
        ndsize = []

        for ictx, ctx in enumerate(self.cl_ctx):
            # nCUw = 0.25 if self.cl_ctx[ictx].devices[0].type == \
            #    cl.device_type.CPU else 1.0
            nCUw = 1.0
            nCU.extend([self.cl_ctx[ictx].devices[0].max_compute_units*nCUw])

        totalCUs = sum(nCU)
        for ictx, ctx in enumerate(self.cl_ctx):
            ev_h2d.extend([[]])
            kernel_bufs.extend([[]])
            if scalarArgs is not None:
                kernel_bufs[ictx].extend(scalarArgs)
            ndstart.extend([sum(ndsize)])

            if dimension > 1:
                if ictx < nctx - 1:
                    ndsize.extend([np.floor(dimension*nCU[ictx]/totalCUs)])
                else:
                    ndsize.extend([dimension-ndstart[ictx]])
                ndslice.extend([slice(ndstart[ictx],
                                      ndstart[ictx]+ndsize[ictx])])
            else:
                ndslice.extend([0])

# In case each photon has an array of input/output data we define a second
# dimension
            if slicedROArgs is not None and dimension > 1:
                for iarg, arg in enumerate(slicedROArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg[iSlice])])

            if nonSlicedROArgs is not None:
                for iarg, arg in enumerate(nonSlicedROArgs):
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg)])

            if slicedRWArgs is not None:
                for iarg, arg in enumerate(slicedRWArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.WRITE_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg[iSlice])])
                global_size.extend([(np.int(ndsize[ictx]),)])
#                global_size.extend([slicedRWArgs[0][ndslice[ictx]].shape])
            if nonSlicedRWArgs is not None:
                for iarg, arg in enumerate(nonSlicedRWArgs):
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.WRITE_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg)])
                global_size.extend([np.array([1]).shape])

        local_size = None

        for ictx, ctx in enumerate(self.cl_ctx):
            kernel = getattr(self.cl_program[ictx], kernelName)
            ev_run.extend([kernel(
                self.cl_queue[ictx],
                global_size[ictx],
                local_size,
                *kernel_bufs[ictx])])

        for iev, ev in enumerate(ev_run):
            status = cl.command_execution_status.to_string(
                ev.command_execution_status)
            if _DEBUG > 20:
                print("ctx status {0} {1}".format(iev, status))

        ret = ()

        if slicedRWArgs is not None:
            for ictx, ctx in enumerate(self.cl_ctx):
                for iarg, arg in enumerate(slicedRWArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    cl.enqueue_copy(self.cl_queue[ictx],
                                    slicedRWArgs[iarg][iSlice],
                                    kernel_bufs[ictx][iarg + rw_pos],
                                    is_blocking=self.cl_is_blocking)
            ret += tuple(slicedRWArgs)

        if nonSlicedRWArgs is not None:
            for ictx, ctx in enumerate(self.cl_ctx):
                for iarg, arg in enumerate(nonSlicedRWArgs):
                    cl.enqueue_copy(self.cl_queue[ictx],
                                    nonSlicedRWArgs[iarg],
                                    kernel_bufs[ictx][iarg + nsrw_pos],
                                    is_blocking=self.cl_is_blocking)
            ret += tuple(nonSlicedRWArgs)

        return ret
<commit_msg>Optimized openCL device selection procedure<commit_after>
# -*- coding: utf-8 -*-
r"""
Initialization code of pyopencl.
"""
__author__ = "Konstantin Klementiev, Roman Chernikov"
__date__ = "19 Mar 2017"
import numpy as np
import os
from .. import raycing
try:
    import pyopencl as cl
    os.environ['PYOPENCL_COMPILER_OUTPUT'] = '1'
    isOpenCL = True
except ImportError:
    isOpenCL = False

__dir__ = os.path.dirname(__file__)
_DEBUG = 20

class XRT_CL(object):
    def __init__(self, filename=None, targetOpenCL=raycing.targetOpenCL,
                 precisionOpenCL=raycing.precisionOpenCL, kernelsource=None):
        self.kernelsource = kernelsource
        self.cl_filename = filename
        self.lastTargetOpenCL = None
        self.lastPrecisionOpenCL = None
        self.set_cl(targetOpenCL, precisionOpenCL)
        self.cl_is_blocking = True

    def set_cl(self, targetOpenCL=raycing.targetOpenCL,
               precisionOpenCL=raycing.precisionOpenCL):
        if (targetOpenCL == self.lastTargetOpenCL) and\
                (precisionOpenCL == self.lastPrecisionOpenCL):
            return
        self.lastTargetOpenCL = targetOpenCL
        self.lastPrecisionOpenCL = precisionOpenCL
        if not isOpenCL:
            raise EnvironmentError("pyopencl is not available!")
        else:
            try:
                cl_platforms = cl.get_platforms()
            except:
                targetOpenCL = None
                raise EnvironmentError("Unknown error. OpenCL disabled")
            if isinstance(targetOpenCL, (tuple, list)):
                iDevice = []
                targetOpenCL = list(targetOpenCL)
                if isinstance(targetOpenCL[0], int):
                    nPlatform, nDevice = targetOpenCL
                    platform = cl_platforms[nPlatform]
                    try:
                        dev = platform.get_devices()[nDevice]
                        iDevice.extend([dev])
                    except:
                        pass
                else:
                    for target in targetOpenCL:
                        if isinstance(target, (tuple, list)):
                            target = list(target)
                            if len(target) > 1:
                                nPlatform, nDevice = target
                                platform = cl_platforms[nPlatform]
                                try:
                                    iDevice.extend(
                                        [platform.get_devices()[nDevice]])
                                except:
                                    pass
                            else:
                                nPlatform = target[0]
                                platform = cl_platforms[nPlatform]
                                try:
                                    iDevice.extend(platform.get_devices())
                                except:
                                    pass
            elif isinstance(targetOpenCL, int):
                nPlatform = targetOpenCL
                platform = cl_platforms[nPlatform]
                try:
                    iDevice = platform.get_devices()
                except:
                    pass
            elif isinstance(targetOpenCL, str):
                iDeviceCPU = []
                iDeviceGPU = []
                iDeviceAcc = []
                iDevice = []
                for platform in cl_platforms:
                    if 'mesa' in platform.vendor.lower():
                        continue  # for new Linuxes Mesa provides OpenCL 1.1
                    CPUdevices = []
                    GPUdevices = []
                    AccDevices = []
                    try:  # at old pyopencl versions:
                        CPUdevices =\
                            platform.get_devices(
                                device_type=cl.device_type.CPU)
                        GPUdevices =\
                            platform.get_devices(
                                device_type=cl.device_type.GPU)
                        AccDevices =\
                            platform.get_devices(
                                device_type=cl.device_type.ACCELERATOR)
                    except cl.RuntimeError:
                        pass

                    if len(CPUdevices) > 0:
                        if len(iDeviceCPU) > 0:
                            if CPUdevices[0].vendor == \
                                    CPUdevices[0].platform.vendor:
                                try:
                                    tmpctx = cl.Context(devices=CPUdevices)
                                    iDeviceCPU = CPUdevices
                                except:
                                    pass
                        else:
                            try:
                                tmpctx = cl.Context(devices=CPUdevices)
                                iDeviceCPU.extend(CPUdevices)
                            except:
                                pass
                    for GPUDevice in GPUdevices:
                        try:
                            tmpctx = cl.Context(devices=[GPUDevice])
                            if GPUDevice.double_fp_config > 0:
                                iDeviceGPU.extend([GPUDevice])
                        except:
                            pass
                    iDeviceAcc.extend(AccDevices)
                if _DEBUG > 10:
                    print("OpenCL: bulding {0} ...".format(self.cl_filename))
                    print("OpenCL: found {0} CPU{1}".format(
                          len(iDeviceCPU) if len(iDeviceCPU) > 0 else 'none',
                          's' if len(iDeviceCPU) > 1 else ''))
                    print("OpenCL: found {0} GPU{1}".format(
                          len(iDeviceGPU) if len(iDeviceGPU) > 0 else 'none',
                          's' if len(iDeviceGPU) > 1 else ''))
                    print("OpenCL: found {0} other accelerator{1}".format(
                          len(iDeviceAcc) if len(iDeviceAcc) > 0 else 'none',
                          's' if len(iDeviceAcc) > 1 else ''))

                if targetOpenCL.upper().startswith('GPU'):
                    iDevice.extend(iDeviceGPU)
                elif targetOpenCL.upper().startswith('CPU'):
                    iDevice.extend(iDeviceCPU)
                elif targetOpenCL.upper().startswith('ALL'):
                    iDevice.extend(iDeviceGPU)
                    iDevice.extend(iDeviceCPU)
                    iDevice.extend(iDeviceAcc)
                else:  # auto
                    if len(iDeviceGPU) > 0:
                        iDevice = iDeviceGPU
                    elif len(iDeviceAcc) > 0:
                        iDevice = iDeviceAcc
                    else:
                        iDevice = iDeviceCPU
                if len(iDevice) == 0:
                    targetOpenCL = None
                    self.lastTargetOpenCL = targetOpenCL
            else:  # None
                targetOpenCL = None
                self.lastTargetOpenCL = targetOpenCL
        if targetOpenCL is not None:
            if _DEBUG > 10:
                autoStr = "Autos" if isinstance(targetOpenCL, str) else "S"
                for idn, idv in enumerate(iDevice):
                    print("OpenCL for {0}: {1}elected device {2}: {3}".format(
                        self.cl_filename, autoStr, idn, idv.name))

            if self.kernelsource is None:
                cl_file = os.path.join(os.path.dirname(__file__),
                                       self.cl_filename)
                with open(cl_file, 'r') as f:
                    kernelsource = f.read()
            else:
                kernelsource = self.kernelsource
            if precisionOpenCL == 'auto':
                try:
                    for device in iDevice:
                        if device.double_fp_config == 63:
                            precisionOpenCL = 'float64'
                        else:
                            raise AttributeError
                except AttributeError:
                    precisionOpenCL = 'float32'
            if _DEBUG > 10:
                print('precisionOpenCL = {0}'.format(precisionOpenCL))
            if precisionOpenCL == 'float64':
                self.cl_precisionF = np.float64
                self.cl_precisionC = np.complex128
                kernelsource = kernelsource.replace('float', 'double')
            elif precisionOpenCL == 'float32':
                self.cl_precisionF = np.float32
                self.cl_precisionC = np.complex64
            else:
                raise ValueError("Unknown precision")
            self.cl_queue = []
            self.cl_ctx = []
            self.cl_program = []
            for device in iDevice:
                cl_ctx = cl.Context(devices=[device])
                self.cl_queue.extend([cl.CommandQueue(cl_ctx, device)])
                self.cl_program.extend(
                    [cl.Program(cl_ctx, kernelsource).build(
                        options=['-I', __dir__])])
                self.cl_ctx.extend([cl_ctx])

            self.cl_mf = cl.mem_flags

    def run_parallel(self, kernelName='', scalarArgs=None,
                     slicedROArgs=None, nonSlicedROArgs=None,
                     slicedRWArgs=None, nonSlicedRWArgs=None, dimension=0):

        ka_offset = len(scalarArgs) if scalarArgs is not None else 0
        ro_offset = len(slicedROArgs) if slicedROArgs is not None else 0
        ns_offset = len(nonSlicedROArgs) if nonSlicedROArgs is not None else 0
        rw_offset = len(slicedRWArgs) if slicedRWArgs is not None else 0
        rw_pos = ka_offset + ro_offset + ns_offset
        nsrw_pos = ka_offset + ro_offset + ns_offset + rw_offset

        kernel_bufs = []
        global_size = []
        ev_h2d = []
        ev_run = []
        nCU = []
        ndstart = []
        ndslice = []
        ndsize = []

        for ictx, ctx in enumerate(self.cl_ctx):
            nCUw = 1
            nCU.extend([ctx.devices[0].max_compute_units*nCUw])

        totalCUs = sum(nCU)
        work_cl_ctx = self.cl_ctx if dimension > totalCUs else [self.cl_ctx[0]]
        nctx = len(work_cl_ctx)

        for ictx, ctx in enumerate(work_cl_ctx):
            ev_h2d.extend([[]])
            kernel_bufs.extend([[]])
            if scalarArgs is not None:
                kernel_bufs[ictx].extend(scalarArgs)
            ndstart.extend([sum(ndsize)])

            if dimension > 1:
                if ictx < nctx - 1:
                    ndsize.extend([np.floor(dimension*nCU[ictx]/totalCUs)])
                else:
                    ndsize.extend([dimension-ndstart[ictx]])
                ndslice.extend([slice(ndstart[ictx],
                                      ndstart[ictx]+ndsize[ictx])])
            else:
                ndslice.extend([0])
# In case each photon has an array of input/output data we define a second
# dimension
            if slicedROArgs is not None and dimension > 1:
                for iarg, arg in enumerate(slicedROArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg[iSlice])])

            if nonSlicedROArgs is not None:
                for iarg, arg in enumerate(nonSlicedROArgs):
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_ONLY |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg)])

            if slicedRWArgs is not None:
                for iarg, arg in enumerate(slicedRWArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_WRITE |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg[iSlice])])
                global_size.extend([(np.int(ndsize[ictx]),)])
#                global_size.extend([slicedRWArgs[0][ndslice[ictx]].shape])
            if nonSlicedRWArgs is not None:
                for iarg, arg in enumerate(nonSlicedRWArgs):
                    kernel_bufs[ictx].extend([cl.Buffer(
                        self.cl_ctx[ictx], self.cl_mf.READ_WRITE |
                        self.cl_mf.COPY_HOST_PTR, hostbuf=arg)])
                global_size.extend([np.array([1]).shape])

        local_size = None

        for ictx, ctx in enumerate(work_cl_ctx):
            kernel = getattr(self.cl_program[ictx], kernelName)
            ev_run.extend([kernel(
                self.cl_queue[ictx],
                global_size[ictx],
                local_size,
                *kernel_bufs[ictx])])

        for iev, ev in enumerate(ev_run):
            status = cl.command_execution_status.to_string(
                ev.command_execution_status)
            if _DEBUG > 20:
                print("ctx status {0} {1}".format(iev, status))

        ret = ()

        if slicedRWArgs is not None:
            for ictx, ctx in enumerate(work_cl_ctx):
                for iarg, arg in enumerate(slicedRWArgs):
                    secondDim = np.int(len(arg) / dimension)
                    iSlice = slice(ndstart[ictx]*secondDim,
                                   (ndstart[ictx]+ndsize[ictx])*secondDim)
                    cl.enqueue_copy(self.cl_queue[ictx],
                                    slicedRWArgs[iarg][iSlice],
                                    kernel_bufs[ictx][iarg + rw_pos],
                                    is_blocking=self.cl_is_blocking)
            ret += tuple(slicedRWArgs)

        if nonSlicedRWArgs is not None:
            for ictx, ctx in enumerate(work_cl_ctx):
                for iarg, arg in enumerate(nonSlicedRWArgs):
                    cl.enqueue_copy(self.cl_queue[ictx],
                                    nonSlicedRWArgs[iarg],
                                    kernel_bufs[ictx][iarg + nsrw_pos],
                                    is_blocking=self.cl_is_blocking)
            ret += tuple(nonSlicedRWArgs)

        return ret
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import time
from rpython.rlib import streamio as sio
from rpython.rlib.rbigint import rbigint
from rpython.rlib.objectmodel import specialize
from rpython.rlib.rstring import ParseStringError, ParseStringOverflowError
from rpython.rlib.rarithmetic import string_to_int
from pycket import impersonators as imp
from pycket import values
from pycket.cont import continuation, loop_label, call_cont
from pycket import cont
from pycket import values_struct
from pycket import vector as values_vector
from pycket.error import SchemeException
from pycket.prims.expose import unsafe, default, expose, expose_val, procedure, make_call_method
from rpython.rlib import jit
from rpython.rlib.rsre import rsre_re as re

# import for side effects
from pycket.prims import continuation_marks
from pycket.prims import equal as eq_prims
from pycket.prims import hash
from pycket.prims import impersonator
from pycket.prims import numeric
from pycket.prims import random
from pycket.prims import string
from pycket.prims import undefined
from pycket.prims import vector

from rpython.rlib import jit

class Token(object):
    def __init__(self, v):
        self.val = v

class NumberToken(Token): pass
class StringToken(Token): pass
class SymbolToken(Token): pass
class BooleanToken(Token): pass
class LParenToken(Token): pass
class RParenToken(Token): pass
class LVecToken(Token): pass
class RVecToken(Token): pass

def read_number_or_id(f, init):
    sofar = init
    while True:
        (count, c) = f.peek()
        if c == "":
            break
        c = c[0]
        if c.isalnum():
            sofar = sofar + f.read(1)
        else:
            break
    try:
        return NumberToken(values.W_Fixnum.make(string_to_int(sofar)))
    except ParseStringOverflowError:
        val = rbigint.fromdecimalstr(sofar)
        return NumberToken(values.W_Bignum(val))
    except ParseStringError:
        try:
            return NumberToken(values.W_Flonum.make(float(sofar)))
        except:
            return SymbolToken(values.W_Symbol.make(sofar))

def read_token(f):
    while True:
        c = f.read(1) # FIXME: unicode
        if c == " " or c == "\n" or c == "\t":
            continue
        if c == "(" or c == "[" or c == "{":
            return LParenToken(c)
        if c == ")" or c == "]" or c == "}":
            return RParenToken(c)
        if c.isalnum():
            return read_number_or_id(f, c)
        if c == "#":
            c2 = f.read(1)
            if c2 == "t":
                return BooleanToken(values.w_true)
            if c2 == "f":
                return BooleanToken(values.w_false)
            if c2 == "(" or c2 == "[" or c2 == "{":
                return LVecToken(c2)
            raise SchemeException("bad token in read: %"%c2)
        raise SchemeException("bad token in read: %"%c)

@expose("read", [default(values.W_InputPort, None)])
def read(port):
    if port is None:
        stream = sio.fdopen_as_stream(0, "rb")
    else:
        stream = port.file
    token = read_token(stream)
    if isinstance(token, NumberToken):
        return token.val
    if isinstance(token, StringToken):
        return token.val
    if isinstance(token, SymbolToken):
        return token.val
    if isinstance(token, BooleanToken):
        return token.val
    return values.w_false # fail!

@expose("read-line", [default(values.W_InputPort, None)])
def read(port):
    if port is None:
        stream = sio.fdopen_as_stream(0, "rb")
    else:
        stream = port.file
    return values.W_String(stream.readline())

text_sym = values.W_Symbol.make("text")
binary_sym = values.W_Symbol.make("binary")
none_sym = values.W_Symbol.make("none")
error_sym = values.W_Symbol.make("error")

@expose("open-input-file", [values.W_String,
                            default(values.W_Symbol, binary_sym),
                            default(values.W_Symbol, none_sym)])
def open_input_file(str, mode, mod_mode):
    m = "r" if mode is text_sym else "rb"
    f = str.value
    return values.W_FileInputPort(sio.open_file_as_stream(f, mode=m))

@expose("open-output-file", [values.W_String,
                            default(values.W_Symbol, binary_sym),
                            default(values.W_Symbol, error_sym)])
def open_output_file(str):
    m = "w" if mode is text_sym else "wb"
    f = str.value
    return values.W_FileOutputPort(sio.open_file_as_stream(f, mode=m))
<commit_msg>Get input_output.py translating<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
from rpython.rlib             import streamio as sio
from rpython.rlib.rbigint     import rbigint
from rpython.rlib.rstring     import ParseStringError, ParseStringOverflowError
from rpython.rlib.rarithmetic import string_to_int
from pycket                   import values
from pycket.error             import SchemeException
from pycket.prims.expose      import default, expose, expose_val

class Token(object):
    def __init__(self, v):
        self.val = v

class NumberToken(Token): pass
class StringToken(Token): pass
class SymbolToken(Token): pass
class BooleanToken(Token): pass
class LParenToken(Token): pass
class RParenToken(Token): pass
class LVecToken(Token): pass
class RVecToken(Token): pass

def read_number_or_id(f, init):
    sofar = init
    while True:
        (count, c) = f.peek()
        if c == "":
            break
        c = c[0]
        if c.isalnum():
            sofar = sofar + f.read(1)
        else:
            break
    try:
        return NumberToken(values.W_Fixnum.make(string_to_int(sofar)))
    except ParseStringOverflowError:
        val = rbigint.fromdecimalstr(sofar)
        return NumberToken(values.W_Bignum(val))
    except ParseStringError:
        try:
            return NumberToken(values.W_Flonum.make(float(sofar)))
        except:
            return SymbolToken(values.W_Symbol.make(sofar))

def read_token(f):
    while True:
        c = f.read(1) # FIXME: unicode
        if c in [" ", "\n", "\t"]:
            continue
        if c in ["(", "[", "{"]:
            return LParenToken(values.W_String.make(c))
        if c in [")", "]", "}"]:
            return RParenToken(values.W_String.make(c))
        if c.isalnum():
            return read_number_or_id(f, c)
        if c == "#":
            c2 = f.read(1)
            if c2 == "t":
                return BooleanToken(values.w_true)
            if c2 == "f":
                return BooleanToken(values.w_false)
            if c2 in ["(", "[", "{"]:
                return LVecToken(values.W_String.make(c2))
            raise SchemeException("bad token in read: %s" % c2)
        raise SchemeException("bad token in read: %s" % c)

@expose("read", [default(values.W_InputPort, None)])
def read(port):
    if port is None:
        stream = sio.fdopen_as_stream(0, "rb")
    else:
        stream = port.file
    token = read_token(stream)
    if isinstance(token, NumberToken):
        return token.val
    if isinstance(token, StringToken):
        return token.val
    if isinstance(token, SymbolToken):
        return token.val
    if isinstance(token, BooleanToken):
        return token.val
    return values.w_false # fail!

@expose("read-line", [default(values.W_InputPort, None)])
def read(port):
    if port is None:
        stream = sio.fdopen_as_stream(0, "rb")
    else:
        stream = port.file
    return values.W_String(stream.readline())

text_sym   = values.W_Symbol.make("text")
binary_sym = values.W_Symbol.make("binary")
none_sym   = values.W_Symbol.make("none")
error_sym  = values.W_Symbol.make("error")

@expose("open-input-file", [values.W_String,
                            default(values.W_Symbol, binary_sym),
                            default(values.W_Symbol, none_sym)])
def open_input_file(str, mode, mod_mode):
    m = "r" if mode is text_sym else "rb"
    f = str.value
    return values.W_FileInputPort(sio.open_file_as_stream(f, mode=m))

@expose("open-output-file", [values.W_String,
                             default(values.W_Symbol, binary_sym),
                             default(values.W_Symbol, error_sym)])
def open_output_file(str, mode, exists):
    m = "w" if mode is text_sym else "wb"
    f = str.value
    return values.W_FileOutputPort(sio.open_file_as_stream(f, mode=m))

<|endoftext|>


<commit_before>import pandas as pd
import time

def records_to_df(records):
    records = pd.DataFrame.from_records(records)
    records['receive_ts'] = records['send_ts'] + records['interval_real'] / 1e6
    records['receive_sec'] = records.receive_ts.astype(int)
    # TODO: consider configuration for the following:
    records['tag'] = records.tag.str.rsplit('#', 1, expand=True)[0]
    records.set_index(['receive_sec'], inplace=True)
    return records

class BfgReader(object):
    def __init__(self, results):
        self.buffer = ""
        self.stat_buffer = ""
        self.results = results
        self.closed = False

    def next(self):
        if self.closed:
            raise StopIteration
        records = []
        while not self.results.empty():
            records.append(self.results.get(1))
        if records:
            return records_to_df(records)
        return None

    def __iter__(self):
        return self

    def close(self):
        self.closed = True

class BfgStatsReader(object):
    def __init__(self, instance_counter, steps):
        self.closed = False
        self.last_ts = 0
        self.steps = steps
        self.instance_counter = instance_counter
        self.start_time = int(time.time())

    def __iter__(self):
        while not self.closed:
            cur_ts = int(time.time())
            if cur_ts > self.last_ts:
                offset = cur_ts - 1 - self.start_time
                reqps = 0
                if offset >= 0 and offset < len(self.steps):
                    reqps = self.steps[offset][0]
                yield [{'ts': cur_ts,
                        'metrics': {'instances': self.instance_counter.get(),
                                    'reqps': reqps}}]
                self.last_ts = cur_ts
            else:
                yield []

    def close(self):
        self.closed = True
<commit_msg>fix rps stat<commit_after>import pandas as pd
import time

def records_to_df(records):
    records = pd.DataFrame.from_records(records)
    records['receive_ts'] = records['send_ts'] + records['interval_real'] / 1e6
    records['receive_sec'] = records.receive_ts.astype(int)
    # TODO: consider configuration for the following:
    records['tag'] = records.tag.str.rsplit('#', 1, expand=True)[0]
    records.set_index(['receive_sec'], inplace=True)
    return records

class BfgReader(object):
    def __init__(self, results):
        self.buffer = ""
        self.stat_buffer = ""
        self.results = results
        self.closed = False

    def next(self):
        if self.closed:
            raise StopIteration
        records = []
        while not self.results.empty():
            records.append(self.results.get(1))
        if records:
            return records_to_df(records)
        return None

    def __iter__(self):
        return self

    def close(self):
        self.closed = True

class BfgStatsReader(object):
    def __init__(self, instance_counter, steps):
        self.closed = False
        self.last_ts = 0
        self.steps = steps
        self.instance_counter = instance_counter
        self.start_time = int(time.time())

    def __iter__(self):
        while not self.closed:
            cur_ts = int(time.time())
            if cur_ts > self.last_ts:
                offset = cur_ts - self.start_time
                reqps = 0
                if offset >= 0 and offset < len(self.steps):
                    reqps = self.steps[offset][0]
                yield [{'ts': cur_ts,
                        'metrics': {'instances': self.instance_counter.get(),
                                    'reqps': reqps}}]
                self.last_ts = cur_ts
            else:
                yield []

    def close(self):
        self.closed = True
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from nose import with_setup

import os
import shutil
import numpy
import spectral
from pycoal import mineral
import urllib2 as urllib

# test files for filterClasses test
test_filterClasses_Filename = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80.hdr'
test_filterClasses_Image = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80.img'
test_filterClasses_testFilename = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80_filtered.hdr'
test_filterClasses_testImage = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80_filtered.img'

# set up filterClasses test by copying classified image
def _test_filterClasses_setup():
    shutil.copyfile(test_filterClasses_Filename, test_filterClasses_testFilename)
    shutil.copyfile(test_filterClasses_Image, test_filterClasses_testImage)

# tear down filterClasses test by deleting filtered image
def _test_filterClasses_teardown():
    try:
        os.remove(test_filterClasses_testFilename)
        os.remove(test_filterClasses_testImage)
    except OSError:
        pass

# verify that filterClasses removes unused classes and reindexes correctly
@with_setup(_test_filterClasses_setup, _test_filterClasses_teardown)
def test_filterClasses():
    mineral.MineralClassification.filterClasses(test_filterClasses_testFilename)
    original = spectral.open_image(test_filterClasses_Filename)
    filtered = spectral.open_image(test_filterClasses_testFilename)
    assert int(filtered.metadata.get('classes')) == len(set(original.asarray().flatten().tolist()))
    for x in range(original.shape[0]):
        for y in range(original.shape[1]):
            originalClassName = original.metadata.get('class names')[original[x,y,0]]
            filteredClassName = filtered.metadata.get('class names')[filtered[x,y,0]]
            assert originalClassName == filteredClassName

# filename of 1x1 subimage
test_toRGB_imageFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654.hdr'

# filename of 1x1 RGB subimage
test_toRGB_rgbFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb.hdr'

# filenames of temporary files
test_toRGB_testFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb_test.hdr'
test_toRGB_testImage = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb_test.img'

# delete temporary files
def _test_toRGB_teardown():
    try:
        os.remove(test_toRGB_testFilename)
        os.remove(test_toRGB_testImage)
    except OSError:
        pass

# verify that the RGB converter selects the correct bands and updates the metadata
@with_setup(None, _test_toRGB_teardown)
def test_toRGB():
    mineral.MineralClassification.toRGB(test_toRGB_imageFilename, test_toRGB_testFilename)
    expected = spectral.open_image(test_toRGB_rgbFilename)
    actual = spectral.open_image(test_toRGB_testFilename)
    assert numpy.array_equal(expected.asarray(), actual.asarray())
    assert expected.metadata.get('wavelength') == actual.metadata.get('wavelength')
    assert expected.metadata.get('correction factors') == actual.metadata.get('correction factors')
    assert expected.metadata.get('fwhm') == actual.metadata.get('fwhm')
    assert expected.metadata.get('bbl') == actual.metadata.get('bbl')
    assert expected.metadata.get('smoothing factors') == actual.metadata.get('smoothing factors')

# filename of 1x1 subimage
test_toRGB_AVC_imageFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_123_456.hdr'

# filename of 1x1 RGB subimage
test_toRGB_AVC_rgbFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_123_456_rgb.hdr'

# filenames of temporary files
test_toRGB_AVC_testFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_rgb_test.hdr'
test_toRGB_AVC_testImage = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_rgb_test.img'

# delete temporary files
def _test_toRGB_AVC_teardown():
    try:
        os.remove(test_toRGB_AVC_testFilename)
        os.remove(test_toRGB_AVC_testImage)
    except OSError:
        pass

# verify that the RGB converter selects the correct bands and updates the metadata
@with_setup(None, _test_toRGB_AVC_teardown)
def test_toRGB_AVC():
    mineral.MineralClassification.toRGB(test_toRGB_AVC_imageFilename, test_toRGB_AVC_testFilename)
    expected = spectral.open_image(test_toRGB_AVC_rgbFilename)
    actual = spectral.open_image(test_toRGB_AVC_testFilename)
    assert numpy.array_equal(expected.asarray(), actual.asarray())
    assert expected.metadata.get('wavelength') == actual.metadata.get('wavelength')
    assert expected.metadata.get('correction factors') == actual.metadata.get('correction factors')
    assert expected.metadata.get('fwhm') == actual.metadata.get('fwhm')
    assert expected.metadata.get('bbl') == actual.metadata.get('bbl')
    assert expected.metadata.get('smoothing factors') == actual.metadata.get('smoothing factors')

# test files for the classifyImage test
#TODO use an AVIRIS-NG image for two test cases
test_classifyImage_testFilename_1 = "pycoal/tests/ang20140912t192359_corr_v1c_img_400-410_10-20.hdr"
test_classifyImage_testFilename_2 = "pycoal/tests/ang20140912t192359_corr_v1c_img_2580-2590_540-550.hdr"

library_files = ["ftp://ftpext.cr.usgs.gov/pub/cr/co/denver/speclab/pub/spectral.library/splib06.library/Convolved.libraries/s06av95a_envi.hdr",
                 "ftp://ftpext.cr.usgs.gov/pub/cr/co/denver/speclab/pub/spectral.library/splib06.library/Convolved.libraries/s06av95a_envi.sli"]

# delete temporary files
def _test_classifyImage_teardown():
    files = [test_classifyImage_testFilename_1[:-4] + "_class.hdr",
             test_classifyImage_testFilename_1[:-4] + "_class.img",
             test_classifyImage_testFilename_2[:-4] + "_class.hdr",
             test_classifyImage_testFilename_2[:-4] + "_class.img",
             library_files[0][-17:], library_files[1][-17:]]

    for f in files:
        try:
            os.remove(f)
        except OSError:
            pass

# verify that classified images have valid classifications
@with_setup(None, _test_classifyImage_teardown)
def test_classifyImage():
    for f in library_files:
        url = urllib.urlopen(f)

        with open(f[-17:], "wb") as lib_f:
            lib_f.write(url.read())

    lib = "s06av95a_envi.hdr"

    tst_cls = mineral.MineralClassification(lib)

    # make sure library is being opened as such
    assert isinstance(tst_cls.library, spectral.io.envi.SpectralLibrary)

    for f in (test_classifyImage_testFilename_1, test_classifyImage_testFilename_2):
        tst_cls.classifyImage(f, f[:-4] + "_class.hdr")

        classified = spectral.open_image(f[:-4] + "_class.hdr")

        # assert there are no invalid class numbers
        for i in classified.asarray().flatten():
            assert 0 <= i <= len(tst_cls.library.names) + 1
<commit_msg>Add FTP implementation.<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from nose import with_setup

import os
import shutil
import numpy
import spectral
from pycoal import mineral
import ftplib
import urlparse

# test files for filterClasses test
test_filterClasses_Filename = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80.hdr'
test_filterClasses_Image = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80.img'
test_filterClasses_testFilename = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80_filtered.hdr'
test_filterClasses_testImage = 'pycoal/tests/ang20150420t182808_corr_v1e_img_class_4200-4210_70-80_filtered.img'

# set up filterClasses test by copying classified image
def _test_filterClasses_setup():
    shutil.copyfile(test_filterClasses_Filename, test_filterClasses_testFilename)
    shutil.copyfile(test_filterClasses_Image, test_filterClasses_testImage)

# tear down filterClasses test by deleting filtered image
def _test_filterClasses_teardown():
    try:
        os.remove(test_filterClasses_testFilename)
        os.remove(test_filterClasses_testImage)
    except OSError:
        pass

# verify that filterClasses removes unused classes and reindexes correctly
@with_setup(_test_filterClasses_setup, _test_filterClasses_teardown)
def test_filterClasses():
    mineral.MineralClassification.filterClasses(test_filterClasses_testFilename)
    original = spectral.open_image(test_filterClasses_Filename)
    filtered = spectral.open_image(test_filterClasses_testFilename)
    assert int(filtered.metadata.get('classes')) == len(set(original.asarray().flatten().tolist()))
    for x in range(original.shape[0]):
        for y in range(original.shape[1]):
            originalClassName = original.metadata.get('class names')[original[x,y,0]]
            filteredClassName = filtered.metadata.get('class names')[filtered[x,y,0]]
            assert originalClassName == filteredClassName

# filename of 1x1 subimage
test_toRGB_imageFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654.hdr'

# filename of 1x1 RGB subimage
test_toRGB_rgbFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb.hdr'

# filenames of temporary files
test_toRGB_testFilename = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb_test.hdr'
test_toRGB_testImage = 'pycoal/tests/ang20150422t163638_corr_v1e_img_987_654_rgb_test.img'

# delete temporary files
def _test_toRGB_teardown():
    try:
        os.remove(test_toRGB_testFilename)
        os.remove(test_toRGB_testImage)
    except OSError:
        pass

# verify that the RGB converter selects the correct bands and updates the metadata
@with_setup(None, _test_toRGB_teardown)
def test_toRGB():
    mineral.MineralClassification.toRGB(test_toRGB_imageFilename, test_toRGB_testFilename)
    expected = spectral.open_image(test_toRGB_rgbFilename)
    actual = spectral.open_image(test_toRGB_testFilename)
    assert numpy.array_equal(expected.asarray(), actual.asarray())
    assert expected.metadata.get('wavelength') == actual.metadata.get('wavelength')
    assert expected.metadata.get('correction factors') == actual.metadata.get('correction factors')
    assert expected.metadata.get('fwhm') == actual.metadata.get('fwhm')
    assert expected.metadata.get('bbl') == actual.metadata.get('bbl')
    assert expected.metadata.get('smoothing factors') == actual.metadata.get('smoothing factors')

# filename of 1x1 subimage
test_toRGB_AVC_imageFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_123_456.hdr'

# filename of 1x1 RGB subimage
test_toRGB_AVC_rgbFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_123_456_rgb.hdr'

# filenames of temporary files
test_toRGB_AVC_testFilename = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_rgb_test.hdr'
test_toRGB_AVC_testImage = 'pycoal/tests/f080702t01p00r08rdn_c_sc01_ort_img_rgb_test.img'

# delete temporary files
def _test_toRGB_AVC_teardown():
    try:
        os.remove(test_toRGB_AVC_testFilename)
        os.remove(test_toRGB_AVC_testImage)
    except OSError:
        pass

# verify that the RGB converter selects the correct bands and updates the metadata
@with_setup(None, _test_toRGB_AVC_teardown)
def test_toRGB_AVC():
    mineral.MineralClassification.toRGB(test_toRGB_AVC_imageFilename, test_toRGB_AVC_testFilename)
    expected = spectral.open_image(test_toRGB_AVC_rgbFilename)
    actual = spectral.open_image(test_toRGB_AVC_testFilename)
    assert numpy.array_equal(expected.asarray(), actual.asarray())
    assert expected.metadata.get('wavelength') == actual.metadata.get('wavelength')
    assert expected.metadata.get('correction factors') == actual.metadata.get('correction factors')
    assert expected.metadata.get('fwhm') == actual.metadata.get('fwhm')
    assert expected.metadata.get('bbl') == actual.metadata.get('bbl')
    assert expected.metadata.get('smoothing factors') == actual.metadata.get('smoothing factors')

# test files for the classifyImage test
#TODO use an AVIRIS-NG image for two test cases
test_classifyImage_testFilename_1 = "pycoal/tests/ang20140912t192359_corr_v1c_img_400-410_10-20.hdr"
test_classifyImage_testFilename_2 = "pycoal/tests/ang20140912t192359_corr_v1c_img_2580-2590_540-550.hdr"

ftp_url = "ftp://ftpext.cr.usgs.gov/"
library_files = ["s06av95a_envi.hdr", "s06av95a_envi.sli"]

# delete temporary files
def _test_classifyImage_teardown():
    files = [test_classifyImage_testFilename_1[:-4] + "_class.hdr",
             test_classifyImage_testFilename_1[:-4] + "_class.img",
             test_classifyImage_testFilename_2[:-4] + "_class.hdr",
             test_classifyImage_testFilename_2[:-4] + "_class.img",
             "pycoal/tests/" + library_files[0],
             "pycoal/tests/" + library_files[1]]

    for f in files:
        try:
            os.remove(f)
        except OSError:
            pass

# verify that classified images have valid classifications
@with_setup(None, _test_classifyImage_teardown)
def test_classifyImage():
    url = urlparse.urlparse(ftp_url)
    ftp = ftplib.FTP(url.netloc)
    ftp.login()

    ftp.cwd("pub/cr/co/denver/speclab/pub/spectral.library/splib06.library/Convolved.libraries/")

    for f in library_files:
        print f
        with open("pycoal/tests/" + f, "wb") as lib_f:
            ftp.retrbinary('RETR %s' % f, lib_f.write)

    lib = "pycoal/tests/s06av95a_envi.hdr"

    tst_cls = mineral.MineralClassification(lib)

    # make sure library is being opened as such
    assert isinstance(tst_cls.library, spectral.io.envi.SpectralLibrary)

    for f in (test_classifyImage_testFilename_1, test_classifyImage_testFilename_2):
        tst_cls.classifyImage(f, f[:-4] + "_class.hdr")

        classified = spectral.open_image(f[:-4] + "_class.hdr")

        # assert there are no invalid class numbers
        for i in classified.asarray().flatten():
            assert 0 <= i <= len(tst_cls.library.names) + 1
<|endoftext|>