<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import functools
import inspect
import pickle

from ..decorators import (deprecated_attribute, deprecated, wraps,
                          sharedmethod, classproperty, add_docstring)
from ..exceptions import AstropyDeprecationWarning
from ...extern import six
from ...tests.helper import pytest, catch_warnings

def test_wraps():
    """
    Tests the compatibility replacement for functools.wraps which supports
    argument preservation across all supported Python versions.
    """

    def foo(a, b, c=1, d=2, e=3, **kwargs):
        """A test function."""

        return a, b, c, d, e, kwargs

    @wraps(foo)
    def bar(*args, **kwargs):
        return ('test',) + foo(*args, **kwargs)

    expected = ('test', 1, 2, 3, 4, 5, {'f': 6, 'g': 7})
    assert bar(1, 2, 3, 4, 5, f=6, g=7) == expected
    assert bar.__name__ == 'foo'

    if foo.__doc__ is not None:
        # May happen if using optimized opcode
        assert bar.__doc__ == "A test function."

    if hasattr(foo, '__qualname__'):
        assert bar.__qualname__ == foo.__qualname__

    if six.PY2:
        argspec = inspect.getargspec(bar)
        assert argspec.keywords == 'kwargs'
    else:
        argspec = inspect.getfullargspec(bar)
        assert argspec.varkw == 'kwargs'

    assert argspec.args == ['a', 'b', 'c', 'd', 'e']
    assert argspec.defaults == (1, 2, 3)

def test_wraps_exclude_names():
    """
    Test the optional ``exclude_names`` argument to the wraps decorator.
    """

    # This particular test demonstrates wrapping an instance method
    # as a function and excluding the "self" argument:

    class TestClass(object):
        def method(self, a, b, c=1, d=2, **kwargs):
            return (a, b, c, d, kwargs)

    test = TestClass()

    @wraps(test.method, exclude_args=('self',))
    def func(*args, **kwargs):
        return test.method(*args, **kwargs)

    if six.PY2:
        argspec = inspect.getargspec(func)
    else:
        argspec = inspect.getfullargspec(func)
    assert argspec.args == ['a', 'b', 'c', 'd']

    assert func('a', 'b', e=3) == ('a', 'b', 1, 2, {'e': 3})

def test_wraps_keep_orig_name():
    """
    Test that when __name__ is excluded from the ``assigned`` argument
    to ``wrap`` that the function being wrapped keeps its original name.

    Regression test for https://github.com/astropy/astropy/pull/4016
    """

    def foo():
        pass

    assigned = list(functools.WRAPPER_ASSIGNMENTS)
    assigned.remove('__name__')

    def bar():
        pass

    orig_bar = bar

    bar = wraps(foo, assigned=assigned)(bar)

    assert bar is not orig_bar
    assert bar.__name__ == 'bar'

def test_deprecated_attribute():
    class DummyClass:
        def __init__(self):
            self._foo = 42

        def set_private(self):
            self._foo = 100

        foo = deprecated_attribute('foo', '0.2')

    dummy = DummyClass()

    with catch_warnings(AstropyDeprecationWarning) as w:
        x = dummy.foo

    assert len(w) == 1
    assert str(w[0].message) == ("The foo attribute is deprecated and may be "
                                 "removed in a future version.")

    with catch_warnings() as w:
        dummy.set_private()

    assert len(w) == 0

# This needs to be defined outside of the test function, because we
# want to try to pickle it.
@deprecated('100.0')
class TA(object):
    """
    This is the class docstring.
    """
    def __init__(self):
        """
        This is the __init__ docstring
        """
        pass

class TMeta(type):
    metaclass_attr = 1

@deprecated('100.0')
@six.add_metaclass(TMeta)
class TB(object):
    pass

def test_deprecated_class():
    orig_A = TA.__bases__[0]

    # The only thing that should be different about the new class
    # is __doc__, __init__, __bases__ and __subclasshook__.
    for x in dir(orig_A):
        if x not in ('__doc__', '__init__', '__bases__', '__dict__',
                     '__subclasshook__'):
            assert getattr(TA, x) == getattr(orig_A, x)

    with catch_warnings(AstropyDeprecationWarning) as w:
        TA()

    assert len(w) == 1
    if TA.__doc__ is not None:
        assert 'function' not in TA.__doc__
        assert 'deprecated' in TA.__doc__
        assert 'function' not in TA.__init__.__doc__
        assert 'deprecated' in TA.__init__.__doc__

    # Make sure the object is picklable
    pickle.dumps(TA)

def test_deprecated_class_with_super():
    """
    Regression test for an issue where classes that used `super()` in their
    ``__init__`` did not actually call the correct class's ``__init__`` in the
    MRO.
    """

    @deprecated('100.0')
    class TB(object):
        def __init__(self, a, b):
            super(TB, self).__init__()

    with catch_warnings(AstropyDeprecationWarning) as w:
        TB(1, 2)

    assert len(w) == 1
    if TB.__doc__ is not None:
        assert 'function' not in TB.__doc__
        assert 'deprecated' in TB.__doc__
        assert 'function' not in TB.__init__.__doc__
        assert 'deprecated' in TB.__init__.__doc__

def test_deprecated_class_with_custom_metaclass():
    """
    Regression test for an issue where deprecating a class with a metaclass
    other than type did not restore the metaclass properly (at least not
    on Python 3).
    """

    with catch_warnings(AstropyDeprecationWarning) as w:
        TB()

    assert len(w) == 1
    assert type(TB) is TMeta
    assert TB.metaclass_attr == 1

def test_deprecated_static_and_classmethod():
    """
    Regression test for issue introduced by
    https://github.com/astropy/astropy/pull/2811 and mentioned also here:
    https://github.com/astropy/astropy/pull/2580#issuecomment-51049969
    where it appears that deprecated staticmethods didn't work on Python 2.6.
    """

    class A(object):
        """Docstring"""

        @deprecated('1.0')
        @staticmethod
        def B():
            pass

        @deprecated('1.0')
        @classmethod
        def C(cls):
            pass

    with catch_warnings(AstropyDeprecationWarning) as w:
        A.B()

    assert len(w) == 1
    if A.__doc__ is not None:
        assert 'deprecated' in A.B.__doc__

    with catch_warnings(AstropyDeprecationWarning) as w:
        A.C()

    assert len(w) == 1
    if A.__doc__ is not None:
        assert 'deprecated' in A.C.__doc__

@pytest.mark.skipif('six.PY3')
def test_sharedmethod_imfunc():
    """
    Test that the im_func of a sharedmethod always points to the correct
    underlying function.

    This only applies to Python 2 as Python 3 does not have an im_func
    attribute on methods.
    """

    # The original function
    def foo(self): pass
    actual_foo = foo

    class Bar(object):
        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo
    assert Bar().foo.im_func is actual_foo

    # Now test the case where there the metaclass has a separate
    # implementation
    def foo(cls): pass
    actual_foo_2 = foo

    class MetaBar(type):
        foo = actual_foo_2

    class Bar(object):
        __metaclass__ = MetaBar

        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo_2
    assert Bar().foo.im_func is actual_foo

    # Finally, test case where the metaclass also has an attribute called
    # 'foo', but it is not a method (hence sharedmethod should ignore it)
    class MetaBar(type):
        foo = None

    class Bar(object):
        __metaclass__ = MetaBar

        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo
    assert Bar().foo.im_func is actual_foo

def test_sharedmethod_reuse_on_subclasses():
    """
    Regression test for an issue where sharedmethod would bind to one class
    for all time, causing the same method not to work properly on other
    subclasses of that class.

    It has the same problem when the same sharedmethod is called on different
    instances of some class as well.
    """

    class AMeta(type):
        def foo(cls):
            return cls.x

    six.add_metaclass(AMeta)
    class A(object):
        x = 3

        def __init__(self, x):
            self.x = x

        @sharedmethod
        def foo(self):
            return self.x

    a1 = A(1)
    a2 = A(2)

    assert a1.foo() == 1
    assert a2.foo() == 2

    # Similar test now, but for multiple subclasses using the same sharedmethod
    # as a classmethod
    assert A.foo() == 3

    class B(A):
        x = 5

    assert B.foo() == 5

def test_classproperty_docstring():
    """
    Tests that the docstring is set correctly on classproperties.

    This failed previously due to a bug in Python that didn't always
    set __doc__ properly on instances of property subclasses.
    """

    class A(object):
        # Inherits docstring from getter
        @classproperty
        def foo(cls):
            """The foo."""

            return 1

    assert A.__dict__['foo'].__doc__ == "The foo."

    class B(object):
        # Use doc passed to classproperty constructor
        def _get_foo(cls): return 1

        foo = classproperty(_get_foo, doc="The foo.")

    assert B.__dict__['foo'].__doc__ == "The foo."

def test_add_docstring_function():
    docstring = 'test'

    @add_docstring(docstring)
    def testfunc():
        pass

    @add_docstring(docstring, replace=False)
    def testfunc2():
        """this is a """
        pass

    assert inspect.getdoc(testfunc) == docstring
    assert inspect.getdoc(testfunc2) == 'this is a test'

def test_add_docstring_method():
    docstring = 'test'

    class TestClass(object):
        @add_docstring(docstring)
        def testfunc():
            pass

        @add_docstring(docstring, replace=False)
        def testfunc2():
            """this is a """
            pass

    assert inspect.getdoc(TestClass.testfunc) == docstring
    assert inspect.getdoc(TestClass.testfunc2) == 'this is a test'
    instance = TestClass()
    assert inspect.getdoc(instance.testfunc) == docstring
    assert inspect.getdoc(instance.testfunc2) == 'this is a test'

@pytest.mark.skipif('six.PY2')
def test_add_docstring_class():
    docstring = 'test'

    @add_docstring(docstring)
    class TestClass(object):
        pass

    @add_docstring(docstring, replace=False)
    class TestClass2(object):
        """this is a """
        pass

    assert inspect.getdoc(TestClass) == docstring
    assert inspect.getdoc(TestClass2) == 'this is a test'
    instance = TestClass()
    instance2 = TestClass2()
    assert inspect.getdoc(instance) == docstring
    assert inspect.getdoc(instance2) == 'this is a test'
<commit_msg>Added Tests where docstring is really replaced<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import functools
import inspect
import pickle

from ..decorators import (deprecated_attribute, deprecated, wraps,
                          sharedmethod, classproperty, add_docstring)
from ..exceptions import AstropyDeprecationWarning
from ...extern import six
from ...tests.helper import pytest, catch_warnings

def test_wraps():
    """
    Tests the compatibility replacement for functools.wraps which supports
    argument preservation across all supported Python versions.
    """

    def foo(a, b, c=1, d=2, e=3, **kwargs):
        """A test function."""

        return a, b, c, d, e, kwargs

    @wraps(foo)
    def bar(*args, **kwargs):
        return ('test',) + foo(*args, **kwargs)

    expected = ('test', 1, 2, 3, 4, 5, {'f': 6, 'g': 7})
    assert bar(1, 2, 3, 4, 5, f=6, g=7) == expected
    assert bar.__name__ == 'foo'

    if foo.__doc__ is not None:
        # May happen if using optimized opcode
        assert bar.__doc__ == "A test function."

    if hasattr(foo, '__qualname__'):
        assert bar.__qualname__ == foo.__qualname__

    if six.PY2:
        argspec = inspect.getargspec(bar)
        assert argspec.keywords == 'kwargs'
    else:
        argspec = inspect.getfullargspec(bar)
        assert argspec.varkw == 'kwargs'

    assert argspec.args == ['a', 'b', 'c', 'd', 'e']
    assert argspec.defaults == (1, 2, 3)

def test_wraps_exclude_names():
    """
    Test the optional ``exclude_names`` argument to the wraps decorator.
    """

    # This particular test demonstrates wrapping an instance method
    # as a function and excluding the "self" argument:

    class TestClass(object):
        def method(self, a, b, c=1, d=2, **kwargs):
            return (a, b, c, d, kwargs)

    test = TestClass()

    @wraps(test.method, exclude_args=('self',))
    def func(*args, **kwargs):
        return test.method(*args, **kwargs)

    if six.PY2:
        argspec = inspect.getargspec(func)
    else:
        argspec = inspect.getfullargspec(func)
    assert argspec.args == ['a', 'b', 'c', 'd']

    assert func('a', 'b', e=3) == ('a', 'b', 1, 2, {'e': 3})

def test_wraps_keep_orig_name():
    """
    Test that when __name__ is excluded from the ``assigned`` argument
    to ``wrap`` that the function being wrapped keeps its original name.

    Regression test for https://github.com/astropy/astropy/pull/4016
    """

    def foo():
        pass

    assigned = list(functools.WRAPPER_ASSIGNMENTS)
    assigned.remove('__name__')

    def bar():
        pass

    orig_bar = bar

    bar = wraps(foo, assigned=assigned)(bar)

    assert bar is not orig_bar
    assert bar.__name__ == 'bar'

def test_deprecated_attribute():
    class DummyClass:
        def __init__(self):
            self._foo = 42

        def set_private(self):
            self._foo = 100

        foo = deprecated_attribute('foo', '0.2')

    dummy = DummyClass()

    with catch_warnings(AstropyDeprecationWarning) as w:
        x = dummy.foo

    assert len(w) == 1
    assert str(w[0].message) == ("The foo attribute is deprecated and may be "
                                 "removed in a future version.")

    with catch_warnings() as w:
        dummy.set_private()

    assert len(w) == 0

# This needs to be defined outside of the test function, because we
# want to try to pickle it.
@deprecated('100.0')
class TA(object):
    """
    This is the class docstring.
    """
    def __init__(self):
        """
        This is the __init__ docstring
        """
        pass

class TMeta(type):
    metaclass_attr = 1

@deprecated('100.0')
@six.add_metaclass(TMeta)
class TB(object):
    pass

def test_deprecated_class():
    orig_A = TA.__bases__[0]

    # The only thing that should be different about the new class
    # is __doc__, __init__, __bases__ and __subclasshook__.
    for x in dir(orig_A):
        if x not in ('__doc__', '__init__', '__bases__', '__dict__',
                     '__subclasshook__'):
            assert getattr(TA, x) == getattr(orig_A, x)

    with catch_warnings(AstropyDeprecationWarning) as w:
        TA()

    assert len(w) == 1
    if TA.__doc__ is not None:
        assert 'function' not in TA.__doc__
        assert 'deprecated' in TA.__doc__
        assert 'function' not in TA.__init__.__doc__
        assert 'deprecated' in TA.__init__.__doc__

    # Make sure the object is picklable
    pickle.dumps(TA)

def test_deprecated_class_with_super():
    """
    Regression test for an issue where classes that used `super()` in their
    ``__init__`` did not actually call the correct class's ``__init__`` in the
    MRO.
    """

    @deprecated('100.0')
    class TB(object):
        def __init__(self, a, b):
            super(TB, self).__init__()

    with catch_warnings(AstropyDeprecationWarning) as w:
        TB(1, 2)

    assert len(w) == 1
    if TB.__doc__ is not None:
        assert 'function' not in TB.__doc__
        assert 'deprecated' in TB.__doc__
        assert 'function' not in TB.__init__.__doc__
        assert 'deprecated' in TB.__init__.__doc__

def test_deprecated_class_with_custom_metaclass():
    """
    Regression test for an issue where deprecating a class with a metaclass
    other than type did not restore the metaclass properly (at least not
    on Python 3).
    """

    with catch_warnings(AstropyDeprecationWarning) as w:
        TB()

    assert len(w) == 1
    assert type(TB) is TMeta
    assert TB.metaclass_attr == 1

def test_deprecated_static_and_classmethod():
    """
    Regression test for issue introduced by
    https://github.com/astropy/astropy/pull/2811 and mentioned also here:
    https://github.com/astropy/astropy/pull/2580#issuecomment-51049969
    where it appears that deprecated staticmethods didn't work on Python 2.6.
    """

    class A(object):
        """Docstring"""

        @deprecated('1.0')
        @staticmethod
        def B():
            pass

        @deprecated('1.0')
        @classmethod
        def C(cls):
            pass

    with catch_warnings(AstropyDeprecationWarning) as w:
        A.B()

    assert len(w) == 1
    if A.__doc__ is not None:
        assert 'deprecated' in A.B.__doc__

    with catch_warnings(AstropyDeprecationWarning) as w:
        A.C()

    assert len(w) == 1
    if A.__doc__ is not None:
        assert 'deprecated' in A.C.__doc__

@pytest.mark.skipif('six.PY3')
def test_sharedmethod_imfunc():
    """
    Test that the im_func of a sharedmethod always points to the correct
    underlying function.

    This only applies to Python 2 as Python 3 does not have an im_func
    attribute on methods.
    """

    # The original function
    def foo(self): pass
    actual_foo = foo

    class Bar(object):
        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo
    assert Bar().foo.im_func is actual_foo

    # Now test the case where there the metaclass has a separate
    # implementation
    def foo(cls): pass
    actual_foo_2 = foo

    class MetaBar(type):
        foo = actual_foo_2

    class Bar(object):
        __metaclass__ = MetaBar

        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo_2
    assert Bar().foo.im_func is actual_foo

    # Finally, test case where the metaclass also has an attribute called
    # 'foo', but it is not a method (hence sharedmethod should ignore it)
    class MetaBar(type):
        foo = None

    class Bar(object):
        __metaclass__ = MetaBar

        foo = sharedmethod(actual_foo)

    assert Bar.foo.im_func is actual_foo
    assert Bar().foo.im_func is actual_foo

def test_sharedmethod_reuse_on_subclasses():
    """
    Regression test for an issue where sharedmethod would bind to one class
    for all time, causing the same method not to work properly on other
    subclasses of that class.

    It has the same problem when the same sharedmethod is called on different
    instances of some class as well.
    """

    class AMeta(type):
        def foo(cls):
            return cls.x

    six.add_metaclass(AMeta)
    class A(object):
        x = 3

        def __init__(self, x):
            self.x = x

        @sharedmethod
        def foo(self):
            return self.x

    a1 = A(1)
    a2 = A(2)

    assert a1.foo() == 1
    assert a2.foo() == 2

    # Similar test now, but for multiple subclasses using the same sharedmethod
    # as a classmethod
    assert A.foo() == 3

    class B(A):
        x = 5

    assert B.foo() == 5

def test_classproperty_docstring():
    """
    Tests that the docstring is set correctly on classproperties.

    This failed previously due to a bug in Python that didn't always
    set __doc__ properly on instances of property subclasses.
    """

    class A(object):
        # Inherits docstring from getter
        @classproperty
        def foo(cls):
            """The foo."""

            return 1

    assert A.__dict__['foo'].__doc__ == "The foo."

    class B(object):
        # Use doc passed to classproperty constructor
        def _get_foo(cls): return 1

        foo = classproperty(_get_foo, doc="The foo.")

    assert B.__dict__['foo'].__doc__ == "The foo."

def test_add_docstring_function():
    docstring = 'test'

    @add_docstring(docstring)
    def testfunc1():
        pass

    @add_docstring(docstring)
    def testfunc2():
        """this is a """
        pass

    @add_docstring(docstring, replace=False)
    def testfunc3():
        """this is a """
        pass

    assert inspect.getdoc(testfunc1) == docstring
    assert inspect.getdoc(testfunc2) == docstring
    assert inspect.getdoc(testfunc3) == 'this is a test'

def test_add_docstring_method():
    docstring = 'test'

    class TestClass(object):
        @add_docstring(docstring)
        def testfunc1():
            pass

        @add_docstring(docstring)
        def testfunc2():
            """this is a """
            pass

        @add_docstring(docstring, replace=False)
        def testfunc3():
            """this is a """
            pass

    assert inspect.getdoc(TestClass.testfunc1) == docstring
    assert inspect.getdoc(TestClass.testfunc2) == docstring
    assert inspect.getdoc(TestClass.testfunc3) == 'this is a test'
    instance = TestClass()
    assert inspect.getdoc(instance.testfunc1) == docstring
    assert inspect.getdoc(instance.testfunc2) == docstring
    assert inspect.getdoc(instance.testfunc3) == 'this is a test'

@pytest.mark.skipif('six.PY2')
def test_add_docstring_class():
    docstring = 'test'

    @add_docstring(docstring)
    class TestClass1(object):
        pass

    @add_docstring(docstring)
    class TestClass2(object):
        """this is a """
        pass

    @add_docstring(docstring, replace=False)
    class TestClass3(object):
        """this is a """
        pass

    assert inspect.getdoc(TestClass1) == docstring
    assert inspect.getdoc(TestClass2) == docstring
    assert inspect.getdoc(TestClass3) == 'this is a test'
    instance1 = TestClass1()
    instance2 = TestClass2()
    instance3 = TestClass3()
    assert inspect.getdoc(instance1) == docstring
    assert inspect.getdoc(instance2) == docstring
    assert inspect.getdoc(instance3) == 'this is a test'
<|endoftext|>


<commit_before>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.

Convert a help markdown file a man page and store it to /help.1 in the image
so that 'atomic help' could display it.
This is accomplished by appending an ADD command to it.

Example configuration:
{
    'name': 'add_help',
    'args': {'help_file': 'help.md'}
}
"""

import os
from subprocess import check_output, CalledProcessError, STDOUT
from atomic_reactor.plugin import PreBuildPlugin
from atomic_reactor.util import df_parser

DEFAULT_HELP_FILENAME = "help.md"

class AddHelpPlugin(PreBuildPlugin):
    key = "add_help"
    man_filename = "help.1"

    def __init__(self, tasker, workflow, help_file=DEFAULT_HELP_FILENAME):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        :param help_file: filename of the markdown help file
        """
        # call parent constructor
        super(AddHelpPlugin, self).__init__(tasker, workflow)
        self.help_file = help_file

    def run(self):
        """
        run the plugin
        """

        help_path = os.path.join(self.workflow.builder.df_dir, self.help_file)

        if not os.path.exists(help_path):
            self.log.info("File %s not found", help_path)
            return

        man_path = os.path.join(self.workflow.builder.df_dir, self.man_filename)

        go_md2man_cmd = ['go-md2man', '-in={}'.format(help_path), '-out={}'.format(man_path)]

        try:
            check_output(go_md2man_cmd, stderr=STDOUT)
        except CalledProcessError as e:
            if e.returncode == 127:
                raise RuntimeError(
                    "Help file is available, but go-md2man is not present in a buildroot")
            raise RuntimeError("Error running %s: %r, exit code: %s, output: '%s'" % (
                e.cmd, e, e.returncode, e.output))

        if not os.path.exists(man_path):
            raise RuntimeError("go-md2man run complete, but man file is not found")

        # Include the help file in the docker file
        dockerfile = df_parser(self.workflow.builder.df_path, workflow=self.workflow)
        lines = dockerfile.lines

        content = 'ADD {0} /{0}'.format(self.man_filename)
        # put it before last instruction
        lines.insert(-1, content + '\n')

        dockerfile.lines = lines

        self.log.info("added %s", man_path)

        return content
<commit_msg>Fix add_help (from unit testing)<commit_after>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.

Convert a help markdown file a man page and store it to /help.1 in the image
so that 'atomic help' could display it.
This is accomplished by appending an ADD command to it.

Example configuration:
{
    'name': 'add_help',
    'args': {'help_file': 'help.md'}
}
"""

import errno
import os
from subprocess import check_output, CalledProcessError, STDOUT
from atomic_reactor.plugin import PreBuildPlugin
from atomic_reactor.util import df_parser

DEFAULT_HELP_FILENAME = "help.md"

class AddHelpPlugin(PreBuildPlugin):
    key = "add_help"
    man_filename = "help.1"

    def __init__(self, tasker, workflow, help_file=DEFAULT_HELP_FILENAME):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        :param help_file: filename of the markdown help file
        """
        # call parent constructor
        super(AddHelpPlugin, self).__init__(tasker, workflow)
        self.help_file = help_file

    def run(self):
        """
        run the plugin
        """

        help_path = os.path.join(self.workflow.builder.df_dir, self.help_file)

        if not os.path.exists(help_path):
            self.log.info("File %s not found", help_path)
            return

        man_path = os.path.join(self.workflow.builder.df_dir, self.man_filename)

        go_md2man_cmd = ['go-md2man', '-in={}'.format(help_path), '-out={}'.format(man_path)]

        try:
            check_output(go_md2man_cmd, stderr=STDOUT)
        except OSError as e:
            if e.errno == errno.ENOENT:
                raise RuntimeError(
                    "Help file is available, but go-md2man is not present in a buildroot")

            raise
        except CalledProcessError as e:
            raise RuntimeError("Error running %s: %r, exit code: %s, output: '%s'" % (
                e.cmd, e, e.returncode, e.output))

        if not os.path.exists(man_path):
            raise RuntimeError("go-md2man run complete, but man file is not found")

        # Include the help file in the docker file
        dockerfile = df_parser(self.workflow.builder.df_path, workflow=self.workflow)
        lines = dockerfile.lines

        content = 'ADD {0} /{0}'.format(self.man_filename)
        # put it before last instruction
        lines.insert(-1, content + '\n')

        dockerfile.lines = lines

        self.log.info("added %s", man_path)

        return content
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import json
import argparse
import tarfile
import StringIO

try:
    from bottle import Bottle, route, run, request, server_names, ServerAdapter, hook, response, HTTPError
except ImportError:
    sys.exit("ERROR: Bottle.py library is missing")

sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))

from lib.cuckoo.common.constants import CUCKOO_ROOT
from lib.cuckoo.common.utils import store_temp_file, delete_folder
from lib.cuckoo.core.database import Database

# Global DB pointer.
db = Database()

def jsonize(data):
    """Converts data dict to JSON.
    @param data: data dict
    @return: JSON formatted data
    """
    response.content_type = "application/json; charset=UTF-8"
    return json.dumps(data, sort_keys=False, indent=4)

@hook("after_request")
def custom_headers():
    """Set some custom headers across all HTTP responses."""
    response.headers["Server"] = "Machete Server"
    response.headers["X-Content-Type-Options"] = "nosniff"
    response.headers["X-Frame-Options"] = "DENY"
    response.headers["X-XSS-Protection"] = "1; mode=block"
    response.headers["Pragma"] = "no-cache"
    response.headers["Cache-Control"] = "no-cache"
    response.headers["Expires"] = "0"

@route("/tasks/create/file", method="POST")
def tasks_create_file():
    response = {}

    data = request.files.file
    package = request.forms.get("package", "")
    timeout = request.forms.get("timeout", "")
    priority = request.forms.get("priority", 1)
    options = request.forms.get("options", "")
    machine = request.forms.get("machine", "")
    platform = request.forms.get("platform", "")
    custom = request.forms.get("custom", "")
    memory = request.forms.get("memory", False)
    clock = request.forms.get("clock", None)
    if memory:
        memory = True
    enforce_timeout = request.forms.get("enforce_timeout", False)
    if enforce_timeout:
        enforce_timeout = True

    temp_file_path = store_temp_file(data.file.read(), data.filename)
    task_id = db.add_path(file_path=temp_file_path,
                          package=package,
                          timeout=timeout,
                          priority=priority,
                          options=options,
                          machine=machine,
                          platform=platform,
                          custom=custom,
                          memory=memory,
                          enforce_timeout=enforce_timeout,
                          clock=clock)

    response["task_id"] = task_id
    return jsonize(response)

@route("/tasks/create/url", method="POST")
def tasks_create_url():
    response = {}

    url = request.forms.get("url")
    package = request.forms.get("package", "")
    timeout = request.forms.get("timeout", "")
    priority = request.forms.get("priority", 1)
    options = request.forms.get("options", "")
    machine = request.forms.get("machine", "")
    platform = request.forms.get("platform", "")
    custom = request.forms.get("custom", "")
    memory = request.forms.get("memory", False)
    if memory:
        memory = True
    enforce_timeout = request.forms.get("enforce_timeout", False)
    if enforce_timeout:
        enforce_timeout = True
    clock = request.forms.get("clock", None)

    task_id = db.add_url(url=url,
                         package=package,
                         timeout=timeout,
                         options=options,
                         priority=priority,
                         machine=machine,
                         platform=platform,
                         custom=custom,
                         memory=memory,
                         enforce_timeout=enforce_timeout,
                         clock=clock)

    response["task_id"] = task_id
    return jsonize(response)

@route("/tasks/list", method="GET")
@route("/tasks/list/<limit:int>", method="GET")
@route("/tasks/list/<limit:int>/<offset:int>", method="GET")
def tasks_list(limit=None, offset=None):
    response = {}

    response["tasks"] = []

    for row in db.list_tasks(limit=limit, details=True, offset=offset):
        task = row.to_dict()
        task["guest"] = {}
        if row.guest:
            task["guest"] = row.guest.to_dict()

        task["errors"] = []
        for error in row.errors:
            task["errors"].append(error.message)

        response["tasks"].append(task)

    return jsonize(response)

@route("/tasks/view/<task_id>", method="GET")
def tasks_view(task_id):
    response = {}

    task = db.view_task(task_id, details=True)
    if task:
        entry = task.to_dict()
        entry["guest"] = {}
        if task.guest:
            entry["guest"] = task.guest.to_dict()

        entry["errors"] = []
        for error in task.errors:
            entry["errors"].append(error.message)

        response["task"] = entry
    else:
        return HTTPError(404, "Task not found")

    return jsonize(response)

@route("/tasks/delete/<task_id>", method="GET")
def tasks_delete(task_id):
    response = {}

    task = db.view_task(task_id)
    if task:
        if task.status == "processing":
            return HTTPError(500, "The task is currently being processed, cannot delete")

        if db.delete_task(task_id):
            delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id))
            response["status"] = "OK"
        else:
            return HTTPError(500, "An error occurred while trying to delete the task")
    else:
        return HTTPError(404, "Task not found")

    return jsonize(response)

@route("/tasks/report/<task_id>", method="GET")
@route("/tasks/report/<task_id>/<report_format>", method="GET")
def tasks_report(task_id, report_format="json"):
    response = {}

    formats = {
        "json" : "report.json",
        "html" : "report.html",
        "maec" : "report.maec-1.1.xml",
        "metadata" : "report.metadata.xml"
    }

    bz_formats = {
        "all": {"type": "-", "files": ["memory.dmp"]},
        "dropped": {"type": "+", "files": ["files"]},
    }

    if report_format.lower() in formats:
        report_path = os.path.join(CUCKOO_ROOT,
                                   "storage",
                                   "analyses",
                                   task_id,
                                   "reports",
                                   formats[report_format.lower()])
    elif report_format.lower() in bz_formats:
            bzf = bz_formats[report_format.lower()]
            srcdir = os.path.join(CUCKOO_ROOT,
                                   "storage",
                                   "analyses",
                                   task_id)
            s = StringIO.StringIO()
            tar = tarfile.open(fileobj=s, mode="w:bz2")
            for filedir in os.listdir(srcdir):
                if bzf["type"] == "-" and not filedir in bzf["files"]:
                    tar.add(os.path.join(srcdir, filedir), arcname=filedir)
                if bzf["type"] == "+" and filedir in bzf["files"]:
                    tar.add(os.path.join(srcdir, filedir), arcname=filedir)
            tar.close()
            return s.getvalue()
    else:
        return HTTPError(400, "Invalid report format")

    if os.path.exists(report_path):
        return open(report_path, "rb").read()
    else:
        return HTTPError(404, "Report not found")

@route("/files/view/md5/<md5>", method="GET")
@route("/files/view/sha256/<sha256>", method="GET")
@route("/files/view/id/<sample_id>", method="GET")
def files_view(md5=None, sha256=None, sample_id=None):
    response = {}

    if md5:
        sample = db.find_sample(md5=md5)
    elif sha256:
        sample = db.find_sample(sha256=sha256)
    elif sample_id:
        sample = db.view_sample(sample_id)
    else:
        return HTTPError(400, "Invalid lookup term")

    if sample:
        response["sample"] = sample.to_dict()
    else:
        return HTTPError(404, "File not found")

    return jsonize(response)

@route("/files/get/<sha256>", method="GET")
def files_get(sha256):
    file_path = os.path.join(CUCKOO_ROOT, "storage", "binaries", sha256)
    if os.path.exists(file_path):
        response.content_type = "application/octet-stream; charset=UTF-8"
        return open(file_path, "rb").read()
    else:
        return HTTPError(404, "File not found")

@route("/machines/list", method="GET")
def machines_list():
    response = {}

    machines = db.list_machines()

    response["machines"] = []
    for row in machines:
        response["machines"].append(row.to_dict())

    return jsonize(response)

@route("/machines/view/<name>", method="GET")
def machines_view(name=None):
    response = {}

    machine = db.view_machine(name=name)
    if machine:
        response["machine"] = machine.to_dict()
    else:
        return HTTPError(404, "Machine not found")

    return jsonize(response)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-H", "--host", help="Host to bind the API server on", default="localhost", action="store", required=False)
    parser.add_argument("-p", "--port", help="Port to bind the API server on", default=8090, action="store", required=False)
    args = parser.parse_args()

    run(host=args.host, port=args.port)
<commit_msg>Added proper content type otherwise it will be displayed as a text page (Refs: #112)<commit_after>#!/usr/bin/env python
# Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import json
import argparse
import tarfile
import StringIO

try:
    from bottle import Bottle, route, run, request, server_names, ServerAdapter, hook, response, HTTPError
except ImportError:
    sys.exit("ERROR: Bottle.py library is missing")

sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."))

from lib.cuckoo.common.constants import CUCKOO_ROOT
from lib.cuckoo.common.utils import store_temp_file, delete_folder
from lib.cuckoo.core.database import Database

# Global DB pointer.
db = Database()

def jsonize(data):
    """Converts data dict to JSON.
    @param data: data dict
    @return: JSON formatted data
    """
    response.content_type = "application/json; charset=UTF-8"
    return json.dumps(data, sort_keys=False, indent=4)

@hook("after_request")
def custom_headers():
    """Set some custom headers across all HTTP responses."""
    response.headers["Server"] = "Machete Server"
    response.headers["X-Content-Type-Options"] = "nosniff"
    response.headers["X-Frame-Options"] = "DENY"
    response.headers["X-XSS-Protection"] = "1; mode=block"
    response.headers["Pragma"] = "no-cache"
    response.headers["Cache-Control"] = "no-cache"
    response.headers["Expires"] = "0"

@route("/tasks/create/file", method="POST")
def tasks_create_file():
    response = {}

    data = request.files.file
    package = request.forms.get("package", "")
    timeout = request.forms.get("timeout", "")
    priority = request.forms.get("priority", 1)
    options = request.forms.get("options", "")
    machine = request.forms.get("machine", "")
    platform = request.forms.get("platform", "")
    custom = request.forms.get("custom", "")
    memory = request.forms.get("memory", False)
    clock = request.forms.get("clock", None)
    if memory:
        memory = True
    enforce_timeout = request.forms.get("enforce_timeout", False)
    if enforce_timeout:
        enforce_timeout = True

    temp_file_path = store_temp_file(data.file.read(), data.filename)
    task_id = db.add_path(file_path=temp_file_path,
                          package=package,
                          timeout=timeout,
                          priority=priority,
                          options=options,
                          machine=machine,
                          platform=platform,
                          custom=custom,
                          memory=memory,
                          enforce_timeout=enforce_timeout,
                          clock=clock)

    response["task_id"] = task_id
    return jsonize(response)

@route("/tasks/create/url", method="POST")
def tasks_create_url():
    response = {}

    url = request.forms.get("url")
    package = request.forms.get("package", "")
    timeout = request.forms.get("timeout", "")
    priority = request.forms.get("priority", 1)
    options = request.forms.get("options", "")
    machine = request.forms.get("machine", "")
    platform = request.forms.get("platform", "")
    custom = request.forms.get("custom", "")
    memory = request.forms.get("memory", False)
    if memory:
        memory = True
    enforce_timeout = request.forms.get("enforce_timeout", False)
    if enforce_timeout:
        enforce_timeout = True
    clock = request.forms.get("clock", None)

    task_id = db.add_url(url=url,
                         package=package,
                         timeout=timeout,
                         options=options,
                         priority=priority,
                         machine=machine,
                         platform=platform,
                         custom=custom,
                         memory=memory,
                         enforce_timeout=enforce_timeout,
                         clock=clock)

    response["task_id"] = task_id
    return jsonize(response)

@route("/tasks/list", method="GET")
@route("/tasks/list/<limit:int>", method="GET")
@route("/tasks/list/<limit:int>/<offset:int>", method="GET")
def tasks_list(limit=None, offset=None):
    response = {}

    response["tasks"] = []

    for row in db.list_tasks(limit=limit, details=True, offset=offset):
        task = row.to_dict()
        task["guest"] = {}
        if row.guest:
            task["guest"] = row.guest.to_dict()

        task["errors"] = []
        for error in row.errors:
            task["errors"].append(error.message)

        response["tasks"].append(task)

    return jsonize(response)

@route("/tasks/view/<task_id>", method="GET")
def tasks_view(task_id):
    response = {}

    task = db.view_task(task_id, details=True)
    if task:
        entry = task.to_dict()
        entry["guest"] = {}
        if task.guest:
            entry["guest"] = task.guest.to_dict()

        entry["errors"] = []
        for error in task.errors:
            entry["errors"].append(error.message)

        response["task"] = entry
    else:
        return HTTPError(404, "Task not found")

    return jsonize(response)

@route("/tasks/delete/<task_id>", method="GET")
def tasks_delete(task_id):
    response = {}

    task = db.view_task(task_id)
    if task:
        if task.status == "processing":
            return HTTPError(500, "The task is currently being processed, cannot delete")

        if db.delete_task(task_id):
            delete_folder(os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id))
            response["status"] = "OK"
        else:
            return HTTPError(500, "An error occurred while trying to delete the task")
    else:
        return HTTPError(404, "Task not found")

    return jsonize(response)

@route("/tasks/report/<task_id>", method="GET")
@route("/tasks/report/<task_id>/<report_format>", method="GET")
def tasks_report(task_id, report_format="json"):
    formats = {
        "json" : "report.json",
        "html" : "report.html",
        "maec" : "report.maec-1.1.xml",
        "metadata" : "report.metadata.xml"
    }

    bz_formats = {
        "all": {"type": "-", "files": ["memory.dmp"]},
        "dropped": {"type": "+", "files": ["files"]},
    }

    if report_format.lower() in formats:
        report_path = os.path.join(CUCKOO_ROOT,
                                   "storage",
                                   "analyses",
                                   task_id,
                                   "reports",
                                   formats[report_format.lower()])
    elif report_format.lower() in bz_formats:
            bzf = bz_formats[report_format.lower()]
            srcdir = os.path.join(CUCKOO_ROOT,
                                   "storage",
                                   "analyses",
                                   task_id)
            s = StringIO.StringIO()
            tar = tarfile.open(fileobj=s, mode="w:bz2")
            for filedir in os.listdir(srcdir):
                if bzf["type"] == "-" and not filedir in bzf["files"]:
                    tar.add(os.path.join(srcdir, filedir), arcname=filedir)
                if bzf["type"] == "+" and filedir in bzf["files"]:
                    tar.add(os.path.join(srcdir, filedir), arcname=filedir)
            tar.close()
            response.content_type = "application/x-tar; charset=UTF-8"
            return s.getvalue()
    else:
        return HTTPError(400, "Invalid report format")

    if os.path.exists(report_path):
        return open(report_path, "rb").read()
    else:
        return HTTPError(404, "Report not found")

@route("/files/view/md5/<md5>", method="GET")
@route("/files/view/sha256/<sha256>", method="GET")
@route("/files/view/id/<sample_id>", method="GET")
def files_view(md5=None, sha256=None, sample_id=None):
    response = {}

    if md5:
        sample = db.find_sample(md5=md5)
    elif sha256:
        sample = db.find_sample(sha256=sha256)
    elif sample_id:
        sample = db.view_sample(sample_id)
    else:
        return HTTPError(400, "Invalid lookup term")

    if sample:
        response["sample"] = sample.to_dict()
    else:
        return HTTPError(404, "File not found")

    return jsonize(response)

@route("/files/get/<sha256>", method="GET")
def files_get(sha256):
    file_path = os.path.join(CUCKOO_ROOT, "storage", "binaries", sha256)
    if os.path.exists(file_path):
        response.content_type = "application/octet-stream; charset=UTF-8"
        return open(file_path, "rb").read()
    else:
        return HTTPError(404, "File not found")

@route("/machines/list", method="GET")
def machines_list():
    response = {}

    machines = db.list_machines()

    response["machines"] = []
    for row in machines:
        response["machines"].append(row.to_dict())

    return jsonize(response)

@route("/machines/view/<name>", method="GET")
def machines_view(name=None):
    response = {}

    machine = db.view_machine(name=name)
    if machine:
        response["machine"] = machine.to_dict()
    else:
        return HTTPError(404, "Machine not found")

    return jsonize(response)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-H", "--host", help="Host to bind the API server on", default="localhost", action="store", required=False)
    parser.add_argument("-p", "--port", help="Port to bind the API server on", default=8090, action="store", required=False)
    args = parser.parse_args()

    run(host=args.host, port=args.port)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
"""Test scriptworker.config
"""
from copy import deepcopy
from frozendict import frozendict
import json
import mock
import os
import pytest
import scriptworker.config as config
from scriptworker.constants import DEFAULT_CONFIG

# constants helpers and fixtures {{{1
ENV_CREDS_PARAMS = ((
    {
        'TASKCLUSTER_ACCESS_TOKEN': 'x',
        'TASKCLUSTER_CLIENT_ID': 'y',
    }, {
        "accessToken": 'x',
        "clientId": 'y',
    }
), (
    {
        'TASKCLUSTER_ACCESS_TOKEN': 'x',
        'TASKCLUSTER_CLIENT_ID': 'y',
        'TASKCLUSTER_CERTIFICATE': 'z',
    }, {
        "accessToken": 'x',
        "clientId": 'y',
        "certificate": 'z',
    }
))

@pytest.fixture(scope='function')
def t_config():
    return dict(deepcopy(DEFAULT_CONFIG))

@pytest.fixture(scope='function')
def t_env():
    env = {}
    for key, value in os.environ.items():
        if not key.startswith('TASKCLUSTER_'):
            env[key] = value
    return env

# freeze_values {{{1
def test_freeze_values():
    d = {'a': [1, 2, 3], 'b': {'c': 'd'}}
    config.freeze_values(d)
    assert d == {'a': (1, 2, 3), 'b': frozendict({'c': 'd'})}

# check_config {{{1
def test_check_config_invalid_key(t_config):
    t_config['invalid_key_for_testing'] = 1
    messages = config.check_config(t_config, "test_path")
    assert "Unknown key" in "\n".join(messages)

def test_check_config_invalid_type(t_config):
    t_config['log_dir'] = tuple(t_config['log_dir'])
    messages = config.check_config(t_config, "test_path")
    assert "log_dir: type" in "\n".join(messages)

def test_check_config_bad_keyring(t_config):
    t_config['gpg_secret_keyring'] = 'foo{}'.format(t_config['gpg_secret_keyring'])
    messages = config.check_config(t_config, "test_path")
    assert "needs to start with %(gpg_home)s/" in "\n".join(messages)

@pytest.mark.parametrize("params", ("provisioner_id", "worker_group", "worker_type", "worker_id"))
def test_check_config_invalid_ids(params, t_config):
    t_config[params] = 'twenty-three-characters'
    messages = config.check_config(t_config, "test_path")
    assert '{} doesn\'t match "^[a-zA-Z0-9-_]{{1,22}}$" (required by Taskcluster)'.format(params) in messages

def test_check_config_good(t_config):
    for key, value in t_config.items():
        if value == "...":
            t_config[key] = "filled_in"
    messages = config.check_config(t_config, "test_path")
    assert messages == []

# create_config {{{1
def test_create_config_missing_file():
    with pytest.raises(SystemExit):
        config.create_config(config_path="this_file_does_not_exist.json")

def test_create_config_bad_config():
    path = os.path.join(os.path.dirname(__file__), "data", "bad.json")
    with pytest.raises(SystemExit):
        config.create_config(config_path=path)

def test_create_config_good(t_config):
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    with open(path, "r") as fh:
        contents = json.load(fh)
    t_config.update(contents)
    test_creds = t_config['credentials']
    del(t_config['credentials'])
    generated_config, generated_creds = config.create_config(config_path=path)
    assert generated_config == t_config
    assert generated_creds == test_creds
    assert isinstance(generated_config, frozendict)
    assert isinstance(generated_creds, frozendict)

# credentials {{{1
def test_bad_worker_creds():
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    with mock.patch.object(config, 'CREDS_FILES', new=(path, )):
        assert config.read_worker_creds(key="nonexistent_key") is None

def test_good_worker_creds(mocker):
    path = os.path.join(os.path.dirname(__file__), "data", "client_credentials.json")
    mocker.patch.object(config, 'CREDS_FILES', new=(path, ))
    assert config.read_worker_creds()

def test_no_creds_in_config():
    fake_creds = {"foo": "bar"}

    def fake_read(*args, **kwargs):
        return deepcopy(fake_creds)

    path = os.path.join(os.path.dirname(__file__), "data", "no_creds.json")
    with mock.patch.object(config, 'read_worker_creds', new=fake_read):
        _, creds = config.create_config(config_path=path)
    assert creds == fake_creds

def test_missing_creds(t_env):
    with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
        with mock.patch.object(os, 'environ', new=t_env):
            assert config.read_worker_creds() is None

@pytest.mark.parametrize("params", ENV_CREDS_PARAMS)
def test_environ_creds(params, t_env):
    t_env.update(params[0])
    with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
        with mock.patch.object(os, 'environ', new=t_env):
            assert config.read_worker_creds() == params[1]

# get_context_from_cmdln {{{1
def test_get_context_from_cmdln(t_config):
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    c = deepcopy(dict(DEFAULT_CONFIG))
    with open(path) as fh:
        c.update(json.load(fh))
    expected_creds = frozendict(c['credentials'])
    del(c['credentials'])
    expected_config = frozendict(c)

    def noop(*args, **kwargs):
        pass

    context, credentials = config.get_context_from_cmdln([path])
    assert credentials == expected_creds
    assert context.config == expected_config

@pytest.mark.parametrize("args", ([
    "1", "2", "3", "4"
], [
    "x", os.path.join(os.path.dirname(__file__), "data", "bad.json")
], []))
def test_get_context_from_cmdln_exception(args):
    with pytest.raises(SystemExit):
        config.get_context_from_cmdln(args)
<commit_msg>Add test<commit_after>#!/usr/bin/env python
# coding=utf-8
"""Test scriptworker.config
"""
from copy import deepcopy
from frozendict import frozendict
import json
import mock
import os
import pytest
import scriptworker.config as config
from scriptworker.constants import DEFAULT_CONFIG

# constants helpers and fixtures {{{1
ENV_CREDS_PARAMS = ((
    {
        'TASKCLUSTER_ACCESS_TOKEN': 'x',
        'TASKCLUSTER_CLIENT_ID': 'y',
    }, {
        "accessToken": 'x',
        "clientId": 'y',
    }
), (
    {
        'TASKCLUSTER_ACCESS_TOKEN': 'x',
        'TASKCLUSTER_CLIENT_ID': 'y',
        'TASKCLUSTER_CERTIFICATE': 'z',
    }, {
        "accessToken": 'x',
        "clientId": 'y',
        "certificate": 'z',
    }
))

@pytest.fixture(scope='function')
def t_config():
    return dict(deepcopy(DEFAULT_CONFIG))

@pytest.fixture(scope='function')
def t_env():
    env = {}
    for key, value in os.environ.items():
        if not key.startswith('TASKCLUSTER_'):
            env[key] = value
    return env

# freeze_values {{{1
def test_freeze_values():
    d = {'a': [1, 2, 3], 'b': {'c': 'd'}}
    config.freeze_values(d)
    assert d == {'a': (1, 2, 3), 'b': frozendict({'c': 'd'})}

# check_config {{{1
def test_check_config_invalid_key(t_config):
    t_config['invalid_key_for_testing'] = 1
    messages = config.check_config(t_config, "test_path")
    assert "Unknown key" in "\n".join(messages)

def test_check_config_missing_key(t_config):
    t_config = _fill_missing_values(t_config)

    # The current implementation of create_config() doesn't delete t_config['credentials']
    # if no secrets are present in any file.
    t_config['credentials'] = None
    messages = config.check_config(t_config, "test_path")
    assert "credentials needs to be defined!" in "\n".join(messages)

def _fill_missing_values(config):
    return {
        key: 'filled_in' if value == '...' else value
        for key, value in config.items()
    }

def test_check_config_invalid_type(t_config):
    t_config['log_dir'] = tuple(t_config['log_dir'])
    messages = config.check_config(t_config, "test_path")
    assert "log_dir: type" in "\n".join(messages)

def test_check_config_bad_keyring(t_config):
    t_config['gpg_secret_keyring'] = 'foo{}'.format(t_config['gpg_secret_keyring'])
    messages = config.check_config(t_config, "test_path")
    assert "needs to start with %(gpg_home)s/" in "\n".join(messages)

@pytest.mark.parametrize("params", ("provisioner_id", "worker_group", "worker_type", "worker_id"))
def test_check_config_invalid_ids(params, t_config):
    t_config[params] = 'twenty-three-characters'
    messages = config.check_config(t_config, "test_path")
    assert '{} doesn\'t match "^[a-zA-Z0-9-_]{{1,22}}$" (required by Taskcluster)'.format(params) in messages

def test_check_config_good(t_config):
    t_config = _fill_missing_values(t_config)
    messages = config.check_config(t_config, "test_path")
    assert messages == []

# create_config {{{1
def test_create_config_missing_file():
    with pytest.raises(SystemExit):
        config.create_config(config_path="this_file_does_not_exist.json")

def test_create_config_bad_config():
    path = os.path.join(os.path.dirname(__file__), "data", "bad.json")
    with pytest.raises(SystemExit):
        config.create_config(config_path=path)

def test_create_config_good(t_config):
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    with open(path, "r") as fh:
        contents = json.load(fh)
    t_config.update(contents)
    test_creds = t_config['credentials']
    del(t_config['credentials'])
    generated_config, generated_creds = config.create_config(config_path=path)
    assert generated_config == t_config
    assert generated_creds == test_creds
    assert isinstance(generated_config, frozendict)
    assert isinstance(generated_creds, frozendict)

# credentials {{{1
def test_bad_worker_creds():
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    with mock.patch.object(config, 'CREDS_FILES', new=(path, )):
        assert config.read_worker_creds(key="nonexistent_key") is None

def test_good_worker_creds(mocker):
    path = os.path.join(os.path.dirname(__file__), "data", "client_credentials.json")
    mocker.patch.object(config, 'CREDS_FILES', new=(path, ))
    assert config.read_worker_creds()

def test_no_creds_in_config():
    fake_creds = {"foo": "bar"}

    def fake_read(*args, **kwargs):
        return deepcopy(fake_creds)

    path = os.path.join(os.path.dirname(__file__), "data", "no_creds.json")
    with mock.patch.object(config, 'read_worker_creds', new=fake_read):
        _, creds = config.create_config(config_path=path)
    assert creds == fake_creds

def test_missing_creds(t_env):
    with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
        with mock.patch.object(os, 'environ', new=t_env):
            assert config.read_worker_creds() is None

@pytest.mark.parametrize("params", ENV_CREDS_PARAMS)
def test_environ_creds(params, t_env):
    t_env.update(params[0])
    with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
        with mock.patch.object(os, 'environ', new=t_env):
            assert config.read_worker_creds() == params[1]

# get_context_from_cmdln {{{1
def test_get_context_from_cmdln(t_config):
    path = os.path.join(os.path.dirname(__file__), "data", "good.json")
    c = deepcopy(dict(DEFAULT_CONFIG))
    with open(path) as fh:
        c.update(json.load(fh))
    expected_creds = frozendict(c['credentials'])
    del(c['credentials'])
    expected_config = frozendict(c)

    def noop(*args, **kwargs):
        pass

    context, credentials = config.get_context_from_cmdln([path])
    assert credentials == expected_creds
    assert context.config == expected_config

@pytest.mark.parametrize("args", ([
    "1", "2", "3", "4"
], [
    "x", os.path.join(os.path.dirname(__file__), "data", "bad.json")
], []))
def test_get_context_from_cmdln_exception(args):
    with pytest.raises(SystemExit):
        config.get_context_from_cmdln(args)
<|endoftext|>


<commit_before># @ 2019 Akretion - www.akretion.com.br -
#   Magno Costa <magno.costa@akretion.com.br>
#   Renato Lima <renato.lima@akretion.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo.addons.l10n_br_purchase.tests import test_l10n_br_purchase

class L10nBrPurchaseStockBase(test_l10n_br_purchase.L10nBrPurchaseBaseTest):

    def setUp(self):
        super().setUp()
        self.invoice_model = self.env['account.invoice']
        self.invoice_wizard = self.env['stock.invoice.onshipping']

    def _picking_purchase_order(self, order):
        self.assertEqual(
            order.picking_count, 1,
            'Purchase: one picking should be created"'
        )

        picking = order.picking_ids[0]
        for move_line in picking.move_line_ids:
            move_line.write({'qty_done': move_line.move_id.product_uom_qty})
        picking.with_context(tracking_disable=True).button_validate()
        self.assertEqual(
            order.order_line.mapped('qty_received'), [4.0, 2.0],
            'Purchase: all products should be received"')

    def test_l10n_br_purchase_products(self):
        super().test_l10n_br_purchase_products()
        self._picking_purchase_order(self.po_products)

    def test_grouping_pickings(self):
        """
        Test the invoice generation grouped by partner/product with 2
        picking and 2 moves per picking.
        """
        purchase_1 = self.env.ref(
            'l10n_br_purchase_stock.main_po_only_products_1')
        purchase_1.button_confirm()
        picking_1 = purchase_1.picking_ids
        self.assertEqual(picking_1.invoice_state, '2binvoiced',
                         'Error to inform Invoice State.')

        picking_1.action_confirm()
        picking_1.action_assign()
        # Force product availability
        for move in picking_1.move_ids_without_package:
            move.quantity_done = move.product_uom_qty
        picking_1.button_validate()
        self.assertEqual(picking_1.state, 'done')

        self.assertEquals(
            purchase_1.invoice_status, 'to invoice',
            "Error in compute field invoice_status,"
            " before create invoice by Picking."
        )

        purchase_2 = self.env.ref(
            'l10n_br_purchase_stock.main_po_only_products_2')
        purchase_2.button_confirm()
        picking_2 = purchase_2.picking_ids
        picking_2.action_confirm()
        picking_2.action_assign()
        # Force product availability
        for move in picking_2.move_ids_without_package:
            move.quantity_done = move.product_uom_qty
        picking_2.button_validate()

        pickings = picking_1 | picking_2
        wizard_obj = self.invoice_wizard.with_context(
            active_ids=pickings.ids,
            active_model=pickings._name,
        )
        fields_list = wizard_obj.fields_get().keys()
        wizard_values = wizard_obj.default_get(fields_list)
        # One invoice per partner but group products
        wizard_values.update({
            'group': 'partner_product',
        })
        wizard = wizard_obj.create(wizard_values)
        wizard.onchange_group()
        wizard.action_generate()
        domain = [('picking_ids', 'in', (picking_1.id, picking_2.id))]
        invoice = self.invoice_model.search(domain)
        # Fatura Agrupada
        self.assertEquals(len(invoice), 1)
        self.assertEqual(picking_1.invoice_state, 'invoiced')
        self.assertEqual(picking_2.invoice_state, 'invoiced')

        self.assertIn(invoice, picking_1.invoice_ids)
        self.assertIn(picking_1, invoice.picking_ids)
        self.assertIn(invoice, picking_2.invoice_ids)
        self.assertIn(picking_2, invoice.picking_ids)

        # Validar o price_unit usado
        for inv_line in invoice.invoice_line_ids:
            self.assertTrue(
                inv_line.invoice_line_tax_ids,
                'Error to map Purchase Tax in invoice.line.')
            # Preço usado na Linha da Invoice deve ser o mesmo
            # informado no Pedido de Compra
            self.assertEqual(inv_line.price_unit,
                             inv_line.purchase_line_id.price_unit)
<commit_msg>[FIX] Check fields used to mapping Fiscal values in the tests.<commit_after># @ 2019 Akretion - www.akretion.com.br -
#   Magno Costa <magno.costa@akretion.com.br>
#   Renato Lima <renato.lima@akretion.com.br>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo.addons.l10n_br_purchase.tests import test_l10n_br_purchase

class L10nBrPurchaseStockBase(test_l10n_br_purchase.L10nBrPurchaseBaseTest):

    def setUp(self):
        super().setUp()
        self.invoice_model = self.env['account.invoice']
        self.invoice_wizard = self.env['stock.invoice.onshipping']

    def _picking_purchase_order(self, order):
        self.assertEqual(
            order.picking_count, 1,
            'Purchase: one picking should be created"'
        )

        picking = order.picking_ids[0]
        for move_line in picking.move_line_ids:
            move_line.write({'qty_done': move_line.move_id.product_uom_qty})
        picking.with_context(tracking_disable=True).button_validate()
        self.assertEqual(
            order.order_line.mapped('qty_received'), [4.0, 2.0],
            'Purchase: all products should be received"')

    def test_l10n_br_purchase_products(self):
        super().test_l10n_br_purchase_products()
        self._picking_purchase_order(self.po_products)

    def test_grouping_pickings(self):
        """
        Test the invoice generation grouped by partner/product with 2
        picking and 2 moves per picking.
        """
        purchase_1 = self.env.ref(
            'l10n_br_purchase_stock.main_po_only_products_1')
        purchase_1.button_confirm()
        picking_1 = purchase_1.picking_ids
        self.assertEqual(picking_1.invoice_state, '2binvoiced',
                         'Error to inform Invoice State.')

        picking_1.action_confirm()
        picking_1.action_assign()
        # Force product availability
        for move in picking_1.move_ids_without_package:
            move.quantity_done = move.product_uom_qty
        picking_1.button_validate()
        self.assertEqual(picking_1.state, 'done')

        self.assertEquals(
            purchase_1.invoice_status, 'to invoice',
            "Error in compute field invoice_status,"
            " before create invoice by Picking."
        )

        purchase_2 = self.env.ref(
            'l10n_br_purchase_stock.main_po_only_products_2')
        purchase_2.button_confirm()
        picking_2 = purchase_2.picking_ids
        picking_2.action_confirm()
        picking_2.action_assign()
        # Force product availability
        for move in picking_2.move_ids_without_package:
            move.quantity_done = move.product_uom_qty
        picking_2.button_validate()

        pickings = picking_1 | picking_2
        wizard_obj = self.invoice_wizard.with_context(
            active_ids=pickings.ids,
            active_model=pickings._name,
        )
        fields_list = wizard_obj.fields_get().keys()
        wizard_values = wizard_obj.default_get(fields_list)
        # One invoice per partner but group products
        wizard_values.update({
            'group': 'partner_product',
        })
        wizard = wizard_obj.create(wizard_values)
        wizard.onchange_group()
        wizard.action_generate()
        domain = [('picking_ids', 'in', (picking_1.id, picking_2.id))]
        invoice = self.invoice_model.search(domain)
        # Fatura Agrupada
        self.assertEquals(len(invoice), 1)
        self.assertEqual(picking_1.invoice_state, 'invoiced')
        self.assertEqual(picking_2.invoice_state, 'invoiced')

        self.assertIn(invoice, picking_1.invoice_ids)
        self.assertIn(picking_1, invoice.picking_ids)
        self.assertIn(invoice, picking_2.invoice_ids)
        self.assertIn(picking_2, invoice.picking_ids)

        # Validar o price_unit usado
        for inv_line in invoice.invoice_line_ids:
            self.assertTrue(
                inv_line.invoice_line_tax_ids,
                'Error to map Purchase Tax in invoice.line.')
            # Preço usado na Linha da Invoice deve ser o mesmo
            # informado no Pedido de Compra
            self.assertEqual(
                inv_line.price_unit,
                inv_line.purchase_line_id.price_unit)
            # Valida presença dos campos principais para o mapeamento Fiscal
            self.assertTrue(
                inv_line.fiscal_operation_id,
                'Missing Fiscal Operation.')
            self.assertTrue(
                inv_line.fiscal_operation_line_id,
                'Missing Fiscal Operation Line.')
<|endoftext|>


<commit_before># encoding: utf-8
from django.utils.translation import gettext_lazy as _
from django.db import models
import logging.config
from xbaydns.tools import nsupdate

log = logging.getLogger('xbaydnsweb.web.tests')
logging.basicConfig(level=logging.DEBUG)

class Acl(models.Model):
    """Acl Model"""
    aclName = models.CharField(maxlength=100)

    class Admin:
        list_display = ('aclName',)
        search_fields = ('aclName',)
    class Meta:
        ordering = ('aclName',)
        verbose_name = _('acl_verbose_name')
        verbose_name_plural = '1.1 '+str(_('acl_verbose_name_plural'))

    def __str__(self):
        return self.aclName

class AclMatch(models.Model):
    """AclMatch Model"""
    acl = models.ForeignKey(Acl)
    aclMatch = models.CharField(maxlength=100)

    class Admin:
        list_display = ('acl','aclMatch')
        #search_fields = ('',)
    class Meta:
        verbose_name = _('aclmatch_verbose_name')
        verbose_name_plural = '1.2 '+str(_('aclmatch_verbose_name_plural'))

    def __str__(self):
        return ' '.join([str(self.acl),self.aclMatch])

class View(models.Model):
    """View Model"""
    viewName = models.CharField(maxlength=100,verbose_name=str(_('view_verbose_name'))+'名称')
    #viewgroup = models.ForeignKey("ViewGroup")
    viewgroup = models.ManyToManyField("ViewGroup")
    aclmatch = models.ManyToManyField(AclMatch,verbose_name=_('aclmatch_verbose_name'))

    class Admin:
        list_display = ('viewName','showviewgroup','showacls')
        #search_fields = ('viewName',)
    class Meta:
        ordering = ('viewName',)
        verbose_name = _('view_verbose_name')
        verbose_name_plural = '2.2 '+str(_('view_verbose_name_plural'))
    def showacls(self):
        return ','.join(map(lambda x:str(x),self.aclmatch.all()))
    showacls.short_description = 'Acl'
    showacls.allow_tags = True
    def showviewgroup(self):
        return ','.join(map(lambda x:str(x),self.viewgroup.all()))
    showviewgroup.short_description = 'View Group'
    showviewgroup.allow_tags = True

    def __str__(self):
        return self.viewName

class Domain(models.Model):
    """Domain Model"""
    zone = models.CharField(maxlength=100)

    class Admin:
        list_display = ('zone',)
        search_fields = ('zone',)
    class Meta:
        verbose_name = _('domain_verbose_name')
        verbose_name_plural = '4.1 '+str(_('domain_verbose_name_plural'))

    def __str__(self):
        return self.zone

class ViewGroup(models.Model):
    """ViewGourp"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = _('viewgroup_verbose_name')
        verbose_name_plural = '2.1 '+str(_('viewgroup_verbose_name_plural'))

    def __str__(self):
        return self.name

class Record(models.Model):
    """Record"""
    domain = models.ForeignKey(Domain)
    record = models.CharField(maxlength=100)
    ttl = models.CharField(maxlength=100,default='600')
    ip = models.CharField(maxlength=100)
    rdclass = models.CharField(maxlength=100,default='IN')
    rdtype = models.ForeignKey("RecordType",default='1')
    recordgroup = models.ForeignKey("RecordGroup")

    def save(self):
        for view in self.getRecordViews():
            nsupobj = nsupdate.NSUpdate('127.0.0.1',str(self.domain),view=view)
            #['foo', 3600, 'IN', 'A', ['192.168.1.1', '172.16.1.1']]#record style
            add_data=[[self.record,int(self.ttl),self.rdclass,str(self.rdtype),[self.ip,]],]
            if self.id!=None:
                old_r=Record.objects.get(id=self.id)
                del_data=[[old_r.record,int(old_r.ttl),old_r.rdclass,str(old_r.rdtype),[old_r.ip,]],]
                nsupobj.removeRecord(del_data)

            nsupobj.addRecord(add_data)
            nsupobj.commitChanges()

        super(Record,self).save()
    def delete(self):
        for view in self.getRecordViews():
            nsupobj = nsupdate.NSUpdate('127.0.0.1',str(self.domain),view=view)
            del_data=[[self.record,int(self.ttl),self.rdclass,str(self.rdtype),[self.ip,]],]
            #['foo', 3600, 'IN', 'A', ['192.168.1.1', '172.16.1.1']]#record style
            nsupobj.removeRecord(del_data)
            nsupobj.commitChanges()

        super(Record,self).delete()

    class Admin:
        list_display = ('domain','record','rdtype','ttl','ip','recordgroup','showviews')
        #search_fields = ('domain','record')
    class Meta:
        verbose_name = _('record_verbose_name')
        verbose_name_plural = '4.3 '+str(_('record_verbose_name_plural'))

    def getRecordViews(self):
        views=[]
        for vm in ViewMatch.objects.filter(recordgroup=self.recordgroup):
            for vg in vm.viewgroup.all():
                for view in View.objects.filter(viewgroup=vg):
                    views.append(view.viewName)
        return views
    def showviews(self):
        return ','.join(self.getRecordViews())
    showviews.short_description = _('view_verbose_name')
    showviews.allow_tags = True

    def __str__(self):
        return '%s IN %s'%(self.domain,self.rdtype)

class RecordGroup(models.Model):
    """RecordGroup"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = _('recordgroup_verbose_name')
        verbose_name_plural = '3.1 '+str(_('recordgroup_verbose_name_plural'))

    def __str__(self):
        return self.name

class ViewMatch(models.Model):
    """ViewMatch"""
    name = models.CharField(maxlength=100)
    viewgroup = models.ManyToManyField(ViewGroup)
    recordgroup = models.ManyToManyField(RecordGroup)

    class Admin:
        list_display = ('name','showViewGroups','showRecordGroups')
        search_fields = ('name',)
    class Meta:
        verbose_name = _('viewmatch_verbose_name')
        verbose_name_plural = '3.2 '+str(_('viewmatch_verbose_name_plural'))
    def showViewGroups(self):
        return ','.join(map(lambda x:x.name,self.viewgroup.all()))
    showViewGroups.short_description = _('viewgroup_verbose_name')
    showViewGroups.allow_tags = True
    def showRecordGroups(self):
        return ','.join(map(lambda x:x.name,self.recordgroup.all()))
    showRecordGroups.short_description = _('recordgroup_verbose_name')
    showRecordGroups.allow_tags = True

    def __str__(self):
        return self.name

class RecordType(models.Model):
    """RecordType"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = 'RecordType'
        verbose_name_plural = '4.2 Record Type管理'

    def __str__(self):
        return self.name
<commit_msg>log的namespace写错了，同时把DEBUG关闭<commit_after># encoding: utf-8
from django.utils.translation import gettext_lazy as _
from django.db import models
import logging.config
from xbaydns.tools import nsupdate

log = logging.getLogger('xbaydnsweb.web.models')

class Acl(models.Model):
    """Acl Model"""
    aclName = models.CharField(maxlength=100)

    class Admin:
        list_display = ('aclName',)
        search_fields = ('aclName',)
    class Meta:
        ordering = ('aclName',)
        verbose_name = _('acl_verbose_name')
        verbose_name_plural = '1.1 '+str(_('acl_verbose_name_plural'))

    def __str__(self):
        return self.aclName

class AclMatch(models.Model):
    """AclMatch Model"""
    acl = models.ForeignKey(Acl)
    aclMatch = models.CharField(maxlength=100)

    class Admin:
        list_display = ('acl','aclMatch')
        #search_fields = ('',)
    class Meta:
        verbose_name = _('aclmatch_verbose_name')
        verbose_name_plural = '1.2 '+str(_('aclmatch_verbose_name_plural'))

    def __str__(self):
        return ' '.join([str(self.acl),self.aclMatch])

class View(models.Model):
    """View Model"""
    viewName = models.CharField(maxlength=100,verbose_name=str(_('view_verbose_name'))+'名称')
    #viewgroup = models.ForeignKey("ViewGroup")
    viewgroup = models.ManyToManyField("ViewGroup")
    aclmatch = models.ManyToManyField(AclMatch,verbose_name=_('aclmatch_verbose_name'))

    class Admin:
        list_display = ('viewName','showviewgroup','showacls')
        #search_fields = ('viewName',)
    class Meta:
        ordering = ('viewName',)
        verbose_name = _('view_verbose_name')
        verbose_name_plural = '2.2 '+str(_('view_verbose_name_plural'))
    def showacls(self):
        return ','.join(map(lambda x:str(x),self.aclmatch.all()))
    showacls.short_description = 'Acl'
    showacls.allow_tags = True
    def showviewgroup(self):
        return ','.join(map(lambda x:str(x),self.viewgroup.all()))
    showviewgroup.short_description = 'View Group'
    showviewgroup.allow_tags = True

    def __str__(self):
        return self.viewName

class Domain(models.Model):
    """Domain Model"""
    zone = models.CharField(maxlength=100)

    class Admin:
        list_display = ('zone',)
        search_fields = ('zone',)
    class Meta:
        verbose_name = _('domain_verbose_name')
        verbose_name_plural = '4.1 '+str(_('domain_verbose_name_plural'))

    def __str__(self):
        return self.zone

class ViewGroup(models.Model):
    """ViewGourp"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = _('viewgroup_verbose_name')
        verbose_name_plural = '2.1 '+str(_('viewgroup_verbose_name_plural'))

    def __str__(self):
        return self.name

class Record(models.Model):
    """Record"""
    domain = models.ForeignKey(Domain)
    record = models.CharField(maxlength=100)
    ttl = models.CharField(maxlength=100,default='600')
    ip = models.CharField(maxlength=100)
    rdclass = models.CharField(maxlength=100,default='IN')
    rdtype = models.ForeignKey("RecordType",default='1')
    recordgroup = models.ForeignKey("RecordGroup")

    def save(self):
        for view in self.getRecordViews():
            nsupobj = nsupdate.NSUpdate('127.0.0.1',str(self.domain),view=view)
            #['foo', 3600, 'IN', 'A', ['192.168.1.1', '172.16.1.1']]#record style
            add_data=[[self.record,int(self.ttl),self.rdclass,str(self.rdtype),[self.ip,]],]
            if self.id!=None:
                old_r=Record.objects.get(id=self.id)
                del_data=[[old_r.record,int(old_r.ttl),old_r.rdclass,str(old_r.rdtype),[old_r.ip,]],]
                nsupobj.removeRecord(del_data)

            nsupobj.addRecord(add_data)
            nsupobj.commitChanges()

        super(Record,self).save()
    def delete(self):
        for view in self.getRecordViews():
            nsupobj = nsupdate.NSUpdate('127.0.0.1',str(self.domain),view=view)
            del_data=[[self.record,int(self.ttl),self.rdclass,str(self.rdtype),[self.ip,]],]
            #['foo', 3600, 'IN', 'A', ['192.168.1.1', '172.16.1.1']]#record style
            nsupobj.removeRecord(del_data)
            nsupobj.commitChanges()

        super(Record,self).delete()

    class Admin:
        list_display = ('domain','record','rdtype','ttl','ip','recordgroup','showviews')
        #search_fields = ('domain','record')
    class Meta:
        verbose_name = _('record_verbose_name')
        verbose_name_plural = '4.3 '+str(_('record_verbose_name_plural'))

    def getRecordViews(self):
        views=[]
        for vm in ViewMatch.objects.filter(recordgroup=self.recordgroup):
            for vg in vm.viewgroup.all():
                for view in View.objects.filter(viewgroup=vg):
                    views.append(view.viewName)
        return views
    def showviews(self):
        return ','.join(self.getRecordViews())
    showviews.short_description = _('view_verbose_name')
    showviews.allow_tags = True

    def __str__(self):
        return '%s IN %s'%(self.domain,self.rdtype)

class RecordGroup(models.Model):
    """RecordGroup"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = _('recordgroup_verbose_name')
        verbose_name_plural = '3.1 '+str(_('recordgroup_verbose_name_plural'))

    def __str__(self):
        return self.name

class ViewMatch(models.Model):
    """ViewMatch"""
    name = models.CharField(maxlength=100)
    viewgroup = models.ManyToManyField(ViewGroup)
    recordgroup = models.ManyToManyField(RecordGroup)

    class Admin:
        list_display = ('name','showViewGroups','showRecordGroups')
        search_fields = ('name',)
    class Meta:
        verbose_name = _('viewmatch_verbose_name')
        verbose_name_plural = '3.2 '+str(_('viewmatch_verbose_name_plural'))
    def showViewGroups(self):
        return ','.join(map(lambda x:x.name,self.viewgroup.all()))
    showViewGroups.short_description = _('viewgroup_verbose_name')
    showViewGroups.allow_tags = True
    def showRecordGroups(self):
        return ','.join(map(lambda x:x.name,self.recordgroup.all()))
    showRecordGroups.short_description = _('recordgroup_verbose_name')
    showRecordGroups.allow_tags = True

    def __str__(self):
        return self.name

class RecordType(models.Model):
    """RecordType"""
    name = models.CharField(maxlength=100)

    class Admin:
        list_display = ('name',)
        search_fields = ('name',)
    class Meta:
        verbose_name = 'RecordType'
        verbose_name_plural = '4.2 Record Type管理'

    def __str__(self):
        return self.name
<|endoftext|>


<commit_before>from .moments import moments
from .fractions import print_fraction_of_abundances

def read_loom_layers(file_name, backup_url=None):
    """Read loom file and return AnnData object

    Arguments
    ---------
    loom_filepath: str
        directory of loom file

    backup_url: str

    all_loom_layers: bool, default=False
        whether to load loom layers 'loom_obs' and 'loom_var' into AnnData object

    Returns
    -------
    Updates the attributes .X, .var, .obsm['Xs'], .obsm['Xs']
    """
    import os
    import loompy
    from scanpy.api import AnnData
    from collections import OrderedDict

    if not os.path.exists(file_name):
        try:
            from urllib.request import urlretrieve
            urlretrieve(backup_url, file_name)
        except OSError:
            print("OS error: {0}".format(OSError))

    with loompy.connect(file_name, 'r') as lc:
        X = lc.layer['spliced'].sparse().T.tocsr()

        layers = OrderedDict()
        layers['spliced'] = lc.layer["spliced"].sparse().T.tocsr()
        layers['unspliced'] = lc.layer["unspliced"].sparse().T.tocsr()

        obs = dict(lc.col_attrs)
        obs['obs_names'] = obs.pop('CellID')

        var = dict(lc.row_attrs)
        var['var_names'] = var.pop('Gene')

        adata = AnnData(X, obs=obs, var=var, layers=layers)

    return adata

def recipe_velocity(adata, min_counts=10, n_top_genes=3000, copy=False):
    from scanpy.api.pp import \
        filter_genes, filter_genes_dispersion, normalize_per_cell, pca, neighbors
    filter_genes(adata, min_counts=min_counts)
    filter_genes_dispersion(adata, n_top_genes=n_top_genes)
    normalize_per_cell(adata, layers='all')
    pca(adata, n_comps=30)
    neighbors(adata, n_neighbors=30, use_rep='X_pca')

    moments(adata)
    return adata if copy else None
<commit_msg>add n_neighbors attr to recipe_velocity<commit_after>from .moments import moments
from .fractions import print_fraction_of_abundances

def read_loom_layers(file_name, backup_url=None):
    """Read loom file and return AnnData object

    Arguments
    ---------
    loom_filepath: str
        directory of loom file

    backup_url: str

    all_loom_layers: bool, default=False
        whether to load loom layers 'loom_obs' and 'loom_var' into AnnData object

    Returns
    -------
    Updates the attributes .X, .var, .obsm['Xs'], .obsm['Xs']
    """
    import os
    import loompy
    from scanpy.api import AnnData
    from collections import OrderedDict

    if not os.path.exists(file_name):
        try:
            from urllib.request import urlretrieve
            urlretrieve(backup_url, file_name)
        except OSError:
            print("OS error: {0}".format(OSError))

    with loompy.connect(file_name, 'r') as lc:
        X = lc.layer['spliced'].sparse().T.tocsr()

        layers = OrderedDict()
        layers['spliced'] = lc.layer["spliced"].sparse().T.tocsr()
        layers['unspliced'] = lc.layer["unspliced"].sparse().T.tocsr()

        obs = dict(lc.col_attrs)
        obs['obs_names'] = obs.pop('CellID')

        var = dict(lc.row_attrs)
        var['var_names'] = var.pop('Gene')

        adata = AnnData(X, obs=obs, var=var, layers=layers)

    return adata

def recipe_velocity(adata, min_counts=10, n_top_genes=3000, n_neighbors=15, copy=False):
    from scanpy.api.pp import \
        filter_genes, filter_genes_dispersion, normalize_per_cell, pca, neighbors
    filter_genes(adata, min_counts=min_counts)
    filter_genes_dispersion(adata, n_top_genes=n_top_genes)
    normalize_per_cell(adata, layers='all')
    pca(adata, n_comps=30)
    neighbors(adata, n_neighbors=n_neighbors, use_rep='X_pca')

    moments(adata)
    return adata if copy else None
<|endoftext|>


<commit_before># __BEGIN_LICENSE__
# Copyright (C) 2008-2010 United States Government as represented by
# the Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# __END_LICENSE__

# import logging

#TODO: write tests that do more than check pages with GET requests.

from django.test import TestCase
from django.core.urlresolvers import reverse

class TestMaps(TestCase):
    fixtures = ['xgds_map_server_testing.json',
                'xgds_map_server_auth.json']
    urls = "xgds_map_server.testing_urls"

    def test_index(self):
        response = self.client.get(reverse('xgds_map_server_index'))
        self.assertEqual(response.status_code, 200)

    def test_list(self):
        response = self.client.get(reverse('mapList'))
        self.assertEqual(response.status_code, 200)

    def test_mapList(self):
        response = self.client.get(reverse('mapList'))
        self.assertEqual(response.status_code, 200)

    def test_mapTree(self):
        response = self.client.get(reverse('mapTree'))
        self.assertEqual(response.status_code, 200)

    def test_mapListJSON(self):
        response = self.client.get(reverse('mapListJSON'))
        self.assertEqual(response.status_code, 200)

    def test_mapDetail(self):
        mapId = '1'
        response = self.client.get(reverse('mapDetail', args=[mapId]))
        self.assertEqual(response.status_code, 200)

    def test_folderDetail(self):
        groupId = '1'
        response = self.client.get(reverse('folderDetail', args=[groupId]))
        self.assertEqual(response.status_code, 200)

    def test_folderDelete(self):
        groupId = '1'
        response = self.client.get(reverse('folderDelete', args=[groupId]))
        self.assertEqual(response.status_code, 200)

    def test_folderAdd(self):
        response = self.client.get(reverse('folderAdd'))
        self.assertEqual(response.status_code, 200)

    def test_addMap(self):
        response = self.client.get(reverse('addMap'))
        self.assertEqual(response.status_code, 200)

    def test_mapDelete(self):
        mapId = '1'
        response = self.client.get(reverse('mapDelete', args=[mapId]))
        self.assertEqual(response.status_code, 200)

    def test_deletedMaps(self):
        response = self.client.get(reverse('deletedMaps'))
        self.assertEqual(response.status_code, 200)

    # def test_jsonMove(self):
    #     response = self.client.get(reverse('jsonMove'))
    #     self.assertEqual(response.status_code, 200)

    # The xgds_map_server_static function was a hack I think I (Jerome) created because
    # I didn't know how to work the static file settings. Indeed it isn't used.

    def test_feed(self):
        # currently the only feed possible is the master feed
        feedName = ''
        response = self.client.get(reverse('xgds_map_server_feed', args=[feedName]))
        self.assertEqual(response.status_code, 200)
<commit_msg>Update tests so they work<commit_after># __BEGIN_LICENSE__
# Copyright (C) 2008-2010 United States Government as represented by
# the Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# __END_LICENSE__

# import logging

# TODO: write tests that do more than check pages with GET requests.

from django.test import TestCase
from django.core.urlresolvers import reverse
from unittest import skip

class TestMaps(TestCase):
    fixtures = ['xgds_map_server_testing.json',
                'xgds_map_server_auth.json']
    urls = "xgds_map_server.testing_urls"

    def test_index(self):
        response = self.client.get(reverse('xgds_map_server_index'))
        self.assertEqual(response.status_code, 200)

    @skip('Url disabled in urls.py')
    def test_mapList(self):
        response = self.client.get(reverse('mapList'))
        self.assertEqual(response.status_code, 200)

    def test_mapTree(self):
        response = self.client.get(reverse('mapTree'))
        self.assertEqual(response.status_code, 200)

    def test_mapListJSON(self):
        response = self.client.get(reverse('mapListJSON'))
        self.assertEqual(response.status_code, 200)

    def test_mapDetail(self):
        mapId = '1'
        response = self.client.get(reverse('mapDetail', args=[mapId]))
        self.assertEqual(response.status_code, 200)

    def test_folderDetail(self):
        groupId = '1'
        response = self.client.get(reverse('folderDetail', args=[groupId]))
        self.assertEqual(response.status_code, 200)

    def test_folderDelete(self):
        groupId = '1'
        response = self.client.get(reverse('folderDelete', args=[groupId]))
        self.assertEqual(response.status_code, 200)

    def test_folderAdd(self):
        response = self.client.get(reverse('folderAdd'))
        self.assertEqual(response.status_code, 200)

    def test_addMap(self):
        response = self.client.get(reverse('addMap'))
        self.assertEqual(response.status_code, 200)

    def test_mapDelete(self):
        mapId = '1'
        response = self.client.get(reverse('mapDelete', args=[mapId]))
        self.assertEqual(response.status_code, 200)

    def test_deletedMaps(self):
        response = self.client.get(reverse('deletedMaps'))
        self.assertEqual(response.status_code, 200)

    @skip('Test is broken')
    def test_jsonMove(self):
        response = self.client.get(reverse('jsonMove'))
        self.assertEqual(response.status_code, 200)

    # The xgds_map_server_static function was a hack I think I (Jerome) created because
    # I didn't know how to work the static file settings. Indeed it isn't used.

    def test_feed(self):
        # currently the only feed possible is the master feed
        feedName = ''
        response = self.client.get(reverse('xgds_map_server_feed', args=[feedName]))
        self.assertEqual(response.status_code, 200)
<|endoftext|>


<commit_before>import time
from unittest import TestCase

class Sample(TestCase):
    def test_sameple_with_big_timeout(self):
        print("Testing false assert")
        self.assertEquals(1, 2)
<commit_msg>Fix deprecated use of function on sample test<commit_after>from unittest import TestCase

class Sample(TestCase):
    def test_sameple_with_big_timeout(self):
        print("Testing false assert")
        self.assertEqual(1, 2)
<|endoftext|>


<commit_before>import logging
from collections import defaultdict
from typing import Optional

import numpy as np
import torch
from torch.nn import Linear

from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import examples_per_class, get_last_fc_layer, \
    get_layer_by_name, freeze_everything, unfreeze_everything

class CWRStarPlugin(StrategyPlugin):

    def __init__(self, model, cwr_layer_name=None, freeze_remaining_model=True):
        """
        CWR* Strategy.
        This plugin does not use task identities.

        :param model: the model.
        :param cwr_layer_name: name of the last fully connected layer. Defaults
            to None, which means that the plugin will attempt an automatic
            detection.
        :param freeze_remaining_model: If True, the plugin will freeze (set
            layers in eval mode and disable autograd for parameters) all the
            model except the cwr layer. Defaults to True.
        """
        super().__init__()
        self.log = logging.getLogger("avalanche")
        self.model = model
        self.cwr_layer_name = cwr_layer_name
        self.freeze_remaining_model = freeze_remaining_model

        # Model setup
        self.model.saved_weights = {}
        self.model.past_j = defaultdict(int)
        self.model.cur_j = defaultdict(int)

        # to be updated
        self.cur_class = None

    def after_training_exp(self, strategy, **kwargs):
        self.consolidate_weights()
        self.set_consolidate_weights()

    def before_training_exp(self, strategy, **kwargs):
        if self.freeze_remaining_model and strategy.training_exp_counter > 0:
            self.freeze_other_layers()

        # Count current classes and number of samples for each of them.
        data = strategy.experience.dataset
        self.model.cur_j = examples_per_class(data.targets)
        self.cur_class = [cls for cls in set(self.model.cur_j.keys()) if
                          self.model.cur_j[cls] > 0]

        self.reset_weights(self.cur_class)

    def consolidate_weights(self):
        """ Mean-shift for the target layer weights"""

        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            globavg = np.average(cwr_layer.weight.detach()
                                 .cpu().numpy()[self.cur_class])
            for c in self.cur_class:
                w = cwr_layer.weight.detach().cpu().numpy()[c]

                if c in self.cur_class:
                    new_w = w - globavg
                    if c in self.model.saved_weights.keys():
                        wpast_j = np.sqrt(self.model.past_j[c] /
                                          self.model.cur_j[c])
                        # wpast_j = model.past_j[c] / model.cur_j[c]
                        self.model.saved_weights[c] = \
                            (self.model.saved_weights[c] * wpast_j + new_w) / \
                            (wpast_j + 1)
                    else:
                        self.model.saved_weights[c] = new_w

    def set_consolidate_weights(self):
        """ set trained weights """

        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            for c, w in self.model.saved_weights.items():
                cwr_layer.weight[c].copy_(
                    torch.from_numpy(self.model.saved_weights[c])
                )

    def reset_weights(self, cur_clas):
        """ reset weights"""
        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            cwr_layer.weight.fill_(0.0)
            for c, w in self.model.saved_weights.items():
                if c in cur_clas:
                    cwr_layer.weight[c].copy_(
                        torch.from_numpy(self.model.saved_weights[c])
                    )

    def get_cwr_layer(self) -> Optional[Linear]:
        result = None
        if self.cwr_layer_name is None:
            last_fc = get_last_fc_layer(self.model)
            if last_fc is not None:
                result = last_fc[1]
        else:
            result = get_layer_by_name(self.model, self.cwr_layer_name)

        return result

    def freeze_other_layers(self):
        cwr_layer = self.get_cwr_layer()
        if cwr_layer is None:
            raise RuntimeError('Can\'t find a the Linear layer')
        freeze_everything(self.model)
        unfreeze_everything(cwr_layer)
<commit_msg>Updated pastj after experience<commit_after>import logging
from collections import defaultdict
from typing import Optional

import numpy as np
import torch
from torch.nn import Linear

from avalanche.training.plugins.strategy_plugin import StrategyPlugin
from avalanche.training.utils import examples_per_class, get_last_fc_layer, \
    get_layer_by_name, freeze_everything, unfreeze_everything

class CWRStarPlugin(StrategyPlugin):

    def __init__(self, model, cwr_layer_name=None, freeze_remaining_model=True):
        """
        CWR* Strategy.
        This plugin does not use task identities.

        :param model: the model.
        :param cwr_layer_name: name of the last fully connected layer. Defaults
            to None, which means that the plugin will attempt an automatic
            detection.
        :param freeze_remaining_model: If True, the plugin will freeze (set
            layers in eval mode and disable autograd for parameters) all the
            model except the cwr layer. Defaults to True.
        """
        super().__init__()
        self.log = logging.getLogger("avalanche")
        self.model = model
        self.cwr_layer_name = cwr_layer_name
        self.freeze_remaining_model = freeze_remaining_model

        # Model setup
        self.model.saved_weights = {}
        self.model.past_j = defaultdict(int)
        self.model.cur_j = defaultdict(int)

        # to be updated
        self.cur_class = None

    def after_training_exp(self, strategy, **kwargs):
        self.consolidate_weights()
        self.set_consolidate_weights()

    def before_training_exp(self, strategy, **kwargs):
        if self.freeze_remaining_model and strategy.training_exp_counter > 0:
            self.freeze_other_layers()

        # Count current classes and number of samples for each of them.
        data = strategy.experience.dataset
        self.model.cur_j = examples_per_class(data.targets)
        self.cur_class = [cls for cls in set(self.model.cur_j.keys()) if
                          self.model.cur_j[cls] > 0]

        self.reset_weights(self.cur_class)

    def consolidate_weights(self):
        """ Mean-shift for the target layer weights"""

        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            globavg = np.average(cwr_layer.weight.detach()
                                 .cpu().numpy()[self.cur_class])
            for c in self.cur_class:
                w = cwr_layer.weight.detach().cpu().numpy()[c]

                if c in self.cur_class:
                    new_w = w - globavg
                    if c in self.model.saved_weights.keys():
                        wpast_j = np.sqrt(self.model.past_j[c] /
                                          self.model.cur_j[c])
                        # wpast_j = model.past_j[c] / model.cur_j[c]
                        self.model.saved_weights[c] = \
                            (self.model.saved_weights[c] * wpast_j + new_w) / \
                            (wpast_j + 1)
                        self.model.past_j[c] += self.model.cur_j[c]
                    else:
                        self.model.saved_weights[c] = new_w

    def set_consolidate_weights(self):
        """ set trained weights """

        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            for c, w in self.model.saved_weights.items():
                cwr_layer.weight[c].copy_(
                    torch.from_numpy(self.model.saved_weights[c])
                )

    def reset_weights(self, cur_clas):
        """ reset weights"""
        with torch.no_grad():
            cwr_layer = self.get_cwr_layer()
            cwr_layer.weight.fill_(0.0)
            for c, w in self.model.saved_weights.items():
                if c in cur_clas:
                    cwr_layer.weight[c].copy_(
                        torch.from_numpy(self.model.saved_weights[c])
                    )

    def get_cwr_layer(self) -> Optional[Linear]:
        result = None
        if self.cwr_layer_name is None:
            last_fc = get_last_fc_layer(self.model)
            if last_fc is not None:
                result = last_fc[1]
        else:
            result = get_layer_by_name(self.model, self.cwr_layer_name)

        return result

    def freeze_other_layers(self):
        cwr_layer = self.get_cwr_layer()
        if cwr_layer is None:
            raise RuntimeError('Can\'t find a the Linear layer')
        freeze_everything(self.model)
        unfreeze_everything(cwr_layer)
<|endoftext|>