<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: Pedro Algarvio (pedro@algarvio.me)

    tests.unit.states.pip_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging

# Import Salt Testing libs
from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch

# Import salt libs
import salt.states.pip_state as pip_state
import salt.utils.versions

# Import 3rd-party libs
try:
    import pip
    HAS_PIP = True
except ImportError:
    HAS_PIP = False

log = logging.getLogger(__name__)

@skipIf(NO_MOCK, NO_MOCK_REASON)
@skipIf(not HAS_PIP,
        'The \'pip\' library is not importable(installed system-wide)')
class PipStateTest(TestCase, SaltReturnAssertsMixin, LoaderModuleMockMixin):

    def setup_loader_modules(self):
        return {
            pip_state: {
                '__env__': 'base',
                '__opts__': {'test': False},
                '__salt__': {'cmd.which_bin': lambda _: 'pip'}
            }
        }

    def test_install_requirements_parsing(self):
        log.debug("Real pip version is %s", pip.__version__)
        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
        pip_list = MagicMock(return_value={'pep8': '1.3.3'})
        pip_version = pip.__version__
        mock_pip_version = MagicMock(return_value=pip_version)
        with patch.dict(pip_state.__salt__, {'pip.version': mock_pip_version}):
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    log.debug(
                        'pip_state._from_line globals: %s',
                        [name for name in pip_state._from_line.__globals__]
                    )
                    ret = pip_state.installed('pep8=1.3.2')
                    self.assertSaltFalseReturn({'test': ret})
                    self.assertInSaltComment(
                        'Invalid version specification in package pep8=1.3.2. '
                        '\'=\' is not supported, use \'==\' instead.',
                        {'test': ret})

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.3'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>=1.3.2')
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>=1.3.2 was already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.3'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8<1.3.2')
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8<1.3.2 is set to be installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.2'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>1.3.1,<1.3.3')
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>1.3.1,<1.3.3 was already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>1.3.1,<1.3.3')
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>1.3.1,<1.3.3 is set to be installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting>=0.5.1'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package git+https://github.com/saltstack/'
                        'salt-testing.git#egg=SaltTesting>=0.5.1 is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package git+https://github.com/saltstack/'
                        'salt-testing.git#egg=SaltTesting is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'https://pypi.python.org/packages/source/S/SaltTesting/'
                        'SaltTesting-0.5.0.tar.gz'
                        '#md5=e6760af92b7165f8be53b5763e40bc24'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package https://pypi.python.org/packages/source/'
                        'S/SaltTesting/SaltTesting-0.5.0.tar.gz'
                        '#md5=e6760af92b7165f8be53b5763e40bc24 is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Downloading/unpacking https://pypi.python.org/packages'
                          '/source/S/SaltTesting/SaltTesting-0.5.0.tar.gz\n  '
                          'Downloading SaltTesting-0.5.0.tar.gz\n  Running '
                          'setup.py egg_info for package from '
                          'https://pypi.python.org/packages/source/S/SaltTesting/'
                          'SaltTesting-0.5.0.tar.gz\n    \nCleaning up...'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                ret = pip_state.installed(
                    'https://pypi.python.org/packages/source/S/SaltTesting/'
                    'SaltTesting-0.5.0.tar.gz'
                    '#md5=e6760af92b7165f8be53b5763e40bc24'
                )
                self.assertSaltTrueReturn({'test': ret})
                self.assertInSaltComment('All packages were successfully installed',
                    {'test': ret}
                )
                self.assertInSaltReturn(
                    'Installed',
                    {'test': ret},
                    ('changes', 'https://pypi.python.org/packages/source/S/'
                                'SaltTesting/SaltTesting-0.5.0.tar.gz'
                                '#md5=e6760af92b7165f8be53b5763e40bc24==???')
                )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Cloned!'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'packages are already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'arbitrary ID that should be ignored due to requirements specified',
                        requirements='/tmp/non-existing-requirements.txt'
                    )
                    self.assertSaltTrueReturn({'test': ret})

            # Test VCS installations using git+git://
            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Cloned!'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'git+git://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'packages are already installed',
                        {'test': ret}
                    )

            # Test VCS installations with version info like >= 0.1
            with patch.object(pip, '__version__', MagicMock(side_effect=AttributeError(
                                                        'Faked missing __version__ attribute'))):
                mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
                pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
                pip_install = MagicMock(return_value={
                    'retcode': 0,
                    'stderr': '',
                    'stdout': 'Cloned!'
                })
                with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                     'pip.list': pip_list,
                                                     'pip.install': pip_install}):
                    with patch.dict(pip_state.__opts__, {'test': False}):
                        ret = pip_state.installed(
                            'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting>=0.5.0'
                        )
                        self.assertSaltTrueReturn({'test': ret})
                        self.assertInSaltComment(
                            'packages are already installed',
                            {'test': ret}
                        )

    def test_install_in_editable_mode(self):
        '''
        Check that `name` parameter containing bad characters is not parsed by
        pip when package is being installed in editable mode.
        For more information, see issue #21890.
        '''
        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
        pip_list = MagicMock(return_value={})
        pip_install = MagicMock(return_value={
            'retcode': 0,
            'stderr': '',
            'stdout': 'Cloned!'
        })
        pip_version = MagicMock(return_value='10.0.1')
        with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                             'pip.list': pip_list,
                                             'pip.install': pip_install,
                                             'pip.version': pip_version}):
            ret = pip_state.installed('state@name',
                                      cwd='/path/to/project',
                                      editable=['.'])
            self.assertSaltTrueReturn({'test': ret})
            self.assertInSaltComment(
                'successfully installed',
                {'test': ret}
            )
<commit_msg>Fix unused import<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: Pedro Algarvio (pedro@algarvio.me)

    tests.unit.states.pip_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging

# Import Salt Testing libs
from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin
from tests.support.unit import skipIf, TestCase
from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch

# Import salt libs
import salt.states.pip_state as pip_state

# Import 3rd-party libs
try:
    import pip
    HAS_PIP = True
except ImportError:
    HAS_PIP = False

log = logging.getLogger(__name__)

@skipIf(NO_MOCK, NO_MOCK_REASON)
@skipIf(not HAS_PIP,
        'The \'pip\' library is not importable(installed system-wide)')
class PipStateTest(TestCase, SaltReturnAssertsMixin, LoaderModuleMockMixin):

    def setup_loader_modules(self):
        return {
            pip_state: {
                '__env__': 'base',
                '__opts__': {'test': False},
                '__salt__': {'cmd.which_bin': lambda _: 'pip'}
            }
        }

    def test_install_requirements_parsing(self):
        log.debug("Real pip version is %s", pip.__version__)
        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
        pip_list = MagicMock(return_value={'pep8': '1.3.3'})
        pip_version = pip.__version__
        mock_pip_version = MagicMock(return_value=pip_version)
        with patch.dict(pip_state.__salt__, {'pip.version': mock_pip_version}):
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    log.debug(
                        'pip_state._from_line globals: %s',
                        [name for name in pip_state._from_line.__globals__]
                    )
                    ret = pip_state.installed('pep8=1.3.2')
                    self.assertSaltFalseReturn({'test': ret})
                    self.assertInSaltComment(
                        'Invalid version specification in package pep8=1.3.2. '
                        '\'=\' is not supported, use \'==\' instead.',
                        {'test': ret})

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.3'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>=1.3.2')
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>=1.3.2 was already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.3'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8<1.3.2')
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8<1.3.2 is set to be installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.2'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>1.3.1,<1.3.3')
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>1.3.1,<1.3.3 was already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed('pep8>1.3.1,<1.3.3')
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package pep8>1.3.1,<1.3.3 is set to be installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting>=0.5.1'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package git+https://github.com/saltstack/'
                        'salt-testing.git#egg=SaltTesting>=0.5.1 is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package git+https://github.com/saltstack/'
                        'salt-testing.git#egg=SaltTesting is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list}):
                with patch.dict(pip_state.__opts__, {'test': True}):
                    ret = pip_state.installed(
                        'https://pypi.python.org/packages/source/S/SaltTesting/'
                        'SaltTesting-0.5.0.tar.gz'
                        '#md5=e6760af92b7165f8be53b5763e40bc24'
                    )
                    self.assertSaltNoneReturn({'test': ret})
                    self.assertInSaltComment(
                        'Python package https://pypi.python.org/packages/source/'
                        'S/SaltTesting/SaltTesting-0.5.0.tar.gz'
                        '#md5=e6760af92b7165f8be53b5763e40bc24 is set to be '
                        'installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Downloading/unpacking https://pypi.python.org/packages'
                          '/source/S/SaltTesting/SaltTesting-0.5.0.tar.gz\n  '
                          'Downloading SaltTesting-0.5.0.tar.gz\n  Running '
                          'setup.py egg_info for package from '
                          'https://pypi.python.org/packages/source/S/SaltTesting/'
                          'SaltTesting-0.5.0.tar.gz\n    \nCleaning up...'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                ret = pip_state.installed(
                    'https://pypi.python.org/packages/source/S/SaltTesting/'
                    'SaltTesting-0.5.0.tar.gz'
                    '#md5=e6760af92b7165f8be53b5763e40bc24'
                )
                self.assertSaltTrueReturn({'test': ret})
                self.assertInSaltComment('All packages were successfully installed',
                    {'test': ret}
                )
                self.assertInSaltReturn(
                    'Installed',
                    {'test': ret},
                    ('changes', 'https://pypi.python.org/packages/source/S/'
                                'SaltTesting/SaltTesting-0.5.0.tar.gz'
                                '#md5=e6760af92b7165f8be53b5763e40bc24==???')
                )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Cloned!'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'packages are already installed',
                        {'test': ret}
                    )

            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'pep8': '1.3.1'})
            pip_install = MagicMock(return_value={'retcode': 0})
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'arbitrary ID that should be ignored due to requirements specified',
                        requirements='/tmp/non-existing-requirements.txt'
                    )
                    self.assertSaltTrueReturn({'test': ret})

            # Test VCS installations using git+git://
            mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
            pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
            pip_install = MagicMock(return_value={
                'retcode': 0,
                'stderr': '',
                'stdout': 'Cloned!'
            })
            with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                 'pip.list': pip_list,
                                                 'pip.install': pip_install}):
                with patch.dict(pip_state.__opts__, {'test': False}):
                    ret = pip_state.installed(
                        'git+git://github.com/saltstack/salt-testing.git#egg=SaltTesting'
                    )
                    self.assertSaltTrueReturn({'test': ret})
                    self.assertInSaltComment(
                        'packages are already installed',
                        {'test': ret}
                    )

            # Test VCS installations with version info like >= 0.1
            with patch.object(pip, '__version__', MagicMock(side_effect=AttributeError(
                                                        'Faked missing __version__ attribute'))):
                mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
                pip_list = MagicMock(return_value={'SaltTesting': '0.5.0'})
                pip_install = MagicMock(return_value={
                    'retcode': 0,
                    'stderr': '',
                    'stdout': 'Cloned!'
                })
                with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                                     'pip.list': pip_list,
                                                     'pip.install': pip_install}):
                    with patch.dict(pip_state.__opts__, {'test': False}):
                        ret = pip_state.installed(
                            'git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting>=0.5.0'
                        )
                        self.assertSaltTrueReturn({'test': ret})
                        self.assertInSaltComment(
                            'packages are already installed',
                            {'test': ret}
                        )

    def test_install_in_editable_mode(self):
        '''
        Check that `name` parameter containing bad characters is not parsed by
        pip when package is being installed in editable mode.
        For more information, see issue #21890.
        '''
        mock = MagicMock(return_value={'retcode': 0, 'stdout': ''})
        pip_list = MagicMock(return_value={})
        pip_install = MagicMock(return_value={
            'retcode': 0,
            'stderr': '',
            'stdout': 'Cloned!'
        })
        pip_version = MagicMock(return_value='10.0.1')
        with patch.dict(pip_state.__salt__, {'cmd.run_all': mock,
                                             'pip.list': pip_list,
                                             'pip.install': pip_install,
                                             'pip.version': pip_version}):
            ret = pip_state.installed('state@name',
                                      cwd='/path/to/project',
                                      editable=['.'])
            self.assertSaltTrueReturn({'test': ret})
            self.assertInSaltComment(
                'successfully installed',
                {'test': ret}
            )
<|endoftext|>


<commit_before>import pytest

from matchzoo import tasks

def test_list_available_task_types():
    assert tasks.list_available_task_types()

@pytest.mark.parametrize("task_type", [
    tasks.Ranking, tasks.Classification
])
def test_task_listings(task_type):
    assert task_type.list_available_losses()
    assert task_type.list_available_metrics()

@pytest.mark.parametrize("arg", [None, -1, 0, 1])
def test_classification_instantiation_failure(arg):
    with pytest.raises(Exception):
        tasks.Classification(num_classes=arg)

@pytest.mark.parametrize("arg", [2, 10, 2048])
def test_classification_num_classes(arg):
    task = tasks.Classification(num_classes=arg)
    assert task.num_classes == arg

def test_list_available_tasks():
    assert tasks.list_available_task_types()
<commit_msg>Update test, 100% coverage<commit_after>import pytest

from matchzoo import tasks

def test_list_available_task_types():
    assert tasks.list_available_task_types()

@pytest.mark.parametrize("task_type", [
    tasks.Ranking, tasks.Classification
])
def test_task_listings(task_type):
    assert task_type.list_available_losses()
    assert task_type.list_available_metrics()

@pytest.mark.parametrize("arg", [None, -1, 0, 1])
def test_classification_instantiation_failure(arg):
    with pytest.raises(Exception):
        tasks.Classification(num_classes=arg)

@pytest.mark.parametrize("arg", [2, 10, 2048])
def test_classification_num_classes(arg):
    task = tasks.Classification(num_classes=arg)
    assert task.num_classes == arg

def test_list_available_tasks():
    assert tasks.list_available_task_types()

@pytest.mark.parametrize("task_type", [
    tasks.Ranking, tasks.Classification
])
def test_make_output_layers(task_type):
    assert task_type().make_output_layer()
<|endoftext|>


<commit_before>def mask(bits):
    return (1 << bits) - 1

def bitField(_from, to):
    """
    _from 0 to 1 -> '1'
    """
    w = to - _from
    return mask(w) << _from

def extendWithSet(mask, actualWidth, toWidth):
    return bitField(actualWidth - 1, toWidth) | mask

def selectBit(val, bitNo):
    """
    select bit from integer
    """
    return (val >> bitNo) & 1

def selectBitRange(val, bitsStart, bitsLen):
    val >>= bitsStart
    return val & mask(bitsLen)

def clean(val, bitNo):
    return val & ~(1 << bitNo)

def setBit(val, bitNo):
    return val | (1 << bitNo)

def toogle(val, bitNo):
    return val ^ (1 << bitNo)

def setBitRange(val, bitStart, bitsLen, newBits):
    _mask = mask(bitsLen)
    newBits &= _mask

    _mask <<= bitStart
    newBits <<= bitStart

    return (val & ~_mask) | newBits

def bitSetTo(val, bitNo, bitVal):
    if bitVal == 0:
        return clean(val, bitNo)
    elif bitVal == 1:
        return setBit(val, bitNo)
    else:
        raise NotImplementedError()

def align(val, lowerBitCntToAlign):
    val = val >> lowerBitCntToAlign
    return val << lowerBitCntToAlign
<commit_msg>bitmask:rm unused extendWithSet<commit_after>def mask(bits):
    return (1 << bits) - 1

def bitField(_from, to):
    """
    _from 0 to 1 -> '1'
    """
    w = to - _from
    return mask(w) << _from

def selectBit(val, bitNo):
    """
    select bit from integer
    """
    return (val >> bitNo) & 1

def selectBitRange(val, bitsStart, bitsLen):
    val >>= bitsStart
    return val & mask(bitsLen)

def clean(val, bitNo):
    return val & ~(1 << bitNo)

def setBit(val, bitNo):
    return val | (1 << bitNo)

def toogle(val, bitNo):
    return val ^ (1 << bitNo)

def setBitRange(val, bitStart, bitsLen, newBits):
    _mask = mask(bitsLen)
    newBits &= _mask

    _mask <<= bitStart
    newBits <<= bitStart

    return (val & ~_mask) | newBits

def bitSetTo(val, bitNo, bitVal):
    if bitVal == 0:
        return clean(val, bitNo)
    elif bitVal == 1:
        return setBit(val, bitNo)
    else:
        raise NotImplementedError()

def align(val, lowerBitCntToAlign):
    val = val >> lowerBitCntToAlign
    return val << lowerBitCntToAlign
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import datetime
import json

import apiclient.discovery
import apiclient.http
import gdata.gauth
import gdata.spreadsheets.client
import gdata.spreadsheets.data
import httplib2
import oauth2client.client

from . import util

GOOGLE_SPREADSHEET_SCOPES = (
    'https://spreadsheets.google.com/feeds',
    'https://www.googleapis.com/auth/drive',
)

SPREADSHEET_URL = (
    'https://spreadsheets.google.com/feeds/spreadsheets/private/full/%s')

class Collection(util.LazyOrderedDictionary):
  def __init__(self, client, drive):
    super(Collection, self).__init__(
        self._spreadsheet_enumerator,
        self._spreadsheet_constructor)
    self.client = client
    self.drive = drive

  @classmethod
  def login(cls, json_path=None, json_text=None):
    if json_text is None:
      with open(json_path, 'rb') as f:
        json_text = f.read()
    json_data = json.loads(json_text)
    if '_module' in json_data:
      credentials = oauth2client.client.Credentials.new_from_json(json_text)
    elif 'private_key' in json_data:
      credentials = oauth2client.client.SignedJwtAssertionCredentials(
          service_account_name=json_data['client_email'],
          private_key=json_data['private_key'],
          scope=GOOGLE_SPREADSHEET_SCOPES)
    else:
      raise ValueError('unrecognized credential format')
    # Don't use auth_token= argument. It does not refresh tokens.
    client = gdata.spreadsheets.client.SpreadsheetsClient()
    auth_http = httplib2.Http()
    auth_token = gdata.gauth.OAuth2TokenFromCredentials(credentials)
    auth_token.authorize(client)
    auth_http = credentials.authorize(auth_http)
    drive = apiclient.discovery.build('drive', 'v2', http=auth_http)
    return cls(client, drive)

  def create_spreadsheet(self, title, rows=1000, cols=26):
    body = {
        'title': title,
        'mimeType': 'application/vnd.google-apps.spreadsheet',
    }
    response = self.drive.files().insert(body=body).execute()
    key = response['id']
    self.refresh()
    spreadsheet = self[key]
    if (rows, cols) != (1000, 26):
      spreadsheet[0].set_size(rows, cols)
    return spreadsheet

  def _spreadsheet_enumerator(self):
    feed = self.client.get_spreadsheets()
    for entry in feed.entry:
      key = entry.get_spreadsheet_key()
      yield (key, Spreadsheet(self, self.client, self.drive, key, entry))

  def _spreadsheet_constructor(self, key):
    # TODO: Upstream to gdata.
    entry = self.client.get_feed(
        SPREADSHEET_URL % key,
        desired_class=gdata.spreadsheets.data.Spreadsheet)
    key = entry.get_spreadsheet_key()
    return Spreadsheet(self, self.client, self.drive, key, entry)

class Spreadsheet(util.LazyOrderedDictionary):
  def __init__(self, collection, client, drive, key, entry):
    super(Spreadsheet, self).__init__(self._worksheet_enumerator, None)
    self.collection = collection
    self.client = client
    self.drive = drive
    self.key = key
    self._entry = entry

  def refresh(self):
    super(Spreadsheet, self).refresh()
    # TODO: Upstream to gdata.
    self._entry = self.client.get_feed(
        SPREADSHEET_URL % self.key,
        desired_class=gdata.spreadsheets.data.Spreadsheet)

  def add_worksheet(self, title, rows=1000, cols=26):
    self.client.add_worksheet(self.key, title, rows=rows, cols=cols)
    self.refresh()
    return self[title]

  def delete_worksheet(self, title):
    worksheet = self[title]
    url = gdata.spreadsheets.client.WORKSHEET_URL % (self.key, worksheet.key)
    self.client.delete(url, force=True)
    self.refresh()

  @property
  def url(self):
    return 'https://docs.google.com/spreadsheets/d/%s/edit' % self.key

  @property
  def title(self):
    return self._entry.title.text

  @title.setter
  def title(self, new_title):
    body = {'title': new_title}
    response = self.drive.files().update(fileId=self.key, body=body).execute()
    self.refresh()

  @property
  def updated(self):
    return datetime.datetime.strptime(
        self._entry.updated.text, '%Y-%m-%dT%H:%M:%S.%fZ')

  def _worksheet_enumerator(self):
    feed = self.client.get_worksheets(self.key)
    for entry in feed.entry:
      key = entry.get_worksheet_id()
      worksheet = Worksheet(self, self.client, key, entry)
      yield (worksheet.title, worksheet)

class WorksheetView(object):
  def __init__(self, worksheet, client, start_row, end_row, start_col, end_col):
    self.worksheet = worksheet
    self.client = client
    self._reset_size(start_row, end_row, start_col, end_col)
    self._input_value_map = {}
    self._cells_fetched = False
    self._queued_updates = []

  def refresh(self):
    self._input_value_map.clear()
    self._cells_fetched = False
    del self._queued_updates[:]

  def _reset_size(self, start_row, end_row, start_col, end_col):
    self.start_row = start_row
    self.end_row = end_row
    self.start_col = start_col
    self.end_col = end_col
    self._view_rows = [
        WorksheetViewRow(self, row, start_col, end_col)
        for row in xrange(start_row, end_row)]

  def _ensure_cells_fetched(self):
    if self._cells_fetched:
      return
    query = gdata.spreadsheets.client.CellQuery(
        min_row=(self.start_row + 1),
        max_row=self.end_row,
        min_col=(self.start_col + 1),
        max_col=self.end_col,
        return_empty=False)
    feed = self.client.get_cells(
        self.worksheet.spreadsheet.key, self.worksheet.key, query=query)
    self._input_value_map = {}
    for entry in feed.entry:
      cell = entry.cell
      self._input_value_map.setdefault(
          (int(cell.row) - 1, int(cell.col) - 1),
          cell.input_value)
    self._cells_fetched = True

  def commit(self):
    if not self._queued_updates:
      return
    feed = gdata.spreadsheets.data.build_batch_cells_update(
        self.worksheet.spreadsheet.key, self.worksheet.key)
    for row, col, new_value in self._queued_updates:
      feed.add_set_cell(row + 1, col + 1, new_value)
    self.client.batch(feed, force=True)
    del self._queued_updates[:]

  def __nonzero__(self):
    return len(self) > 0

  def __getitem__(self, index):
    return self._view_rows[index]

  def __len__(self):
    return self.rows

  def __iter__(self):
    for row in self._view_rows:
      yield row

  def __repr__(self):
    return repr(self._view_rows)

  def __enter__(self):
    return self

  def __exit__(self, exc_type, exc_value, traceback):
    self.commit()

  @property
  def rows(self):
    return self.end_row - self.start_row

  @property
  def cols(self):
    return self.end_col - self.start_col

class WorksheetViewRow(util.CustomMutableFixedList):
  def __init__(self, view, row, start_col, end_col):
    self._view = view
    self._row = row
    self._start_col = start_col
    self._end_col = end_col

  def __nonzero__(self):
    return len(self) > 0

  def __getitem__(self, index):
    if isinstance(index, slice):
      start, stop, step = index.indices(len(self))
      assert step == 1, 'slicing with step is not supported'
      if stop < start:
        stop = start
      return WorksheetViewRow(
          self._view, self._row,
          self._start_col + start, self._end_col + stop)
    assert isinstance(index, int)
    col = self._start_col + index
    if not (self._start_col <= col < self._end_col):
      raise IndexError()
    if (self._row, col) not in self._view._input_value_map:
      self._view._ensure_cells_fetched()
    return self._view._input_value_map.get((self._row, col), '')

  def __setitem__(self, index, new_value):
    assert isinstance(index, int)
    col = self._start_col + index
    if not (self._start_col <= col < self._end_col):
      raise IndexError()
    if new_value is None:
      pass
    elif isinstance(new_value, int):
      new_value = '%d' % new_value
    elif isinstance(new_value, float):
      # Do best not to lose precision...
      new_value = '%20e' % new_value
    elif isinstance(new_value, str):
      # May raise UnicodeDecodeError.
      new_value.decode('ascii')
    elif not isinstance(new_value, unicode):
      new_value = unicode(new_value)
    self._view._input_value_map[(self._row, col)] = new_value
    self._view._queued_updates.append((self._row, col, new_value))

  def __len__(self):
    return self._end_col - self._start_col

  def __iter__(self):
    self._view._ensure_cells_fetched()
    for col in xrange(self._start_col, self._end_col):
      yield self._view._input_value_map.get((self._row, col), '')

  def __repr__(self):
    return repr([self[i] for i in xrange(len(self))])

class Worksheet(WorksheetView):
  def __init__(self, spreadsheet, client, key, entry):
    self.spreadsheet = spreadsheet
    self.client = client
    self.key = key
    self._entry = entry
    super(Worksheet, self).__init__(self, client, 0, self.rows, 0, self.cols)

  def refresh(self):
    self._entry = self.client.get_worksheet(self.spreadsheet.key, self.key)
    self._reset_size(0, self.rows, 0, self.cols)
    super(Worksheet, self).refresh()

  def view(self, start_row=None, end_row=None, start_col=None, end_col=None):
    if start_row is None:
      start_row = 0
    if end_row is None:
      end_row = self.rows
    if start_col is None:
      start_col = 0
    if end_col is None:
      end_col = self.cols
    if not (0 <= start_row <= end_row <= self.rows):
      raise IndexError()
    if not (0 <= start_col <= end_col <= self.cols):
      raise IndexError()
    return WorksheetView(
        self, self.client,
        start_row=start_row, end_row=end_row,
        start_col=start_col, end_col=end_col)

  def set_size(self, rows, cols):
    assert isinstance(rows, int) and rows > 0
    assert isinstance(cols, int) and cols > 0
    self._entry.row_count.text = str(rows)
    self._entry.col_count.text = str(cols)
    self._update()

  @property
  def title(self):
    return self._entry.title.text

  @title.setter
  def title(self, new_title):
    self._entry.title.text = new_title
    self._update()

  @property
  def rows(self):
    return int(self._entry.row_count.text)

  @rows.setter
  def rows(self, rows):
    assert isinstance(rows, int) and rows > 0
    self._entry.row_count.text = str(rows)
    self._update()

  @property
  def cols(self):
    return int(self._entry.col_count.text)

  @cols.setter
  def cols(self, cols):
    assert isinstance(cols, int) and cols > 0
    self._entry.col_count.text = str(cols)
    self._update()

  def _update(self):
    url = gdata.spreadsheets.client.WORKSHEET_URL % (
        self.spreadsheet.key, self.key)
    # TODO: Use returned entry to speed up
    unused_entry = self.client.update(self._entry, uri=url, force=True)
    self.refresh()
<commit_msg>Allow negative indexing.<commit_after># Copyright 2015 Google Inc. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import datetime
import json

import apiclient.discovery
import apiclient.http
import gdata.gauth
import gdata.spreadsheets.client
import gdata.spreadsheets.data
import httplib2
import oauth2client.client

from . import util

GOOGLE_SPREADSHEET_SCOPES = (
    'https://spreadsheets.google.com/feeds',
    'https://www.googleapis.com/auth/drive',
)

SPREADSHEET_URL = (
    'https://spreadsheets.google.com/feeds/spreadsheets/private/full/%s')

class Collection(util.LazyOrderedDictionary):
  def __init__(self, client, drive):
    super(Collection, self).__init__(
        self._spreadsheet_enumerator,
        self._spreadsheet_constructor)
    self.client = client
    self.drive = drive

  @classmethod
  def login(cls, json_path=None, json_text=None):
    if json_text is None:
      with open(json_path, 'rb') as f:
        json_text = f.read()
    json_data = json.loads(json_text)
    if '_module' in json_data:
      credentials = oauth2client.client.Credentials.new_from_json(json_text)
    elif 'private_key' in json_data:
      credentials = oauth2client.client.SignedJwtAssertionCredentials(
          service_account_name=json_data['client_email'],
          private_key=json_data['private_key'],
          scope=GOOGLE_SPREADSHEET_SCOPES)
    else:
      raise ValueError('unrecognized credential format')
    # Don't use auth_token= argument. It does not refresh tokens.
    client = gdata.spreadsheets.client.SpreadsheetsClient()
    auth_http = httplib2.Http()
    auth_token = gdata.gauth.OAuth2TokenFromCredentials(credentials)
    auth_token.authorize(client)
    auth_http = credentials.authorize(auth_http)
    drive = apiclient.discovery.build('drive', 'v2', http=auth_http)
    return cls(client, drive)

  def create_spreadsheet(self, title, rows=1000, cols=26):
    body = {
        'title': title,
        'mimeType': 'application/vnd.google-apps.spreadsheet',
    }
    response = self.drive.files().insert(body=body).execute()
    key = response['id']
    self.refresh()
    spreadsheet = self[key]
    if (rows, cols) != (1000, 26):
      spreadsheet[0].set_size(rows, cols)
    return spreadsheet

  def _spreadsheet_enumerator(self):
    feed = self.client.get_spreadsheets()
    for entry in feed.entry:
      key = entry.get_spreadsheet_key()
      yield (key, Spreadsheet(self, self.client, self.drive, key, entry))

  def _spreadsheet_constructor(self, key):
    # TODO: Upstream to gdata.
    entry = self.client.get_feed(
        SPREADSHEET_URL % key,
        desired_class=gdata.spreadsheets.data.Spreadsheet)
    key = entry.get_spreadsheet_key()
    return Spreadsheet(self, self.client, self.drive, key, entry)

class Spreadsheet(util.LazyOrderedDictionary):
  def __init__(self, collection, client, drive, key, entry):
    super(Spreadsheet, self).__init__(self._worksheet_enumerator, None)
    self.collection = collection
    self.client = client
    self.drive = drive
    self.key = key
    self._entry = entry

  def refresh(self):
    super(Spreadsheet, self).refresh()
    # TODO: Upstream to gdata.
    self._entry = self.client.get_feed(
        SPREADSHEET_URL % self.key,
        desired_class=gdata.spreadsheets.data.Spreadsheet)

  def add_worksheet(self, title, rows=1000, cols=26):
    self.client.add_worksheet(self.key, title, rows=rows, cols=cols)
    self.refresh()
    return self[title]

  def delete_worksheet(self, title):
    worksheet = self[title]
    url = gdata.spreadsheets.client.WORKSHEET_URL % (self.key, worksheet.key)
    self.client.delete(url, force=True)
    self.refresh()

  @property
  def url(self):
    return 'https://docs.google.com/spreadsheets/d/%s/edit' % self.key

  @property
  def title(self):
    return self._entry.title.text

  @title.setter
  def title(self, new_title):
    body = {'title': new_title}
    response = self.drive.files().update(fileId=self.key, body=body).execute()
    self.refresh()

  @property
  def updated(self):
    return datetime.datetime.strptime(
        self._entry.updated.text, '%Y-%m-%dT%H:%M:%S.%fZ')

  def _worksheet_enumerator(self):
    feed = self.client.get_worksheets(self.key)
    for entry in feed.entry:
      key = entry.get_worksheet_id()
      worksheet = Worksheet(self, self.client, key, entry)
      yield (worksheet.title, worksheet)

class WorksheetView(object):
  def __init__(self, worksheet, client, start_row, end_row, start_col, end_col):
    self.worksheet = worksheet
    self.client = client
    self._reset_size(start_row, end_row, start_col, end_col)
    self._input_value_map = {}
    self._cells_fetched = False
    self._queued_updates = []

  def refresh(self):
    self._input_value_map.clear()
    self._cells_fetched = False
    del self._queued_updates[:]

  def _reset_size(self, start_row, end_row, start_col, end_col):
    self.start_row = start_row
    self.end_row = end_row
    self.start_col = start_col
    self.end_col = end_col
    self._view_rows = [
        WorksheetViewRow(self, row, start_col, end_col)
        for row in xrange(start_row, end_row)]

  def _ensure_cells_fetched(self):
    if self._cells_fetched:
      return
    query = gdata.spreadsheets.client.CellQuery(
        min_row=(self.start_row + 1),
        max_row=self.end_row,
        min_col=(self.start_col + 1),
        max_col=self.end_col,
        return_empty=False)
    feed = self.client.get_cells(
        self.worksheet.spreadsheet.key, self.worksheet.key, query=query)
    self._input_value_map = {}
    for entry in feed.entry:
      cell = entry.cell
      self._input_value_map.setdefault(
          (int(cell.row) - 1, int(cell.col) - 1),
          cell.input_value)
    self._cells_fetched = True

  def commit(self):
    if not self._queued_updates:
      return
    feed = gdata.spreadsheets.data.build_batch_cells_update(
        self.worksheet.spreadsheet.key, self.worksheet.key)
    for row, col, new_value in self._queued_updates:
      feed.add_set_cell(row + 1, col + 1, new_value)
    self.client.batch(feed, force=True)
    del self._queued_updates[:]

  def __nonzero__(self):
    return len(self) > 0

  def __getitem__(self, index):
    return self._view_rows[index]

  def __len__(self):
    return self.rows

  def __iter__(self):
    for row in self._view_rows:
      yield row

  def __repr__(self):
    return repr(self._view_rows)

  def __enter__(self):
    return self

  def __exit__(self, exc_type, exc_value, traceback):
    self.commit()

  @property
  def rows(self):
    return self.end_row - self.start_row

  @property
  def cols(self):
    return self.end_col - self.start_col

class WorksheetViewRow(util.CustomMutableFixedList):
  def __init__(self, view, row, start_col, end_col):
    self._view = view
    self._row = row
    self._start_col = start_col
    self._end_col = end_col

  def __nonzero__(self):
    return len(self) > 0

  def __getitem__(self, index):
    if isinstance(index, slice):
      start, stop, step = index.indices(len(self))
      assert step == 1, 'slicing with step is not supported'
      if stop < start:
        stop = start
      return WorksheetViewRow(
          self._view, self._row,
          self._start_col + start, self._end_col + stop)
    assert isinstance(index, int)
    if index < 0:
      col = self._end_col + index
    else:
      col = self._start_col + index
    if not (self._start_col <= col < self._end_col):
      raise IndexError()
    if (self._row, col) not in self._view._input_value_map:
      self._view._ensure_cells_fetched()
    return self._view._input_value_map.get((self._row, col), '')

  def __setitem__(self, index, new_value):
    assert isinstance(index, int)
    if index < 0:
      col = self._end_col + index
    else:
      col = self._start_col + index
    if not (self._start_col <= col < self._end_col):
      raise IndexError()
    if new_value is None:
      pass
    elif isinstance(new_value, int):
      new_value = '%d' % new_value
    elif isinstance(new_value, float):
      # Do best not to lose precision...
      new_value = '%20e' % new_value
    elif isinstance(new_value, str):
      # May raise UnicodeDecodeError.
      new_value.decode('ascii')
    elif not isinstance(new_value, unicode):
      new_value = unicode(new_value)
    self._view._input_value_map[(self._row, col)] = new_value
    self._view._queued_updates.append((self._row, col, new_value))

  def __len__(self):
    return self._end_col - self._start_col

  def __iter__(self):
    self._view._ensure_cells_fetched()
    for col in xrange(self._start_col, self._end_col):
      yield self._view._input_value_map.get((self._row, col), '')

  def __repr__(self):
    return repr([self[i] for i in xrange(len(self))])

class Worksheet(WorksheetView):
  def __init__(self, spreadsheet, client, key, entry):
    self.spreadsheet = spreadsheet
    self.client = client
    self.key = key
    self._entry = entry
    super(Worksheet, self).__init__(self, client, 0, self.rows, 0, self.cols)

  def refresh(self):
    self._entry = self.client.get_worksheet(self.spreadsheet.key, self.key)
    self._reset_size(0, self.rows, 0, self.cols)
    super(Worksheet, self).refresh()

  def view(self, start_row=None, end_row=None, start_col=None, end_col=None):
    if start_row is None:
      start_row = 0
    if end_row is None:
      end_row = self.rows
    if start_col is None:
      start_col = 0
    if end_col is None:
      end_col = self.cols
    if not (0 <= start_row <= end_row <= self.rows):
      raise IndexError()
    if not (0 <= start_col <= end_col <= self.cols):
      raise IndexError()
    return WorksheetView(
        self, self.client,
        start_row=start_row, end_row=end_row,
        start_col=start_col, end_col=end_col)

  def set_size(self, rows, cols):
    assert isinstance(rows, int) and rows > 0
    assert isinstance(cols, int) and cols > 0
    self._entry.row_count.text = str(rows)
    self._entry.col_count.text = str(cols)
    self._update()

  @property
  def title(self):
    return self._entry.title.text

  @title.setter
  def title(self, new_title):
    self._entry.title.text = new_title
    self._update()

  @property
  def rows(self):
    return int(self._entry.row_count.text)

  @rows.setter
  def rows(self, rows):
    assert isinstance(rows, int) and rows > 0
    self._entry.row_count.text = str(rows)
    self._update()

  @property
  def cols(self):
    return int(self._entry.col_count.text)

  @cols.setter
  def cols(self, cols):
    assert isinstance(cols, int) and cols > 0
    self._entry.col_count.text = str(cols)
    self._update()

  def _update(self):
    url = gdata.spreadsheets.client.WORKSHEET_URL % (
        self.spreadsheet.key, self.key)
    # TODO: Use returned entry to speed up
    unused_entry = self.client.update(self._entry, uri=url, force=True)
    self.refresh()
<|endoftext|>


<commit_before>import struct
import logging
from collections import namedtuple

import six

logger = logging.getLogger(__name__)

class TAGS:
    '''
    via: https://www.hex-rays.com/products/ida/support/sdkdoc/group__nn__res.html#gaedcc558fe55e19ebc6e304ba7ad8c4d6
    '''
    ALTVAL = 'A'
    SUPVAL = 'S'
    CHARVAL = 'C'  # this is just a guess...
    HASHVAL = 'H'
    VALUE = 'V'
    NAME = 'N'
    LINK = 'L'

def make_key(nodeid, tag=None, index=None, wordsize=4):
    '''

    Example::

        k = make_key('Root Node')

    Example::

        k = make_key(0x401000, 'X')

    Example::

        k = make_key(0x401000, 'X', 0x4010A24)
    '''
    if wordsize == 4:
        wordformat = 'I'
    elif wordsize == 8:
        wordformat = 'Q'
    else:
        raise ValueError('unexpected wordsize')

    if isinstance(nodeid, six.string_types):
        return b'N' + nodeid.encode('utf-8')

    elif isinstance(nodeid, six.integer_types):
        if tag is None:
            raise ValueError('tag required')
        if not isinstance(tag, str):
            raise ValueError('tag must be a string')
        if len(tag) != 1:
            raise ValueError('tag must be a single character string')

        tag = tag.encode('ascii')

        if index is None:
            return b'.' + struct.pack('>' + wordformat + 'c', nodeid, tag)
        else:
            return b'.' + struct.pack('>' + wordformat + 'c' + wordformat.lower(), nodeid, tag, index)
    else:
        raise ValueError('unexpected type of nodeid: ' + str(type(nodeid)))

ComplexKey = namedtuple('ComplexKey', ['nodeid', 'tag', 'index'])

TAG_LENGTH = 1
KEY_HEADER_LENGTH = 1

def parse_key(buf, wordsize=4):
    if six.indexbytes(buf, 0x0) != 0x2E:
        raise ValueError('buf is not a complex key')

    if wordsize == 4:
        wordformat = 'I'
    elif wordsize == 8:
        wordformat = 'Q'
    else:
        raise ValueError('unexpected wordsize')

    nodeid, tag = struct.unpack_from('>' + wordformat + 'c', buf, 1)
    tag = tag.decode('ascii')
    if len(buf) > TAG_LENGTH + wordsize + KEY_HEADER_LENGTH:
        offset = TAG_LENGTH + KEY_HEADER_LENGTH + wordsize
        index = struct.unpack_from('>' + wordformat.lower(), buf, offset)[0]
    else:
        index = None

    return ComplexKey(nodeid, tag, index)

def as_uint(buf, wordsize=None):
    if len(buf) == 1:
        return struct.unpack('<B', buf)[0]
    elif len(buf) == 2:
        return struct.unpack('<H', buf)[0]
    elif len(buf) == 4:
        return struct.unpack('<L', buf)[0]
    elif len(buf) == 8:
        return struct.unpack('<Q', buf)[0]
    else:
        return RuntimeError('unexpected buf size')

def as_int(buf, wordsize=None):
    if len(buf) == 1:
        return struct.unpack('<b', buf)[0]
    elif len(buf) == 2:
        return struct.unpack('<h', buf)[0]
    elif len(buf) == 4:
        return struct.unpack('<l', buf)[0]
    elif len(buf) == 8:
        return struct.unpack('<q', buf)[0]
    else:
        return RuntimeError('unexpected buf size')

def as_string(buf, wordsize=None):
    return bytes(buf).rstrip(b'\x00').decode('utf-8').rstrip('\x00')

# try to implement the methods here:
#
#   https://www.hex-rays.com/products/ida/support/sdkdoc/classnetnode.html

Entry = namedtuple('Entry', ['key', 'parsed_key', 'value'])

class Netnode(object):
    def __init__(self, db, nodeid):
        '''
        Args:
          db (idb.IDB): the IDA Pro database.
          nodeid (Union[str, int]): the node id used to identify the netnode.

        Example::

            nn = Netnode("Root Node")
            print(nn.supval(1303))  # --> "6.95"

        Example::

            nn = Netnode(0x401000)
            for xref in nn.alts(tag='X'):
                print(xref)

        TODO: how to address the following keys:
          - $ MAX LINK
          - $ MAX NODE
          - $ MAX DESC

        these are unaddressable via IDA Pro netnodes, too.
        '''
        self.idb = db
        self.wordsize = self.idb.wordsize
        if self.wordsize == 4:
            self.nodebase = 0xFF000000
        elif self.wordsize == 8:
            self.nodebase = 0xFF00000000000000
        else:
            raise RuntimeError('unexpected wordsize')

        if isinstance(nodeid, six.string_types):
            key = make_key(nodeid, wordsize=self.wordsize)
            cursor = self.idb.id0.find(key)
            self.nodeid = as_uint(cursor.value)
            logger.info('resolved string netnode %s to %x', nodeid, self.nodeid)
        elif isinstance(nodeid, six.integer_types):
            self.nodeid = nodeid
        else:
            raise ValueError('unexpected type for nodeid')

    @staticmethod
    def get_nodebase(db):
        if db.wordsize == 4:
            return 0xFF000000
        elif db.wordsize == 8:
            return 0xFF00000000000000

    def name(self):
        '''
        fetch the name associated with the netnode.
        basically supval(tag='N')

        Returns:
          str: the name stored in the netnode.

        Raises:
          KeyError: if the name for the netnode does not exist.
        '''
        key = make_key(self.nodeid, TAGS.NAME, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return as_string(cursor.value)

    def get_tag_entries(self, tag=TAGS.SUPVAL):
        '''
        generate the entries for the given tag in this netnode.

        this replaces:
          - *1st
          - *nxt
          - *last
          - *prev

        Yields:
          Entry: an entry (with key and value) under the given tag in this netnode.
        '''
        key = make_key(self.nodeid, tag, wordsize=self.wordsize)

        cursor = self.idb.id0.find_prefix(key)
        while bytes(cursor.key).startswith(key):
            parsed_key = parse_key(cursor.key, wordsize=self.idb.wordsize)
            yield Entry(cursor.key, parsed_key, cursor.value)
            try:
                cursor.next()
            except IndexError:
                break

    def get_val(self, index, tag=TAGS.SUPVAL):
        '''
        fetch a sup/alt/hash/etc value from the netnode.
        the nodeid for this netnode must be an integer/effective address.

        Args:
          index (int): the index of the data to fetch.
          tag (str): single character tag.

        Returns:
          bytes: the raw data.
        '''
        key = make_key(self.nodeid, tag, index, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return bytes(cursor.value)

    def supval(self, index, tag=TAGS.SUPVAL):
        return self.get_val(index, tag)

    def supstr(self, index, tag=TAGS.SUPVAL):
        return as_string(self.supval(index, tag))

    def sups(self, tag=TAGS.SUPVAL):
        '''
        this replaces:
          - sup1st
          - supnxt
          - suplast
          - supprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def supentries(self, tag=TAGS.SUPVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield entry

    def altval(self, index, tag=TAGS.ALTVAL):
        return as_int(self.get_val(index, tag))

    def alts(self, tag=TAGS.ALTVAL):
        '''
        this replaces:
          - alt1st
          - altnxt
          - altlast
          - altprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def altentries(self, tag=TAGS.ALTVAL):
        for entry in self.get_tag_entries(tag=tag):
            # TODO: cast the value?
            yield entry

    def charval(self, index, tag=TAGS.CHARVAL):
        return as_int(self.get_val(index, tag))

    def chars(self, tag=TAGS.ALTVAL):
        '''
        this replaces:
          - char1st
          - charnxt
          - charlast
          - charprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def charentries(self, tag=TAGS.CHARVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield Entry(entry.key, entry.parsed_key, as_int(entry.value))

    def hashval(self, index, tag=TAGS.HASHVAL):
        '''
        TODO: how is this different from a supval?
        '''
        return self.get_val(index, tag)

    def hashes(self, tag=TAGS.HASHVAL):
        '''
        this replaces:
          - hash1st
          - hashnxt
          - hashlast
          - hashprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def hashentries(self, tag=TAGS.HASHVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield entry

    def valobj(self):
        '''
        fetch the default netnode value.
        this is basically supval(tag='V').
        '''
        key = make_key(self.nodeid, TAGS.VALUE, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return bytes(cursor.value)

    def valstr(self):
        return as_string(self.valobj())

    def value_exists(self):
        try:
            return self.valobj() is not None
        except KeyError:
            return False

    def long_value(self):
        return as_uint(self.valobj())

    def blobsize(self):
        '''
        TODO: how is this arbitrary data stored?
        '''
        raise NotImplementedError()

    def getblob(self):
        raise NotImplementedError()
<commit_msg>netnode: correctly handle signed netnode index when emitting the key<commit_after>import struct
import logging
from collections import namedtuple

import six

logger = logging.getLogger(__name__)

class TAGS:
    '''
    via: https://www.hex-rays.com/products/ida/support/sdkdoc/group__nn__res.html#gaedcc558fe55e19ebc6e304ba7ad8c4d6
    '''
    ALTVAL = 'A'
    SUPVAL = 'S'
    CHARVAL = 'C'  # this is just a guess...
    HASHVAL = 'H'
    VALUE = 'V'
    NAME = 'N'
    LINK = 'L'

def make_key(nodeid, tag=None, index=None, wordsize=4):
    '''

    Example::

        k = make_key('Root Node')

    Example::

        k = make_key(0x401000, 'X')

    Example::

        k = make_key(0x401000, 'X', 0x4010A24)
    '''
    if wordsize == 4:
        wordformat = 'I'
    elif wordsize == 8:
        wordformat = 'Q'
    else:
        raise ValueError('unexpected wordsize')

    if isinstance(nodeid, six.string_types):
        return b'N' + nodeid.encode('utf-8')

    elif isinstance(nodeid, six.integer_types):
        if tag is None:
            raise ValueError('tag required')
        if not isinstance(tag, str):
            raise ValueError('tag must be a string')
        if len(tag) != 1:
            raise ValueError('tag must be a single character string')

        tag = tag.encode('ascii')

        if index is None:
            return b'.' + struct.pack('>' + wordformat + 'c', nodeid, tag)
        elif index < 0:
            return b'.' + struct.pack('>' + wordformat + 'c' + wordformat.lower(), nodeid, tag, index)
        else:
            return b'.' + struct.pack('>' + wordformat + 'c' + wordformat, nodeid, tag, index)
    else:
        raise ValueError('unexpected type of nodeid: ' + str(type(nodeid)))

ComplexKey = namedtuple('ComplexKey', ['nodeid', 'tag', 'index'])

TAG_LENGTH = 1
KEY_HEADER_LENGTH = 1

def parse_key(buf, wordsize=4):
    if six.indexbytes(buf, 0x0) != 0x2E:
        raise ValueError('buf is not a complex key')

    if wordsize == 4:
        wordformat = 'I'
    elif wordsize == 8:
        wordformat = 'Q'
    else:
        raise ValueError('unexpected wordsize')

    nodeid, tag = struct.unpack_from('>' + wordformat + 'c', buf, 1)
    tag = tag.decode('ascii')
    if len(buf) > TAG_LENGTH + wordsize + KEY_HEADER_LENGTH:
        offset = TAG_LENGTH + KEY_HEADER_LENGTH + wordsize
        index = struct.unpack_from('>' + wordformat.lower(), buf, offset)[0]
    else:
        index = None

    return ComplexKey(nodeid, tag, index)

def as_uint(buf, wordsize=None):
    if len(buf) == 1:
        return struct.unpack('<B', buf)[0]
    elif len(buf) == 2:
        return struct.unpack('<H', buf)[0]
    elif len(buf) == 4:
        return struct.unpack('<L', buf)[0]
    elif len(buf) == 8:
        return struct.unpack('<Q', buf)[0]
    else:
        return RuntimeError('unexpected buf size')

def as_int(buf, wordsize=None):
    if len(buf) == 1:
        return struct.unpack('<b', buf)[0]
    elif len(buf) == 2:
        return struct.unpack('<h', buf)[0]
    elif len(buf) == 4:
        return struct.unpack('<l', buf)[0]
    elif len(buf) == 8:
        return struct.unpack('<q', buf)[0]
    else:
        return RuntimeError('unexpected buf size')

def as_string(buf, wordsize=None):
    return bytes(buf).rstrip(b'\x00').decode('utf-8').rstrip('\x00')

# try to implement the methods here:
#
#   https://www.hex-rays.com/products/ida/support/sdkdoc/classnetnode.html

Entry = namedtuple('Entry', ['key', 'parsed_key', 'value'])

class Netnode(object):
    def __init__(self, db, nodeid):
        '''
        Args:
          db (idb.IDB): the IDA Pro database.
          nodeid (Union[str, int]): the node id used to identify the netnode.

        Example::

            nn = Netnode("Root Node")
            print(nn.supval(1303))  # --> "6.95"

        Example::

            nn = Netnode(0x401000)
            for xref in nn.alts(tag='X'):
                print(xref)

        TODO: how to address the following keys:
          - $ MAX LINK
          - $ MAX NODE
          - $ MAX DESC

        these are unaddressable via IDA Pro netnodes, too.
        '''
        self.idb = db
        self.wordsize = self.idb.wordsize
        if self.wordsize == 4:
            self.nodebase = 0xFF000000
        elif self.wordsize == 8:
            self.nodebase = 0xFF00000000000000
        else:
            raise RuntimeError('unexpected wordsize')

        if isinstance(nodeid, six.string_types):
            key = make_key(nodeid, wordsize=self.wordsize)
            cursor = self.idb.id0.find(key)
            self.nodeid = as_uint(cursor.value)
            logger.info('resolved string netnode %s to %x', nodeid, self.nodeid)
        elif isinstance(nodeid, six.integer_types):
            self.nodeid = nodeid
        else:
            raise ValueError('unexpected type for nodeid')

    @staticmethod
    def get_nodebase(db):
        if db.wordsize == 4:
            return 0xFF000000
        elif db.wordsize == 8:
            return 0xFF00000000000000

    def name(self):
        '''
        fetch the name associated with the netnode.
        basically supval(tag='N')

        Returns:
          str: the name stored in the netnode.

        Raises:
          KeyError: if the name for the netnode does not exist.
        '''
        key = make_key(self.nodeid, TAGS.NAME, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return as_string(cursor.value)

    def get_tag_entries(self, tag=TAGS.SUPVAL):
        '''
        generate the entries for the given tag in this netnode.

        this replaces:
          - *1st
          - *nxt
          - *last
          - *prev

        Yields:
          Entry: an entry (with key and value) under the given tag in this netnode.
        '''
        key = make_key(self.nodeid, tag, wordsize=self.wordsize)

        cursor = self.idb.id0.find_prefix(key)
        while bytes(cursor.key).startswith(key):
            parsed_key = parse_key(cursor.key, wordsize=self.idb.wordsize)
            yield Entry(cursor.key, parsed_key, cursor.value)
            try:
                cursor.next()
            except IndexError:
                break

    def get_val(self, index, tag=TAGS.SUPVAL):
        '''
        fetch a sup/alt/hash/etc value from the netnode.
        the nodeid for this netnode must be an integer/effective address.

        Args:
          index (int): the index of the data to fetch.
          tag (str): single character tag.

        Returns:
          bytes: the raw data.
        '''
        key = make_key(self.nodeid, tag, index, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return bytes(cursor.value)

    def supval(self, index, tag=TAGS.SUPVAL):
        return self.get_val(index, tag)

    def supstr(self, index, tag=TAGS.SUPVAL):
        return as_string(self.supval(index, tag))

    def sups(self, tag=TAGS.SUPVAL):
        '''
        this replaces:
          - sup1st
          - supnxt
          - suplast
          - supprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def supentries(self, tag=TAGS.SUPVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield entry

    def altval(self, index, tag=TAGS.ALTVAL):
        return as_int(self.get_val(index, tag))

    def alts(self, tag=TAGS.ALTVAL):
        '''
        this replaces:
          - alt1st
          - altnxt
          - altlast
          - altprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def altentries(self, tag=TAGS.ALTVAL):
        for entry in self.get_tag_entries(tag=tag):
            # TODO: cast the value?
            yield entry

    def charval(self, index, tag=TAGS.CHARVAL):
        return as_int(self.get_val(index, tag))

    def chars(self, tag=TAGS.ALTVAL):
        '''
        this replaces:
          - char1st
          - charnxt
          - charlast
          - charprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def charentries(self, tag=TAGS.CHARVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield Entry(entry.key, entry.parsed_key, as_int(entry.value))

    def hashval(self, index, tag=TAGS.HASHVAL):
        '''
        TODO: how is this different from a supval?
        '''
        return self.get_val(index, tag)

    def hashes(self, tag=TAGS.HASHVAL):
        '''
        this replaces:
          - hash1st
          - hashnxt
          - hashlast
          - hashprev
        '''
        for entry in self.get_tag_entries(tag=tag):
            yield entry.parsed_key.index

    def hashentries(self, tag=TAGS.HASHVAL):
        for entry in self.get_tag_entries(tag=tag):
            yield entry

    def valobj(self):
        '''
        fetch the default netnode value.
        this is basically supval(tag='V').
        '''
        key = make_key(self.nodeid, TAGS.VALUE, wordsize=self.wordsize)
        cursor = self.idb.id0.find(key)
        return bytes(cursor.value)

    def valstr(self):
        return as_string(self.valobj())

    def value_exists(self):
        try:
            return self.valobj() is not None
        except KeyError:
            return False

    def long_value(self):
        return as_uint(self.valobj())

    def blobsize(self):
        '''
        TODO: how is this arbitrary data stored?
        '''
        raise NotImplementedError()

    def getblob(self):
        raise NotImplementedError()
<|endoftext|>