<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Python Libs
import os

# Import Salt Libs
import integration

# Import Salt Testing Libs
from salttesting import skipIf
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

class OutputReturnTest(integration.ShellCase):
    '''
    Integration tests to ensure outputters return their expected format.
    Tests against situations where the loader might not be returning the
    right outputter even though it was explicitly requested.
    '''

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_json(self):
        '''
        Tests the return of json-formatted data
        '''
        expected = ['{', '    "local": true', '}']
        ret = self.run_call('test.ping --out=json')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_nested(self):
        '''
        Tests the return of nested-formatted data
        '''
        expected = ['local:', '    True']
        ret = self.run_call('test.ping --out=nested')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_quiet(self):
        '''
        Tests the return of an out=quiet query
        '''
        expected = []
        ret = self.run_call('test.ping --out=quiet')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_pprint(self):
        '''
        Tests the return of pprint-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=pprint')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_raw(self):
        '''
        Tests the return of raw-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=raw')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_txt(self):
        '''
        Tests the return of txt-formatted data
        '''
        expected = ['local: True']
        ret = self.run_call('test.ping --out=txt')
        self.assertEqual(ret, expected)

    @skipIf(os.geteuid() != 0, 'You must be logged in as root to run this test')
    def test_output_yaml(self):
        '''
        Tests the return of yaml-formatted data
        '''
        expected = ['local: true']
        ret = self.run_call('test.ping --out=yaml')
        self.assertEqual(ret, expected)
<commit_msg>No need to be root to run these tests<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Salt Libs
import integration

# Import Salt Testing Libs
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

class OutputReturnTest(integration.ShellCase):
    '''
    Integration tests to ensure outputters return their expected format.
    Tests against situations where the loader might not be returning the
    right outputter even though it was explicitly requested.
    '''

    def test_output_json(self):
        '''
        Tests the return of json-formatted data
        '''
        expected = ['{', '    "local": true', '}']
        ret = self.run_call('test.ping --out=json')
        self.assertEqual(ret, expected)

    def test_output_nested(self):
        '''
        Tests the return of nested-formatted data
        '''
        expected = ['local:', '    True']
        ret = self.run_call('test.ping --out=nested')
        self.assertEqual(ret, expected)

    def test_output_quiet(self):
        '''
        Tests the return of an out=quiet query
        '''
        expected = []
        ret = self.run_call('test.ping --out=quiet')
        self.assertEqual(ret, expected)

    def test_output_pprint(self):
        '''
        Tests the return of pprint-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=pprint')
        self.assertEqual(ret, expected)

    def test_output_raw(self):
        '''
        Tests the return of raw-formatted data
        '''
        expected = ["{'local': True}"]
        ret = self.run_call('test.ping --out=raw')
        self.assertEqual(ret, expected)

    def test_output_txt(self):
        '''
        Tests the return of txt-formatted data
        '''
        expected = ['local: True']
        ret = self.run_call('test.ping --out=txt')
        self.assertEqual(ret, expected)

    def test_output_yaml(self):
        '''
        Tests the return of yaml-formatted data
        '''
        expected = ['local: true']
        ret = self.run_call('test.ping --out=yaml')
        self.assertEqual(ret, expected)
<|endoftext|>


<commit_before>import os
import re
import subprocess
import sys

from .base import Base

from deoplete.util import charpos2bytepos
from deoplete.util import error

try:
    current_dir = os.path.dirname(os.path.abspath(__file__))
    ujson_dir = os.path.dirname(current_dir)
    sys.path.insert(0, ujson_dir)
    from ujson import loads
except ImportError:
    from json import loads

class Source(Base):

    def __init__(self, vim):
        Base.__init__(self, vim)

        self.name = 'go'
        self.mark = '[go]'
        self.filetypes = ['go']
        self.input_pattern = r'(?:\b[^\W\d]\w*|[\]\)])\.(?:[^\W\d]\w*)?'
        self.rank = 500

        self.align_class = self.vim.vars['deoplete#sources#go#align_class']
        self.gocode_binary = self.vim.vars['deoplete#sources#go#gocode_binary']
        self.package_dot = self.vim.vars['deoplete#sources#go#package_dot']
        self.sort_class = self.vim.vars['deoplete#sources#go#sort_class']

    def get_complete_position(self, context):
        m = re.search(r'\w*$|(?<=")[./\w]*$', context['input'])
        return m.start() if m else -1

    def gather_candidates(self, context):
        line = self.vim.current.window.cursor[0]
        column = context['complete_position']

        buf = self.vim.current.buffer
        offset = self.vim.call('line2byte', line) + \
            charpos2bytepos(self.vim, context['input'][: column], column) - 1
        source = '\n'.join(buf).encode()

        process = subprocess.Popen([self.GoCodeBinary(),
                                    '-f=json',
                                    'autocomplete',
                                    buf.name,
                                    str(offset)],
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   start_new_session=True)
        process.stdin.write(source)
        stdout_data, stderr_data = process.communicate()
        result = loads(stdout_data.decode())

        if self.sort_class:
            # TODO(zchee): Why not work with this?
            #              class_dict = {}.fromkeys(self.sort_class, [])
            class_dict = {
                'package': [],
                'func': [],
                'type': [],
                'var': [],
                'const': [],
            }
        try:
            out = []
            sep = ' '
            if result[1][0]['class'] == 'PANIC':
                error(self.vim, 'gocode panicked')
                return []
            for complete in result[1]:
                _class = complete['class']
                word = complete['name']
                info = complete['type']

                if _class not in ('package', 'import') and self.align_class:
                    abbr = '{:<6}'.format(_class) + \
                        str(word + sep + info).replace(' func', '')
                else:
                    abbr = _class + sep + word

                if _class == 'package' and self.package_dot:
                    word += '.'

                candidates = dict(word=word,
                                  abbr=abbr,
                                  info=info,
                                  dup=1
                                  )
                if not self.sort_class or _class == 'import':
                    out.append(candidates)
                else:
                    class_dict[_class].append(candidates)

            # append with sort by complete['class']
            if self.sort_class:
                for c in self.sort_class:
                    for x in class_dict[c]:
                        out.append(x)

            return out
        except Exception:
            return []

    def GoCodeBinary(self):
        try:
            if os.path.isfile(self.gocode_binary):
                return self.gocode_binary
            else:
                raise
        except Exception:
            return self.FindBinaryPath('gocode')

    def FindBinaryPath(self, cmd):
        def is_exec(fpath):
            return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

        fpath, fname = os.path.split(cmd)
        if fpath:
            if is_exec(cmd):
                return cmd
        else:
            for path in os.environ["PATH"].split(os.pathsep):
                path = path.strip('"')
                binary = os.path.join(path, cmd)
                if is_exec(binary):
                    return binary
        return error(self.vim, 'gocode binary not found')
<commit_msg>Remove unnecessary 'str'<commit_after>import os
import re
import subprocess
import sys

from .base import Base

from deoplete.util import charpos2bytepos
from deoplete.util import error

try:
    current_dir = os.path.dirname(os.path.abspath(__file__))
    ujson_dir = os.path.dirname(current_dir)
    sys.path.insert(0, ujson_dir)
    from ujson import loads
except ImportError:
    from json import loads

class Source(Base):

    def __init__(self, vim):
        Base.__init__(self, vim)

        self.name = 'go'
        self.mark = '[go]'
        self.filetypes = ['go']
        self.input_pattern = r'(?:\b[^\W\d]\w*|[\]\)])\.(?:[^\W\d]\w*)?'
        self.rank = 500

        self.align_class = self.vim.vars['deoplete#sources#go#align_class']
        self.gocode_binary = self.vim.vars['deoplete#sources#go#gocode_binary']
        self.package_dot = self.vim.vars['deoplete#sources#go#package_dot']
        self.sort_class = self.vim.vars['deoplete#sources#go#sort_class']

    def get_complete_position(self, context):
        m = re.search(r'\w*$|(?<=")[./\w]*$', context['input'])
        return m.start() if m else -1

    def gather_candidates(self, context):
        line = self.vim.current.window.cursor[0]
        column = context['complete_position']

        buf = self.vim.current.buffer
        offset = self.vim.call('line2byte', line) + \
            charpos2bytepos(self.vim, context['input'][: column], column) - 1
        source = '\n'.join(buf).encode()

        process = subprocess.Popen([self.GoCodeBinary(),
                                    '-f=json',
                                    'autocomplete',
                                    buf.name,
                                    str(offset)],
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   start_new_session=True)
        process.stdin.write(source)
        stdout_data, stderr_data = process.communicate()
        result = loads(stdout_data.decode())

        if self.sort_class:
            # TODO(zchee): Why not work with this?
            #              class_dict = {}.fromkeys(self.sort_class, [])
            class_dict = {
                'package': [],
                'func': [],
                'type': [],
                'var': [],
                'const': [],
            }
        try:
            out = []
            sep = ' '
            if result[1][0]['class'] == 'PANIC':
                error(self.vim, 'gocode panicked')
                return []
            for complete in result[1]:
                _class = complete['class']
                word = complete['name']
                info = complete['type']

                if _class not in ('package', 'import') and self.align_class:
                    abbr = '{:<6}'.format(_class) + \
                        (word + sep + info).replace(' func', '')
                else:
                    abbr = _class + sep + word

                if _class == 'package' and self.package_dot:
                    word += '.'

                candidates = dict(word=word,
                                  abbr=abbr,
                                  info=info,
                                  dup=1
                                  )
                if not self.sort_class or _class == 'import':
                    out.append(candidates)
                else:
                    class_dict[_class].append(candidates)

            # append with sort by complete['class']
            if self.sort_class:
                for c in self.sort_class:
                    for x in class_dict[c]:
                        out.append(x)

            return out
        except Exception:
            return []

    def GoCodeBinary(self):
        try:
            if os.path.isfile(self.gocode_binary):
                return self.gocode_binary
            else:
                raise
        except Exception:
            return self.FindBinaryPath('gocode')

    def FindBinaryPath(self, cmd):
        def is_exec(fpath):
            return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

        fpath, fname = os.path.split(cmd)
        if fpath:
            if is_exec(cmd):
                return cmd
        else:
            for path in os.environ["PATH"].split(os.pathsep):
                path = path.strip('"')
                binary = os.path.join(path, cmd)
                if is_exec(binary):
                    return binary
        return error(self.vim, 'gocode binary not found')
<|endoftext|>


<commit_before>from collections import MutableMapping, OrderedDict

import itertools

import re
import json

__all__ = ['ParseError',
           'InvalidSectionNameError', 'InvalidOptionNameError',
           'NoSectionError', 'NoOptionError',
           'DuplicateSectionError', 'DuplicateOptionError',
           'JSONConfigParser']

DEFAULT_SECT = 'DEFAULT'
_UNSET = object()

class ParseError(BaseException):
    def __init__(self, message, **kwargs):

        info = []
        if 'filename' in kwargs:
            info.append('file: %s' % kwargs['filename'])
        if 'section' in kwargs:
            info.append('section: %s' % kwargs['section'])
        if 'lineno' in kwargs:
            info.append('line: %s' % kwargs['lineno'])

        if len(info):
            message.append(', '.join(info) + '\n')

        if 'line' in kwargs:
            message.append(kwargs['line'])

        BaseException.__init__(self, message)

class MissingSectionHeaderError(ParseError):
    """Raised if an option occurs before the first header
    """
    def __init__(self, filename, lineno, line):
        BaseException.__init__(
            self,
            'No section header before first option.\n' +
            'file: %s, line: %d\n%r' %
            (filename, lineno, line))

class InvalidSectionNameError(ParseError):
    def __init__(self, section, **kwargs):
        msg = 'Invalid section name: %s' % repr(section)
        ParseError.__init__(self, msg, **kwargs)

class InvalidOptionNameError(ParseError):
    def __init__(self, option, **kwargs):
        msg = 'Invalid option name: %s' % repr(option)
        ParseError.__init__(self, msg, **kwargs)

class NoSectionError(KeyError):
    pass

class NoOptionError(KeyError):
    pass

class DuplicateSectionError(ParseError):
    """Raised when a section is repeated in an input source.

    Possible repetitions that raise this exception are: multiple creation
    using the API or when a section is found more than once in a single input
    file, string or dictionary.
    """
    def __init__(self, section, **kwargs):
        msg = 'Section %s already declared' % repr(section)
        ParseError.__init__(self, msg, **kwargs)

class DuplicateOptionError(ParseError):
    """Raised by parser when an option is repeated in an input source.

    Current implementation raises this exception only when an option is found
    more than once in a single file, string or dictionary.
    """

    def __init__(self, option, **kwargs):
        msg = 'Duplicate definition of option: %s' % repr(option)
        ParseError.__init__(self, msg, **kwargs)

class JSONConfigParser(MutableMapping):

    _BLANK_TMPL = r"""
        ^
        (\#[^\n\r]*)?                # optional comment
        [\n\r]+                      #eol
        """

    _HEADER_TMPL = r"""
        #^
        \[
        (?P<section>[\-\w]+)
        \]
        [\n\r]+                      #eol
        """

    _KEY_TMPL = r"""
        (?P<key>[\-\w]+)            # at least one letter underscore or hyphen
        \s*                         # optional whitespace
        =                           # followed by '='
        \s*                         # optional whitespace
        """

    _EOL_TMPL = r"""
        [\n\r]+                     # eol
        """

    _blank_re = re.compile(_BLANK_TMPL, re.VERBOSE | re.MULTILINE)
    _header_re = re.compile(_HEADER_TMPL, re.VERBOSE | re.MULTILINE)
    _key_re = re.compile(_KEY_TMPL, re.VERBOSE | re.MULTILINE)
    _eol_re = re.compile(_EOL_TMPL, re.VERBOSE | re.MULTILINE)

    _json_decoder = json.JSONDecoder()

    def __init__(self, defaults=None, *,
                 dict_type=OrderedDict, default_section=DEFAULT_SECT):
        self._dict = dict_type
        self._default_section = default_section
        self._sections = self._dict()
        self._defaults = self._dict()
        self._proxies = self._dict()
        self._proxies[default_section] = SectionProxy(self, default_section)
        if defaults:
            self.read_dict({default_section: defaults})

    def sections(self):
        """Return a list of section names, excluding [DEFAULT]"""
        return self._sections.keys()

    def add_section(self, section):
        """Create a new section in the configuration.

        Raise DuplicateSectionError if a section by the specified name
        already exists. Raise ValueError if name is DEFAULT.
        """
        if section == self.default_section:
            raise ValueError('Invalid section name: %r' % section)

        if section in self._sections:
            raise DuplicateSectionError(section)
        self._sections[section] = self._dict()
        self._proxies[section] = SectionProxy(self, section)

    def has_section(self, section):
        return section in self._sections

    def remove_section(self, section):
        """Delete a section.

        Returns True if the section existed previously, False otherwise.
        """
        existed = section in self._sections
        if existed:
            del self._sections[section]
            del self._proxies[section]
        return existed

    def __getitem__(self, key):
        if key != self.default_section and not self.has_section(key):
            raise KeyError(key)
        return self._proxies[key]

    def __setitem__(self, key, value):
        # To conform with the mapping protocol, overwrites existing values in
        # the section.
        if key == self.default_section:
            self._defaults.clear()
        elif key in self._sections:
            self._sections[key].clear()
        self.read_dict({key: value})

    def __delitem__(self, key):
        if key == self.default_section:
            raise ValueError("Cannot remove the default section.")
        if not self.remove_section(key):
            raise KeyError(key)

    def __contains__(self, key):
        return key == self.default_section or key in self.sections

    def __len__(self):
        return len(self._sections) + 1

    def __iter__(self):
        return itertools.chain((self.default_section,), self._sections.keys())

    def options(self, section):
        """Return a list of option names for the given section name."""
        try:
            opts = iter(set(self._sections[section].keys() +
                            self._defaults.keys()))
        except KeyError:
            raise NoSectionError(section)
        return opts

    def get(self, section, option, fallback=_UNSET, *, vars=None):
        """Get an option value for a given section.

        If `vars' is provided, it must be a dictionary. The option is looked up
        in `vars' (if provided), `section', and in `DEFAULTSECT' in that order.
        If the key is not found and `fallback' is provided, it is used as
        a fallback value. `None' can be provided as a `fallback' value.

        The section DEFAULT is special.
        """
        if not self.has_section(section):
            raise NoSectionError(section)

        if vars is not None and option in vars:
            return vars[option]

        if section is not self.default_section:
            if option in self._sections[section]:
                return self._sections[section][option]

        if option in self._defaults:
            return self._defaults[option]

        if fallback is _UNSET:
            raise KeyError(option)

        return fallback

    def has_option(self, section, option):
        if not section or section == self.default_section:
            return option in self._defaults
        elif section in self._sections:
            return (option in self._sections[section] or
                    option in self._defaults)
        else:
            return False

    def set(self, section, option, value=None):
        if not section or section == self.default_section:
            sectdict = self._defaults
        else:
            try:
                sectdict = self._sections[section]
            except KeyError:
                raise NoSectionError(section)
        sectdict[option] = value

    def read(self, filnames, encoding=None):
        pass

    def read_file(self, fp, filename=None):
        pass

    def read_dict(self, dictionary):
        pass

    def read_string(self, string, fpname=None):
        sections_added = set()
        entries_added = set()

        sectname = None
        cursect = None

        idx = 0
        lineno = 0

        while idx < len(string):
            mo = self._header_re.match(string, idx)
            if mo:
                sectname = mo.group('section')

                # check that section has not occured in this file before
                if sectname in sections_added:
                    raise DuplicateSectionError(sectname)
                sections_added.add(sectname)
                entries_added = set()

                # find or create the section
                if sectname == self.default_section:
                    cursect = self._defaults
                elif sectname in self._sections:
                    cursect = self._sections[sectname]
                else:
                    cursect = self._dict()
                    self._sections[sectname] = cursect
                    self._proxies[sectname] = SectionProxy(self, sectname)

                idx = mo.end()
            else:
                # consume blank lines and comments
                mo = self._blank_re.match(string, idx)
                if mo:
                    idx = mo.end()
                else:
                    mo = self._key_re.match(string, idx)
                    if not mo:
                        raise ParseError(
                            "expected section, comment or newline")

                    # read key
                    optname = mo.group('key')
                    idx = mo.end()
                    if optname in entries_added:
                        raise DuplicateOptionError(sectname, optname,
                                                   fpname, lineno)
                    entries_added.add(optname)

                    # read value
                    # TODO increment lineno
                    optval, idx = self._json_decoder.raw_decode(string, idx)
                    cursect[optname] = optval

                    # consume remaining comments and whitespace
                    mo = self._eol_re.match(string, idx)
                    if not mo:
                        raise ParseError(repr(string[idx:]))
                    idx = mo.end()

    @property
    def default_section(self):
        # default section should be read-only
        return self._default_section

class SectionProxy(MutableMapping):
    """A proxy for a single section from a parser."""

    def __init__(self, parser, name):
        """Creates a view on a section of the specified `name` in `parser`."""
        self._parser = parser
        self._name = name

    def __repr__(self):
        return '<Section: {}>'.format(self._name)

    def __getitem__(self, key):
        if not self._parser.has_option(self._name, key):
            raise KeyError(key)
        return self._parser.get(self._name, key)

    def __setitem__(self, key, value):
        self._parser._validate_value_types(option=key, value=value)
        return self._parser.set(self._name, key, value)

    def __delitem__(self, key):
        if not (self._parser.has_option(self._name, key) and
                self._parser.remove_option(self._name, key)):
            raise KeyError(key)

    def __contains__(self, key):
        return self._parser.has_option(self._name, key)

    def __len__(self):
        return len(self._options())

    def __iter__(self):
        return self._options().__iter__()

    def _options(self):
        if self._name != self._parser.default_section:
            return self._parser.options(self._name)
        else:
            return self._parser.defaults()

    def get(self, option, *args, **kwargs):
        return self._parser.get(self._name, option, *args, **kwargs)

    @property
    def parser(self):
        # The parser object of the proxy is read-only.
        return self._parser

    @property
    def name(self):
        # The name of the section on a proxy is read-only.
        return self._name
<commit_msg>Add basic support for reading from files<commit_after>from collections import MutableMapping, OrderedDict

import itertools

import re
import json

__all__ = ['ParseError',
           'InvalidSectionNameError', 'InvalidOptionNameError',
           'NoSectionError', 'NoOptionError',
           'DuplicateSectionError', 'DuplicateOptionError',
           'JSONConfigParser']

DEFAULT_SECT = 'DEFAULT'
_UNSET = object()

class ParseError(BaseException):
    def __init__(self, message, **kwargs):

        info = []
        if 'filename' in kwargs:
            info.append('file: %s' % kwargs['filename'])
        if 'section' in kwargs:
            info.append('section: %s' % kwargs['section'])
        if 'lineno' in kwargs:
            info.append('line: %s' % kwargs['lineno'])

        if len(info):
            message.append(', '.join(info) + '\n')

        if 'line' in kwargs:
            message.append(kwargs['line'])

        BaseException.__init__(self, message)

class MissingSectionHeaderError(ParseError):
    """Raised if an option occurs before the first header
    """
    def __init__(self, filename, lineno, line):
        BaseException.__init__(
            self,
            'No section header before first option.\n' +
            'file: %s, line: %d\n%r' %
            (filename, lineno, line))

class InvalidSectionNameError(ParseError):
    def __init__(self, section, **kwargs):
        msg = 'Invalid section name: %s' % repr(section)
        ParseError.__init__(self, msg, **kwargs)

class InvalidOptionNameError(ParseError):
    def __init__(self, option, **kwargs):
        msg = 'Invalid option name: %s' % repr(option)
        ParseError.__init__(self, msg, **kwargs)

class NoSectionError(KeyError):
    pass

class NoOptionError(KeyError):
    pass

class DuplicateSectionError(ParseError):
    """Raised when a section is repeated in an input source.

    Possible repetitions that raise this exception are: multiple creation
    using the API or when a section is found more than once in a single input
    file, string or dictionary.
    """
    def __init__(self, section, **kwargs):
        msg = 'Section %s already declared' % repr(section)
        ParseError.__init__(self, msg, **kwargs)

class DuplicateOptionError(ParseError):
    """Raised by parser when an option is repeated in an input source.

    Current implementation raises this exception only when an option is found
    more than once in a single file, string or dictionary.
    """

    def __init__(self, option, **kwargs):
        msg = 'Duplicate definition of option: %s' % repr(option)
        ParseError.__init__(self, msg, **kwargs)

class JSONConfigParser(MutableMapping):

    _BLANK_TMPL = r"""
        ^
        (\#[^\n\r]*)?                # optional comment
        [\n\r]+                      #eol
        """

    _HEADER_TMPL = r"""
        #^
        \[
        (?P<section>[\-\w]+)
        \]
        [\n\r]+                      #eol
        """

    _KEY_TMPL = r"""
        (?P<key>[\-\w]+)            # at least one letter underscore or hyphen
        \s*                         # optional whitespace
        =                           # followed by '='
        \s*                         # optional whitespace
        """

    _EOL_TMPL = r"""
        [\n\r]+                     # eol
        """

    _blank_re = re.compile(_BLANK_TMPL, re.VERBOSE | re.MULTILINE)
    _header_re = re.compile(_HEADER_TMPL, re.VERBOSE | re.MULTILINE)
    _key_re = re.compile(_KEY_TMPL, re.VERBOSE | re.MULTILINE)
    _eol_re = re.compile(_EOL_TMPL, re.VERBOSE | re.MULTILINE)

    _json_decoder = json.JSONDecoder()

    def __init__(self, defaults=None, *,
                 dict_type=OrderedDict, default_section=DEFAULT_SECT):
        self._dict = dict_type
        self._default_section = default_section
        self._sections = self._dict()
        self._defaults = self._dict()
        self._proxies = self._dict()
        self._proxies[default_section] = SectionProxy(self, default_section)
        if defaults:
            self.read_dict({default_section: defaults})

    def sections(self):
        """Return a list of section names, excluding [DEFAULT]"""
        return self._sections.keys()

    def add_section(self, section):
        """Create a new section in the configuration.

        Raise DuplicateSectionError if a section by the specified name
        already exists. Raise ValueError if name is DEFAULT.
        """
        if section == self.default_section:
            raise ValueError('Invalid section name: %r' % section)

        if section in self._sections:
            raise DuplicateSectionError(section)
        self._sections[section] = self._dict()
        self._proxies[section] = SectionProxy(self, section)

    def has_section(self, section):
        return section in self._sections

    def remove_section(self, section):
        """Delete a section.

        Returns True if the section existed previously, False otherwise.
        """
        existed = section in self._sections
        if existed:
            del self._sections[section]
            del self._proxies[section]
        return existed

    def __getitem__(self, key):
        if key != self.default_section and not self.has_section(key):
            raise KeyError(key)
        return self._proxies[key]

    def __setitem__(self, key, value):
        # To conform with the mapping protocol, overwrites existing values in
        # the section.
        if key == self.default_section:
            self._defaults.clear()
        elif key in self._sections:
            self._sections[key].clear()
        self.read_dict({key: value})

    def __delitem__(self, key):
        if key == self.default_section:
            raise ValueError("Cannot remove the default section.")
        if not self.remove_section(key):
            raise KeyError(key)

    def __contains__(self, key):
        return key == self.default_section or key in self.sections

    def __len__(self):
        return len(self._sections) + 1

    def __iter__(self):
        return itertools.chain((self.default_section,), self._sections.keys())

    def options(self, section):
        """Return a list of option names for the given section name."""
        try:
            opts = iter(set(self._sections[section].keys() +
                            self._defaults.keys()))
        except KeyError:
            raise NoSectionError(section)
        return opts

    def get(self, section, option, fallback=_UNSET, *, vars=None):
        """Get an option value for a given section.

        If `vars' is provided, it must be a dictionary. The option is looked up
        in `vars' (if provided), `section', and in `DEFAULTSECT' in that order.
        If the key is not found and `fallback' is provided, it is used as
        a fallback value. `None' can be provided as a `fallback' value.

        The section DEFAULT is special.
        """
        if not self.has_section(section):
            raise NoSectionError(section)

        if vars is not None and option in vars:
            return vars[option]

        if section is not self.default_section:
            if option in self._sections[section]:
                return self._sections[section][option]

        if option in self._defaults:
            return self._defaults[option]

        if fallback is _UNSET:
            raise KeyError(option)

        return fallback

    def has_option(self, section, option):
        if not section or section == self.default_section:
            return option in self._defaults
        elif section in self._sections:
            return (option in self._sections[section] or
                    option in self._defaults)
        else:
            return False

    def set(self, section, option, value=None):
        if not section or section == self.default_section:
            sectdict = self._defaults
        else:
            try:
                sectdict = self._sections[section]
            except KeyError:
                raise NoSectionError(section)
        sectdict[option] = value

    def read(self, filenames, encoding=None):
        if isinstance(filenames, str):
            filenames = [filenames]
        for f in filenames:
            try:
                with open(f, 'r') as fp:
                    self.read_file(fp)
            except OSError:
                # TODO other exceptions leave cfg object in an inconsitant
                # state and are basically unrecoverable
                continue

    def read_file(self, fp, filename=None):
        self.read_string(fp.read())

    def read_dict(self, dictionary):
        raise NotImplementedError()

    def read_string(self, string, fpname=None):
        sections_added = set()
        entries_added = set()

        sectname = None
        cursect = None

        idx = 0
        lineno = 0

        while idx < len(string):
            mo = self._header_re.match(string, idx)
            if mo:
                sectname = mo.group('section')

                # check that section has not occured in this file before
                if sectname in sections_added:
                    raise DuplicateSectionError(sectname)
                sections_added.add(sectname)
                entries_added = set()

                # find or create the section
                if sectname == self.default_section:
                    cursect = self._defaults
                elif sectname in self._sections:
                    cursect = self._sections[sectname]
                else:
                    cursect = self._dict()
                    self._sections[sectname] = cursect
                    self._proxies[sectname] = SectionProxy(self, sectname)

                idx = mo.end()
            else:
                # consume blank lines and comments
                mo = self._blank_re.match(string, idx)
                if mo:
                    idx = mo.end()
                else:
                    mo = self._key_re.match(string, idx)
                    if not mo:
                        raise ParseError(
                            "expected section, comment or newline")

                    # read key
                    optname = mo.group('key')
                    idx = mo.end()
                    if optname in entries_added:
                        raise DuplicateOptionError(sectname, optname,
                                                   fpname, lineno)
                    entries_added.add(optname)

                    # read value
                    # TODO increment lineno
                    optval, idx = self._json_decoder.raw_decode(string, idx)
                    cursect[optname] = optval

                    # consume remaining comments and whitespace
                    mo = self._eol_re.match(string, idx)
                    if not mo:
                        raise ParseError("unexpected symbol or whitespace")
                    idx = mo.end()

    @property
    def default_section(self):
        # default section should be read-only
        return self._default_section

class SectionProxy(MutableMapping):
    """A proxy for a single section from a parser."""

    def __init__(self, parser, name):
        """Creates a view on a section of the specified `name` in `parser`."""
        self._parser = parser
        self._name = name

    def __repr__(self):
        return '<Section: {}>'.format(self._name)

    def __getitem__(self, key):
        if not self._parser.has_option(self._name, key):
            raise KeyError(key)
        return self._parser.get(self._name, key)

    def __setitem__(self, key, value):
        self._parser._validate_value_types(option=key, value=value)
        return self._parser.set(self._name, key, value)

    def __delitem__(self, key):
        if not (self._parser.has_option(self._name, key) and
                self._parser.remove_option(self._name, key)):
            raise KeyError(key)

    def __contains__(self, key):
        return self._parser.has_option(self._name, key)

    def __len__(self):
        return len(self._options())

    def __iter__(self):
        return self._options().__iter__()

    def _options(self):
        if self._name != self._parser.default_section:
            return self._parser.options(self._name)
        else:
            return self._parser.defaults()

    def get(self, option, *args, **kwargs):
        return self._parser.get(self._name, option, *args, **kwargs)

    @property
    def parser(self):
        # The parser object of the proxy is read-only.
        return self._parser

    @property
    def name(self):
        # The name of the section on a proxy is read-only.
        return self._name
<|endoftext|>


<commit_before>from django.test import TestCase
from django.test.client import RequestFactory
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.settings import api_settings
from rest_framework.views import APIView

factory = RequestFactory()

class BasicView(APIView):
    def get(self, request, *args, **kwargs):
        return Response({'method': 'GET'})

    def post(self, request, *args, **kwargs):
        return Response({'method': 'POST', 'data': request.DATA})

@api_view(['GET', 'POST', 'PUT'])
def basic_view(request):
    if request.method == 'GET':
        return {'method': 'GET'}
    elif request.method == 'POST':
        return {'method': 'POST', 'data': request.DATA}
    elif request.method == 'PUT':
        return {'method': 'PUT', 'data': request.DATA}

class ClassBasedViewIntegrationTests(TestCase):
    def setUp(self):
        self.view = BasicView.as_view()

    def test_400_parse_error(self):
        request = factory.post('/', 'f00bar', content_type='application/json')
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(response.data, expected)

    def test_400_parse_error_tunneled_content(self):
        content = 'f00bar'
        content_type = 'application/json'
        form_data = {
            api_settings.FORM_CONTENT_OVERRIDE: content,
            api_settings.FORM_CONTENTTYPE_OVERRIDE: content_type
        }
        request = factory.post('/', form_data)
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(response.data, expected)

class FunctionBasedViewIntegrationTests(TestCase):
    def setUp(self):
        self.view = basic_view

    def test_400_parse_error(self):
        request = factory.post('/', 'f00bar', content_type='application/json')
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(response.data, expected)

    def test_400_parse_error_tunneled_content(self):
        content = 'f00bar'
        content_type = 'application/json'
        form_data = {
            api_settings.FORM_CONTENT_OVERRIDE: content,
            api_settings.FORM_CONTENTTYPE_OVERRIDE: content_type
        }
        request = factory.post('/', form_data)
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(response.data, expected)
<commit_msg>Sanitise JSON error messages<commit_after>import copy
from django.test import TestCase
from django.test.client import RequestFactory
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.settings import api_settings
from rest_framework.views import APIView

factory = RequestFactory()

class BasicView(APIView):
    def get(self, request, *args, **kwargs):
        return Response({'method': 'GET'})

    def post(self, request, *args, **kwargs):
        return Response({'method': 'POST', 'data': request.DATA})

@api_view(['GET', 'POST', 'PUT'])
def basic_view(request):
    if request.method == 'GET':
        return {'method': 'GET'}
    elif request.method == 'POST':
        return {'method': 'POST', 'data': request.DATA}
    elif request.method == 'PUT':
        return {'method': 'PUT', 'data': request.DATA}

def sanitise_json_error(error_dict):
    """
    Exact contents of JSON error messages depend on the installed version
    of json.
    """
    ret = copy.copy(error_dict)
    chop = len('JSON parse error - No JSON object could be decoded')
    ret['detail'] = ret['detail'][:chop]
    return ret

class ClassBasedViewIntegrationTests(TestCase):
    def setUp(self):
        self.view = BasicView.as_view()

    def test_400_parse_error(self):
        request = factory.post('/', 'f00bar', content_type='application/json')
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(sanitise_json_error(response.data), expected)

    def test_400_parse_error_tunneled_content(self):
        content = 'f00bar'
        content_type = 'application/json'
        form_data = {
            api_settings.FORM_CONTENT_OVERRIDE: content,
            api_settings.FORM_CONTENTTYPE_OVERRIDE: content_type
        }
        request = factory.post('/', form_data)
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(sanitise_json_error(response.data), expected)

class FunctionBasedViewIntegrationTests(TestCase):
    def setUp(self):
        self.view = basic_view

    def test_400_parse_error(self):
        request = factory.post('/', 'f00bar', content_type='application/json')
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(sanitise_json_error(response.data), expected)

    def test_400_parse_error_tunneled_content(self):
        content = 'f00bar'
        content_type = 'application/json'
        form_data = {
            api_settings.FORM_CONTENT_OVERRIDE: content,
            api_settings.FORM_CONTENTTYPE_OVERRIDE: content_type
        }
        request = factory.post('/', form_data)
        response = self.view(request)
        expected = {
            'detail': u'JSON parse error - No JSON object could be decoded'
        }
        self.assertEquals(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEquals(sanitise_json_error(response.data), expected)
<|endoftext|>


<commit_before># Import python libs
import sys

# Import salt libs
from saltunittest import TestLoader, TextTestRunner
import integration
from integration import TestDaemon

class MatchTest(integration.ShellCase):
    '''
    Test salt matchers
    '''
    def test_list(self):
        '''
        test salt -L matcher
        '''
        data = self.run_salt('-L minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-L minion,sub_minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_glob(self):
        '''
        test salt glob matcher
        '''
        data = self.run_salt('minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('"*" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_regex(self):
        '''
        test salt regex matcher
        '''
        data = self.run_salt('-E "^minion$" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-E ".*" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_grain(self):
        '''
        test salt grain matcher
        '''
        data = self.run_salt('-t 1 -G "test_grain:cheese" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-G "test_grain:spam" test.ping')
        data = '\n'.join(data)
        self.assertIn('sub_minion', data)
        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))

if __name__ == "__main__":
    loader = TestLoader()
    tests = loader.loadTestsFromTestCase(KeyTest)
    print('Setting up Salt daemons to execute tests')
    with TestDaemon():
        runner = TextTestRunner(verbosity=1).run(tests)
        sys.exit(runner.wasSuccessful())
<commit_msg>Add regex grain match tests<commit_after># Import python libs
import sys

# Import salt libs
from saltunittest import TestLoader, TextTestRunner
import integration
from integration import TestDaemon

class MatchTest(integration.ShellCase):
    '''
    Test salt matchers
    '''
    def test_list(self):
        '''
        test salt -L matcher
        '''
        data = self.run_salt('-L minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-L minion,sub_minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_glob(self):
        '''
        test salt glob matcher
        '''
        data = self.run_salt('minion test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('"*" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_regex(self):
        '''
        test salt regex matcher
        '''
        data = self.run_salt('-E "^minion$" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-E ".*" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertIn('sub_minion', data)

    def test_grain(self):
        '''
        test salt grain matcher
        '''
        data = self.run_salt('-t 1 -G "test_grain:cheese" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('-G "test_grain:spam" test.ping')
        data = '\n'.join(data)
        self.assertIn('sub_minion', data)
        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))

    def test_regrain(self):
        '''
        test salt grain matcher
        '''
        data = self.run_salt('-t 1 --grain-pcre "test_grain:^cheese$" test.ping')
        data = '\n'.join(data)
        self.assertIn('minion', data)
        self.assertNotIn('sub_minion', data)
        data = self.run_salt('--grain-pcre "test_grain:.*am$" test.ping')
        data = '\n'.join(data)
        self.assertIn('sub_minion', data)
        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))

if __name__ == "__main__":
    loader = TestLoader()
    tests = loader.loadTestsFromTestCase(KeyTest)
    print('Setting up Salt daemons to execute tests')
    with TestDaemon():
        runner = TextTestRunner(verbosity=1).run(tests)
        sys.exit(runner.wasSuccessful())
<|endoftext|>


<commit_before>from django.conf.urls import url

from .views import (
    change_password,
    login,
    logout,
    profile,
    register,
    register_email,
    reset_password,
    send_reset_password_link,
    verify_email,
    verify_registration
)

app_name = 'rest_registration'
urlpatterns = [
    url('register/$', register, name='register'),
    url('verify-registration/$', verify_registration, name='verify-registration'),

    url('send-reset-password-link/$', send_reset_password_link, name='send-reset-password-link'),
    url('reset-password/$', reset_password, name='reset-password'),

    url('login/$', login, name='login'),
    url('logout/$', logout, name='logout'),

    url('profile/$', profile, name='profile'),

    url('change-password/$', change_password, name='change_password'),

    url('register-email/$', register_email, name='register-email'),
    url('verify-email/$', verify_email, name='verify-email'),
]  # noqa
<commit_msg>flake8 normalization<commit_after>from django.conf.urls import url

from .views import (
    change_password,
    login,
    logout,
    profile,
    register,
    register_email,
    reset_password,
    send_reset_password_link,
    verify_email,
    verify_registration
)

app_name = 'rest_registration'
urlpatterns = [
    url('register/$', register, name='register'),
    url('verify-registration/$', verify_registration,
        name='verify-registration'),

    url('send-reset-password-link/$', send_reset_password_link,
        name='send-reset-password-link'),
    url('reset-password/$', reset_password, name='reset-password'),

    url('login/$', login, name='login'),
    url('logout/$', logout, name='logout'),

    url('profile/$', profile, name='profile'),

    url('change-password/$', change_password, name='change_password'),

    url('register-email/$', register_email, name='register-email'),
    url('verify-email/$', verify_email, name='verify-email'),
]  # noqa
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Base class for product tests.  Handles setting up a docker cluster and has
other utilities
"""

import distutils.core
import fnmatch
import json
import os
import shutil
import subprocess
import sys
import errno

from docker.errors import APIError
from docker import Client

import prestoadmin
from tests import utils

LOCAL_MOUNT_POINT = os.path.join(prestoadmin.main_dir, "tmp/docker-pa/%s")
LOCAL_RESOURCES_DIR = prestoadmin.main_dir + "/tests/product/resources/"
DOCKER_MOUNT_POINT = "/mnt/presto-admin"

class BaseProductTestCase(utils.BaseTestCase):
    client = Client()
    slaves = ["slave1", "slave2", "slave3"]
    master = "master"

    def setUp(self):
        self.capture_stdout_stderr()
        self.check_if_docker_exists()
        self.create_docker_cluster()

    def tearDown(self):
        self.restore_stdout_stderr_keep_open()
        self.tear_down_docker_cluster()

    def check_if_docker_exists(self):
        try:
            subprocess.call(['docker', '--version'])
        except OSError:
            sys.exit('Docker is not installed. Try installing it with '
                     'presto-admin/bin/install-docker.sh.')

    def create_host_mount_dirs(self):
        for container_name in self.all_hosts():
            try:
                os.makedirs(LOCAL_MOUNT_POINT % container_name)
            except OSError as e:
                # file exists
                if e.errno == errno.EEXIST:
                    pass

    def create_docker_cluster(self):
        self.tear_down_docker_cluster()
        self.create_host_mount_dirs()

        if not self.client.images("jdeathe/centos-ssh"):
            self._execute_and_wait(self.client.pull, "jdeathe/centos-ssh")
        self._execute_and_wait(self.client.build,
                               path=os.path.join(prestoadmin.main_dir,
                                                 "tests/product/resources/"
                                                 "centos6-ssh-test"),
                               tag="teradatalabs/centos6-ssh-test", rm=True)

        for container_name in self.slaves:
            self._execute_and_wait(self.client.create_container,
                                   "teradatalabs/centos6-ssh-test",
                                   detach=True,
                                   name=container_name,
                                   volumes=LOCAL_MOUNT_POINT %
                                   container_name)

            self.client.start(container_name,
                              binds={LOCAL_MOUNT_POINT % container_name:
                                     {"bind": DOCKER_MOUNT_POINT,
                                      "ro": False}})

        self._execute_and_wait(self.client.create_container,
                               "teradatalabs/centos6-ssh-test",
                               detach=True,
                               name=self.master,
                               hostname=self.master,
                               volumes=LOCAL_MOUNT_POINT % self.master)

        self.client.start(self.master,
                          binds={LOCAL_MOUNT_POINT % self.master:
                                 {"bind": DOCKER_MOUNT_POINT,
                                  "ro": False}},
                          links=zip(self.slaves, self.slaves))

    def _execute_and_wait(self, func, *args, **kwargs):
        ret = func(*args, **kwargs)
        # go through all lines in returned stream to ensure func finishes
        for line in ret:
            pass

    def remove_host_mount_dirs(self):
        for container_name in self.all_hosts():
            try:
                shutil.rmtree(LOCAL_MOUNT_POINT % container_name)
            except OSError as e:
                # no such file or directory
                if e.errno == errno.ENOENT:
                    pass

        try:
            os.removedirs(os.path.dirname(LOCAL_MOUNT_POINT))
        except OSError as e:
            if e.errno == errno.ENOENT:
                pass

    def tear_down_docker_cluster(self):
        for container in self.all_hosts():
            try:
                self.client.stop(container)
                self.client.wait(container)
                self.client.remove_container(container)
            except APIError as e:
                # container does not exist
                if e.response.status_code == 404:
                    pass

        self.remove_host_mount_dirs()

    def copy_to_master(self, source):
        shutil.copy(source, LOCAL_MOUNT_POINT % self.master)

    def install_presto_admin(self):
        dist_dir = os.path.join(prestoadmin.main_dir, "dist")
        if not os.path.exists(dist_dir) or not fnmatch.filter(
                os.listdir(dist_dir), "prestoadmin-*.tar.bz2"):
            # setup.py expects you to be in the main directory
            saved_path = os.getcwd()
            os.chdir(prestoadmin.main_dir)
            distutils.core.run_setup("setup.py",
                                     ["bdist_prestoadmin"]).run_commands()
            os.chdir(saved_path)
        for dist_file in os.listdir(dist_dir):
            if fnmatch.fnmatch(dist_file, "prestoadmin-*.tar.bz2"):
                self.copy_to_master(os.path.join(dist_dir, dist_file))
        self.copy_to_master(LOCAL_RESOURCES_DIR + "/install-admin.sh")
        self.exec_create_start(self.master,
                               DOCKER_MOUNT_POINT + "/install-admin.sh")

    def exec_create_start(self, host, command):
        ex = self.client.exec_create(host, command)
        output = self.client.exec_start(ex['Id'])
        exit_code = self.client.exec_inspect(ex['Id'])['ExitCode']
        if exit_code:
            raise OSError(output)

        return output

    def upload_topology(self, topology=None):
        if not topology:
            topology = {"coordinator": "master",
                        "workers": ["slave1", "slave2", "slave3"]}
        with open(os.path.join(LOCAL_MOUNT_POINT % self.master,
                               "config.json"), "w") as conf_file:
            json.dump(topology, conf_file)

        self.exec_create_start(self.master, "cp %s /etc/opt/presto-admin/" %
                               os.path.join(DOCKER_MOUNT_POINT, "config.json"))

    def run_prestoadmin(self, command):
        return self.exec_create_start(self.master,
                                      "/opt/prestoadmin/presto-admin %s"
                                      % command)

    def all_hosts(self):
        return self.slaves[:] + [self.master]
<commit_msg>Fix presto-admin directory name for unit test.<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Base class for product tests.  Handles setting up a docker cluster and has
other utilities
"""

import distutils.core
import fnmatch
import json
import os
import shutil
import subprocess
import sys
import errno

from docker.errors import APIError
from docker import Client

import prestoadmin
from tests import utils

LOCAL_MOUNT_POINT = os.path.join(prestoadmin.main_dir, "tmp/docker-pa/%s")
LOCAL_RESOURCES_DIR = prestoadmin.main_dir + "/tests/product/resources/"
DOCKER_MOUNT_POINT = "/mnt/presto-admin"

class BaseProductTestCase(utils.BaseTestCase):
    client = Client()
    slaves = ["slave1", "slave2", "slave3"]
    master = "master"

    def setUp(self):
        self.capture_stdout_stderr()
        self.check_if_docker_exists()
        self.create_docker_cluster()

    def tearDown(self):
        self.restore_stdout_stderr_keep_open()
        self.tear_down_docker_cluster()

    def check_if_docker_exists(self):
        try:
            subprocess.call(['docker', '--version'])
        except OSError:
            sys.exit('Docker is not installed. Try installing it with '
                     'presto-admin/bin/install-docker.sh.')

    def create_host_mount_dirs(self):
        for container_name in self.all_hosts():
            try:
                os.makedirs(LOCAL_MOUNT_POINT % container_name)
            except OSError as e:
                # file exists
                if e.errno == errno.EEXIST:
                    pass

    def create_docker_cluster(self):
        self.tear_down_docker_cluster()
        self.create_host_mount_dirs()

        if not self.client.images("jdeathe/centos-ssh"):
            self._execute_and_wait(self.client.pull, "jdeathe/centos-ssh")
        self._execute_and_wait(self.client.build,
                               path=os.path.join(prestoadmin.main_dir,
                                                 "tests/product/resources/"
                                                 "centos6-ssh-test"),
                               tag="teradatalabs/centos6-ssh-test", rm=True)

        for container_name in self.slaves:
            self._execute_and_wait(self.client.create_container,
                                   "teradatalabs/centos6-ssh-test",
                                   detach=True,
                                   name=container_name,
                                   volumes=LOCAL_MOUNT_POINT %
                                   container_name)

            self.client.start(container_name,
                              binds={LOCAL_MOUNT_POINT % container_name:
                                     {"bind": DOCKER_MOUNT_POINT,
                                      "ro": False}})

        self._execute_and_wait(self.client.create_container,
                               "teradatalabs/centos6-ssh-test",
                               detach=True,
                               name=self.master,
                               hostname=self.master,
                               volumes=LOCAL_MOUNT_POINT % self.master)

        self.client.start(self.master,
                          binds={LOCAL_MOUNT_POINT % self.master:
                                 {"bind": DOCKER_MOUNT_POINT,
                                  "ro": False}},
                          links=zip(self.slaves, self.slaves))

    def _execute_and_wait(self, func, *args, **kwargs):
        ret = func(*args, **kwargs)
        # go through all lines in returned stream to ensure func finishes
        for line in ret:
            pass

    def remove_host_mount_dirs(self):
        for container_name in self.all_hosts():
            try:
                shutil.rmtree(LOCAL_MOUNT_POINT % container_name)
            except OSError as e:
                # no such file or directory
                if e.errno == errno.ENOENT:
                    pass

        try:
            os.removedirs(os.path.dirname(LOCAL_MOUNT_POINT))
        except OSError as e:
            if e.errno == errno.ENOENT:
                pass

    def tear_down_docker_cluster(self):
        for container in self.all_hosts():
            try:
                self.client.stop(container)
                self.client.wait(container)
                self.client.remove_container(container)
            except APIError as e:
                # container does not exist
                if e.response.status_code == 404:
                    pass

        self.remove_host_mount_dirs()

    def copy_to_master(self, source):
        shutil.copy(source, LOCAL_MOUNT_POINT % self.master)

    def install_presto_admin(self):
        dist_dir = os.path.join(prestoadmin.main_dir, "dist")
        if not os.path.exists(dist_dir) or not fnmatch.filter(
                os.listdir(dist_dir), "prestoadmin-*.tar.bz2"):
            # setup.py expects you to be in the main directory
            saved_path = os.getcwd()
            os.chdir(prestoadmin.main_dir)
            distutils.core.run_setup("setup.py",
                                     ["bdist_prestoadmin"]).run_commands()
            os.chdir(saved_path)
        for dist_file in os.listdir(dist_dir):
            if fnmatch.fnmatch(dist_file, "prestoadmin-*.tar.bz2"):
                self.copy_to_master(os.path.join(dist_dir, dist_file))
        self.copy_to_master(LOCAL_RESOURCES_DIR + "/install-admin.sh")
        self.exec_create_start(self.master,
                               DOCKER_MOUNT_POINT + "/install-admin.sh")

    def exec_create_start(self, host, command):
        ex = self.client.exec_create(host, command)
        output = self.client.exec_start(ex['Id'])
        exit_code = self.client.exec_inspect(ex['Id'])['ExitCode']
        if exit_code:
            raise OSError(output)

        return output

    def upload_topology(self, topology=None):
        if not topology:
            topology = {"coordinator": "master",
                        "workers": ["slave1", "slave2", "slave3"]}
        with open(os.path.join(LOCAL_MOUNT_POINT % self.master,
                               "config.json"), "w") as conf_file:
            json.dump(topology, conf_file)

        self.exec_create_start(self.master, "cp %s /etc/opt/presto-admin/" %
                               os.path.join(DOCKER_MOUNT_POINT, "config.json"))

    def run_prestoadmin(self, command):
        return self.exec_create_start(self.master,
                                      "/opt/presto-admin/presto-admin %s"
                                      % command)

    def all_hosts(self):
        return self.slaves[:] + [self.master]
<|endoftext|>


<commit_before>'''
scrape_nominatim_special_phrases.py
-----------------------------------

Simple script to scrape https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases
for category-related phrases sometimes found in geocoder input.

Populates a per-language CSV with (phrase, OSM key, OSM value, plural):

OSM keys/values are like:

amenity=restaurant
tourism=museum
shop=books

Using these phrases, it is possible to construct queries like "restaurants in Brooklyn"
'''

import csv
import os
import re
import requests
import six
import sys
import time
import yaml

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

from geodata.encoding import safe_decode, safe_encode

DEFAULT_CATEGORIES_DIR = os.path.join(this_dir, os.pardir, os.pardir, os.pardir,
                                      'resources', 'categories')

# Use Special:Export to get wiki markup
WIKI_BASE_URL = 'https://wiki.openstreetmap.org/wiki/Special:Export/'
NOMINATIM_SPECIAL_PHRASES_PREFIX = 'Nominatim/Special Phrases'
NOMINATIM_SPECIAL_PHRASES_URL = WIKI_BASE_URL + NOMINATIM_SPECIAL_PHRASES_PREFIX.replace(' ', '_')

phrase_table_re = re.compile('\| ([^|]+) \|\| ([^|]+) \|\| ([^|]+) \|\| ([^|]+) \|\| ([\-YN])', re.I)
wiki_link_re = re.compile('(?:\[\[([^\|\]]+(?<=\S))[\s]*(?:\|[\s]*)?(?:([^\]]+))?\]\])')

IGNORE_PLURAL_LANGUAGES = {
    # For Japanese, seems to just put an s on the end, which doesn't seem right
    # Need input from a native speaker on that one
    'ja',
}

# Wait this many seconds between page fetches
POLITENESS_DELAY = 5.0

def scrape_nominatim_category_page(url, ignore_plurals=False):
    result = requests.get(url)

    if not result or not result.content:
        return

    for phrase, key, value, operator, plural in phrase_table_re.findall(result.content):
        if operator and operator != '-':
            continue

        is_plural = plural == 'Y'
        if is_plural and ignore_plurals:
            continue

        yield safe_decode(phrase).lower(), key, value, is_plural

def scrape_all_nominatim_category_pages(url=NOMINATIM_SPECIAL_PHRASES_URL):
    print('Fetching main page')
    result = requests.get(url)
    languages = {}
    if not result or not result.content:
        return languages

    time.sleep(POLITENESS_DELAY)

    for entity, anchor_text in wiki_link_re.findall(result.content):
        if not entity.startswith(NOMINATIM_SPECIAL_PHRASES_PREFIX):
            continue

        lang = entity.rstrip('/').rsplit('/')[-1].lower()

        link = WIKI_BASE_URL + entity.replace(' ', '_')

        ignore_plurals = lang in IGNORE_PLURAL_LANGUAGES

        print('Doing {}'.format(lang))
        phrases = list(scrape_nominatim_category_page(link, ignore_plurals=ignore_plurals))
        time.sleep(POLITENESS_DELAY)

        if not phrases:
            continue

        languages[lang] = phrases

    return languages

def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR):
    languages = scrape_all_nominatim_category_pages(url=url)
    for lang, phrases in six.iteritems(languages):
        filename = os.path.join(output_dir, '{}.yml'.format(lang.lower()))
        with open(filename, 'w') as f:
            phrase_data = [
                {
                    'phrase': phrase,
                    'key': key,
                    'value': value,
                    'is_plural': is_plural
                }
                for phrase, key, value, is_plural in phrases
            ]

            yaml.dump(phrase_data, f, allow_unicode=True, default_flow_style=False)

    print('Done')

if __name__ == '__main__':
    main()
<commit_msg>[fix] changing extension to .yaml<commit_after>'''
scrape_nominatim_special_phrases.py
-----------------------------------

Simple script to scrape https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases
for category-related phrases sometimes found in geocoder input.

Populates a per-language CSV with (phrase, OSM key, OSM value, plural):

OSM keys/values are like:

amenity=restaurant
tourism=museum
shop=books

Using these phrases, it is possible to construct queries like "restaurants in Brooklyn"
'''

import os
import re
import requests
import six
import sys
import time
import yaml

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

from geodata.encoding import safe_decode, safe_encode

DEFAULT_CATEGORIES_DIR = os.path.join(this_dir, os.pardir, os.pardir, os.pardir,
                                      'resources', 'categories')

# Use Special:Export to get wiki markup
WIKI_BASE_URL = 'https://wiki.openstreetmap.org/wiki/Special:Export/'
NOMINATIM_SPECIAL_PHRASES_PREFIX = 'Nominatim/Special Phrases'
NOMINATIM_SPECIAL_PHRASES_URL = WIKI_BASE_URL + NOMINATIM_SPECIAL_PHRASES_PREFIX.replace(' ', '_')

phrase_table_re = re.compile('\| ([^|]+) \|\| ([^|]+) \|\| ([^|]+) \|\| ([^|]+) \|\| ([\-YN])', re.I)
wiki_link_re = re.compile('(?:\[\[([^\|\]]+(?<=\S))[\s]*(?:\|[\s]*)?(?:([^\]]+))?\]\])')

IGNORE_PLURAL_LANGUAGES = {
    # For Japanese, seems to just put an s on the end, which doesn't seem right
    # Need input from a native speaker on that one
    'ja',
}

# Wait this many seconds between page fetches
POLITENESS_DELAY = 5.0

def scrape_nominatim_category_page(url, ignore_plurals=False):
    result = requests.get(url)

    if not result or not result.content:
        return

    for phrase, key, value, operator, plural in phrase_table_re.findall(result.content):
        if operator and operator != '-':
            continue

        is_plural = plural == 'Y'
        if is_plural and ignore_plurals:
            continue

        yield safe_decode(phrase).lower(), key, value, is_plural

def scrape_all_nominatim_category_pages(url=NOMINATIM_SPECIAL_PHRASES_URL):
    print('Fetching main page')
    result = requests.get(url)
    languages = {}
    if not result or not result.content:
        return languages

    time.sleep(POLITENESS_DELAY)

    for entity, anchor_text in wiki_link_re.findall(result.content):
        if not entity.startswith(NOMINATIM_SPECIAL_PHRASES_PREFIX):
            continue

        lang = entity.rstrip('/').rsplit('/')[-1].lower()

        link = WIKI_BASE_URL + entity.replace(' ', '_')

        ignore_plurals = lang in IGNORE_PLURAL_LANGUAGES

        print('Doing {}'.format(lang))
        phrases = list(scrape_nominatim_category_page(link, ignore_plurals=ignore_plurals))
        time.sleep(POLITENESS_DELAY)

        if not phrases:
            continue

        languages[lang] = phrases

    return languages

def main(url=NOMINATIM_SPECIAL_PHRASES_URL, output_dir=DEFAULT_CATEGORIES_DIR):
    languages = scrape_all_nominatim_category_pages(url=url)
    for lang, phrases in six.iteritems(languages):
        filename = os.path.join(output_dir, '{}.yaml'.format(lang.lower()))
        with open(filename, 'w') as f:
            phrase_data = [
                {
                    'phrase': phrase,
                    'key': key,
                    'value': value,
                    'is_plural': is_plural
                }
                for phrase, key, value, is_plural in phrases
            ]

            yaml.dump(phrase_data, f, allow_unicode=True, default_flow_style=False)

    print('Done')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import numpy as np
from CGvsPhoto import image_loader as il

from multiprocessing import Pool

from functools import partial

from sklearn.svm import SVC
from sklearn.metrics import accuracy_score

def compute_code(minipatch, mode = 'ltc'):

	s = np.sign(minipatch - minipatch[1,1])
	# print(s)
	if mode == 'lbp':
		s[s == -1] = 0
		# print(s)
		binary = array_to_bin(s)
		# print(binary)

		return(binary)
	if mode == 'ltc':
		return((str(int(np.sum(s[s==1])))+ 'u', str(-int(np.sum(s[s == -1]))) + 'l'))
	# return('0b100000000')

def get_classes(mode = 'ltc'):
	classes = dict()
	if mode == 'lbp':
		n = 0
		for i in range(511):
			b = '{:08b}'.format(i)
			A = [[int(b[0]), int(b[1]), int(b[2])],
				 [int(b[7]), 0, int(b[3])],
				 [int(b[6]), int(b[5]), int(b[4])]]

			b = array_to_bin(np.array(A))
			if b not in classes:
				classes[b] = n
				n+=1

	if mode == 'ltc':
		n = 0
		for i in range(9):
			classes[str(i) + 'l'] = n
			classes[str(i) + 'u'] = n + 1
			n+=2

	print(str(n) + ' classes')
	return(classes)

def compute_error_image(image):
	prediction = np.empty([image.shape[0] - 1, image.shape[1] - 1])
	for i in range(image.shape[0]-1):
		for j in range(image.shape[1]-1):
			a = image[i, j+1]
			b = image[i+1, j]
			c = image[i+1, j+1]

			if c <= min(a,b):
				prediction[i,j] = max(a,b)
			else:
				if c >= max(a,b):
					prediction[i,j] = min(a,b)
				else:
					prediction[i,j] = a + b -c
	# print(prediction.shape)
	# print(image.shape)
	error = image[:image.shape[0]-1, :image.shape[1]-1, 0] - prediction
	# print(error.shape)
	return(error)

def array_to_bin(A):

	T = [int(A[0,0]), int(A[0,1]), int(A[0,2]), int(A[1,2]), int(A[2,2]),
	     int(A[2,1]), int(A[2,0]), int(A[1,0])]

	nb_c = 0
	for i in range(1,8):
		if T[i-1] != T[i]:
			nb_c += 1

	if nb_c > 2:
		binary = '0b100000000'
	else:
		binary = '0b' + str(T[0]) + str(T[1]) + str(T[2]) + str(T[3]) + str(T[4]) + str(T[5]) + str(T[6]) + str(T[7])

	return(binary)

def compute_hist(image, mode = 'ltc'):

	hist_1 = dict()
	hist_2 = dict()
	# hist_error = dict()
	for i in classes.keys():
		hist_1[i] = 0
		hist_2[i] = 0
		# hist_error[i] = 0

	# error = compute_error_image(image)

	for i in range(1, image.shape[0] - 2):
		for j in range(1, image.shape[1] - 2):
			if mode == 'lbp':
				b = compute_code(image[i-1:i+2, j-1:j+2,0], mode)
				hist_1[b] += 1
				b = compute_code(image[i-1:i+2, j-1:j+2,1], mode)
				hist_2[b] += 1

			if mode == 'ltc':
				b = compute_code(image[i-1:i+2, j-1:j+2,0], mode)
				hist_1[b[0]] += 1
				hist_1[b[1]] += 1
				b = compute_code(image[i-1:i+2, j-1:j+2,1], mode)
				hist_2[b[0]] += 1
				hist_2[b[1]] += 1
			# b_error = compute_code(error[i-1:i+2, j-1:j+2])
			# hist_error[b_error] += 1

	F = []
	N = (image.shape[0] - 3)*(image.shape[1] - 3)
	for i in hist_1.keys():
		F.append(hist_1[i]/N)
		F.append(hist_2[i]/N)
		# F.append(hist_error[i])

	return(np.array(F))

def compute_features(data, i, batch_size, nb_batch, mode = 'ltc'):

	print('Compute features for batch ' + str(i+1) + '/' + str(nb_batch))
	images, labels = data[0], data[1]
	features = []
	y_train = []
	for i in range(batch_size):
		features.append(compute_hist(images, mode))
		y_train.append(labels[0])

	print(features[0])
	return(features, y_train)

def compute_testing_features(i, batch_size, nb_test_batch, data):

	print('Compute features for testing batch ' + str(i+1) + '/' + str(nb_test_batch))
	images, labels = data.get_batch_test(batch_size = batch_size,
											   crop = False)

	features = []
	y_test = []
	for i in range(batch_size):
		features.append(compute_hist(images))
		y_test.append(labels[0])

	return(features, y_test)

if __name__ == '__main__':

	data_directory = '/work/smg/v-nicolas/level-design_raise_100_color/'
	image_size = None

	data = il.Database_loader(directory = data_directory,
							  size = image_size, only_green = False)

	mode = 'lbp'

	classes = get_classes(mode)

	nb_train_batch = 1000
	batch_size = 1

	print('Training...')
	features_train = np.empty([nb_train_batch*batch_size, 2*len(classes.keys())])
	y_train = np.empty([nb_train_batch*batch_size,])

	data_train = []
	for i in range(nb_train_batch):
		print('Getting batch ' + str(i+1) + '/' + str(nb_train_batch))
		images_batch, y_batch = data.get_next_train(crop = False)
		data_train.append([images_batch, y_batch])

	pool = Pool()

	to_compute = [i for i in range(nb_train_batch)]
	result = pool.starmap(partial(compute_features,
							  batch_size = batch_size,
							  nb_batch = nb_train_batch,
							  mode = mode),
							  zip(data_train, to_compute))

	del(data_train)

	index = 0
	for i in range(len(result)):
		features_train[index:index+batch_size] = result[i][0]
		y_train[index:index+batch_size] = result[i][1]

		index+=batch_size

	del(result)

	clf = SVC()

	print('Fitting SVM...')

	clf.fit(features_train, y_train)

	print('Testing...')

	nb_test_batch = 500

	features_test = np.empty([nb_test_batch*batch_size, 2*len(classes.keys())])
	y_test = np.empty([nb_test_batch*batch_size,])

	data_test = []
	for i in range(nb_test_batch):
		print('Getting batch ' + str(i+1) + '/' + str(nb_test_batch))
		images_batch, y_batch = data.get_next_test(crop = False)
		data_test.append([images_batch, y_batch])
	pool = Pool()

	to_compute = [i for i in range(nb_test_batch)]
	result = pool.starmap(partial(compute_features,
							  batch_size = batch_size,
							  nb_batch = nb_test_batch,
							  mode = mode),
							  zip(data_test, to_compute))

	del(data_test)
	index = 0
	for i in range(len(result)):
		features_test[index:index+batch_size] = result[i][0]
		y_test[index:index+batch_size] = result[i][1]

		index+=batch_size

	del(result)

	print('Prediction...')
	y_pred = clf.predict(features_test)

	score = accuracy_score(y_pred,y_test)

	print("Accuracy : " + str(score))

<commit_msg>Last parameters<commit_after>import numpy as np
from CGvsPhoto import image_loader as il

from multiprocessing import Pool

from functools import partial

from sklearn.svm import SVC
from sklearn.metrics import accuracy_score

def compute_code(minipatch, mode = 'ltc'):

	s = np.sign(minipatch - minipatch[1,1])
	# print(s)
	if mode == 'lbp':
		s[s == -1] = 0
		# print(s)
		binary = array_to_bin(s)
		# print(binary)

		return(binary)
	if mode == 'ltc':
		return((str(int(np.sum(s[s==1])))+ 'u', str(-int(np.sum(s[s == -1]))) + 'l'))
	# return('0b100000000')

def get_classes(mode = 'ltc'):
	classes = dict()
	if mode == 'lbp':
		n = 0
		for i in range(511):
			b = '{:08b}'.format(i)
			A = [[int(b[0]), int(b[1]), int(b[2])],
				 [int(b[7]), 0, int(b[3])],
				 [int(b[6]), int(b[5]), int(b[4])]]

			b = array_to_bin(np.array(A))
			if b not in classes:
				classes[b] = n
				n+=1

	if mode == 'ltc':
		n = 0
		for i in range(9):
			classes[str(i) + 'l'] = n
			classes[str(i) + 'u'] = n + 1
			n+=2

	print(str(n) + ' classes')
	return(classes)

def compute_error_image(image):
	prediction = np.empty([image.shape[0] - 1, image.shape[1] - 1])
	for i in range(image.shape[0]-1):
		for j in range(image.shape[1]-1):
			a = image[i, j+1]
			b = image[i+1, j]
			c = image[i+1, j+1]

			if c <= min(a,b):
				prediction[i,j] = max(a,b)
			else:
				if c >= max(a,b):
					prediction[i,j] = min(a,b)
				else:
					prediction[i,j] = a + b -c
	# print(prediction.shape)
	# print(image.shape)
	error = image[:image.shape[0]-1, :image.shape[1]-1, 0] - prediction
	# print(error.shape)
	return(error)

def array_to_bin(A):

	T = [int(A[0,0]), int(A[0,1]), int(A[0,2]), int(A[1,2]), int(A[2,2]),
	     int(A[2,1]), int(A[2,0]), int(A[1,0])]

	nb_c = 0
	for i in range(1,8):
		if T[i-1] != T[i]:
			nb_c += 1

	if nb_c > 2:
		binary = '0b100000000'
	else:
		binary = '0b' + str(T[0]) + str(T[1]) + str(T[2]) + str(T[3]) + str(T[4]) + str(T[5]) + str(T[6]) + str(T[7])

	return(binary)

def compute_hist(image, mode = 'ltc'):

	hist_1 = dict()
	hist_2 = dict()
	# hist_error = dict()
	for i in classes.keys():
		hist_1[i] = 0
		hist_2[i] = 0
		# hist_error[i] = 0

	# error = compute_error_image(image)

	for i in range(1, image.shape[0] - 2):
		for j in range(1, image.shape[1] - 2):
			if mode == 'lbp':
				b = compute_code(image[i-1:i+2, j-1:j+2,0], mode)
				hist_1[b] += 1
				b = compute_code(image[i-1:i+2, j-1:j+2,1], mode)
				hist_2[b] += 1

			if mode == 'ltc':
				b = compute_code(image[i-1:i+2, j-1:j+2,0], mode)
				hist_1[b[0]] += 1
				hist_1[b[1]] += 1
				b = compute_code(image[i-1:i+2, j-1:j+2,1], mode)
				hist_2[b[0]] += 1
				hist_2[b[1]] += 1
			# b_error = compute_code(error[i-1:i+2, j-1:j+2])
			# hist_error[b_error] += 1

	F = []
	N = (image.shape[0] - 3)*(image.shape[1] - 3)
	for i in hist_1.keys():
		F.append(hist_1[i]/N)
		F.append(hist_2[i]/N)
		# F.append(hist_error[i])

	return(np.array(F))

def compute_features(data, i, batch_size, nb_batch, mode = 'ltc'):

	print('Compute features for batch ' + str(i+1) + '/' + str(nb_batch))
	images, labels = data[0], data[1]
	features = []
	y_train = []
	for i in range(batch_size):
		features.append(compute_hist(images, mode))
		y_train.append(labels[0])

	print(features[0])
	return(features, y_train)

def compute_testing_features(i, batch_size, nb_test_batch, data):

	print('Compute features for testing batch ' + str(i+1) + '/' + str(nb_test_batch))
	images, labels = data.get_batch_test(batch_size = batch_size,
											   crop = False)

	features = []
	y_test = []
	for i in range(batch_size):
		features.append(compute_hist(images))
		y_test.append(labels[0])

	return(features, y_test)

if __name__ == '__main__':

	data_directory = '/work/smg/v-nicolas/level-design_raise/'
	image_size = None

	data = il.Database_loader(directory = data_directory,
							  size = image_size, only_green = False)

	mode = 'lbp'

	classes = get_classes(mode)

	nb_train_batch = 1000
	batch_size = 1

	print('Training...')
	features_train = np.empty([nb_train_batch*batch_size, 2*len(classes.keys())])
	y_train = np.empty([nb_train_batch*batch_size,])

	data_train = []
	for i in range(nb_train_batch):
		print('Getting batch ' + str(i+1) + '/' + str(nb_train_batch))
		images_batch, y_batch = data.get_next_train(crop = False)
		data_train.append([images_batch, y_batch])

	pool = Pool()

	to_compute = [i for i in range(nb_train_batch)]
	result = pool.starmap(partial(compute_features,
							  batch_size = batch_size,
							  nb_batch = nb_train_batch,
							  mode = mode),
							  zip(data_train, to_compute))

	del(data_train)

	index = 0
	for i in range(len(result)):
		features_train[index:index+batch_size] = result[i][0]
		y_train[index:index+batch_size] = result[i][1]

		index+=batch_size

	del(result)

	clf = SVC()

	print('Fitting SVM...')

	clf.fit(features_train, y_train)

	print('Testing...')

	nb_test_batch = 500

	features_test = np.empty([nb_test_batch*batch_size, 2*len(classes.keys())])
	y_test = np.empty([nb_test_batch*batch_size,])

	data_test = []
	for i in range(nb_test_batch):
		print('Getting batch ' + str(i+1) + '/' + str(nb_test_batch))
		images_batch, y_batch = data.get_next_test(crop = False)
		data_test.append([images_batch, y_batch])
	pool = Pool()

	to_compute = [i for i in range(nb_test_batch)]
	result = pool.starmap(partial(compute_features,
							  batch_size = batch_size,
							  nb_batch = nb_test_batch,
							  mode = mode),
							  zip(data_test, to_compute))

	del(data_test)
	index = 0
	for i in range(len(result)):
		features_test[index:index+batch_size] = result[i][0]
		y_test[index:index+batch_size] = result[i][1]

		index+=batch_size

	del(result)

	print('Prediction...')
	y_pred = clf.predict(features_test)

	score = accuracy_score(y_pred,y_test)

	print("Accuracy : " + str(score))

<|endoftext|>