<commit_before># Copyright © 2013-2017 Jakub Wilk <jwilk@jwilk.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
the command-line interface
'''

import argparse
import io
import sys
import functools

import enchant.tokenize

class lib:
    from . import colors
    from . import data
    from . import extdict
    from . import intdict
    from . import ns
    from . import pager
    from . import text

__version__ = '0.7.4'

class VersionAction(argparse.Action):

    def __init__(self, option_strings, dest=argparse.SUPPRESS):
        super().__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=0,
            help="show program's version information and exit"
        )

    def __call__(self, parser, namespace, values, option_string=None):
        print('{prog} {0}'.format(__version__, prog=parser.prog))
        print('+ Python {0}.{1}.{2}'.format(*sys.version_info))
        print('+ PyEnchant {0}'.format(__version__))
        try:
            enchant_version = enchant.get_enchant_version()
        except AttributeError:
            pass
        else:
            if isinstance(enchant_version, bytes):
                enchant_version = enchant_version.decode('ASCII', 'replace')
            print('  + Enchant {0}'.format(enchant_version))
        print('+ regex {0}'.format(lib.intdict.re.__version__))
        parser.exit()

def main():
    ap = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
    ap.add_argument('--version', action=VersionAction)
    ap.add_argument('files', metavar='FILE', nargs='*', default=['-'],
        help='file to process (default: stdin)')
    ap.add_argument('-l', '--language', metavar='LANG', default='en',
        help='spell-check for this language (default: "en")')
    ap.add_argument('--list-languages', nargs=0, action=list_languages,
        help='print list of available languages')
    ap.add_argument('--blacklist', metavar='FILE', action='append', default=[],
        help='use misspelling dictionary')
    ap.add_argument('--camel-case', action='store_true',
        help='split camel-cased compound words')
    ap.add_argument('--input-encoding', metavar='ENC', default='UTF-8:replace',
        help='assume input encoding ENC (default: "UTF-8:replace")')
    default_output_format = 'color' if sys.stdout.isatty() else 'plain'
    ap.add_argument('-f', '--output-format', choices=('plain', 'color'), default=default_output_format,
        help=(
            '"plain" = use "^" to emphasize words\n'
            '"color" = highlight words in color (default on tty)\n'
        )
    )
    ap.add_argument('-r', '--reverse', action='store_true',
        help='print most frequent words first')
    ap.add_argument('--compact', action='store_true',
        help='omit blank lines in output')
    ap.add_argument('--limit', metavar='N', type=int, default=1e999,
        help='skip words that have >N instances')
    ap.add_argument('--max-context-width', metavar='N', default=30,
        help='limit context width to N chars')
    ap.add_argument('--suggest', metavar='N', type=int, default=0,
        help='suggest up to N corrections')
    ap.add_argument('--debug-dict', action='store_true', help=argparse.SUPPRESS)
    options = ap.parse_args()
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, 'UTF-8')
    try:
        split_words = enchant.tokenize.get_tokenizer(options.language)
    except enchant.errors.TokenizerNotFoundError:
        split_words = enchant.tokenize.get_tokenizer(None)
    if options.camel_case:
        split_words = lib.text.camel_case_tokenizer(split_words)
    if options.language == 'und':
        dictionary = None
        spellcheck = ''.__gt__  # always returns False
        options.suggest = 0
    else:
        dictionary = enchant.Dict(options.language)
        spellcheck = functools.lru_cache(maxsize=None)(
            dictionary.check
        )
    if options.debug_dict:
        if dictionary is None:
            dictvars = {}
        else:
            dictvars = vars(dictionary).items()
        for key, value in sorted(dictvars):
            print('{k} = {v!r}'.format(k=key, v=value))
        sys.exit(0)
    intdict = lib.intdict.Dictionary(options.language)
    extdict = lib.extdict.Dictionary(*options.blacklist)
    misspellings = lib.data.Misspellings()
    encoding = options.input_encoding
    enc_errors = 'strict'
    if ':' in encoding:
        [encoding, enc_errors] = encoding.rsplit(':', 1)
    ctxt = lib.ns.Namespace(
        dictionary=dictionary,
        intdict=intdict,
        extdict=extdict,
        split_words=split_words,
        spellcheck=spellcheck,
        misspellings=misspellings,
        options=options,
    )
    for path in options.files:
        if path == '-':
            file = io.TextIOWrapper(
                sys.stdin.buffer,
                encoding=encoding,
                errors=enc_errors,
            )
        else:
            file = open(
                path, 'rt',
                encoding=encoding,
                errors=enc_errors,
            )
        with file:
            spellcheck_file(ctxt, file)
    if not misspellings:
        return
    raw_cc = options.output_format == 'color'
    with lib.pager.autopager(raw_control_chars=raw_cc):
        print_misspellings(ctxt)

def spellcheck_file(ctxt, file):
    force_ucs2 = (
        ctxt.dictionary is not None and
        ctxt.dictionary.provider.name == 'myspell'
    )
    for line in file:
        if force_ucs2:
            # https://github.com/rfk/pyenchant/issues/58
            line = ''.join(c if c <= '\uFFFF' else '\uFFFD' for c in line)
        line = line.strip()
        line = line.expandtabs()
        taken = bytearray(len(line))
        for word, pos in ctxt.split_words(line):
            assert len(word) >= 1
            if word in ctxt.extdict:
                certainty = 1
            elif ctxt.spellcheck(word):
                continue
            elif ctxt.intdict.is_whitelisted(word):
                continue
            else:
                certainty = 0
            for i, ch in enumerate(word, start=pos):  # pylint: disable=unused-variable
                taken[i] = True
            ctxt.misspellings.add(word, line, pos, certainty)
        for word, pos in ctxt.intdict.find(line):
            assert len(word) >= 1
            for i, ch in enumerate(word, start=pos):
                if taken[i]:
                    break
            else:
                ctxt.misspellings.add(word, line, pos, 1)

def print_misspellings(ctxt):
    rare_misspellings = lib.data.Misspellings()
    for word, occurrences in ctxt.misspellings.sorted_words():
        if len(occurrences) == 1:
            [(word, line, positions)] = occurrences
            for pos, certainty in positions.items():
                rare_misspellings.add(word, line, pos, certainty)
    ctxt.rare_misspellings = rare_misspellings
    if ctxt.options.reverse:
        print_common_misspellings(ctxt)
        print_rare_misspellings(ctxt)
    else:
        print_rare_misspellings(ctxt)
        print_common_misspellings(ctxt)

def print_common_misspellings(ctxt):
    options = ctxt.options
    for word, occurrences in ctxt.misspellings.sorted_words(reverse=options.reverse):
        if len(occurrences) == 1:
            continue
        if occurrences.count() > options.limit:
            continue
        extra = ''
        if options.suggest > 0:
            suggestions = ctxt.dictionary.suggest(word)[:options.suggest]
            if suggestions:
                extra = ' ({sug})'.format(sug=', '.join(suggestions))
        print(word + extra + ':')
        highlight_color = 'error' if occurrences.certainty > 0 else 'warn'
        occurrences = [
            (
                lib.text.ltrim(lcontext, options.max_context_width),
                word,
                lib.text.rtrim(rcontext, options.max_context_width),
            )
            for lcontext, word, rcontext
            in occurrences.sorted_context()
        ]
        lwidth = max(len(lcontext) for lcontext, _, _, in occurrences)
        for lcontext, word, rcontext in occurrences:
            lcontext = lcontext.rjust(lwidth)
            if options.output_format == 'color':
                lcontext = lib.colors.escape(lcontext)
                word = lib.colors.highlight(word, highlight_color)
                rcontext = lib.colors.escape(rcontext)
                print(lib.colors.dim('|'), end=' ')
            else:
                print('|', end=' ')
            print('{lc}{word}{rc}'.format(
                lc=lcontext,
                word=word,
                rc=rcontext,
            ))
        if options.output_format != 'color':
            print('', ' ' * lwidth, '^' * len(word))
        if not options.compact:
            print()

def print_rare_misspellings(ctxt):
    options = ctxt.options
    use_color = options.output_format == 'color'
    for line, occurrences in ctxt.rare_misspellings.sorted_lines(reverse=options.reverse):
        header = []
        underline = bytearray(b' ' * len(line))
        for word, line, positions in sorted(occurrences):
            if use_color and (max(positions.values()) > 0):
                underline_char = b'!'
            else:
                underline_char = b'^'
            if len(positions) > options.limit:
                continue
            extra = ''
            if options.suggest > 0:
                suggestions = ctxt.dictionary.suggest(word)[:options.suggest]
                if suggestions:
                    extra = ' ({sug})'.format(sug=', '.join(suggestions))
            header += [word + extra]
            for x in positions:
                underline[x : x + len(word)] = underline_char * len(word)
        if not header:
            continue
        print(', '.join(header) + ':')
        underline = underline.decode()
        lwidth = len(underline) - len(underline.lstrip())
        rwidth = len(underline) - len(underline.rstrip())
        lexceed = lwidth - options.max_context_width
        rexceed = rwidth - options.max_context_width
        if lexceed > 0:
            lwidth = len(line) - lexceed
            line = lib.text.ltrim(line, lwidth)
            underline = lib.text.ltrim(underline, lwidth, char=' ')
        if rexceed > 0:
            rwidth = len(line) - rexceed
            line = lib.text.rtrim(line, rwidth)
            underline = lib.text.rtrim(underline, rwidth, char=' ')
        if use_color:
            hline = lib.colors.highlight(
                line, (
                    'warn' if u == '^' else
                    'error' if u == '!' else
                    'off'
                    for u in underline
                )
            )
            print(lib.colors.dim('|'), hline)
        else:
            print('|', line)
            print(' ', underline.rstrip())
        if not options.compact:
            print()

class list_languages(argparse.Action):
    def __call__(self, *args, **kwargs):
        for lang in sorted(enchant.list_languages()):
            print(lang)
        sys.exit(0)

__all__ = ['main']

# vim:ts=4 sts=4 sw=4 et
<commit_msg>lib.cli: silence some pylint warnings.<commit_after># Copyright © 2013-2017 Jakub Wilk <jwilk@jwilk.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

'''
the command-line interface
'''

import argparse
import io
import sys
import functools

import enchant.tokenize

class lib:
    from . import colors
    from . import data
    from . import extdict
    from . import intdict
    from . import ns
    from . import pager
    from . import text

__version__ = '0.7.4'

class VersionAction(argparse.Action):

    def __init__(self, option_strings, dest=argparse.SUPPRESS):
        super().__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=0,
            help="show program's version information and exit"
        )

    def __call__(self, parser, namespace, values, option_string=None):
        print('{prog} {0}'.format(__version__, prog=parser.prog))
        print('+ Python {0}.{1}.{2}'.format(*sys.version_info))
        print('+ PyEnchant {0}'.format(__version__))
        try:
            enchant_version = enchant.get_enchant_version()
        except AttributeError:
            pass
        else:
            if isinstance(enchant_version, bytes):
                enchant_version = enchant_version.decode('ASCII', 'replace')
            print('  + Enchant {0}'.format(enchant_version))
        print('+ regex {0}'.format(lib.intdict.re.__version__))
        parser.exit()

def main():
    ap = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
    ap.add_argument('--version', action=VersionAction)
    ap.add_argument('files', metavar='FILE', nargs='*', default=['-'],
        help='file to process (default: stdin)')
    ap.add_argument('-l', '--language', metavar='LANG', default='en',
        help='spell-check for this language (default: "en")')
    ap.add_argument('--list-languages', nargs=0, action=list_languages,
        help='print list of available languages')
    ap.add_argument('--blacklist', metavar='FILE', action='append', default=[],
        help='use misspelling dictionary')
    ap.add_argument('--camel-case', action='store_true',
        help='split camel-cased compound words')
    ap.add_argument('--input-encoding', metavar='ENC', default='UTF-8:replace',
        help='assume input encoding ENC (default: "UTF-8:replace")')
    default_output_format = 'color' if sys.stdout.isatty() else 'plain'
    ap.add_argument('-f', '--output-format', choices=('plain', 'color'), default=default_output_format,
        help=(
            '"plain" = use "^" to emphasize words\n'
            '"color" = highlight words in color (default on tty)\n'
        )
    )
    ap.add_argument('-r', '--reverse', action='store_true',
        help='print most frequent words first')
    ap.add_argument('--compact', action='store_true',
        help='omit blank lines in output')
    ap.add_argument('--limit', metavar='N', type=int, default=1e999,
        help='skip words that have >N instances')
    ap.add_argument('--max-context-width', metavar='N', default=30,
        help='limit context width to N chars')
    ap.add_argument('--suggest', metavar='N', type=int, default=0,
        help='suggest up to N corrections')
    ap.add_argument('--debug-dict', action='store_true', help=argparse.SUPPRESS)
    options = ap.parse_args()
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, 'UTF-8')
    try:
        split_words = enchant.tokenize.get_tokenizer(options.language)
    except enchant.errors.TokenizerNotFoundError:
        split_words = enchant.tokenize.get_tokenizer(None)
    if options.camel_case:
        split_words = lib.text.camel_case_tokenizer(split_words)
    if options.language == 'und':
        dictionary = None
        spellcheck = ''.__gt__  # always returns False
        options.suggest = 0
    else:
        dictionary = enchant.Dict(options.language)
        spellcheck = functools.lru_cache(maxsize=None)(
            dictionary.check
        )
    if options.debug_dict:
        if dictionary is None:
            dictvars = {}
        else:
            dictvars = vars(dictionary).items()
        for key, value in sorted(dictvars):
            print('{k} = {v!r}'.format(k=key, v=value))
        sys.exit(0)
    intdict = lib.intdict.Dictionary(options.language)
    extdict = lib.extdict.Dictionary(*options.blacklist)
    misspellings = lib.data.Misspellings()
    encoding = options.input_encoding
    enc_errors = 'strict'
    if ':' in encoding:
        [encoding, enc_errors] = encoding.rsplit(':', 1)
    ctxt = lib.ns.Namespace(
        dictionary=dictionary,
        intdict=intdict,
        extdict=extdict,
        split_words=split_words,
        spellcheck=spellcheck,
        misspellings=misspellings,
        options=options,
    )
    for path in options.files:
        if path == '-':
            file = io.TextIOWrapper(
                sys.stdin.buffer,
                encoding=encoding,
                errors=enc_errors,
            )
        else:
            file = open(
                path, 'rt',
                encoding=encoding,
                errors=enc_errors,
            )
        with file:
            spellcheck_file(ctxt, file)
    if not misspellings:
        return
    raw_cc = options.output_format == 'color'
    with lib.pager.autopager(raw_control_chars=raw_cc):
        print_misspellings(ctxt)

def spellcheck_file(ctxt, file):
    force_ucs2 = (
        ctxt.dictionary is not None and
        ctxt.dictionary.provider.name == 'myspell'
    )
    for line in file:
        if force_ucs2:
            # https://github.com/rfk/pyenchant/issues/58
            line = ''.join(c if c <= '\uFFFF' else '\uFFFD' for c in line)
        line = line.strip()
        line = line.expandtabs()
        taken = bytearray(len(line))
        for word, pos in ctxt.split_words(line):
            assert len(word) >= 1
            if word in ctxt.extdict:
                certainty = 1
            elif ctxt.spellcheck(word):
                continue
            elif ctxt.intdict.is_whitelisted(word):
                continue
            else:
                certainty = 0
            for i, ch in enumerate(word, start=pos):  # pylint: disable=unused-variable
                taken[i] = True
            ctxt.misspellings.add(word, line, pos, certainty)
        for word, pos in ctxt.intdict.find(line):
            assert len(word) >= 1
            for i, ch in enumerate(word, start=pos):
                if taken[i]:
                    break
            else:
                ctxt.misspellings.add(word, line, pos, 1)

def print_misspellings(ctxt):
    rare_misspellings = lib.data.Misspellings()
    for word, occurrences in ctxt.misspellings.sorted_words():
        if len(occurrences) == 1:
            [(word, line, positions)] = occurrences
            for pos, certainty in positions.items():
                rare_misspellings.add(word, line, pos, certainty)
    ctxt.rare_misspellings = rare_misspellings
    if ctxt.options.reverse:
        print_common_misspellings(ctxt)
        print_rare_misspellings(ctxt)
    else:
        print_rare_misspellings(ctxt)
        print_common_misspellings(ctxt)

def print_common_misspellings(ctxt):
    options = ctxt.options
    for word, occurrences in ctxt.misspellings.sorted_words(reverse=options.reverse):
        if len(occurrences) == 1:
            continue
        if occurrences.count() > options.limit:
            continue
        extra = ''
        if options.suggest > 0:
            suggestions = ctxt.dictionary.suggest(word)[:options.suggest]
            if suggestions:
                extra = ' ({sug})'.format(sug=', '.join(suggestions))
        print(word + extra + ':')
        highlight_color = 'error' if occurrences.certainty > 0 else 'warn'
        occurrences = [
            (
                lib.text.ltrim(lcontext, options.max_context_width),
                word,
                lib.text.rtrim(rcontext, options.max_context_width),
            )
            for lcontext, word, rcontext
            in occurrences.sorted_context()
        ]
        lwidth = max(len(lcontext) for lcontext, _, _, in occurrences)
        for lcontext, word, rcontext in occurrences:  # pylint: disable=redefined-outer-name
            lcontext = lcontext.rjust(lwidth)
            if options.output_format == 'color':
                lcontext = lib.colors.escape(lcontext)
                word = lib.colors.highlight(word, highlight_color)
                rcontext = lib.colors.escape(rcontext)
                print(lib.colors.dim('|'), end=' ')
            else:
                print('|', end=' ')
            print('{lc}{word}{rc}'.format(
                lc=lcontext,
                word=word,
                rc=rcontext,
            ))
        if options.output_format != 'color':
            print('', ' ' * lwidth, '^' * len(word))
        if not options.compact:
            print()

def print_rare_misspellings(ctxt):
    options = ctxt.options
    use_color = options.output_format == 'color'
    for line, occurrences in ctxt.rare_misspellings.sorted_lines(reverse=options.reverse):
        header = []
        underline = bytearray(b' ' * len(line))
        for word, line, positions in sorted(occurrences):  # pylint: disable=redefined-outer-name
            if use_color and (max(positions.values()) > 0):
                underline_char = b'!'
            else:
                underline_char = b'^'
            if len(positions) > options.limit:
                continue
            extra = ''
            if options.suggest > 0:
                suggestions = ctxt.dictionary.suggest(word)[:options.suggest]
                if suggestions:
                    extra = ' ({sug})'.format(sug=', '.join(suggestions))
            header += [word + extra]
            for x in positions:
                underline[x : x + len(word)] = underline_char * len(word)
        if not header:
            continue
        print(', '.join(header) + ':')
        underline = underline.decode()
        lwidth = len(underline) - len(underline.lstrip())
        rwidth = len(underline) - len(underline.rstrip())
        lexceed = lwidth - options.max_context_width
        rexceed = rwidth - options.max_context_width
        if lexceed > 0:
            lwidth = len(line) - lexceed
            line = lib.text.ltrim(line, lwidth)
            underline = lib.text.ltrim(underline, lwidth, char=' ')
        if rexceed > 0:
            rwidth = len(line) - rexceed
            line = lib.text.rtrim(line, rwidth)
            underline = lib.text.rtrim(underline, rwidth, char=' ')
        if use_color:
            hline = lib.colors.highlight(
                line, (
                    'warn' if u == '^' else
                    'error' if u == '!' else
                    'off'
                    for u in underline
                )
            )
            print(lib.colors.dim('|'), hline)
        else:
            print('|', line)
            print(' ', underline.rstrip())
        if not options.compact:
            print()

class list_languages(argparse.Action):
    def __call__(self, *args, **kwargs):  # pylint: disable=arguments-differ
        for lang in sorted(enchant.list_languages()):
            print(lang)
        sys.exit(0)

__all__ = ['main']

# vim:ts=4 sts=4 sw=4 et
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: David BEAL, Copyright Akretion, 2014
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{'name': 'Connector Base Product',
 'version': '1.0',
 'author': 'Openerp Connector Core Editors',
 'website': 'http://openerp-connector.com',
 'license': 'AGPL-3',
 'category': 'Connector',
 'description': """
Connector Base Product
======================

Add 'Connector' tab to product view
""",
 'depends': [
     'connector',
     'product',
 ],
 'data': [
     'product_view.xml'
 ],
 'installable': False,
 }
<commit_msg>[FIX] connector_base_product: Make it installable<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: David BEAL, Copyright Akretion, 2014
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{'name': 'Connector Base Product',
 'version': '1.0',
 'author': 'Openerp Connector Core Editors',
 'website': 'http://openerp-connector.com',
 'license': 'AGPL-3',
 'category': 'Connector',
 'description': """
Connector Base Product
======================

Add 'Connector' tab to product view
""",
 'depends': [
     'connector',
     'product',
 ],
 'data': [
     'product_view.xml'
 ],
 'installable': True,
 }
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# marathon_group.py: generate a Marathon Service Group out of a list of containers
#
# Author: Fernando Sanchez [ fernando at mesosphere.com ]

import json
import sys
import argparse
import subprocess
import os
import uuid

def create_group ( name, containers ):
	"""
	Creates a marathon group taking a list of containers as a parameter.
	If the list has a single member if returns the member.
	"""
	output = '{ 			\
	  "id": "'+name+'",		\
	  "apps": '+containers+'\
	  }'

	return str(output)

def create_external_volume( external_volume_name ):
	"""
	Create an RBD external volume. Assumes RBD is working in the host
	"""

	#check that the volume exists
	#command = "rbd ls | grep "+external_volume_name
	command = "docker volume ls | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	if external_volume_name in out.decode('utf-8'):
		print('**INFO: volume {0} already exists'.format( external_volume_name ))
		return out.decode('utf-8')
	else:
		command = "docker volume create --driver=rexray --name="+external_volume_name
		proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()

	#check whether the volume is already mapped
	command = "rbd showmapped | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	if external_volume_name in out.decode('utf-8'):
		external_volume_device=out.decode('utf-8').split(' ')[-1]
		print('**INFO: volume {0} already mapped to {1}'.format( external_volume_name, external_volume_device  ))
	else:
		#if not, map it
		command = "rbd map "+external_volume_name
		proc = subprocess.Popen( command, stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()
		#TODO error checking
		external_volume_device = out.decode('utf-8')

	print("**DEBUG: external_volume_MAPPING {}".format(out.decode('utf-8')))

	#format the volume
	command = "mkfs.xfs -f "+external_volume_device
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	return out.decode('utf-8')

def	copy_content_to_external_volume( external_volume_name, source_path, dest_path ):
	"""
	Copy recursively the content in localhost under "path" to the external volume.
	Assumes the path exists. Assumes the external_volume exists
	"""
	#check that the volume exists
	command = "rbd ls | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	if not external_volume_name in out.decode('utf-8'):
		print('**ERROR: volume {0} to copy content into not found'.format( external_volume_name ))
		return False

	#check that the local path exists
	if not os.path.isdir( source_path ):
		print('**ERROR: directory {0} to copy content from not found'.format( source_path ))
		return False

	#check whether the volume is already mapped
	command = "rbd showmapped | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	if external_volume_name in out.decode('utf-8'):
		external_volume_device=out.decode('utf-8').split(' ')[-1]
		print('**INFO: volume {0} already mapped to {1}'.format( external_volume_name, external_volume_device  ))
	else:
		#if not, map it
		print('**INFO: mapping volume {0} for copy'.format( external_volume_name, out.decode('utf-8')  ))
		command = "rbd map "+external_volume_name
		proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()
		#TODO error checking
		external_volume_device = out.decode('utf-8')

	#TODO error checking
	print('**INFO: volume {0} mapped to {1} for copy'.format( external_volume_name, external_volume_device ))

	#print("**DEBUG: external_volume_device {}".format(external_volume_device))
	#create mount point
	mount_point="/tmp/"+external_volume_name
	command = "mkdir -p "+mount_point
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	print("**DEBUG: mount_point {}".format(mount_point))

	#mount the volume
	command = "mount --source "+external_volume_device+" --target "+mount_point
	print("**DEBUG: MOUNT COMMAND {}".format(command))

	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	print("**DEBUG: MOUNT command result {}".format(out.decode('utf-8')))

	#create path
	#command = "mkdir -p "+mount_point+"/"+dest_path
	#proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	#(out, err) = proc.communicate()

	#recursively copy the content
	command = "cp -R "+source_path+"/ "+mount_point
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	#umount the volume
	command = "umount "+external_volume_device
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	#unmap rbd
	command = "rbd unmap "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	#print("**DEBUG: output of rbd map is {}".format(out.decode('utf-8')))

	return out.decode('utf-8')

def modify_volume_for_external ( volume, app_name ):
	"""
	Receives a Marathon volume definition as a dictionary that includes a local directory to load.
	Creates an external persistent volume in a docker external volume (assumed to be mounted).
	Creates the local volume structure on the external volume, and copies the contents.
	Modifies the volume to use the newly create external volume.
	Returns the new volume dictionary.
	"""

	#get firstPartOfHostPath, etc.
	host_path = volume['hostPath'][2:] 					#app
	#first_part_of_host_path = host_path.split('/' , 1)[0]	#app
	#last_part_of_host_path = host_path.split('/' , 1)[1]		#NULL
	container_path = volume['containerPath']							#/src/app
	first_part_of_container_path = container_path[1:].split('/', 1)[0]	#src
	last_part_of_container_path = container_path[1:].split('/', 1)[1]	#app
	#create a volume
	external_volume_name = app_name+'-'+host_path.replace('/','_')
	create_external_volume( external_volume_name ) #nextcloud_apps_UUID
	#copy content from volume[hostPath] to volume
	copy_content_to_external_volume( external_volume_name, volume['hostPath'], "/"+last_part_of_container_path)
	#modify volume
	volume['external'] = { 						#mount it as external volume
		'name': external_volume_name,
		'provider': 'dvdi',
		'options': {
		'dvdi/driver': 'rexray'
		}
	}
	#change containerPath to the firstPiece only
	volume['containerPath'] = first_part_of_container_path	#src
	del( volume['hostPath'] )							#external volumes do not use hostpath

	return volume

def modify_group ( group ):
	"""
	Modifies a marathon group received as a printable string to adapt the apps inside it
	to the desired parameters.
	Adds AcceptedResourceRoles = "*"
	Deletes any hostPort values to have Marathon assign them automatically.
	Adds a label HAPROXY_GROUP=external for all hostPort values.
	If the apps mounts any volume that is local for compose (localhost), it mounts it as "external/rexray"
	Returns the group as a modified string.
	"""

	group_dict = json.loads( group )

	for app in group_dict['apps']:
		app['acceptedResourceRoles']=["*"]
		for portMapping in app.get('container',{}).get('docker',{}).get('portMappings',{}):
			if portMapping.get('hostPort',{}): 	#delete ANY hostPort values
				portMapping['hostPort'] = 0
				if 'labels' in app:
					app['labels'].update( {"HAPROXY_GROUP": "external"} )# if there was a hostPort add to MLB
				else:
					app['labels'] = { "HAPROXY_GROUP": "external" }
		#modify all volumes in the groups apps so that "this directory" volumes become external
		for volume in app.get('container', {}).get('volumes', {}):
			if volume['hostPath'][:2] == "./":				#if the volume is "this dir" for compose
				volume = modify_volume_for_external( volume, app['id'] )

	return json.dumps( group_dict )

if __name__ == "__main__":

	#parse command line arguments
	parser = argparse.ArgumentParser(description='Convert a list of containers to a Marathon Service Group.', \
		usage='marathon_group.py -i [container_list_filename] -n [group_name]'
		)
	parser.add_argument('-i', '--input', help='name of the file including the list of containers', required=True)
	parser.add_argument('-n', '--name', help='name to be given to the Marathon Service Group', required=True)
	args = vars( parser.parse_args() )

	#remove the trailing \n from file
	containers = ""
	for line in open( args['input'], 'r' ):
		containers += line.rstrip()
	#detect if it's just one app - if so, get in list
	if containers[0]=="{":
		containers="["+containers+"]"
	group = create_group( args['name'], containers )
	modified_group = modify_group( group )
	output_file=open( "./group.json", "w")
	print( modified_group, file=output_file )

	sys.exit(0)

<commit_msg>debug<commit_after>#!/usr/bin/env python3
#
# marathon_group.py: generate a Marathon Service Group out of a list of containers
#
# Author: Fernando Sanchez [ fernando at mesosphere.com ]

import json
import sys
import argparse
import subprocess
import os
import uuid

def create_group ( name, containers ):
	"""
	Creates a marathon group taking a list of containers as a parameter.
	If the list has a single member if returns the member.
	"""
	output = '{ 			\
	  "id": "'+name+'",		\
	  "apps": '+containers+'\
	  }'

	return str(output)

def create_external_volume( external_volume_name ):
	"""
	Create an RBD external volume. Assumes RBD is working in the host
	"""

	#check that the volume exists
	#command = "rbd ls | grep "+external_volume_name
	command = "docker volume ls | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	if external_volume_name in out.decode('utf-8'):
		print('**INFO: volume {0} already exists'.format( external_volume_name ))
		return out.decode('utf-8')
	else:
		command = "docker volume create --driver=rexray --name="+external_volume_name
		proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()

	#check whether the volume is already mapped
	command = "rbd showmapped | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	if external_volume_name in out.decode('utf-8'):
		external_volume_device=out.decode('utf-8').split(' ')[-1]
		print('**INFO: volume {0} already mapped to {1}'.format( external_volume_name, external_volume_device  ))
	else:
		#if not, map it
		command = "rbd map "+external_volume_name
		proc = subprocess.Popen( command, stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()
		#TODO error checking
		external_volume_device = out.decode('utf-8')

	print("**DEBUG: external_volume_MAPPING {}".format(out.decode('utf-8')))

	#format the volume
	command = "mkfs.xfs -f "+external_volume_device
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	return out.decode('utf-8')

def	copy_content_to_external_volume( external_volume_name, source_path, dest_path ):
	"""
	Copy recursively the content in localhost under "path" to the external volume.
	Assumes the path exists. Assumes the external_volume exists
	"""
	#check that the volume exists
	command = "rbd ls | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	if not external_volume_name in out.decode('utf-8'):
		print('**ERROR: volume {0} to copy content into not found'.format( external_volume_name ))
		return False

	#check that the local path exists
	if not os.path.isdir( source_path ):
		print('**ERROR: directory {0} to copy content from not found'.format( source_path ))
		return False

	#check whether the volume is already mapped
	command = "rbd showmapped | grep "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	if external_volume_name in out.decode('utf-8'):
		external_volume_device=out.decode('utf-8').split(' ')[-1]
		print('**INFO: volume {0} already mapped to {1}'.format( external_volume_name, external_volume_device  ))
	else:
		#if not, map it
		print('**INFO: mapping volume {0} for copy'.format( external_volume_name, out.decode('utf-8')  ))
		command = "rbd map "+external_volume_name
		proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
		(out, err) = proc.communicate()
		#TODO error checking
		external_volume_device = out.decode('utf-8')

	#TODO error checking
	print('**INFO: volume {0} mapped to {1} for copy'.format( external_volume_name, external_volume_device ))

	#print("**DEBUG: external_volume_device {}".format(external_volume_device))
	#create mount point
	mount_point="/tmp/"+external_volume_name
	command = "mkdir -p "+mount_point
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	print("**DEBUG: mount_point {}".format(mount_point))

	#mount the volume
	command = "mount "+external_volume_device+" "+mount_point
	print("**DEBUG: MOUNT COMMAND {}".format(command))

	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	print("**DEBUG: MOUNT command result {}".format(out.decode('utf-8')))

	#create path
	#command = "mkdir -p "+mount_point+"/"+dest_path
	#proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	#(out, err) = proc.communicate()

	#recursively copy the content
	command = "cp -R "+source_path+"/ "+mount_point
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	#umount the volume
	command = "umount "+external_volume_device
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()

	#unmap rbd
	command = "rbd unmap "+external_volume_name
	proc = subprocess.Popen( [command], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	#print("**DEBUG: output of rbd map is {}".format(out.decode('utf-8')))

	return out.decode('utf-8')

def modify_volume_for_external ( volume, app_name ):
	"""
	Receives a Marathon volume definition as a dictionary that includes a local directory to load.
	Creates an external persistent volume in a docker external volume (assumed to be mounted).
	Creates the local volume structure on the external volume, and copies the contents.
	Modifies the volume to use the newly create external volume.
	Returns the new volume dictionary.
	"""

	#get firstPartOfHostPath, etc.
	host_path = volume['hostPath'][2:] 					#app
	#first_part_of_host_path = host_path.split('/' , 1)[0]	#app
	#last_part_of_host_path = host_path.split('/' , 1)[1]		#NULL
	container_path = volume['containerPath']							#/src/app
	first_part_of_container_path = container_path[1:].split('/', 1)[0]	#src
	last_part_of_container_path = container_path[1:].split('/', 1)[1]	#app
	#create a volume
	external_volume_name = app_name+'-'+host_path.replace('/','_')
	create_external_volume( external_volume_name ) #nextcloud_apps_UUID
	#copy content from volume[hostPath] to volume
	copy_content_to_external_volume( external_volume_name, volume['hostPath'], "/"+last_part_of_container_path)
	#modify volume
	volume['external'] = { 						#mount it as external volume
		'name': external_volume_name,
		'provider': 'dvdi',
		'options': {
		'dvdi/driver': 'rexray'
		}
	}
	#change containerPath to the firstPiece only
	volume['containerPath'] = first_part_of_container_path	#src
	del( volume['hostPath'] )							#external volumes do not use hostpath

	return volume

def modify_group ( group ):
	"""
	Modifies a marathon group received as a printable string to adapt the apps inside it
	to the desired parameters.
	Adds AcceptedResourceRoles = "*"
	Deletes any hostPort values to have Marathon assign them automatically.
	Adds a label HAPROXY_GROUP=external for all hostPort values.
	If the apps mounts any volume that is local for compose (localhost), it mounts it as "external/rexray"
	Returns the group as a modified string.
	"""

	group_dict = json.loads( group )

	for app in group_dict['apps']:
		app['acceptedResourceRoles']=["*"]
		for portMapping in app.get('container',{}).get('docker',{}).get('portMappings',{}):
			if portMapping.get('hostPort',{}): 	#delete ANY hostPort values
				portMapping['hostPort'] = 0
				if 'labels' in app:
					app['labels'].update( {"HAPROXY_GROUP": "external"} )# if there was a hostPort add to MLB
				else:
					app['labels'] = { "HAPROXY_GROUP": "external" }
		#modify all volumes in the groups apps so that "this directory" volumes become external
		for volume in app.get('container', {}).get('volumes', {}):
			if volume['hostPath'][:2] == "./":				#if the volume is "this dir" for compose
				volume = modify_volume_for_external( volume, app['id'] )

	return json.dumps( group_dict )

if __name__ == "__main__":

	#parse command line arguments
	parser = argparse.ArgumentParser(description='Convert a list of containers to a Marathon Service Group.', \
		usage='marathon_group.py -i [container_list_filename] -n [group_name]'
		)
	parser.add_argument('-i', '--input', help='name of the file including the list of containers', required=True)
	parser.add_argument('-n', '--name', help='name to be given to the Marathon Service Group', required=True)
	args = vars( parser.parse_args() )

	#remove the trailing \n from file
	containers = ""
	for line in open( args['input'], 'r' ):
		containers += line.rstrip()
	#detect if it's just one app - if so, get in list
	if containers[0]=="{":
		containers="["+containers+"]"
	group = create_group( args['name'], containers )
	modified_group = modify_group( group )
	output_file=open( "./group.json", "w")
	print( modified_group, file=output_file )

	sys.exit(0)

<|endoftext|>


<commit_before>
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from configure.lib.storage_plugin.plugin import StoragePlugin
from configure.lib.storage_plugin.resource import StorageResource, ScannableId, GlobalId, ScannableResource

from configure.lib.storage_plugin import attributes
from configure.lib.storage_plugin import base_resources
from configure.lib.storage_plugin import alert_conditions
from configure.lib.storage_plugin import statistics

# This plugin is special, it uses Hydra's built-in infrastructure
# in a way that third party plugins can't/shouldn't/mustn't
from configure.lib.agent import Agent
from configure.models import ManagedHost

class HydraHostProxy(StorageResource, ScannableResource):
    identifier = GlobalId('host_id')

    host_id = attributes.Integer()
    def human_string(self, parent = None):
        host = ManagedHost._base_manager.get(pk=self.host_id)
        return "%s" % host

# Things to offer the user when formatting
# all of the below must NOT be offered if they have an LVM VG descendent
# or a NonLustreFilesystem descendent
# * ScsiDevice (i.e. shared LUNs like DDN VDs) IF it has no LV descendents
# * UnsharedDeviceNode (i.e. IDE or virtio devices)
# * LvmVolume
#
# For any of the above, we must also work out their leaf device nodes, which
# should be just the leaf resources.

class ScsiDevice(base_resources.LogicalDrive):
    identifier = GlobalId('serial')

    serial = attributes.String(subscribe = True)
    size = attributes.Bytes()

    human_name = "SCSI device"

    test_stat = statistics.Gauge()
    test_hist = statistics.BytesHistogram(bins = [(0,256), (257,512), (513, 2048), (2049, 8192)])

    def human_string(self, ancestors = []):
        return self.serial

class UnsharedDeviceNode(base_resources.DeviceNode):
    """A device node whose underlying device has no SCSI ID
    and is therefore assumed to be unshared"""
    identifier = ScannableId('path')

    human_name = "Local disk"

    def human_string(self, ancestors = []):
        if self.path.startswith("/dev/"):
            return self.path[5:]
        else:
            return self.path

class UnsharedDevice(base_resources.LogicalDrive):
    identifier = ScannableId('path')
    # Annoying duplication of this from the node, but it really
    # is the closest thing we have to a real ID.
    path = attributes.PosixPath()

class ScsiDeviceNode(base_resources.DeviceNode):
    """SCSI in this context is a catch-all to refer to
    block devices which look like real disks to the host OS"""
    identifier = ScannableId('path')
    human_name = "SCSI device node"

class MultipathDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "Multipath device node"

class LvmDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "LVM device node"

# FIXME: partitions should really be GlobalIds (they can be seen from more than
# one host) where the ID is their number plus the a foreign key to the parent
# ScsiDevice or UnsharedDevice(HYD-272)
    # TODO: foreign key to VG instead of copied value
class Partition(base_resources.LogicalDrive):
    identifier = ScannableId('path')
    human_name = "Linux partition"
    path = attributes.PosixPath()

class PartitionDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "Linux partition"

class LocalMount(StorageResource):
    """A local filesystem consuming a storage resource -- reported so that
       hydra knows not to try and use the consumed resource for Lustre e.g.
       minor things like your root partition."""
    identifier = ScannableId('mount_point')

    fstype = attributes.String()
    mount_point = attributes.String()

class Linux(StoragePlugin):
    def __init__(self, *args, **kwargs):
        super(Linux, self).__init__(*args, **kwargs)
        self.agent = Agent(log = self.log)

        self._scsi_devices = set()

    # TODO: need to document that initial_scan may not kick off async operations, because
    # the caller looks at overall resource state at exit of function.  If they eg
    # want to kick off an async update thread they should do it at the first
    # call to update_scan, or maybe we could give them a separate function for that.
    def initial_scan(self, root_resource):
        host = ManagedHost.objects.get(pk=root_resource.host_id)
        devices = self.agent.invoke(host, "device-scan")

        lv_block_devices = set()
        for vg, lv_list in devices['lvs'].items():
            for lv_name, lv in lv_list.items():
                try:
                    lv_block_devices.add(lv['block_device'])
                except KeyError:
                    # An inactive LV has no block device
                    pass
        mpath_block_devices = set()
        for mp_name, mp in devices['mpath'].items():
            mpath_block_devices.add(mp['block_device'])

        dm_block_devices = lv_block_devices | mpath_block_devices

        # List of BDs with serial numbers that aren't devicemapper BDs
        devs_by_serial = {}
        for bdev in devices['devs'].values():
            serial = bdev['serial']
            if not bdev['major_minor'] in dm_block_devices:
                if serial != None and not serial in devs_by_serial:
                    # NB it's okay to have multiple block devices with the same
                    # serial (multipath): we just store the serial+size once
                    devs_by_serial[serial] = {
                            'serial': serial,
                            'size': bdev['size']
                            }

        # Resources for devices with serial numbers
        res_by_serial = {}
        for dev in devs_by_serial.values():
            res, created = self.update_or_create(ScsiDevice, serial = dev['serial'], size = dev['size'])
            self._scsi_devices.add(res)
            res_by_serial[dev['serial']] = res

        bdev_to_resource = {}
        for bdev in devices['devs'].values():
            # Partitions: we will do these in a second pass once their
            # parents are in bdev_to_resource
            if bdev['parent'] != None:
                continue

            # DM devices: we will do these later
            if bdev['major_minor'] in dm_block_devices:
                continue

            if bdev['serial'] != None:
                lun_resource = res_by_serial[bdev['serial']]
                res, created = self.update_or_create(ScsiDeviceNode,
                                    parents = [lun_resource],
                                    path = bdev['path'])
            else:
                res, created = self.update_or_create(UnsharedDevice,
                        path = bdev['path'],
                        size = bdev['size'])
                res, created = self.update_or_create(UnsharedDeviceNode,
                        parents = [res],
                        path = bdev['path'])
            bdev_to_resource[bdev['major_minor']] = res

        # Okay, now we've got ScsiDeviceNodes, time to build the devicemapper ones
        # on top of them.  These can come in any order and be nested to any depth.
        # So we have to build a graph and then traverse it to populate our resources.
        edges = []
        for bdev in devices['devs'].values():
            if bdev['major_minor'] in lv_block_devices:
                res,created = self.update_or_create(LvmDeviceNode,
                                    path = bdev['path'])
            elif bdev['major_minor'] in mpath_block_devices:
                res,created = self.update_or_create(MultipathDeviceNode,
                                    path = bdev['path'])
            elif bdev['parent']:
                res, created = self.update_or_create(PartitionDeviceNode,
                        path = bdev['path'])
            else:
                continue

            bdev_to_resource[bdev['major_minor']] = res

        for bdev in devices['devs'].values():
            if bdev['parent'] == None:
                continue

            this_node = bdev_to_resource[bdev['major_minor']]
            parent_resource = bdev_to_resource[bdev['parent']]

            partition, created = self.update_or_create(Partition,
                    parents = [parent_resource],
                    size = devices['devs'][bdev['parent']]['size'],
                    path = bdev['path'])

            this_node.add_parent(partition)

        # Now all the LUNs and device nodes are in, create the links between
        # the DM block devices and their parent entities.
        vg_uuid_to_resource = {}
        for vg in devices['vgs'].values():
            # Create VG resource
            vg_resource, created = self.update_or_create(LvmGroup,
                    uuid = vg['uuid'],
                    name = vg['name'],
                    size = vg['size'])
            vg_uuid_to_resource[vg['uuid']] = vg_resource

            # Add PV block devices as parents of VG
            for pv_bdev in vg['pvs_major_minor']:
                if pv_bdev in bdev_to_resource:
                    vg_resource.add_parent(bdev_to_resource[pv_bdev])

        for vg, lv_list in devices['lvs'].items():
            for lv_name, lv in lv_list.items():
                vg_info = devices['vgs'][vg]
                vg_resource = vg_uuid_to_resource[vg_info['uuid']]

                # Make the LV a parent of its device node on this host
                lv_resource, created = self.update_or_create(LvmVolume,
                        parents = [vg_resource],
                        uuid = lv['uuid'],
                        name = lv['name'],
                        vg_uuid = vg_info['uuid'],
                        size = lv['size'])

                try:
                    lv_bdev = bdev_to_resource[lv['block_device']]
                    lv_bdev.add_parent(lv_resource)
                except KeyError:
                    # Inactive LVs have no block device
                    pass

        for mpath_alias, mpath in devices['mpath'].items():
            mpath_bdev = bdev_to_resource[mpath['block_device']]
            mpath_parents = [bdev_to_resource[n['major_minor']] for n in mpath['nodes']]
            for p in mpath_parents:
                mpath_bdev.add_parent(p)

        for bdev, (mntpnt, fstype) in devices['local_fs'].items():
            bdev_resource = bdev_to_resource[bdev]
            self.update_or_create(LocalMount,
                    parents=[bdev_resource],
                    mount_point = mntpnt,
                    fstype = fstype)

    def update_scan(self, scannable_resource):
        for scsi_dev in list(self._scsi_devices):
            import random
            num = random.randint(10, 20)
            scsi_dev.test_stat = num
            scsi_dev.test_hist = [random.randint(50,100) for r in range(0,4)]

class LvmGroup(base_resources.StoragePool):
    identifier = GlobalId('uuid')

    uuid = attributes.Uuid()
    name = attributes.String()
    size = attributes.Bytes()

    icon = 'lvm_vg'
    human_name = 'VG'

    def human_string(self, parent = None):
        return self.name

class LvmVolume(base_resources.LogicalDrive):
    # Q: Why is this identified by LV UUID and VG UUID rather than just
    #    LV UUID?  Isn't the LV UUID unique enough?
    # A: We're matching LVM2's behaviour.  If you e.g. image a machine that
    #    has some VGs and LVs, then if you want to disambiguate them you run
    #    'vgchange -u' to get a new VG UUID.  However, there is no equivalent
    #    command to reset LV uuid, because LVM finds two LVs with the same UUID
    #    in VGs with different UUIDs to be unique enough.
    identifier = GlobalId('uuid', 'vg_uuid')

    # TODO: foreign key to VG instead of copied value (HYD-272)
    vg_uuid = attributes.Uuid()
    uuid = attributes.Uuid()
    name = attributes.String()

    icon = 'lvm_lv'
    human_name = 'LV'

    def human_string(self, ancestors = []):
        return self.name

<commit_msg>Roll back accidental commit of testing code<commit_after>
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from configure.lib.storage_plugin.plugin import StoragePlugin
from configure.lib.storage_plugin.resource import StorageResource, ScannableId, GlobalId, ScannableResource

from configure.lib.storage_plugin import attributes
from configure.lib.storage_plugin import base_resources
from configure.lib.storage_plugin import alert_conditions

# This plugin is special, it uses Hydra's built-in infrastructure
# in a way that third party plugins can't/shouldn't/mustn't
from configure.lib.agent import Agent
from configure.models import ManagedHost

class HydraHostProxy(StorageResource, ScannableResource):
    identifier = GlobalId('host_id')

    host_id = attributes.Integer()
    def human_string(self, parent = None):
        host = ManagedHost._base_manager.get(pk=self.host_id)
        return "%s" % host

# Things to offer the user when formatting
# all of the below must NOT be offered if they have an LVM VG descendent
# or a NonLustreFilesystem descendent
# * ScsiDevice (i.e. shared LUNs like DDN VDs) IF it has no LV descendents
# * UnsharedDeviceNode (i.e. IDE or virtio devices)
# * LvmVolume
#
# For any of the above, we must also work out their leaf device nodes, which
# should be just the leaf resources.

class ScsiDevice(base_resources.LogicalDrive):
    identifier = GlobalId('serial')

    serial = attributes.String(subscribe = True)

    human_name = "SCSI device"

    def human_string(self, ancestors = []):
        if self.serial[0] == 'S':
            return self.serial[1:]
        else:
            return self.serial

class UnsharedDeviceNode(base_resources.DeviceNode):
    """A device node whose underlying device has no SCSI ID
    and is therefore assumed to be unshared"""
    identifier = ScannableId('path')

    human_name = "Local disk"

    def human_string(self, ancestors = []):
        if self.path.startswith("/dev/"):
            return self.path[5:]
        else:
            return self.path

class UnsharedDevice(base_resources.LogicalDrive):
    identifier = ScannableId('path')
    # Annoying duplication of this from the node, but it really
    # is the closest thing we have to a real ID.
    path = attributes.PosixPath()

    def human_string(self):
        return self.path

class ScsiDeviceNode(base_resources.DeviceNode):
    """SCSI in this context is a catch-all to refer to
    block devices which look like real disks to the host OS"""
    identifier = ScannableId('path')
    human_name = "SCSI device node"

class MultipathDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "Multipath device node"

class LvmDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "LVM device node"

# FIXME: partitions should really be GlobalIds (they can be seen from more than
# one host) where the ID is their number plus the a foreign key to the parent
# ScsiDevice or UnsharedDevice(HYD-272)
# TODO: include containng object human_string in partition human_string
class Partition(base_resources.LogicalDrive):
    identifier = ScannableId('path')
    human_name = "Linux partition"
    path = attributes.PosixPath()

    def human_string(self):
        return self.path

class PartitionDeviceNode(base_resources.DeviceNode):
    identifier = ScannableId('path')
    human_name = "Linux partition"

class LocalMount(StorageResource):
    """A local filesystem consuming a storage resource -- reported so that
       hydra knows not to try and use the consumed resource for Lustre e.g.
       minor things like your root partition."""
    identifier = ScannableId('mount_point')

    fstype = attributes.String()
    mount_point = attributes.String()

class Linux(StoragePlugin):
    def __init__(self, *args, **kwargs):
        super(Linux, self).__init__(*args, **kwargs)
        self.agent = Agent(log = self.log)

    # TODO: need to document that initial_scan may not kick off async operations, because
    # the caller looks at overall resource state at exit of function.  If they eg
    # want to kick off an async update thread they should do it at the first
    # call to update_scan, or maybe we could give them a separate function for that.
    def initial_scan(self, root_resource):
        host = ManagedHost.objects.get(pk=root_resource.host_id)
        devices = self.agent.invoke(host, "device-scan")

        lv_block_devices = set()
        for vg, lv_list in devices['lvs'].items():
            for lv_name, lv in lv_list.items():
                try:
                    lv_block_devices.add(lv['block_device'])
                except KeyError:
                    # An inactive LV has no block device
                    pass
        mpath_block_devices = set()
        for mp_name, mp in devices['mpath'].items():
            mpath_block_devices.add(mp['block_device'])

        dm_block_devices = lv_block_devices | mpath_block_devices

        # List of BDs with serial numbers that aren't devicemapper BDs
        devs_by_serial = {}
        for bdev in devices['devs'].values():
            serial = bdev['serial']
            if not bdev['major_minor'] in dm_block_devices:
                if serial != None and not serial in devs_by_serial:
                    # NB it's okay to have multiple block devices with the same
                    # serial (multipath): we just store the serial+size once
                    devs_by_serial[serial] = {
                            'serial': serial,
                            'size': bdev['size']
                            }

        # Resources for devices with serial numbers
        res_by_serial = {}
        for dev in devs_by_serial.values():
            res, created = self.update_or_create(ScsiDevice, serial = dev['serial'], size = dev['size'])
            res_by_serial[dev['serial']] = res

        bdev_to_resource = {}
        for bdev in devices['devs'].values():
            # Partitions: we will do these in a second pass once their
            # parents are in bdev_to_resource
            if bdev['parent'] != None:
                continue

            # DM devices: we will do these later
            if bdev['major_minor'] in dm_block_devices:
                continue

            if bdev['serial'] != None:
                lun_resource = res_by_serial[bdev['serial']]
                res, created = self.update_or_create(ScsiDeviceNode,
                                    parents = [lun_resource],
                                    path = bdev['path'])
            else:
                res, created = self.update_or_create(UnsharedDevice,
                        path = bdev['path'],
                        size = bdev['size'])
                res, created = self.update_or_create(UnsharedDeviceNode,
                        parents = [res],
                        path = bdev['path'])
            bdev_to_resource[bdev['major_minor']] = res

        # Okay, now we've got ScsiDeviceNodes, time to build the devicemapper ones
        # on top of them.  These can come in any order and be nested to any depth.
        # So we have to build a graph and then traverse it to populate our resources.
        edges = []
        for bdev in devices['devs'].values():
            if bdev['major_minor'] in lv_block_devices:
                res,created = self.update_or_create(LvmDeviceNode,
                                    path = bdev['path'])
            elif bdev['major_minor'] in mpath_block_devices:
                res,created = self.update_or_create(MultipathDeviceNode,
                                    path = bdev['path'])
            elif bdev['parent']:
                res, created = self.update_or_create(PartitionDeviceNode,
                        path = bdev['path'])
            else:
                continue

            bdev_to_resource[bdev['major_minor']] = res

        for bdev in devices['devs'].values():
            if bdev['parent'] == None:
                continue

            this_node = bdev_to_resource[bdev['major_minor']]
            parent_resource = bdev_to_resource[bdev['parent']]

            partition, created = self.update_or_create(Partition,
                    parents = [parent_resource],
                    size = bdev['size'],
                    path = bdev['path'])

            this_node.add_parent(partition)

        # Now all the LUNs and device nodes are in, create the links between
        # the DM block devices and their parent entities.
        vg_uuid_to_resource = {}
        for vg in devices['vgs'].values():
            # Create VG resource
            vg_resource, created = self.update_or_create(LvmGroup,
                    uuid = vg['uuid'],
                    name = vg['name'],
                    size = vg['size'])
            vg_uuid_to_resource[vg['uuid']] = vg_resource

            # Add PV block devices as parents of VG
            for pv_bdev in vg['pvs_major_minor']:
                if pv_bdev in bdev_to_resource:
                    vg_resource.add_parent(bdev_to_resource[pv_bdev])

        for vg, lv_list in devices['lvs'].items():
            for lv_name, lv in lv_list.items():
                vg_info = devices['vgs'][vg]
                vg_resource = vg_uuid_to_resource[vg_info['uuid']]

                # Make the LV a parent of its device node on this host
                lv_resource, created = self.update_or_create(LvmVolume,
                        parents = [vg_resource],
                        uuid = lv['uuid'],
                        name = lv['name'],
                        vg = vg_resource,
                        size = lv['size'])

                try:
                    lv_bdev = bdev_to_resource[lv['block_device']]
                    lv_bdev.add_parent(lv_resource)
                except KeyError:
                    # Inactive LVs have no block device
                    pass

        for mpath_alias, mpath in devices['mpath'].items():
            mpath_bdev = bdev_to_resource[mpath['block_device']]
            mpath_parents = [bdev_to_resource[n['major_minor']] for n in mpath['nodes']]
            for p in mpath_parents:
                mpath_bdev.add_parent(p)

        for bdev, (mntpnt, fstype) in devices['local_fs'].items():
            bdev_resource = bdev_to_resource[bdev]
            self.update_or_create(LocalMount,
                    parents=[bdev_resource],
                    mount_point = mntpnt,
                    fstype = fstype)

class LvmGroup(base_resources.StoragePool):
    identifier = GlobalId('uuid')

    uuid = attributes.Uuid()
    name = attributes.String()
    size = attributes.Bytes()

    icon = 'lvm_vg'
    human_name = 'VG'

    def human_string(self, parent = None):
        return self.name

class LvmVolume(base_resources.LogicalDrive):
    # Q: Why is this identified by LV UUID and VG UUID rather than just
    #    LV UUID?  Isn't the LV UUID unique enough?
    # A: We're matching LVM2's behaviour.  If you e.g. image a machine that
    #    has some VGs and LVs, then if you want to disambiguate them you run
    #    'vgchange -u' to get a new VG UUID.  However, there is no equivalent
    #    command to reset LV uuid, because LVM finds two LVs with the same UUID
    #    in VGs with different UUIDs to be unique enough.
    identifier = GlobalId('uuid', 'vg')

    vg = attributes.ResourceReference()
    uuid = attributes.Uuid()
    name = attributes.String()

    icon = 'lvm_lv'
    human_name = 'LV'

    def human_string(self, ancestors = []):
        return "%s-%s" % (self.vg.name, self.name)

<|endoftext|>


<commit_before>from tastypie.authorization import Authorization
from tastypie_mongoengine import resources
from oms_pds.pds.models import Funf
import datetime
import json

class FunfResource(resources.MongoEngineResource):
    class Meta:
        queryset = Funf.objects.all()
        allowed_methods = ('get', 'post')
        authorization = Authorization()

    def hydrate(self, bundle):
	print bundle.data['value']
        bundle.data['timestamp']=datetime.datetime.fromtimestamp(int(bundle.data['timestamp']))
        return bundle

    def dehydrate(self, bundle):
#        ustr_to_load = unicode(bundle.data['value'], 'latin-1')
#        jvalue=json.loads(bundle.data['value'])
#	print bundle.data['value']
 #       bundle.data['value']=jvalue
        return bundle

<commit_msg>we need a way to include a general json object in the 'value' field of the Funf model.  This is a workaround<commit_after>from tastypie.authorization import Authorization
from tastypie_mongoengine import resources
from oms_pds.pds.models import Funf
import datetime
import json, ast

class FunfResource(resources.MongoEngineResource):
    class Meta:
        queryset = Funf.objects.all()
        allowed_methods = ('get', 'post')
        authorization = Authorization()

    def hydrate(self, bundle):
	#TODO the '.' to '_' translation from the old PDS
        bundle.data['timestamp']=datetime.datetime.fromtimestamp(int(bundle.data['timestamp']))
        return bundle

    def dehydrate(self, bundle):
	# mongoengine is retrieving bundle.data['value'] as a string, and we need to evaluate it as a literal json object
	bundle.data['value'] = ast.literal_eval(bundle.data['value'])
        return bundle

<|endoftext|>


<commit_before>from __future__ import division

from glob import glob
from argparse import ArgumentParser
from itertools import combinations
from os.path import join, dirname
import json

from cairo import ImageSurface, Context, FORMAT_ARGB32
from osgeo import ogr

from . import paths

def make_context(width=960, resolution=1):
    ''' Get Cairo surface, context, and drawing scale.

        World extent: (-19918964.35, -8269767.91) - (19918964.18, 14041770.96)
    '''
    left, top = -18000000, 14050000
    right, bottom = 19500000, -7500000
    aspect = (right - left) / (top - bottom)

    hsize = int(resolution * width)
    vsize = int(hsize / aspect)

    hscale = hsize / (right - left)
    vscale = (hsize / aspect) / (bottom - top)

    hoffset = -left
    voffset = -top

    surface = ImageSurface(FORMAT_ARGB32, hsize, vsize)
    context = Context(surface)
    context.scale(hscale, vscale)
    context.translate(hoffset, voffset)

    return surface, context, hscale

def load_geoids(directory):
    ''' Load a set of GEOIDs that should be rendered.
    '''
    geoids = set()

    for path in glob(join(directory, 'us-*.json')):
        with open(path) as file:
            data = json.load(file)

        if 'geoid' in data.get('coverage', {}).get('US Census', {}):
            geoids.add(data['coverage']['US Census']['geoid'])

    return geoids

def stroke_features(ctx, features):
    '''
    '''
    return stroke_geometries(ctx, [f.GetGeometryRef() for f in features])

def stroke_geometries(ctx, geometries):
    '''
    '''
    for geometry in geometries:
        if geometry.GetGeometryType() in (ogr.wkbMultiPolygon, ogr.wkbMultiLineString):
            parts = geometry
        elif geometry.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbLineString):
            parts = [geometry]
        else:
            continue

        for part in parts:
            if part.GetGeometryType() is ogr.wkbPolygon:
                rings = part
            else:
                rings = [part]

            for ring in rings:
                points = ring.GetPoints()

                if geometry.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
                    start, rest = points[-1], points
                else:
                    start, rest = points[0], points[1:]

                ctx.move_to(*start)
                for point in rest:
                    ctx.line_to(*point)
                ctx.stroke()

def fill_features(ctx, features):
    '''
    '''
    for feature in features:
        geometry = feature.GetGeometryRef()

        if geometry.GetGeometryType() == ogr.wkbMultiPolygon:
            parts = geometry
        elif geometry.GetGeometryType() == ogr.wkbPolygon:
            parts = [geometry]
        else:
            raise NotImplementedError()

        for part in parts:
            for ring in part:
                points = ring.GetPoints()
                ctx.move_to(*points[-1])

                for point in points:
                    ctx.line_to(*point)

            ctx.fill()

parser = ArgumentParser(description='Draw a map of continental U.S. address coverage.')

parser.set_defaults(resolution=1, width=960)

parser.add_argument('--2x', dest='resolution', action='store_const', const=2,
                    help='Draw at double resolution.')

parser.add_argument('--1x', dest='resolution', action='store_const', const=1,
                    help='Draw at normal resolution.')

parser.add_argument('--width', dest='width', type=int,
                    help='Width in pixels.')

parser.add_argument('filename', help='Output PNG filename.')

def main():
    args = parser.parse_args()
    return render(paths.sources, args.width, args.resolution, args.filename)

def render(sources, width, resolution, filename):
    ''' Resolution: 1 for 100%, 2 for 200%, etc.
    '''
    # Prepare output surface
    surface, context, scale = make_context(width, resolution)

    # Load data
    geoids = load_geoids(sources)

    geodata = join(dirname(__file__), 'geodata')
    coastline_ds = ogr.Open(join(geodata, 'ne_50m_coastline-54029.shp'))
    lakes_ds = ogr.Open(join(geodata, 'ne_50m_lakes-54029.shp'))
    countries_ds = ogr.Open(join(geodata, 'ne_50m_admin_0_countries-54029.shp'))
    countries_borders_ds = ogr.Open(join(geodata, 'ne_50m_admin_0_boundary_lines_land-54029.shp'))
    us_state_ds = ogr.Open(join(geodata, 'cb_2013_us_state_20m-54029.shp'))
    us_county_ds = ogr.Open(join(geodata, 'cb_2013_us_county_20m-54029.shp'))

    coastline_features = list(coastline_ds.GetLayer(0))
    lakes_features = [f for f in lakes_ds.GetLayer(0) if f.GetField('scalerank') == 0]
    countries_features = list(countries_ds.GetLayer(0))
    countries_borders_features = list(countries_borders_ds.GetLayer(0))
    us_state_features = list(us_state_ds.GetLayer(0))
    us_county_features = list(us_county_ds.GetLayer(0))
    data_states = [f for f in us_state_features if f.GetFieldAsString('GEOID') in geoids]
    data_counties = [f for f in us_county_features if f.GetFieldAsString('GEOID') in geoids]

    # Draw each border between neighboring states exactly once.
    state_borders = [s1.GetGeometryRef().Intersection(s2.GetGeometryRef())
                     for (s1, s2) in combinations(us_state_features, 2)
                     if s1.GetGeometryRef().Intersects(s2.GetGeometryRef())]

    # Fill countries background
    context.set_source_rgb(0xdd/0xff, 0xdd/0xff, 0xdd/0xff)
    fill_features(context, countries_features)

    # Fill populated U.S. states
    context.set_source_rgb(0x74/0xff, 0xA5/0xff, 0x78/0xff)
    fill_features(context, data_states)

    # Fill populated U.S. counties
    context.set_source_rgb(0x1C/0xff, 0x89/0xff, 0x3F/0xff)
    fill_features(context, data_counties)

    # Outline countries and boundaries, fill lakes
    context.set_source_rgb(0, 0, 0)
    context.set_line_width(.25 * resolution / scale)
    stroke_geometries(context, state_borders)
    stroke_features(context, countries_borders_features)

    context.set_source_rgb(0xff/0xff, 0xff/0xff, 0xff/0xff)
    fill_features(context, lakes_features)

    context.set_source_rgb(0, 0, 0)
    context.set_line_width(.5 * resolution / scale)
    stroke_features(context, coastline_features)

    # Output
    surface.write_to_png(filename)

if __name__ == '__main__':
    exit(main())
<commit_msg>Added support for ISO 3166-1 Alpha 2 codes<commit_after>from __future__ import division

from glob import glob
from argparse import ArgumentParser
from itertools import combinations
from os.path import join, dirname
import json

from cairo import ImageSurface, Context, FORMAT_ARGB32
from osgeo import ogr

from . import paths

def make_context(width=960, resolution=1):
    ''' Get Cairo surface, context, and drawing scale.

        World extent: (-19918964.35, -8269767.91) - (19918964.18, 14041770.96)
    '''
    left, top = -18000000, 14050000
    right, bottom = 19500000, -7500000
    aspect = (right - left) / (top - bottom)

    hsize = int(resolution * width)
    vsize = int(hsize / aspect)

    hscale = hsize / (right - left)
    vscale = (hsize / aspect) / (bottom - top)

    hoffset = -left
    voffset = -top

    surface = ImageSurface(FORMAT_ARGB32, hsize, vsize)
    context = Context(surface)
    context.scale(hscale, vscale)
    context.translate(hoffset, voffset)

    return surface, context, hscale

def load_geoids(directory):
    ''' Load a set of U.S. Census GEOIDs that should be rendered.
    '''
    geoids = set()

    for path in glob(join(directory, 'us-*.json')):
        with open(path) as file:
            data = json.load(file)

        if 'geoid' in data.get('coverage', {}).get('US Census', {}):
            geoids.add(data['coverage']['US Census']['geoid'])

    return geoids

def load_alpha2s(directory):
    ''' Load a set of ISO Alpha 2s that should be rendered.
    '''
    alpha2s = set()

    for path in glob(join(directory, '??.json')):
        with open(path) as file:
            data = json.load(file)

        if 'alpha2' in data.get('coverage', {}).get('ISO', {}):
            alpha2s.add(data['coverage']['ISO']['alpha2'])

    return alpha2s

def stroke_features(ctx, features):
    '''
    '''
    return stroke_geometries(ctx, [f.GetGeometryRef() for f in features])

def stroke_geometries(ctx, geometries):
    '''
    '''
    for geometry in geometries:
        if geometry.GetGeometryType() in (ogr.wkbMultiPolygon, ogr.wkbMultiLineString):
            parts = geometry
        elif geometry.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbLineString):
            parts = [geometry]
        else:
            continue

        for part in parts:
            if part.GetGeometryType() is ogr.wkbPolygon:
                rings = part
            else:
                rings = [part]

            for ring in rings:
                points = ring.GetPoints()

                if geometry.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
                    start, rest = points[-1], points
                else:
                    start, rest = points[0], points[1:]

                ctx.move_to(*start)
                for point in rest:
                    ctx.line_to(*point)
                ctx.stroke()

def fill_features(ctx, features):
    '''
    '''
    for feature in features:
        geometry = feature.GetGeometryRef()

        if geometry.GetGeometryType() == ogr.wkbMultiPolygon:
            parts = geometry
        elif geometry.GetGeometryType() == ogr.wkbPolygon:
            parts = [geometry]
        else:
            raise NotImplementedError()

        for part in parts:
            for ring in part:
                points = ring.GetPoints()
                ctx.move_to(*points[-1])

                for point in points:
                    ctx.line_to(*point)

            ctx.fill()

parser = ArgumentParser(description='Draw a map of continental U.S. address coverage.')

parser.set_defaults(resolution=1, width=960)

parser.add_argument('--2x', dest='resolution', action='store_const', const=2,
                    help='Draw at double resolution.')

parser.add_argument('--1x', dest='resolution', action='store_const', const=1,
                    help='Draw at normal resolution.')

parser.add_argument('--width', dest='width', type=int,
                    help='Width in pixels.')

parser.add_argument('filename', help='Output PNG filename.')

def main():
    args = parser.parse_args()
    return render(paths.sources, args.width, args.resolution, args.filename)

def render(sources, width, resolution, filename):
    ''' Resolution: 1 for 100%, 2 for 200%, etc.
    '''
    # Prepare output surface
    surface, context, scale = make_context(width, resolution)

    # Load data
    geoids = load_geoids(sources)
    alpha2s = load_alpha2s(sources)

    geodata = join(dirname(__file__), 'geodata')
    coastline_ds = ogr.Open(join(geodata, 'ne_50m_coastline-54029.shp'))
    lakes_ds = ogr.Open(join(geodata, 'ne_50m_lakes-54029.shp'))
    countries_ds = ogr.Open(join(geodata, 'ne_50m_admin_0_countries-54029.shp'))
    countries_borders_ds = ogr.Open(join(geodata, 'ne_50m_admin_0_boundary_lines_land-54029.shp'))
    us_state_ds = ogr.Open(join(geodata, 'cb_2013_us_state_20m-54029.shp'))
    us_county_ds = ogr.Open(join(geodata, 'cb_2013_us_county_20m-54029.shp'))

    coastline_features = list(coastline_ds.GetLayer(0))
    lakes_features = [f for f in lakes_ds.GetLayer(0) if f.GetField('scalerank') == 0]
    countries_features = list(countries_ds.GetLayer(0))
    countries_borders_features = list(countries_borders_ds.GetLayer(0))
    us_state_features = list(us_state_ds.GetLayer(0))
    us_county_features = list(us_county_ds.GetLayer(0))
    data_states = [f for f in us_state_features if f.GetFieldAsString('GEOID') in geoids]
    data_counties = [f for f in us_county_features if f.GetFieldAsString('GEOID') in geoids]
    data_countries = [f for f in countries_features if f.GetFieldAsString('iso_a2') in alpha2s]

    # Draw each border between neighboring states exactly once.
    state_borders = [s1.GetGeometryRef().Intersection(s2.GetGeometryRef())
                     for (s1, s2) in combinations(us_state_features, 2)
                     if s1.GetGeometryRef().Intersects(s2.GetGeometryRef())]

    # Fill countries background
    context.set_source_rgb(0xdd/0xff, 0xdd/0xff, 0xdd/0xff)
    fill_features(context, countries_features)

    # Fill populated countries
    context.set_source_rgb(0x74/0xff, 0xA5/0xff, 0x78/0xff)
    fill_features(context, data_countries)

    # Fill populated U.S. states
    context.set_source_rgb(0x74/0xff, 0xA5/0xff, 0x78/0xff)
    fill_features(context, data_states)

    # Fill populated U.S. counties
    context.set_source_rgb(0x1C/0xff, 0x89/0xff, 0x3F/0xff)
    fill_features(context, data_counties)

    # Outline countries and boundaries, fill lakes
    context.set_source_rgb(0, 0, 0)
    context.set_line_width(.25 * resolution / scale)
    stroke_geometries(context, state_borders)
    stroke_features(context, countries_borders_features)

    context.set_source_rgb(0xff/0xff, 0xff/0xff, 0xff/0xff)
    fill_features(context, lakes_features)

    context.set_source_rgb(0, 0, 0)
    context.set_line_width(.5 * resolution / scale)
    stroke_features(context, coastline_features)

    # Output
    surface.write_to_png(filename)

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>#! /usr/bin/python3

import sys #Arguments gathering, exit script, ...
from Bio import SeqIO #Biopython, use to read fasta file sequences
import time
from memory_profiler import memory_usage #Use to get the memory usages. /!\ You may need to install memory_profiler.
import gc #garbage collector. Use to call it only at good moments

'''
usage: ./estimate_mergeScore_2_seq.py <file.fasta>
This script is used to test algorithms to give an estimate percent score
between 2 sequences.
It wil compare each pair of a set of sequences (given by a fasta file)
The main goal is to find a suitable algorithm for a GPU
usage. Therefore, a fairly precise algorithm is required (the reverse
complement is not an obligation as we can re-calculate it),
using as little memory as possible (the computation time is less important).
The ideal algorithm would not care if the beginning of the first read and the
ending of the second one are not merged with the other read if a lot of the
reads are merged, example :
 -read1: CCCGATTCA
 -merge:  ||||| ||
 -read2:  CCGATCCAAT
 -score: high percent
'''

def main(seqs):
	# Correct the seqs
	nbSeqs = len(seqs)
	#print("\n==== Sequences ====")
	for s in range(nbSeqs):
		seqs[s] = seqs[s].upper() #To have only upper cases
		#print("[-- SEQ "+str(s)+" --]\n"+seqs[s])

	#Run algorithms on each couple
	gc.disable() # Disable automatic garbage collector
	#print("\n==== Algorithms run ====")
	print("algo\tseqID1\tseqID2\tscore (%)\tmem (MB)\ttime (s)")

	for i in range(nbSeqs):
		for j in range(nbSeqs):
			coupleDescript = "\t"+str(i)+"\t"+str(j)+"\t"
			#needleman based
			print("needle"+coupleDescript+analyzeFct(needle,seqs[i],seqs[j]))

			#needleman 2 arrays based
			print("need2a"+coupleDescript+analyzeFct(need2arrays,seqs[i],seqs[j]))

			#dot_cut
			print("dot_cut"+coupleDescript+analyzeFct(dot_cut,seqs[i],seqs[j]))

'''
Tool function to run an cmp function and get (and output) its informations
as time, value, etc...
input: function, seq1, seq2
output: string "score(%)\tmem(?)\ttime(s)"
'''
def analyzeFct(fct, seq1, seq2):
	# Execute and get delta time
	t0 = time.time()
	score = fct(seq1, seq2)
	t1 = time.time()

	# Get memory usage during the function exection
	gc.collect() #Run garbage collector
	mem = memory_usage(proc=(fct,(seq1,seq2)),interval=0.1)

	return str(round(score,2))+"\t"+str(round(max(mem),2))+"\t"+str(round(t1-t0,2))

'''
Algorithm 'Needleman&Wunsch'
Based on an needleman principe: each case is the best of
 - score[i-1][j-1] + (seq1[i]==seq2[j]?1:-1) #Match / mismatch
 - score[i-1][j]-1 #Indel
 - score[i][j-1] #Indel
The first nuc of seq2 is either a match of score 1 or a mismatch of score -1: the gap before this on the seq1 doesn't count.
After the end seq1, indels on seq2 doesn't count.
The best score is the best of the last score line and the last column
'''
def needle(seq1, seq2):
	# Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring matrix[len1][len2]
	matrix = [[0 for j in range(len2)] for i in range(len1)]

	# Fill first column: first nuc of seq2 against all seq1 => gap is allowed, only match/mismatch of score 1/-1
	for i in range(len1): matrix[i][0] = 1 if seq1[i]==seq2[0] else -1

	# Fill first line: it is a gap so the -1 increase
	for j in range(1,len2): matrix[0][j] = -j

	#Fill the remaining of the matrix
	#Keep the best possibilities with the end of seq2 (seq2 is inside seq1)
	bestIN=0
	for i in range(1,len1):
		for j in range(1,len2):
			# Match / mismatch ?
			match = matrix[i-1][j-1] + (1 if seq1[i]==seq2[j] else -1)

			# indel ?
			i1 = matrix[i-1][j]-1
			i2 = matrix[i][j-1]-1

			# keep best
			matrix[i][j] = max(match, i1, i2)

		# Best seq2 inside seq 1 ?
		if(matrix[i][len2-1] > bestIN): bestIN = matrix[i][len2-1]

	# Get the list of best concurrents
	best_co = matrix[len1-1][:]
	for i in range(len1):
		best_co.append(matrix[i][-1])
	#Add the se2 is in seq1 possibility
	best_co.append(bestIN)

	# Get score
	return 100*max(best_co)/min(len1,len2)

'''
Algorithm 'Need2Arrays
Same algorithm (needleman&wunsch) but using a score matrix of size (seq1.size(),2)
as we only need the previous line to calculate the current one.
'''
def need2arrays(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring arrays
	previous = [0]*len2
	current = [0]*len2

	# bestIN is the best case of seq2 being INSIDE seq1

	# Calculate the first line (seq2 against seq1[0])
	# Note: we can't 'allow' a gap in seq2 'first nuc'
	for j in range(0,len2): previous[j] = -j
	bestIN = previous[len2-1]

	# Complete the needle
	for i in range(1, len1):
		# Test first nuc, a gap is allowed So it's just a (mis)match test
		current[0] = (1 if seq1[i]==seq2[0] else -1)

		# Finish the line
		for j in range(1,len2):
			scores=[0,0,0]
			# Match/mismatch ?
			scores[0] = previous[j-1] + (1 if seq1[i]==seq2[j] else -1)
			#Indels ?
			scores[1] = previous[j]-1
			scores[2] = current[j-1]-1
			#keep best
			current[j]=max(scores)

		# Best seq2 inside possibility ?
		if(current[j] > bestIN): bestIN=current[j]

		# move buffers
		previous = current
		current = [0]*len2

	# Get the best possibilities score which is inside the last line + the bestIN (pseudo last column)
	previous.append(bestIN) #Previous is last because we moved buffers. We add bestIN to it
	return 100*max(previous)/min(len1,len2) #Percent and normalize by theorical max best score (full match)

'''
Algorithm 'dot_cut'.
The principe is to increase the score when the merge is contiguous (diagonal
comparaison) but an difference make a score=0 :
	IF seq1[i] == seq2[j]: score_actual[j] = score_previous[j-1]+1
	ELSE: score_actual[j] = 0
	...do stuff...
	score_previous = score_actual

Then the final percent score is the sum of higher scores before a difference
compared to a perfect match.
This way you have only to memorize 2 'comparaison' column score and the best
scores.
When a part of seq2 has been 'scored', only check the remaining (on the right)
of seq2 to avoid to count a same parts multiple times and to do "come-backs".
'''

def dot_cut(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Init the scores arrays
	previous = [0]*len2
	actual = [0]*len2
	previous_best = 0
	best_scores =  list()
	begin_len2 = 0 #From which nuc of len2 the cmp start

	# Do cmp
	for i in range(len1):
		#Cmp seq2 to actual nuc of seq1
		for j in range(begin_len2, len2):
			if seq1[i] == seq2[j]:
				if j>0: actual[j] = previous[j-1]+1
				else: actual[j] = 1
			else: actual[j]=0

		# Get the best. Do we have a drop ?
		best = max(actual)

		if best < previous_best:
			#Best found ! (the previous_best ;) )
			best_scores.append(previous_best)

			#Now, start on the next len2 nuc from the last nuc of the
			#previous_best for further check
			begin_len2 = previous.index(previous_best)+1

			#Clear the buffers
			best = 0
			actual = [0]*len2

		previous_best = best

		#Move the scores buffers
		previous = actual
		actual = [0]*len2

	# Add the last best score
	best_scores.append(previous_best)

	# Results
	return 100*sum(best_scores)/min(len1,len2)

'''
Python will run the following part at start.
It gets the sequences then run the main function.
'''

# Run the main function
if __name__ == "__main__":
	# Get the argument (the fasta file)
	if len(sys.argv) != 2:
		print("Expect only one argument : the fasta file of sequences")
		sys.exit()

	#Read fasta file
	seqs = []
	for record in  SeqIO.parse(open(sys.argv[1]), 'fasta'):
		seqs.append(record.seq)

	# Run the main function
	main(seqs)

<commit_msg>Léger correctif sur needle2a: (mis)match de seq1[0] et seq2[0]<commit_after>#! /usr/bin/python3

import sys #Arguments gathering, exit script, ...
from Bio import SeqIO #Biopython, use to read fasta file sequences
import time
from memory_profiler import memory_usage #Use to get the memory usages. /!\ You may need to install memory_profiler.
import gc #garbage collector. Use to call it only at good moments

'''
usage: ./estimate_mergeScore_2_seq.py <file.fasta>
This script is used to test algorithms to give an estimate percent score
between 2 sequences.
It wil compare each pair of a set of sequences (given by a fasta file)
The main goal is to find a suitable algorithm for a GPU
usage. Therefore, a fairly precise algorithm is required (the reverse
complement is not an obligation as we can re-calculate it),
using as little memory as possible (the computation time is less important).
The ideal algorithm would not care if the beginning of the first read and the
ending of the second one are not merged with the other read if a lot of the
reads are merged, example :
 -read1: CCCGATTCA
 -merge:  ||||| ||
 -read2:  CCGATCCAAT
 -score: high percent
'''

def main(seqs):
	# Correct the seqs
	nbSeqs = len(seqs)
	#print("\n==== Sequences ====")
	for s in range(nbSeqs):
		seqs[s] = seqs[s].upper() #To have only upper cases
		#print("[-- SEQ "+str(s)+" --]\n"+seqs[s])

	#Run algorithms on each couple
	gc.disable() # Disable automatic garbage collector
	#print("\n==== Algorithms run ====")
	print("algo\tseqID1\tseqID2\tscore (%)\tmem (MB)\ttime (s)")

	for i in range(nbSeqs):
		for j in range(nbSeqs):
			coupleDescript = "\t"+str(i)+"\t"+str(j)+"\t"
			#needleman based
			print("needle"+coupleDescript+analyzeFct(needle,seqs[i],seqs[j]))

			#needleman 2 arrays based
			print("need2a"+coupleDescript+analyzeFct(need2arrays,seqs[i],seqs[j]))

			#dot_cut
			print("dot_cut"+coupleDescript+analyzeFct(dot_cut,seqs[i],seqs[j]))

'''
Tool function to run an cmp function and get (and output) its informations
as time, value, etc...
input: function, seq1, seq2
output: string "score(%)\tmem(?)\ttime(s)"
'''
def analyzeFct(fct, seq1, seq2):
	# Execute and get delta time
	t0 = time.time()
	score = fct(seq1, seq2)
	t1 = time.time()

	# Get memory usage during the function exection
	gc.collect() #Run garbage collector
	mem = memory_usage(proc=(fct,(seq1,seq2)),interval=0.1)

	return str(round(score,2))+"\t"+str(round(max(mem),2))+"\t"+str(round(t1-t0,2))

'''
Algorithm 'Needleman&Wunsch'
Based on an needleman principe: each case is the best of
 - score[i-1][j-1] + (seq1[i]==seq2[j]?1:-1) #Match / mismatch
 - score[i-1][j]-1 #Indel
 - score[i][j-1] #Indel
The first nuc of seq2 is either a match of score 1 or a mismatch of score -1: the gap before this on the seq1 doesn't count.
After the end seq1, indels on seq2 doesn't count.
The best score is the best of the last score line and the last column
'''
def needle(seq1, seq2):
	# Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring matrix[len1][len2]
	matrix = [[0 for j in range(len2)] for i in range(len1)]

	# Fill first column: first nuc of seq2 against all seq1 => gap is allowed, only match/mismatch of score 1/-1
	for i in range(len1): matrix[i][0] = 1 if seq1[i]==seq2[0] else -1

	# Fill first line: it is a gap so the -1 increase
	for j in range(1,len2): matrix[0][j] = -j

	#Fill the remaining of the matrix
	#Keep the best possibilities with the end of seq2 (seq2 is inside seq1)
	bestIN=0
	for i in range(1,len1):
		for j in range(1,len2):
			# Match / mismatch ?
			match = matrix[i-1][j-1] + (1 if seq1[i]==seq2[j] else -1)

			# indel ?
			i1 = matrix[i-1][j]-1
			i2 = matrix[i][j-1]-1

			# keep best
			matrix[i][j] = max(match, i1, i2)

		# Best seq2 inside seq 1 ?
		if(matrix[i][len2-1] > bestIN): bestIN = matrix[i][len2-1]

	# Get the list of best concurrents
	best_co = matrix[len1-1][:]
	for i in range(len1):
		best_co.append(matrix[i][-1])
	#Add the se2 is in seq1 possibility
	best_co.append(bestIN)

	# Get score
	return 100*max(best_co)/min(len1,len2)

'''
Algorithm 'Need2Arrays
Same algorithm (needleman&wunsch) but using a score matrix of size (seq1.size(),2)
as we only need the previous line to calculate the current one.
'''
def need2arrays(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring arrays
	previous = [0]*len2
	current = [0]*len2

	# bestIN is the best case of seq2 being INSIDE seq1

	# Calculate the first line (seq2 against seq1[0])
	# Note: we can't 'allow' a gap in seq2 'first nuc'
	previous[0] = (1 if seq1[0]==seq2[0] else -1)
	for j in range(1,len2): previous[j] = -j
	bestIN = previous[len2-1]

	# Complete the needle
	for i in range(1, len1):
		# Test first nuc, a gap is allowed So it's just a (mis)match test
		current[0] = (1 if seq1[i]==seq2[0] else -1)

		# Finish the line
		for j in range(1,len2):
			scores=[0,0,0]
			# Match/mismatch ?
			scores[0] = previous[j-1] + (1 if seq1[i]==seq2[j] else -1)
			#Indels ?
			scores[1] = previous[j]-1
			scores[2] = current[j-1]-1
			#keep best
			current[j]=max(scores)

		# Best seq2 inside possibility ?
		if(current[j] > bestIN): bestIN=current[j]

		# move buffers
		previous = current
		current = [0]*len2

	# Get the best possibilities score which is inside the last line + the bestIN (pseudo last column)
	previous.append(bestIN) #Previous is last because we moved buffers. We add bestIN to it
	return 100*max(previous)/min(len1,len2) #Percent and normalize by theorical max best score (full match)

'''
Algorithm 'dot_cut'.
The principe is to increase the score when the merge is contiguous (diagonal
comparaison) but an difference make a score=0 :
	IF seq1[i] == seq2[j]: score_actual[j] = score_previous[j-1]+1
	ELSE: score_actual[j] = 0
	...do stuff...
	score_previous = score_actual

Then the final percent score is the sum of higher scores before a difference
compared to a perfect match.
This way you have only to memorize 2 'comparaison' column score and the best
scores.
When a part of seq2 has been 'scored', only check the remaining (on the right)
of seq2 to avoid to count a same parts multiple times and to do "come-backs".
'''

def dot_cut(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Init the scores arrays
	previous = [0]*len2
	actual = [0]*len2
	previous_best = 0
	best_scores =  list()
	begin_len2 = 0 #From which nuc of len2 the cmp start

	# Do cmp
	for i in range(len1):
		#Cmp seq2 to actual nuc of seq1
		for j in range(begin_len2, len2):
			if seq1[i] == seq2[j]:
				if j>0: actual[j] = previous[j-1]+1
				else: actual[j] = 1
			else: actual[j]=0

		# Get the best. Do we have a drop ?
		best = max(actual)

		if best < previous_best:
			#Best found ! (the previous_best ;) )
			best_scores.append(previous_best)

			#Now, start on the next len2 nuc from the last nuc of the
			#previous_best for further check
			begin_len2 = previous.index(previous_best)+1

			#Clear the buffers
			best = 0
			actual = [0]*len2

		previous_best = best

		#Move the scores buffers
		previous = actual
		actual = [0]*len2

	# Add the last best score
	best_scores.append(previous_best)

	# Results
	return 100*sum(best_scores)/min(len1,len2)

'''
Python will run the following part at start.
It gets the sequences then run the main function.
'''

# Run the main function
if __name__ == "__main__":
	# Get the argument (the fasta file)
	if len(sys.argv) != 2:
		print("Expect only one argument : the fasta file of sequences")
		sys.exit()

	#Read fasta file
	seqs = []
	for record in  SeqIO.parse(open(sys.argv[1]), 'fasta'):
		seqs.append(record.seq)

	# Run the main function
	main(seqs)

<|endoftext|>