<commit_before> #!/usr/bin/env python
'''Functions for converting between the different representations
of NexSON and the NeXML representation.
See https://github.com/OpenTreeOfLife/api.opentreeoflife.org/wiki/NexSON

Most notable functions are:
    write_obj_as_nexml,
    get_ot_study_info_from_nexml,

'''
from peyotl.nexson_syntax.helper import ConversionConfig, \
                                        NexsonConverter, \
                                        get_nexml_el, \
                                        _add_value_to_dict_bf, \
                                        _get_index_list_of_values, \
                                        _index_list_of_values, \
                                        _is_badgerfish_version, \
                                        _is_by_id_hbf, \
                                        _is_direct_hbf, \
                                        BADGER_FISH_NEXSON_VERSION, \
                                        DEFAULT_NEXSON_VERSION, \
                                        DIRECT_HONEY_BADGERFISH, \
                                        NEXML_NEXSON_VERSION, \
                                        BY_ID_HONEY_BADGERFISH, \
                                        SUPPORTED_NEXSON_VERSIONS

from peyotl.nexson_syntax.optimal2direct_nexson import Optimal2DirectNexson

from peyotl.nexson_syntax.direct2optimal_nexson import Direct2OptimalNexson
from peyotl.nexson_syntax.badgerfish2direct_nexson import Badgerfish2DirectNexson
from peyotl.nexson_syntax.direct2badgerfish_nexson import Direct2BadgerfishNexson
from peyotl.nexson_syntax.nexson2nexml import Nexson2Nexml
from peyotl.nexson_syntax.nexml2nexson import Nexml2Nexson
from peyotl.utility import get_logger
from cStringIO import StringIO
import xml.dom.minidom
import codecs
import json

_CONVERTIBLE_FORMATS = frozenset([NEXML_NEXSON_VERSION,
                                  DEFAULT_NEXSON_VERSION,
                                  BADGER_FISH_NEXSON_VERSION,
                                  BY_ID_HONEY_BADGERFISH,
                                  '0.0',
                                  '1.0',
                                  '1.2',
                                  ])
_LOG = get_logger(__name__)

def get_ot_study_info_from_nexml(src=None,
                                 nexml_content=None,
                                 encoding=u'utf8',
                                 nexson_syntax_version=DEFAULT_NEXSON_VERSION):
    '''Converts an XML doc to JSON using the honeybadgerfish convention (see to_honeybadgerfish_dict)
    and then prunes elements not used by open tree of life study curartion.

    If nexml_content is provided, it is interpreted as the contents
    of an NeXML file in utf-8 encoding.

    If nexml_content is None, then the src arg will be used src can be either:
        * a file_object, or
        * a string
    If `src` is a string then it will be treated as a filepath unless it
        begins with http:// or https:// (in which case it will be downloaded
        using peyotl.utility.download)
    Returns a dictionary with the keys/values encoded according to the honeybadgerfish convention
    See https://github.com/OpenTreeOfLife/api.opentreeoflife.org/wiki/HoneyBadgerFish

    Currently:
        removes nexml/characters @TODO: should replace it with a URI for
            where the removed character data can be found.
    '''
    if _is_by_id_hbf(nexson_syntax_version):
        nsv = DIRECT_HONEY_BADGERFISH
    else:
        nsv = nexson_syntax_version
    if nexml_content is None:
        if isinstance(src, str) or isinstance(src, unicode):
            if src.startswith('http://') or src.startswith('https://'):
                from peyotl.utility import download
                nexml_content = download(url=src, encoding=encoding)
            else:
                src = codecs.open(src, 'rU', encoding=encoding)
                nexml_content = src.read().encode('utf-8')
        else:
            nexml_content = src.read().encode('utf-8')
    doc = xml.dom.minidom.parseString(nexml_content)
    doc_root = doc.documentElement

    ccfg = ConversionConfig(output_format=nsv, input_format=NEXML_NEXSON_VERSION)
    converter = Nexml2Nexson(ccfg)
    o = converter.convert(doc_root)
    if _is_by_id_hbf(nexson_syntax_version):
        return convert_nexson_format(o, BY_ID_HONEY_BADGERFISH, current_format=nsv)
    return o

def get_ot_study_info_from_treebase_nexml(src, encoding=u'utf8', nexson_syntax_version=DEFAULT_NEXSON_VERSION):
    '''Just a stub at this point. Intended to normalize treebase-specific metadata
    into the locations where open tree of life software that expects it.

    `src` can be a string (filepath) or a input file object.
    @TODO: need to investigate which metadata should move or be copied
    '''
    o = get_ot_study_info_from_nexml(src, encoding=encoding, nexson_syntax_version=nexson_syntax_version)
    return o

def _nexson_directly_translatable_to_nexml(vers):
    'TEMP: until we refactor nexml writing code to be more general...'
    return (_is_badgerfish_version(vers)
            or _is_direct_hbf(vers)
            or vers == 'nexml')

def write_obj_as_nexml(obj_dict,
                       file_obj,
                       addindent='',
                       newl='',
                       use_default_root_atts=True):
    nsv = detect_nexson_version(obj_dict)
    if not _nexson_directly_translatable_to_nexml(nsv):
        convert_nexson_format(obj_dict, DIRECT_HONEY_BADGERFISH)
        nsv = DIRECT_HONEY_BADGERFISH
    ccfg = ConversionConfig(NEXML_NEXSON_VERSION,
                            input_format=nsv,
                            use_default_root_atts=use_default_root_atts)
    converter = Nexson2Nexml(ccfg)
    doc = converter.convert(obj_dict)
    doc.writexml(file_obj, addindent=addindent, newl=newl, encoding='utf-8')

def detect_nexson_version(blob):
    '''Returns the nexml2json attribute or the default code for badgerfish'''
    n = get_nexml_el(blob)
    assert(isinstance(n, dict))
    return n.get('@nexml2json', BADGER_FISH_NEXSON_VERSION)
def resolve_nexson_format(v):
    if len(v) == 3:
        if v == '1.2':
            return BY_ID_HONEY_BADGERFISH
        if v == '1.0':
            return DIRECT_HONEY_BADGERFISH
        if v == '0.0':
            return BADGER_FISH_NEXSON_VERSION
    else:
        if v in SUPPORTED_NEXSON_VERSIONS:
            return v
    raise NotImplementedError('NexSON version "{v}" not supported.'.format(v=v))
def can_convert_nexson_forms(src_format, dest_format):
    return (dest_format in _CONVERTIBLE_FORMATS) and (src_format in _CONVERTIBLE_FORMATS)

def convert_nexson_format(blob,
                          out_nexson_format,
                          current_format=None,
                          remove_old_structs=True,
                          pristine_if_invalid=False,
                          sort_arbitrary=False):
    '''Take a dict form of NexSON and converts its datastructures to
    those needed to serialize as out_nexson_format.
    If current_format is not specified, it will be inferred.
    If `remove_old_structs` is False and different honeybadgerfish varieties
        are selected, the `blob` will be 'fat" containing both types
        of lookup structures.
    If pristine_if_invalid is False, then the object may be corrupted if it
        is an invalid nexson struct. Setting this to False can result in
        faster translation, but if an exception is raised the object may
        be polluted with partially constructed fields for the out_nexson_format.
    '''
    if not current_format:
        current_format = detect_nexson_version(blob)
    out_nexson_format = resolve_nexson_format(out_nexson_format)
    if current_format == out_nexson_format:
        if sort_arbitrary:
            sort_arbitrarily_ordered_nexson(blob)
        return blob
    if (_is_by_id_hbf(out_nexson_format) and _is_badgerfish_version(current_format)
        or _is_by_id_hbf(current_format) and _is_badgerfish_version(out_nexson_format)):
        # go from 0.0 -> 1.0 then the 1.0->1.2 should succeed without nexml...
        blob = convert_nexson_format(blob,
                                     DIRECT_HONEY_BADGERFISH,
                                     current_format=current_format,
                                     remove_old_structs=remove_old_structs,
                                     pristine_if_invalid=pristine_if_invalid)
        current_format = DIRECT_HONEY_BADGERFISH
    ccdict = {'output_format':out_nexson_format,
              'input_format':current_format,
              'remove_old_structs': remove_old_structs,
              'pristine_if_invalid': pristine_if_invalid}
    ccfg = ConversionConfig(ccdict)
    if _is_badgerfish_version(current_format):
        converter = Badgerfish2DirectNexson(ccfg)
    elif _is_badgerfish_version(out_nexson_format):
        assert(_is_direct_hbf(current_format))
        converter = Direct2BadgerfishNexson(ccfg)
    elif _is_direct_hbf(current_format) and (out_nexson_format == BY_ID_HONEY_BADGERFISH):
        converter = Direct2OptimalNexson(ccfg)
    elif _is_direct_hbf(out_nexson_format) and (current_format == BY_ID_HONEY_BADGERFISH):
        converter = Optimal2DirectNexson(ccfg)
    else:
        raise NotImplementedError('Conversion from {i} to {o}'.format(i=current_format, o=out_nexson_format))
    blob = converter.convert(blob)
    if sort_arbitrary:
        sort_arbitrarily_ordered_nexson(blob)
    return blob

def write_as_json(blob, dest, indent=0, sort_keys=True):
    opened_out = False
    if isinstance(dest, str) or isinstance(dest, unicode):
        out = codecs.open(dest, mode='w', encoding='utf-8')
        opened_out = True
    else:
        out = dest
    try:
        json.dump(blob, out, indent=indent, sort_keys=sort_keys)
        out.write('\n')
    finally:
        out.flush()
        if opened_out:
            out.close()

def read_as_json(infi):
    inpf = codecs.open(infi)
    n = json.load(inpf)
    return(n)

def _recursive_sort_meta(blob, k):
    if isinstance(blob, list):
        for i in blob:
            _recursive_sort_meta(i, k)
    else:
        for inner_k, v in blob.items():
            if inner_k == 'meta' and isinstance(v, list):
                sl = []
                incd = {}
                for el in v:
                    sk = el.get('@property') or el.get('@rel') or ''
                    count = incd.setdefault(sk, 0)
                    incd[sk] = 1 + count
                    sl.append((sk, count, el))
                sl.sort()
                del v[:] # clear out the value in place
                v.extend([i[2] for i in sl]) # replace it with the item from the sorted list
            if isinstance(v, list) or isinstance(v, dict):
                _recursive_sort_meta(v, inner_k)

def sort_meta_elements(blob):
    '''For v0.0 (which has meta values in a list), this
    function recursively walks through the object
    and sorts each meta by @property or @rel values.
    '''
    v = detect_nexson_version(blob)
    if _is_badgerfish_version(v):
        _recursive_sort_meta(blob, '')
    return blob

def _inplace_sort_by_id(unsorted_list):
    '''Takes a list of dicts each of which has an '@id' key,
    sorts the elements in the list by the value of the @id key.
    Assumes that @id is unique or the dicts have a meaningul < operator
    '''
    if not isinstance(unsorted_list, list):
        return
    sorted_list = [(i.get('@id'), i) for i in unsorted_list]
    sorted_list.sort()
    del unsorted_list[:]
    unsorted_list.extend([i[1] for i in sorted_list])

def sort_arbitrarily_ordered_nexson(blob):
    '''Primarily used for testing (getting nice diffs). Calls
    sort_meta_elements and then sorts otu, node and edge list by id
    '''
    # otu, node and edge elements have no necessary orger in v0.0 or v1.0
    v = detect_nexson_version(blob)
    nex = get_nexml_el(blob)
    if _is_by_id_hbf(v):
        return blob
    sort_meta_elements(blob)
    for ob in _get_index_list_of_values(nex, 'otus'):
        _inplace_sort_by_id(ob.get('otu', []))
    for tb in _get_index_list_of_values(nex, 'trees'):
        for tree in _get_index_list_of_values(tb, 'tree'):
            _inplace_sort_by_id(tree.get('node', []))
            _inplace_sort_by_id(tree.get('edge', []))
    return blob

def add_resource_meta(obj, rel, href, version):
    if _is_badgerfish_version(version):
        m = obj.setdefault('meta', [])
        if not isinstance(m, list):
            m = [m]
            obj['meta'] = m
        m.append({'@href':href,
                  '@rel': rel,
                  '@xsi:type': 'nex:ResourceMeta'})
    else:
        k = '^' + rel
        _add_value_to_dict_bf(obj, k, {'@href': href})

def get_empty_nexson(vers='1.2.1'):
    assert(vers == '1.2.1')
    return {
        'nexml': {
        '@about': '#study',
        '@generator': 'Open Tree API',
        '@id': 'study',
        '@nexml2json': vers,
        '@nexmljson': 'http://purl.org/opentree/nexson',
        '@version': '0.9',
        '@xmlns': {
            '$': 'http://www.nexml.org/2009',
            'nex': 'http://www.nexml.org/2009',
            'ot': 'http://purl.org/opentree-terms#',
            'xsd': 'http://www.w3.org/2001/XMLSchema#',
            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
            'xhtml': 'http://www.w3.org/1999/xhtml/vocab#'
        },
        '^xhtml:license': {'@href': 'http://creativecommons.org/publicdomain/zero/1.0/'},
        '^ot:otusElementOrder': [
            'otus1',
        ],
        'otusById': {
            'otus1': {
                'otuById':{},
            },
        },
        '^ot:treesElementOrder': [
            'trees1',
        ],
        'treesById': {
            'trees1': {
                '@otus': 'otus1',
                '^ot:treeElementOrder':[],
                'treeById': {},
                },
            },
        }
    }<commit_msg>bug fix: previous commit removed convertible type from list<commit_after> #!/usr/bin/env python
'''Functions for converting between the different representations
of NexSON and the NeXML representation.
See https://github.com/OpenTreeOfLife/api.opentreeoflife.org/wiki/NexSON

Most notable functions are:
    write_obj_as_nexml,
    get_ot_study_info_from_nexml,

'''
from peyotl.nexson_syntax.helper import ConversionConfig, \
                                        NexsonConverter, \
                                        get_nexml_el, \
                                        _add_value_to_dict_bf, \
                                        _get_index_list_of_values, \
                                        _index_list_of_values, \
                                        _is_badgerfish_version, \
                                        _is_by_id_hbf, \
                                        _is_direct_hbf, \
                                        BADGER_FISH_NEXSON_VERSION, \
                                        DEFAULT_NEXSON_VERSION, \
                                        DIRECT_HONEY_BADGERFISH, \
                                        NEXML_NEXSON_VERSION, \
                                        BY_ID_HONEY_BADGERFISH, \
                                        SUPPORTED_NEXSON_VERSIONS

from peyotl.nexson_syntax.optimal2direct_nexson import Optimal2DirectNexson

from peyotl.nexson_syntax.direct2optimal_nexson import Direct2OptimalNexson
from peyotl.nexson_syntax.badgerfish2direct_nexson import Badgerfish2DirectNexson
from peyotl.nexson_syntax.direct2badgerfish_nexson import Direct2BadgerfishNexson
from peyotl.nexson_syntax.nexson2nexml import Nexson2Nexml
from peyotl.nexson_syntax.nexml2nexson import Nexml2Nexson
from peyotl.utility import get_logger
from cStringIO import StringIO
import xml.dom.minidom
import codecs
import json

_CONVERTIBLE_FORMATS = frozenset([NEXML_NEXSON_VERSION,
                                  DIRECT_HONEY_BADGERFISH,
                                  BADGER_FISH_NEXSON_VERSION,
                                  BY_ID_HONEY_BADGERFISH,
                                  '0.0',
                                  '1.0',
                                  '1.2',
                                  ])
_LOG = get_logger(__name__)

def get_ot_study_info_from_nexml(src=None,
                                 nexml_content=None,
                                 encoding=u'utf8',
                                 nexson_syntax_version=DEFAULT_NEXSON_VERSION):
    '''Converts an XML doc to JSON using the honeybadgerfish convention (see to_honeybadgerfish_dict)
    and then prunes elements not used by open tree of life study curartion.

    If nexml_content is provided, it is interpreted as the contents
    of an NeXML file in utf-8 encoding.

    If nexml_content is None, then the src arg will be used src can be either:
        * a file_object, or
        * a string
    If `src` is a string then it will be treated as a filepath unless it
        begins with http:// or https:// (in which case it will be downloaded
        using peyotl.utility.download)
    Returns a dictionary with the keys/values encoded according to the honeybadgerfish convention
    See https://github.com/OpenTreeOfLife/api.opentreeoflife.org/wiki/HoneyBadgerFish

    Currently:
        removes nexml/characters @TODO: should replace it with a URI for
            where the removed character data can be found.
    '''
    if _is_by_id_hbf(nexson_syntax_version):
        nsv = DIRECT_HONEY_BADGERFISH
    else:
        nsv = nexson_syntax_version
    if nexml_content is None:
        if isinstance(src, str) or isinstance(src, unicode):
            if src.startswith('http://') or src.startswith('https://'):
                from peyotl.utility import download
                nexml_content = download(url=src, encoding=encoding)
            else:
                src = codecs.open(src, 'rU', encoding=encoding)
                nexml_content = src.read().encode('utf-8')
        else:
            nexml_content = src.read().encode('utf-8')
    doc = xml.dom.minidom.parseString(nexml_content)
    doc_root = doc.documentElement

    ccfg = ConversionConfig(output_format=nsv, input_format=NEXML_NEXSON_VERSION)
    converter = Nexml2Nexson(ccfg)
    o = converter.convert(doc_root)
    if _is_by_id_hbf(nexson_syntax_version):
        return convert_nexson_format(o, BY_ID_HONEY_BADGERFISH, current_format=nsv)
    return o

def get_ot_study_info_from_treebase_nexml(src, encoding=u'utf8', nexson_syntax_version=DEFAULT_NEXSON_VERSION):
    '''Just a stub at this point. Intended to normalize treebase-specific metadata
    into the locations where open tree of life software that expects it.

    `src` can be a string (filepath) or a input file object.
    @TODO: need to investigate which metadata should move or be copied
    '''
    o = get_ot_study_info_from_nexml(src, encoding=encoding, nexson_syntax_version=nexson_syntax_version)
    return o

def _nexson_directly_translatable_to_nexml(vers):
    'TEMP: until we refactor nexml writing code to be more general...'
    return (_is_badgerfish_version(vers)
            or _is_direct_hbf(vers)
            or vers == 'nexml')

def write_obj_as_nexml(obj_dict,
                       file_obj,
                       addindent='',
                       newl='',
                       use_default_root_atts=True):
    nsv = detect_nexson_version(obj_dict)
    if not _nexson_directly_translatable_to_nexml(nsv):
        convert_nexson_format(obj_dict, DIRECT_HONEY_BADGERFISH)
        nsv = DIRECT_HONEY_BADGERFISH
    ccfg = ConversionConfig(NEXML_NEXSON_VERSION,
                            input_format=nsv,
                            use_default_root_atts=use_default_root_atts)
    converter = Nexson2Nexml(ccfg)
    doc = converter.convert(obj_dict)
    doc.writexml(file_obj, addindent=addindent, newl=newl, encoding='utf-8')

def detect_nexson_version(blob):
    '''Returns the nexml2json attribute or the default code for badgerfish'''
    n = get_nexml_el(blob)
    assert(isinstance(n, dict))
    return n.get('@nexml2json', BADGER_FISH_NEXSON_VERSION)
def resolve_nexson_format(v):
    if len(v) == 3:
        if v == '1.2':
            return BY_ID_HONEY_BADGERFISH
        if v == '1.0':
            return DIRECT_HONEY_BADGERFISH
        if v == '0.0':
            return BADGER_FISH_NEXSON_VERSION
    else:
        if v in SUPPORTED_NEXSON_VERSIONS:
            return v
    raise NotImplementedError('NexSON version "{v}" not supported.'.format(v=v))
def can_convert_nexson_forms(src_format, dest_format):
    return (dest_format in _CONVERTIBLE_FORMATS) and (src_format in _CONVERTIBLE_FORMATS)

def convert_nexson_format(blob,
                          out_nexson_format,
                          current_format=None,
                          remove_old_structs=True,
                          pristine_if_invalid=False,
                          sort_arbitrary=False):
    '''Take a dict form of NexSON and converts its datastructures to
    those needed to serialize as out_nexson_format.
    If current_format is not specified, it will be inferred.
    If `remove_old_structs` is False and different honeybadgerfish varieties
        are selected, the `blob` will be 'fat" containing both types
        of lookup structures.
    If pristine_if_invalid is False, then the object may be corrupted if it
        is an invalid nexson struct. Setting this to False can result in
        faster translation, but if an exception is raised the object may
        be polluted with partially constructed fields for the out_nexson_format.
    '''
    if not current_format:
        current_format = detect_nexson_version(blob)
    out_nexson_format = resolve_nexson_format(out_nexson_format)
    if current_format == out_nexson_format:
        if sort_arbitrary:
            sort_arbitrarily_ordered_nexson(blob)
        return blob
    if (_is_by_id_hbf(out_nexson_format) and _is_badgerfish_version(current_format)
        or _is_by_id_hbf(current_format) and _is_badgerfish_version(out_nexson_format)):
        # go from 0.0 -> 1.0 then the 1.0->1.2 should succeed without nexml...
        blob = convert_nexson_format(blob,
                                     DIRECT_HONEY_BADGERFISH,
                                     current_format=current_format,
                                     remove_old_structs=remove_old_structs,
                                     pristine_if_invalid=pristine_if_invalid)
        current_format = DIRECT_HONEY_BADGERFISH
    ccdict = {'output_format':out_nexson_format,
              'input_format':current_format,
              'remove_old_structs': remove_old_structs,
              'pristine_if_invalid': pristine_if_invalid}
    ccfg = ConversionConfig(ccdict)
    if _is_badgerfish_version(current_format):
        converter = Badgerfish2DirectNexson(ccfg)
    elif _is_badgerfish_version(out_nexson_format):
        assert(_is_direct_hbf(current_format))
        converter = Direct2BadgerfishNexson(ccfg)
    elif _is_direct_hbf(current_format) and (out_nexson_format == BY_ID_HONEY_BADGERFISH):
        converter = Direct2OptimalNexson(ccfg)
    elif _is_direct_hbf(out_nexson_format) and (current_format == BY_ID_HONEY_BADGERFISH):
        converter = Optimal2DirectNexson(ccfg)
    else:
        raise NotImplementedError('Conversion from {i} to {o}'.format(i=current_format, o=out_nexson_format))
    blob = converter.convert(blob)
    if sort_arbitrary:
        sort_arbitrarily_ordered_nexson(blob)
    return blob

def write_as_json(blob, dest, indent=0, sort_keys=True):
    opened_out = False
    if isinstance(dest, str) or isinstance(dest, unicode):
        out = codecs.open(dest, mode='w', encoding='utf-8')
        opened_out = True
    else:
        out = dest
    try:
        json.dump(blob, out, indent=indent, sort_keys=sort_keys)
        out.write('\n')
    finally:
        out.flush()
        if opened_out:
            out.close()

def read_as_json(infi):
    inpf = codecs.open(infi)
    n = json.load(inpf)
    return(n)

def _recursive_sort_meta(blob, k):
    if isinstance(blob, list):
        for i in blob:
            _recursive_sort_meta(i, k)
    else:
        for inner_k, v in blob.items():
            if inner_k == 'meta' and isinstance(v, list):
                sl = []
                incd = {}
                for el in v:
                    sk = el.get('@property') or el.get('@rel') or ''
                    count = incd.setdefault(sk, 0)
                    incd[sk] = 1 + count
                    sl.append((sk, count, el))
                sl.sort()
                del v[:] # clear out the value in place
                v.extend([i[2] for i in sl]) # replace it with the item from the sorted list
            if isinstance(v, list) or isinstance(v, dict):
                _recursive_sort_meta(v, inner_k)

def sort_meta_elements(blob):
    '''For v0.0 (which has meta values in a list), this
    function recursively walks through the object
    and sorts each meta by @property or @rel values.
    '''
    v = detect_nexson_version(blob)
    if _is_badgerfish_version(v):
        _recursive_sort_meta(blob, '')
    return blob

def _inplace_sort_by_id(unsorted_list):
    '''Takes a list of dicts each of which has an '@id' key,
    sorts the elements in the list by the value of the @id key.
    Assumes that @id is unique or the dicts have a meaningul < operator
    '''
    if not isinstance(unsorted_list, list):
        return
    sorted_list = [(i.get('@id'), i) for i in unsorted_list]
    sorted_list.sort()
    del unsorted_list[:]
    unsorted_list.extend([i[1] for i in sorted_list])

def sort_arbitrarily_ordered_nexson(blob):
    '''Primarily used for testing (getting nice diffs). Calls
    sort_meta_elements and then sorts otu, node and edge list by id
    '''
    # otu, node and edge elements have no necessary orger in v0.0 or v1.0
    v = detect_nexson_version(blob)
    nex = get_nexml_el(blob)
    if _is_by_id_hbf(v):
        return blob
    sort_meta_elements(blob)
    for ob in _get_index_list_of_values(nex, 'otus'):
        _inplace_sort_by_id(ob.get('otu', []))
    for tb in _get_index_list_of_values(nex, 'trees'):
        for tree in _get_index_list_of_values(tb, 'tree'):
            _inplace_sort_by_id(tree.get('node', []))
            _inplace_sort_by_id(tree.get('edge', []))
    return blob

def add_resource_meta(obj, rel, href, version):
    if _is_badgerfish_version(version):
        m = obj.setdefault('meta', [])
        if not isinstance(m, list):
            m = [m]
            obj['meta'] = m
        m.append({'@href':href,
                  '@rel': rel,
                  '@xsi:type': 'nex:ResourceMeta'})
    else:
        k = '^' + rel
        _add_value_to_dict_bf(obj, k, {'@href': href})

def get_empty_nexson(vers='1.2.1'):
    assert(vers == '1.2.1')
    return {
        'nexml': {
        '@about': '#study',
        '@generator': 'Open Tree API',
        '@id': 'study',
        '@nexml2json': vers,
        '@nexmljson': 'http://purl.org/opentree/nexson',
        '@version': '0.9',
        '@xmlns': {
            '$': 'http://www.nexml.org/2009',
            'nex': 'http://www.nexml.org/2009',
            'ot': 'http://purl.org/opentree-terms#',
            'xsd': 'http://www.w3.org/2001/XMLSchema#',
            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
            'xhtml': 'http://www.w3.org/1999/xhtml/vocab#'
        },
        '^xhtml:license': {'@href': 'http://creativecommons.org/publicdomain/zero/1.0/'},
        '^ot:otusElementOrder': [
            'otus1',
        ],
        'otusById': {
            'otus1': {
                'otuById':{},
            },
        },
        '^ot:treesElementOrder': [
            'trees1',
        ],
        'treesById': {
            'trees1': {
                '@otus': 'otus1',
                '^ot:treeElementOrder':[],
                'treeById': {},
                },
            },
        }
    }<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2013 ecdsa@github
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from electroncash.i18n import _

from .util import *
from .qrtextedit import ShowQRTextEdit, ScanQRTextEdit

def seed_warning_msg(seed):
    return ''.join([
        "
",
        _("Please save these %d words on paper (order is important). "),
        _("This seed will allow you to recover your wallet in case "
          "of computer failure."),
        "
",
        "" + _("WARNING") + ":",
        "
",
        "
" + _("Never disclose your seed.") + "
",
        "
" + _("Never type it on a website.") + "
",
        "
" + _("Do not store it electronically.") + "
",
        "
"
    ]) % len(seed.split())

class SeedLayout(QVBoxLayout):
    #options
    is_bip39 = False
    is_ext = False
    is_bip39_145 = False

    def seed_options(self):
        dialog = QDialog()
        vbox = QVBoxLayout(dialog)
        if 'ext' in self.options:
            cb_ext = QCheckBox(_('Extend this seed with custom words') + " " + _("(aka 'passphrase')"))
            cb_ext.setChecked(self.is_ext)
            vbox.addWidget(cb_ext)
        if 'bip39' in self.options:
            def f(b):
                self.is_seed = (lambda x: bool(x)) if b else self.saved_is_seed
                self.is_bip39 = b
                self.on_edit()
                if b:
                    msg = ' '.join([
                        '' + _('Warning') + ':  ',
                        _('BIP39 seeds can be imported in Electron Cash, so that users can access funds locked in other wallets.'),
                        _('However, we do not generate BIP39 seeds, because they do not meet our safety standard.'),
                        _('BIP39 seeds do not include a version number, which compromises compatibility with future software.'),
                        _('We do not guarantee that BIP39 imports will always be supported in Electron Cash.'),
                    ])
                else:
                    msg = ''
                self.seed_warning.setText(msg)
            cb_bip39 = QCheckBox(_('BIP39 seed'))
            cb_bip39.toggled.connect(f)
            cb_bip39.setChecked(self.is_bip39)
            vbox.addWidget(cb_bip39)

        if 'bip39_145' in self.options:
            def f(b):
                self.is_seed = (lambda x: bool(x)) if b else self.saved_is_seed
                self.on_edit()
                self.is_bip39 = b
                if b:
                    msg = ' '.join([
                        '' + _('Warning') + ': BIP39 seeds are dangerous!' + '

',
                        _('BIP39 seeds can be imported in Electron Cash so that users can access funds locked in other wallets.'),
                        _('However, BIP39 seeds do not include a version number, which compromises compatibility with future wallet software.'),
                        '

',
                        _('We do not guarantee that BIP39 imports will always be supported in Electron Cash.'),
                        _('In addition, Electron Cash does not verify the checksum of BIP39 seeds; make sure you type your seed correctly.'),
                    ])
                else:
                    msg = ''
                self.seed_warning.setText(msg)
            cb_bip39_145 = QCheckBox(_('Use Coin Type 145 with bip39'))
            cb_bip39_145.toggled.connect(f)
            cb_bip39_145.setChecked(self.is_bip39_145)
            vbox.addWidget(cb_bip39_145)

        vbox.addLayout(Buttons(OkButton(dialog)))
        if not dialog.exec_():
            return None
        self.is_ext = cb_ext.isChecked() if 'ext' in self.options else False
        self.is_bip39 = cb_bip39.isChecked() if 'bip39' in self.options else False
        self.is_bip39_145 = cb_bip39_145.isChecked() if 'bip39_145' in self.options else False

    def __init__(self, seed=None, title=None, icon=True, msg=None, options=None, is_seed=None, passphrase=None, parent=None, editable=True):
        QVBoxLayout.__init__(self)
        self.parent = parent
        self.options = options
        if title:
            self.addWidget(WWLabel(title))
        self.seed_e = ButtonsTextEdit()
        self.seed_e.setReadOnly(not editable)
        if seed:
            self.seed_e.setText(seed)
        else:
            self.seed_e.setTabChangesFocus(True)
            self.is_seed = is_seed
            self.saved_is_seed = self.is_seed
            self.seed_e.textChanged.connect(self.on_edit)
        self.seed_e.setMaximumHeight(75)
        hbox = QHBoxLayout()
        if icon:
            logo = QLabel()
            logo.setPixmap(QPixmap(":icons/seed.png").scaledToWidth(64))
            logo.setMaximumWidth(60)
            hbox.addWidget(logo)
        hbox.addWidget(self.seed_e)
        self.addLayout(hbox)
        hbox = QHBoxLayout()
        hbox.addStretch(1)
        self.seed_type_label = QLabel('')
        hbox.addWidget(self.seed_type_label)
        if options:
            opt_button = EnterButton(_('Options'), self.seed_options)
            hbox.addWidget(opt_button)
            self.addLayout(hbox)
        if passphrase:
            hbox = QHBoxLayout()
            passphrase_e = QLineEdit()
            passphrase_e.setText(passphrase)
            passphrase_e.setReadOnly(True)
            hbox.addWidget(QLabel(_("Your seed extension is") + ':'))
            hbox.addWidget(passphrase_e)
            self.addLayout(hbox)
        self.addStretch(1)
        self.seed_warning = WWLabel('')
        if msg:
            self.seed_warning.setText(seed_warning_msg(seed))
        self.addWidget(self.seed_warning)

    def get_seed(self):
        text = self.seed_e.text()
        return ' '.join(text.split())

    def on_edit(self):
        from electroncash.bitcoin import seed_type
        s = self.get_seed()
        b = self.is_seed(s)
        if not self.is_bip39:
            t = seed_type(s)
            label = _('Seed Type') + ': ' + t if t else ''
        else:
            from electroncash.keystore import bip39_is_checksum_valid
            is_checksum, is_wordlist = bip39_is_checksum_valid(s)
            status = ('checksum: ' + ('ok' if is_checksum else 'failed')) if is_wordlist else 'unknown wordlist'
            label = 'BIP39' + ' (%s)'%status
        self.seed_type_label.setText(label)
        self.parent.next_button.setEnabled(b)

class KeysLayout(QVBoxLayout):
    def __init__(self, parent=None, title=None, is_valid=None, allow_multi=False):
        QVBoxLayout.__init__(self)
        self.parent = parent
        self.is_valid = is_valid
        self.text_e = ScanQRTextEdit(allow_multi=allow_multi)
        self.text_e.textChanged.connect(self.on_edit)
        self.addWidget(WWLabel(title))
        self.addWidget(self.text_e)

    def get_text(self):
        return self.text_e.text()

    def on_edit(self):
        b = self.is_valid(self.get_text())
        self.parent.next_button.setEnabled(b)

class SeedDialog(WindowModalDialog):

    def __init__(self, parent, seed, passphrase):
        WindowModalDialog.__init__(self, parent, ('Electron Cash - ' + _('Seed')))
        self.setMinimumWidth(400)
        vbox = QVBoxLayout(self)
        title =  _("Your wallet generation seed is:")
        slayout = SeedLayout(title=title, seed=seed, msg=True, passphrase=passphrase, editable=False)
        vbox.addLayout(slayout)
        vbox.addLayout(Buttons(CloseButton(self)))
<commit_msg>Reworded FUD message from BIP39 seed dialog to not be FUD<commit_after>#!/usr/bin/env python3
#
# Electrum - lightweight Bitcoin client
# Copyright (C) 2013 ecdsa@github
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from electroncash.i18n import _

from .util import *
from .qrtextedit import ShowQRTextEdit, ScanQRTextEdit

def seed_warning_msg(seed):
    return ''.join([
        "
",
        _("Please save these %d words on paper (order is important). "),
        _("This seed will allow you to recover your wallet in case "
          "of computer failure."),
        "
",
        "" + _("WARNING") + ":",
        "
",
        "
" + _("Never disclose your seed.") + "
",
        "
" + _("Never type it on a website.") + "
",
        "
" + _("Do not store it electronically.") + "
",
        "
"
    ]) % len(seed.split())

class SeedLayout(QVBoxLayout):
    #options
    is_bip39 = False
    is_ext = False
    is_bip39_145 = False

    def seed_options(self):
        dialog = QDialog()
        vbox = QVBoxLayout(dialog)
        if 'ext' in self.options:
            cb_ext = QCheckBox(_('Extend this seed with custom words') + " " + _("(aka 'passphrase')"))
            cb_ext.setChecked(self.is_ext)
            vbox.addWidget(cb_ext)
        if 'bip39' in self.options:
            def f(b):
                self.is_seed = (lambda x: bool(x)) if b else self.saved_is_seed
                self.is_bip39 = b
                self.on_edit()
                if b:
                    msg = ' '.join([
                        '' + _('About BIP39') + ':  ',
                        _('BIP39 seeds can be imported into Electron Cash so that users can access funds from other wallets.'),
                        _('However, we do not generate BIP39 seeds because our seed format is better at preserving future compatibility.'),
                        _('BIP39 seeds do not include a version number, which makes compatibility with future software more difficult.')
                    ])
                else:
                    msg = ''
                self.seed_warning.setText(msg)
            cb_bip39 = QCheckBox(_('BIP39 seed'))
            cb_bip39.toggled.connect(f)
            cb_bip39.setChecked(self.is_bip39)
            vbox.addWidget(cb_bip39)

        # Note: I grep'd the sources. As of May 2019, this code path cannot
        # be reached.  I'm leaving this here in case it serves some purpose
        # still -- but I cannot see any place in the code where this branch
        # would be triggered.  The below warning message is needlessly
        # FUD-ey.  It should be altered if this code path is ever reinstated.
        # -Calin
        if 'bip39_145' in self.options:
            def f(b):
                self.is_seed = (lambda x: bool(x)) if b else self.saved_is_seed
                self.on_edit()
                self.is_bip39 = b
                if b:
                    msg = ' '.join([
                        '' + _('Warning') + ': BIP39 seeds are dangerous!' + '

',
                        _('BIP39 seeds can be imported in Electron Cash so that users can access funds locked in other wallets.'),
                        _('However, BIP39 seeds do not include a version number, which compromises compatibility with future wallet software.'),
                        '

',
                        _('We do not guarantee that BIP39 imports will always be supported in Electron Cash.'),
                        _('In addition, Electron Cash does not verify the checksum of BIP39 seeds; make sure you type your seed correctly.'),
                    ])
                else:
                    msg = ''
                self.seed_warning.setText(msg)
            cb_bip39_145 = QCheckBox(_('Use Coin Type 145 with bip39'))
            cb_bip39_145.toggled.connect(f)
            cb_bip39_145.setChecked(self.is_bip39_145)
            vbox.addWidget(cb_bip39_145)

        vbox.addLayout(Buttons(OkButton(dialog)))
        if not dialog.exec_():
            return None
        self.is_ext = cb_ext.isChecked() if 'ext' in self.options else False
        self.is_bip39 = cb_bip39.isChecked() if 'bip39' in self.options else False
        self.is_bip39_145 = cb_bip39_145.isChecked() if 'bip39_145' in self.options else False

    def __init__(self, seed=None, title=None, icon=True, msg=None, options=None, is_seed=None, passphrase=None, parent=None, editable=True):
        QVBoxLayout.__init__(self)
        self.parent = parent
        self.options = options
        if title:
            self.addWidget(WWLabel(title))
        self.seed_e = ButtonsTextEdit()
        self.seed_e.setReadOnly(not editable)
        if seed:
            self.seed_e.setText(seed)
        else:
            self.seed_e.setTabChangesFocus(True)
            self.is_seed = is_seed
            self.saved_is_seed = self.is_seed
            self.seed_e.textChanged.connect(self.on_edit)
        self.seed_e.setMaximumHeight(75)
        hbox = QHBoxLayout()
        if icon:
            logo = QLabel()
            logo.setPixmap(QPixmap(":icons/seed.png").scaledToWidth(64))
            logo.setMaximumWidth(60)
            hbox.addWidget(logo)
        hbox.addWidget(self.seed_e)
        self.addLayout(hbox)
        hbox = QHBoxLayout()
        hbox.addStretch(1)
        self.seed_type_label = QLabel('')
        hbox.addWidget(self.seed_type_label)
        if options:
            opt_button = EnterButton(_('Options'), self.seed_options)
            hbox.addWidget(opt_button)
            self.addLayout(hbox)
        if passphrase:
            hbox = QHBoxLayout()
            passphrase_e = QLineEdit()
            passphrase_e.setText(passphrase)
            passphrase_e.setReadOnly(True)
            hbox.addWidget(QLabel(_("Your seed extension is") + ':'))
            hbox.addWidget(passphrase_e)
            self.addLayout(hbox)
        self.addStretch(1)
        self.seed_warning = WWLabel('')
        if msg:
            self.seed_warning.setText(seed_warning_msg(seed))
        self.addWidget(self.seed_warning)

    def get_seed(self):
        text = self.seed_e.text()
        return ' '.join(text.split())

    def on_edit(self):
        from electroncash.bitcoin import seed_type
        s = self.get_seed()
        b = self.is_seed(s)
        if not self.is_bip39:
            t = seed_type(s)
            label = _('Seed Type') + ': ' + t if t else ''
        else:
            from electroncash.keystore import bip39_is_checksum_valid
            is_checksum, is_wordlist = bip39_is_checksum_valid(s)
            status = ('checksum: ' + ('ok' if is_checksum else 'failed')) if is_wordlist else 'unknown wordlist'
            label = 'BIP39' + ' (%s)'%status
        self.seed_type_label.setText(label)
        self.parent.next_button.setEnabled(b)

class KeysLayout(QVBoxLayout):
    def __init__(self, parent=None, title=None, is_valid=None, allow_multi=False):
        QVBoxLayout.__init__(self)
        self.parent = parent
        self.is_valid = is_valid
        self.text_e = ScanQRTextEdit(allow_multi=allow_multi)
        self.text_e.textChanged.connect(self.on_edit)
        self.addWidget(WWLabel(title))
        self.addWidget(self.text_e)

    def get_text(self):
        return self.text_e.text()

    def on_edit(self):
        b = self.is_valid(self.get_text())
        self.parent.next_button.setEnabled(b)

class SeedDialog(WindowModalDialog):

    def __init__(self, parent, seed, passphrase):
        WindowModalDialog.__init__(self, parent, ('Electron Cash - ' + _('Seed')))
        self.setMinimumWidth(400)
        vbox = QVBoxLayout(self)
        title =  _("Your wallet generation seed is:")
        slayout = SeedLayout(title=title, seed=seed, msg=True, passphrase=passphrase, editable=False)
        vbox.addLayout(slayout)
        vbox.addLayout(Buttons(CloseButton(self)))
<|endoftext|>


<commit_before>from __future__ import print_function

import click
import os
import re
import time

from psqlparse import parse
from psqlparse import nodes
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import Terminal256Formatter
from pprint import pformat
from toolz.curried import pipe, map, filter, get
from toolz.itertoolz import count, unique
from toolz.dicttoolz import valmap, valfilter
from toposort import toposort, toposort_flatten

def pprint_color(obj):
    print(highlight(pformat(obj), PythonLexer(), Terminal256Formatter()))

def extract_nodes(content, pattern):
    class NS(object):
        pass
    ns = NS()
    ns.views = set()
    ns.dependencies = set()

    def inner(data, depth):
        if isinstance(data, dict):
            if 'relname' in data and 'schemaname' in data and re.search(pattern, data['relname']):
                entity = '.'.join([
                    data['schemaname'],
                    data['relname']
                ])
                if depth == 1:
                    ns.views.add(entity)
                else:
                    ns.dependencies.add(entity)
            else:
                for key, value in data.items():
                    inner(value, depth)
        elif isinstance(data, list) or isinstance(data, tuple):
            for item in data:
                inner(item, depth + 1)

    parsed_content = parse(content)
    inner(parsed_content, 0)
    return {
        'views': ns.views,
        'dependencies': ns.dependencies - ns.views
    }

def walk_directory_recursively(input_dir):
    file_paths = []
    for root, _, files in os.walk(input_dir):
        for file in files:
            file_paths.append(os.path.join(root, file))
    return file_paths

def process_file(file_name, pattern):
    with open(file_name, 'r') as f:
        file_name = f.name
        content = f.read()
        nodes = extract_nodes(content, pattern)

        return {
            'file_name': file_name,
            'content': content,
            'views': nodes['views'],
            'view_dependencies': nodes['dependencies']
        }

def format_content(entity):
    file_name = entity['file_name']
    content_no_txn = entity['content'] \
        .replace('BEGIN;', '') \
        .replace('COMMIT;', '')
    return "  -- %s%s" % (file_name, content_no_txn)

def generate_script(views, spacer=''):
    return "\n\n".join([
        'BEGIN;',
        spacer.join(views),
        'COMMIT;'
    ])

@click.command()
@click.option('-d', '--dry-run', is_flag=True, help='Analyzes dependencies without actually generating the output files')
@click.option('-i', '--input-dir', default='.', help='The directory for the PostgreSQL scripts to analyze')
@click.option('-I', '--ignore-refresh', default='', help='Regex pattern to match when ignoring refresh on Materialized Views (i.e. "hist" for "users_mv_hist")')
@click.option('-o', '--output-dir', default='.', help='Output directory for the generated creation and refresh scripts')
@click.option('-p', '--pattern', default='_mv', help='Materialized View regex pattern to match (i.e. "_mv" for "users_mv")')
@click.option('-v', '--verbose', is_flag=True, help='Enables verbose logging')
def cli(dry_run, input_dir, ignore_refresh, output_dir, pattern, verbose):

    dag = {}

    file_paths = walk_directory_recursively(input_dir)

    if verbose:
        print('Found %d Scripts in %s' % (len(file_paths), input_dir))

    entities = pipe(file_paths,
        map(lambda file_path: process_file(file_path, pattern)),
        list
    )

    if verbose:
        total_views = pipe(entities,
            map(get('views')),
            map(count),
            sum
        )
        total_deps = pipe(entities,
            map(get('view_dependencies')),
            map(count),
            sum
        )
        print('Identified %d Materialized Views, Containing %d View Dependencies' % (total_views, total_deps))

    view_content = {}
    dag = {}
    for entity in entities:
        view_content.update(
            {view: format_content(entity) for view in entity['views']}
        )
        dag.update(
            {view: entity['view_dependencies'] for view in entity['views']}
        )

    sorted_views = toposort_flatten(dag)

    if verbose:
        print("\nMaterialized View Dependencies:")
        pprint_color(
            valmap(lambda val: list(val), valfilter(lambda val: val, dag))
        )

    create_views = pipe(sorted_views,
        map(lambda view: view_content[view]),
        unique,
        list
    )

    create_script = generate_script(create_views)

    refresh_views = pipe(sorted_views,
        filter(lambda view: re.search(pattern, view) and not (ignore_refresh and re.search(ignore_refresh , view))),
        map(lambda view: '  REFRESH MATERIALIZED VIEW CONCURRENTLY ' + view + ';'),
        list
    )

    if verbose:
        print('Selecting %d Materialized Views for Refresh' % len(refresh_views))

    refresh_script = generate_script(refresh_views, "\n\n")

    if dry_run:
        print('Dry Run Option Enabled - Skipping Script Generation')
        return

    timestr = time.strftime("%Y%m%d-%H%M%S")

    create_script_name = 'create-%s.sql' % timestr
    create_script_path = os.path.join(output_dir, create_script_name)
    with open(create_script_path, 'w') as f:
        f.write(create_script)
        if verbose:
            print('Successfully Saved Creation Script to %s' % create_script_path)

    refresh_script_name = 'refresh-%s.sql' % timestr
    refresh_script_path = os.path.join(output_dir, refresh_script_name)
    with open(refresh_script_path, 'w') as f:
        f.write(refresh_script)
        if verbose:
            print('Successfully Saved Refresh Script to %s' % refresh_script_path)

    pprint_color('Process Complete!')
<commit_msg>Extracted serialize_script method<commit_after>from __future__ import print_function

import click
import os
import re
import time

from psqlparse import parse
from psqlparse import nodes
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import Terminal256Formatter
from pprint import pformat
from toolz.curried import pipe, map, filter, get
from toolz.itertoolz import count, unique
from toolz.dicttoolz import valmap, valfilter
from toposort import toposort, toposort_flatten

def pprint_color(obj):
    print(highlight(pformat(obj), PythonLexer(), Terminal256Formatter()))

def extract_nodes(content, pattern):
    class NS(object):
        pass
    ns = NS()
    ns.views = set()
    ns.dependencies = set()

    def inner(data, depth):
        if isinstance(data, dict):
            if 'relname' in data and 'schemaname' in data and re.search(pattern, data['relname']):
                entity = '.'.join([
                    data['schemaname'],
                    data['relname']
                ])
                if depth == 1:
                    ns.views.add(entity)
                else:
                    ns.dependencies.add(entity)
            else:
                for key, value in data.items():
                    inner(value, depth)
        elif isinstance(data, list) or isinstance(data, tuple):
            for item in data:
                inner(item, depth + 1)

    parsed_content = parse(content)
    inner(parsed_content, 0)
    return {
        'views': ns.views,
        'dependencies': ns.dependencies - ns.views
    }

def walk_directory_recursively(input_dir):
    file_paths = []
    for root, _, files in os.walk(input_dir):
        for file in files:
            file_paths.append(os.path.join(root, file))
    return file_paths

def process_file(file_name, pattern):
    with open(file_name, 'r') as f:
        file_name = f.name
        content = f.read()
        nodes = extract_nodes(content, pattern)

        return {
            'file_name': file_name,
            'content': content,
            'views': nodes['views'],
            'view_dependencies': nodes['dependencies']
        }

def format_content(entity):
    file_name = entity['file_name']
    content_no_txn = entity['content'] \
        .replace('BEGIN;', '') \
        .replace('COMMIT;', '')
    return "  -- %s%s" % (file_name, content_no_txn)

def generate_script(views, spacer=''):
    return "\n\n".join([
        'BEGIN;',
        spacer.join(views),
        'COMMIT;'
    ])

def serialize_script(name, suffix, content, output_dir, verbose):
    script_name = '%s-%s.sql' % (name, suffix)
    script_path = os.path.join(output_dir, script_name)
    with open(script_path, 'w') as f:
        f.write(content)
        if verbose:
            print('Successfully saved "%s script" to "%s"' % (name, script_path))

@click.command()
@click.option('-d', '--dry-run', is_flag=True, help='Analyzes dependencies without actually generating the output files')
@click.option('-i', '--input-dir', default='.', help='The directory for the PostgreSQL scripts to analyze')
@click.option('-I', '--ignore-refresh', default='', help='Regex pattern to match when ignoring refresh on Materialized Views (i.e. "hist" for "users_mv_hist")')
@click.option('-o', '--output-dir', default='.', help='Output directory for the generated creation and refresh scripts')
@click.option('-p', '--pattern', default='_mv', help='Materialized View regex pattern to match (i.e. "_mv" for "users_mv")')
@click.option('-v', '--verbose', is_flag=True, help='Enables verbose logging')
def cli(dry_run, input_dir, ignore_refresh, output_dir, pattern, verbose):

    dag = {}

    file_paths = walk_directory_recursively(input_dir)

    if verbose:
        print('Found %d Scripts in %s' % (len(file_paths), input_dir))

    entities = pipe(file_paths,
        map(lambda file_path: process_file(file_path, pattern)),
        list
    )

    if verbose:
        total_views = pipe(entities,
            map(get('views')),
            map(count),
            sum
        )
        total_deps = pipe(entities,
            map(get('view_dependencies')),
            map(count),
            sum
        )
        print('Identified %d Materialized Views, Containing %d View Dependencies' % (total_views, total_deps))

    view_content = {}
    dag = {}
    for entity in entities:
        view_content.update(
            {view: format_content(entity) for view in entity['views']}
        )
        dag.update(
            {view: entity['view_dependencies'] for view in entity['views']}
        )

    sorted_views = toposort_flatten(dag)

    if verbose:
        print("\nMaterialized View Dependencies:")
        pprint_color(
            valmap(lambda val: list(val), valfilter(lambda val: val, dag))
        )

    create_views = pipe(sorted_views,
        map(lambda view: view_content[view]),
        unique,
        list
    )

    create_script = generate_script(create_views)

    refresh_views = pipe(sorted_views,
        filter(lambda view: re.search(pattern, view) and not (ignore_refresh and re.search(ignore_refresh , view))),
        map(lambda view: '  REFRESH MATERIALIZED VIEW CONCURRENTLY ' + view + ';'),
        list
    )

    if verbose:
        print('Selecting %d Materialized Views for Refresh' % len(refresh_views))

    refresh_script = generate_script(refresh_views, "\n\n")

    if dry_run:
        print('Dry Run Option Enabled - Skipping Script Generation')
        return

    timestr = time.strftime("%Y%m%d-%H%M%S")

    serialize_script('create', timestr, create_script, output_dir, verbose)
    serialize_script('refresh', timestr, refresh_script, output_dir, verbose)

    pprint_color('Process Complete!')
<|endoftext|>


<commit_before>"""
oauthlib.oauth2.rfc6749.tokens
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

This module contains methods for adding two types of access tokens to requests.

- Bearer https://tools.ietf.org/html/rfc6750
- MAC https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01
"""
from __future__ import absolute_import, unicode_literals

import hashlib
import hmac
from binascii import b2a_base64
import warnings

from oauthlib import common
from oauthlib.common import add_params_to_qs, add_params_to_uri, unicode_type

from . import utils

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

class OAuth2Token(dict):

    def __init__(self, params, old_scope=None):
        super(OAuth2Token, self).__init__(params)
        self._new_scope = None
        if 'scope' in params and params['scope']:
            self._new_scope = set(utils.scope_to_list(params['scope']))
        if old_scope is not None:
            self._old_scope = set(utils.scope_to_list(old_scope))
            if self._new_scope is None:
                # the rfc says that if the scope hasn't changed, it's optional
                # in params so set the new scope to the old scope
                self._new_scope = self._old_scope
        else:
            self._old_scope = self._new_scope

    @property
    def scope_changed(self):
        return self._new_scope != self._old_scope

    @property
    def old_scope(self):
        return utils.list_to_scope(self._old_scope)

    @property
    def old_scopes(self):
        return list(self._old_scope)

    @property
    def scope(self):
        return utils.list_to_scope(self._new_scope)

    @property
    def scopes(self):
        return list(self._new_scope)

    @property
    def missing_scopes(self):
        return list(self._old_scope - self._new_scope)

    @property
    def additional_scopes(self):
        return list(self._new_scope - self._old_scope)

def prepare_mac_header(token, uri, key, http_method,
                       nonce=None,
                       headers=None,
                       body=None,
                       ext='',
                       hash_algorithm='hmac-sha-1',
                       issue_time=None,
                       draft=0):
    """Add an `MAC Access Authentication`_ signature to headers.

    Unlike OAuth 1, this HMAC signature does not require inclusion of the
    request payload/body, neither does it use a combination of client_secret
    and token_secret but rather a mac_key provided together with the access
    token.

    Currently two algorithms are supported, "hmac-sha-1" and "hmac-sha-256",
    `extension algorithms`_ are not supported.

    Example MAC Authorization header, linebreaks added for clarity

    Authorization: MAC id="h480djs93hd8",
                       nonce="1336363200:dj83hs9s",
                       mac="bhCQXTVyfj5cmA9uKkPFx1zeOXM="

    .. _`MAC Access Authentication`: https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01
    .. _`extension algorithms`: https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-7.1

    :param token:
    :param uri: Request URI.
    :param key: MAC given provided by token endpoint.
    :param http_method: HTTP Request method.
    :param nonce:
    :param headers: Request headers as a dictionary.
    :param body:
    :param ext:
    :param hash_algorithm: HMAC algorithm provided by token endpoint.
    :param issue_time: Time when the MAC credentials were issued (datetime).
    :param draft: MAC authentication specification version.
    :return: headers dictionary with the authorization field added.
    """
    http_method = http_method.upper()
    host, port = utils.host_from_uri(uri)

    if hash_algorithm.lower() == 'hmac-sha-1':
        h = hashlib.sha1
    elif hash_algorithm.lower() == 'hmac-sha-256':
        h = hashlib.sha256
    else:
        raise ValueError('unknown hash algorithm')

    if draft == 0:
        nonce = nonce or '{0}:{1}'.format(utils.generate_age(issue_time),
                                          common.generate_nonce())
    else:
        ts = common.generate_timestamp()
        nonce = common.generate_nonce()

    sch, net, path, par, query, fra = urlparse(uri)

    if query:
        request_uri = path + '?' + query
    else:
        request_uri = path

    # Hash the body/payload
    if body is not None and draft == 0:
        body = body.encode('utf-8')
        bodyhash = b2a_base64(h(body).digest())[:-1].decode('utf-8')
    else:
        bodyhash = ''

    # Create the normalized base string
    base = []
    if draft == 0:
        base.append(nonce)
    else:
        base.append(ts)
        base.append(nonce)
    base.append(http_method.upper())
    base.append(request_uri)
    base.append(host)
    base.append(port)
    if draft == 0:
        base.append(bodyhash)
    base.append(ext or '')
    base_string = '\n'.join(base) + '\n'

    # hmac struggles with unicode strings - http://bugs.python.org/issue5285
    if isinstance(key, unicode_type):
        key = key.encode('utf-8')
    sign = hmac.new(key, base_string.encode('utf-8'), h)
    sign = b2a_base64(sign.digest())[:-1].decode('utf-8')

    header = []
    header.append('MAC id="%s"' % token)
    if draft != 0:
        header.append('ts="%s"' % ts)
    header.append('nonce="%s"' % nonce)
    if bodyhash:
        header.append('bodyhash="%s"' % bodyhash)
    if ext:
        header.append('ext="%s"' % ext)
    header.append('mac="%s"' % sign)

    headers = headers or {}
    headers['Authorization'] = ', '.join(header)
    return headers

def prepare_bearer_uri(token, uri):
    """Add a `Bearer Token`_ to the request URI.
    Not recommended, use only if client can't use authorization header or body.

    http://www.example.com/path?access_token=h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param uri:
    """
    return add_params_to_uri(uri, [(('access_token', token))])

def prepare_bearer_headers(token, headers=None):
    """Add a `Bearer Token`_ to the request URI.
    Recommended method of passing bearer tokens.

    Authorization: Bearer h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param headers:
    """
    headers = headers or {}
    headers['Authorization'] = 'Bearer %s' % token
    return headers

def prepare_bearer_body(token, body=''):
    """Add a `Bearer Token`_ to the request body.

    access_token=h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param body:
    """
    return add_params_to_qs(body, [(('access_token', token))])

def random_token_generator(request, refresh_token=False):
    """
    :param request: OAuthlib request.
    :type request: oauthlib.common.Request
    :param refresh_token:
    """
    return common.generate_token()

def signed_token_generator(private_pem, **kwargs):
    """
    :param private_pem:
    """
    def signed_token_generator(request):
        request.claims = kwargs
        return common.generate_signed_token(private_pem, request)

    return signed_token_generator

def get_token_from_header(request):
    """
    Helper function to extract a token from the request header.

    :param request: OAuthlib request.
    :type request: oauthlib.common.Request
    :return: Return the token or None if the Authorization header is malformed.
    """
    token = None

    if 'Authorization' in request.headers:
        split_header = request.headers.get('Authorization').split()
        if len(split_header) == 2 and split_header[0] == 'Bearer':
            token = split_header[1]
    else:
        token = request.access_token

    return token

class TokenBase(object):

    def __call__(self, request, refresh_token=False):
        raise NotImplementedError('Subclasses must implement this method.')

    def validate_request(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        raise NotImplementedError('Subclasses must implement this method.')

    def estimate_type(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        raise NotImplementedError('Subclasses must implement this method.')

class BearerToken(TokenBase):
    __slots__ = (
        'request_validator', 'token_generator',
        'refresh_token_generator', 'expires_in'
    )

    def __init__(self, request_validator=None, token_generator=None,
                 expires_in=None, refresh_token_generator=None):
        self.request_validator = request_validator
        self.token_generator = token_generator or random_token_generator
        self.refresh_token_generator = (
            refresh_token_generator or self.token_generator
        )
        self.expires_in = expires_in or 3600

    def create_token(self, request, refresh_token=False, **kwargs):
        """
        Create a BearerToken, by default without refresh token.

        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        :param refresh_token:
        """
        if "save_token" in kwargs:
            warnings.warn("`save_token` has been deprecated, it was not used internally."
                          "If you do, use `request_validator.save_token()` instead.",
                          DeprecationWarning)

        if callable(self.expires_in):
            expires_in = self.expires_in(request)
        else:
            expires_in = self.expires_in

        request.expires_in = expires_in

        token = {
            'access_token': self.token_generator(request),
            'expires_in': expires_in,
            'token_type': 'Bearer',
        }

        # If provided, include - this is optional in some cases https://tools.ietf.org/html/rfc6749#section-3.3 but
        # there is currently no mechanism to coordinate issuing a token for only a subset of the requested scopes so
        # all tokens issued are for the entire set of requested scopes.
        if request.scopes is not None:
            token['scope'] = ' '.join(request.scopes)

        if refresh_token:
            if (request.refresh_token and
                    not self.request_validator.rotate_refresh_token(request)):
                token['refresh_token'] = request.refresh_token
            else:
                token['refresh_token'] = self.refresh_token_generator(request)

        token.update(request.extra_credentials or {})
        return OAuth2Token(token)

    def validate_request(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        token = get_token_from_header(request)
        return self.request_validator.validate_bearer_token(
            token, request.scopes, request)

    def estimate_type(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        if request.headers.get('Authorization', '').split(' ')[0] == 'Bearer':
            return 9
        elif request.access_token is not None:
            return 5
        else:
            return 0
<commit_msg>Add clarity to the deprecation warning<commit_after>"""
oauthlib.oauth2.rfc6749.tokens
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

This module contains methods for adding two types of access tokens to requests.

- Bearer https://tools.ietf.org/html/rfc6750
- MAC https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01
"""
from __future__ import absolute_import, unicode_literals

import hashlib
import hmac
from binascii import b2a_base64
import warnings

from oauthlib import common
from oauthlib.common import add_params_to_qs, add_params_to_uri, unicode_type

from . import utils

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

class OAuth2Token(dict):

    def __init__(self, params, old_scope=None):
        super(OAuth2Token, self).__init__(params)
        self._new_scope = None
        if 'scope' in params and params['scope']:
            self._new_scope = set(utils.scope_to_list(params['scope']))
        if old_scope is not None:
            self._old_scope = set(utils.scope_to_list(old_scope))
            if self._new_scope is None:
                # the rfc says that if the scope hasn't changed, it's optional
                # in params so set the new scope to the old scope
                self._new_scope = self._old_scope
        else:
            self._old_scope = self._new_scope

    @property
    def scope_changed(self):
        return self._new_scope != self._old_scope

    @property
    def old_scope(self):
        return utils.list_to_scope(self._old_scope)

    @property
    def old_scopes(self):
        return list(self._old_scope)

    @property
    def scope(self):
        return utils.list_to_scope(self._new_scope)

    @property
    def scopes(self):
        return list(self._new_scope)

    @property
    def missing_scopes(self):
        return list(self._old_scope - self._new_scope)

    @property
    def additional_scopes(self):
        return list(self._new_scope - self._old_scope)

def prepare_mac_header(token, uri, key, http_method,
                       nonce=None,
                       headers=None,
                       body=None,
                       ext='',
                       hash_algorithm='hmac-sha-1',
                       issue_time=None,
                       draft=0):
    """Add an `MAC Access Authentication`_ signature to headers.

    Unlike OAuth 1, this HMAC signature does not require inclusion of the
    request payload/body, neither does it use a combination of client_secret
    and token_secret but rather a mac_key provided together with the access
    token.

    Currently two algorithms are supported, "hmac-sha-1" and "hmac-sha-256",
    `extension algorithms`_ are not supported.

    Example MAC Authorization header, linebreaks added for clarity

    Authorization: MAC id="h480djs93hd8",
                       nonce="1336363200:dj83hs9s",
                       mac="bhCQXTVyfj5cmA9uKkPFx1zeOXM="

    .. _`MAC Access Authentication`: https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01
    .. _`extension algorithms`: https://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-7.1

    :param token:
    :param uri: Request URI.
    :param key: MAC given provided by token endpoint.
    :param http_method: HTTP Request method.
    :param nonce:
    :param headers: Request headers as a dictionary.
    :param body:
    :param ext:
    :param hash_algorithm: HMAC algorithm provided by token endpoint.
    :param issue_time: Time when the MAC credentials were issued (datetime).
    :param draft: MAC authentication specification version.
    :return: headers dictionary with the authorization field added.
    """
    http_method = http_method.upper()
    host, port = utils.host_from_uri(uri)

    if hash_algorithm.lower() == 'hmac-sha-1':
        h = hashlib.sha1
    elif hash_algorithm.lower() == 'hmac-sha-256':
        h = hashlib.sha256
    else:
        raise ValueError('unknown hash algorithm')

    if draft == 0:
        nonce = nonce or '{0}:{1}'.format(utils.generate_age(issue_time),
                                          common.generate_nonce())
    else:
        ts = common.generate_timestamp()
        nonce = common.generate_nonce()

    sch, net, path, par, query, fra = urlparse(uri)

    if query:
        request_uri = path + '?' + query
    else:
        request_uri = path

    # Hash the body/payload
    if body is not None and draft == 0:
        body = body.encode('utf-8')
        bodyhash = b2a_base64(h(body).digest())[:-1].decode('utf-8')
    else:
        bodyhash = ''

    # Create the normalized base string
    base = []
    if draft == 0:
        base.append(nonce)
    else:
        base.append(ts)
        base.append(nonce)
    base.append(http_method.upper())
    base.append(request_uri)
    base.append(host)
    base.append(port)
    if draft == 0:
        base.append(bodyhash)
    base.append(ext or '')
    base_string = '\n'.join(base) + '\n'

    # hmac struggles with unicode strings - http://bugs.python.org/issue5285
    if isinstance(key, unicode_type):
        key = key.encode('utf-8')
    sign = hmac.new(key, base_string.encode('utf-8'), h)
    sign = b2a_base64(sign.digest())[:-1].decode('utf-8')

    header = []
    header.append('MAC id="%s"' % token)
    if draft != 0:
        header.append('ts="%s"' % ts)
    header.append('nonce="%s"' % nonce)
    if bodyhash:
        header.append('bodyhash="%s"' % bodyhash)
    if ext:
        header.append('ext="%s"' % ext)
    header.append('mac="%s"' % sign)

    headers = headers or {}
    headers['Authorization'] = ', '.join(header)
    return headers

def prepare_bearer_uri(token, uri):
    """Add a `Bearer Token`_ to the request URI.
    Not recommended, use only if client can't use authorization header or body.

    http://www.example.com/path?access_token=h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param uri:
    """
    return add_params_to_uri(uri, [(('access_token', token))])

def prepare_bearer_headers(token, headers=None):
    """Add a `Bearer Token`_ to the request URI.
    Recommended method of passing bearer tokens.

    Authorization: Bearer h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param headers:
    """
    headers = headers or {}
    headers['Authorization'] = 'Bearer %s' % token
    return headers

def prepare_bearer_body(token, body=''):
    """Add a `Bearer Token`_ to the request body.

    access_token=h480djs93hd8

    .. _`Bearer Token`: https://tools.ietf.org/html/rfc6750

    :param token:
    :param body:
    """
    return add_params_to_qs(body, [(('access_token', token))])

def random_token_generator(request, refresh_token=False):
    """
    :param request: OAuthlib request.
    :type request: oauthlib.common.Request
    :param refresh_token:
    """
    return common.generate_token()

def signed_token_generator(private_pem, **kwargs):
    """
    :param private_pem:
    """
    def signed_token_generator(request):
        request.claims = kwargs
        return common.generate_signed_token(private_pem, request)

    return signed_token_generator

def get_token_from_header(request):
    """
    Helper function to extract a token from the request header.

    :param request: OAuthlib request.
    :type request: oauthlib.common.Request
    :return: Return the token or None if the Authorization header is malformed.
    """
    token = None

    if 'Authorization' in request.headers:
        split_header = request.headers.get('Authorization').split()
        if len(split_header) == 2 and split_header[0] == 'Bearer':
            token = split_header[1]
    else:
        token = request.access_token

    return token

class TokenBase(object):

    def __call__(self, request, refresh_token=False):
        raise NotImplementedError('Subclasses must implement this method.')

    def validate_request(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        raise NotImplementedError('Subclasses must implement this method.')

    def estimate_type(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        raise NotImplementedError('Subclasses must implement this method.')

class BearerToken(TokenBase):
    __slots__ = (
        'request_validator', 'token_generator',
        'refresh_token_generator', 'expires_in'
    )

    def __init__(self, request_validator=None, token_generator=None,
                 expires_in=None, refresh_token_generator=None):
        self.request_validator = request_validator
        self.token_generator = token_generator or random_token_generator
        self.refresh_token_generator = (
            refresh_token_generator or self.token_generator
        )
        self.expires_in = expires_in or 3600

    def create_token(self, request, refresh_token=False, **kwargs):
        """
        Create a BearerToken, by default without refresh token.

        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        :param refresh_token:
        """
        if "save_token" in kwargs:
            warnings.warn("`save_token` has been deprecated, it was not called internally."
                          "If you do, call `request_validator.save_token()` instead.",
                          DeprecationWarning)

        if callable(self.expires_in):
            expires_in = self.expires_in(request)
        else:
            expires_in = self.expires_in

        request.expires_in = expires_in

        token = {
            'access_token': self.token_generator(request),
            'expires_in': expires_in,
            'token_type': 'Bearer',
        }

        # If provided, include - this is optional in some cases https://tools.ietf.org/html/rfc6749#section-3.3 but
        # there is currently no mechanism to coordinate issuing a token for only a subset of the requested scopes so
        # all tokens issued are for the entire set of requested scopes.
        if request.scopes is not None:
            token['scope'] = ' '.join(request.scopes)

        if refresh_token:
            if (request.refresh_token and
                    not self.request_validator.rotate_refresh_token(request)):
                token['refresh_token'] = request.refresh_token
            else:
                token['refresh_token'] = self.refresh_token_generator(request)

        token.update(request.extra_credentials or {})
        return OAuth2Token(token)

    def validate_request(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        token = get_token_from_header(request)
        return self.request_validator.validate_bearer_token(
            token, request.scopes, request)

    def estimate_type(self, request):
        """
        :param request: OAuthlib request.
        :type request: oauthlib.common.Request
        """
        if request.headers.get('Authorization', '').split(' ')[0] == 'Bearer':
            return 9
        elif request.access_token is not None:
            return 5
        else:
            return 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Location module.

`author`    :   Quentin Lampin <quentin.lampin@orange.com>
`license`   :   MPL
`date`      :   2015/10/12
Copyright 2015 Orange

"""
from .. import m_common
from . import m_gpsd

from pydispatch import dispatcher

import threading
import bottle

# experiment states
GPS_UNDEFINED = 0
GPS_OFFLINE = 1
GPS_ONLINE = 2
GPS_STATES = ('undefined', 'offline', 'online')

# node_commands
GET_COMMANDS = [m_common.COMMAND_STATUS, m_common.COMMAND_START, m_common.COMMAND_STOP]
POST_COMMANDS = [m_common.COMMAND_SETUP]

# POST request arguments
LATITUDE = 'latitude'
LONGITUDE = 'longitude'

# mandatory configuration arguments
REQUIRED_ARGUMENTS = {
    m_common.COMMAND_SETUP: {'files': [], 'forms': [LATITUDE, LONGITUDE]}
}

COORDINATE_UNDEFINED = 'undefined'
ERROR_ESTIMATE_UNDEFINED = 'undefined'

# exception messages
GPS_ROUTINE_ALREADY_RUNNING = 'GPS routine already running'

class GPS(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.gpsd = m_gpsd.GPS(mode=m_gpsd.WATCH_ENABLE)  # starting the stream of info
        self.running = False
        self.state = GPS_UNDEFINED
        self.latitude = None
        self.longitude = None
        self.altitude = None
        self.speed = None
        self.error_estimate_latitude = None
        self.error_estimate_longitude = None
        self.error_estimate_altitude = None
        self.satellites = []
        self.start()
        # link node_commands to instance methods
        self.commands = {
            m_common.COMMAND_STATUS: self.status,
            m_common.COMMAND_SETUP: self.setup,
            m_common.COMMAND_START: self.start_proxy,
            m_common.COMMAND_STOP: self.stop,
        }

    def run(self):
        self.running = True
        self.state = GPS_OFFLINE
        while self.running:
            # grab EACH set of gpsd info to clear the buffer
            self.gpsd.next()
            # update information
            if self.gpsd.fix.mode == m_gpsd.MODE_3D:
                self.state = GPS_ONLINE
                self.latitude = self.gpsd.fix.latitude
                self.longitude = self.gpsd.fix.longitude
                self.altitude = self.gpsd.fix.altitude
                self.speed = self.gpsd.fix.speed
                self.error_estimate_longitude = self.gpsd.fix.epx
                self.error_estimate_latitude = self.gpsd.fix.epy
                self.error_estimate_altitude = self.gpsd.fix.epv
                self.satellites = self.gpsd.satellites

                dispatcher.send(
                    signal=m_common.LOCATION_UPDATE,
                    sender=self,
                    latitude=self.latitude,
                    longitude=self.longitude,
                    altitude=self.altitude,
                    speed=self.speed,
                    error_estimate_longitude=self.error_estimate_longitude,
                    error_estimate_latitude=self.error_estimate_latitude,
                    error_estimate_altitude=self.error_estimate_altitude
                )
            elif self.gpsd.fix.mode == m_gpsd.MODE_NO_FIX:
                self.state = GPS_OFFLINE
                self.satellites = self.gpsd.satellites

    def status(self):
        return {
            'latitude': self.latitude if self.latitude else COORDINATE_UNDEFINED,
            'longitude': self.longitude if self.longitude else COORDINATE_UNDEFINED,
            'altitude': self.altitude if self.altitude else COORDINATE_UNDEFINED,
            'error_estimate_latitude': self.error_estimate_latitude if self.error_estimate_latitude
            else ERROR_ESTIMATE_UNDEFINED,
            'error_estimate_longitude': self.error_estimate_longitude if self.error_estimate_longitude
            else ERROR_ESTIMATE_UNDEFINED,
            'error_estimate_altitude': self.error_estimate_altitude if self.error_estimate_altitude
            else ERROR_ESTIMATE_UNDEFINED
        }

    def setup(self, latitude, longitude):
        self.stop()
        self.state = GPS_OFFLINE
        self.latitude = latitude
        self.longitude = longitude

    def start_proxy(self):
        if self.running:
            raise m_common.LocationException(
                m_common.ERROR_COMMAND_FORBIDDEN.format(m_common.COMMAND_START, GPS_ROUTINE_ALREADY_RUNNING))
        else:
            threading.Thread.__init__(self)

    def stop(self):
        self.running = False
        self.join()

    @property
    def fix(self):
        return self.gpsd.fix

    @property
    def utc(self):
        return self.gpsd.utc

    def rest_get_command(self, command):
        # check that command exists
        if command not in GET_COMMANDS:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_COMMAND_UNKNOWN.format(command + '(GET)')
        # issue the command and return
        self.commands[command]()
        bottle.response.node_status = m_common.REST_REQUEST_FULFILLED
        return self.status()

    def rest_post_command(self, command):
        # check that command exists
        if command not in POST_COMMANDS:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_COMMAND_UNKNOWN.format(command + '(POST)')
        # node_load arguments
        arguments = {}
        for required_file_argument in REQUIRED_ARGUMENTS[command]['files']:
            arguments[required_file_argument] = bottle.request.files.get(required_file_argument)
        for required_form_argument in REQUIRED_ARGUMENTS[command]['forms']:
            arguments[required_form_argument] = bottle.request.forms.get(required_form_argument)
        # check that all arguments have been filled
        if any(value is None for argument, value in arguments.items()):
            missing_arguments = [argument for argument in arguments.keys() if arguments[argument] is None]
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_MISSING_ARGUMENT_IN_ARCHIVE.format(missing_arguments)
            # issue the command
        try:
            self.commands[command](**arguments)
            bottle.response.node_status = m_common.REST_REQUEST_FULFILLED
            # return the node node_status
            return self.status()
        except m_common.LocationException as e:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_CONFIGURATION_FAIL.format(e.message)
<commit_msg>test2<commit_after># -*- coding: utf-8 -*-

"""
Location module.

`author`    :   Quentin Lampin <quentin.lampin@orange.com>
`license`   :   MPL
`date`      :   2015/10/12
Copyright 2015 Orange

"""
from .. import m_common
from . import m_gpsd

from pydispatch import dispatcher

import threading
import bottle
###
# experiment states
GPS_UNDEFINED = 0
GPS_OFFLINE = 1
GPS_ONLINE = 2
GPS_STATES = ('undefined', 'offline', 'online')

# node_commands
GET_COMMANDS = [m_common.COMMAND_STATUS, m_common.COMMAND_START, m_common.COMMAND_STOP]
POST_COMMANDS = [m_common.COMMAND_SETUP]

# POST request arguments
LATITUDE = 'latitude'
LONGITUDE = 'longitude'

# mandatory configuration arguments
REQUIRED_ARGUMENTS = {
    m_common.COMMAND_SETUP: {'files': [], 'forms': [LATITUDE, LONGITUDE]}
}

COORDINATE_UNDEFINED = 'undefined'
ERROR_ESTIMATE_UNDEFINED = 'undefined'

# exception messages
GPS_ROUTINE_ALREADY_RUNNING = 'GPS routine already running'

class GPS(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.gpsd = m_gpsd.GPS(mode=m_gpsd.WATCH_ENABLE)  # starting the stream of info
        self.running = False
        self.state = GPS_UNDEFINED
        self.latitude = None
        self.longitude = None
        self.altitude = None
        self.speed = None
        self.error_estimate_latitude = None
        self.error_estimate_longitude = None
        self.error_estimate_altitude = None
        self.satellites = []
        self.start()
        # link node_commands to instance methods
        self.commands = {
            m_common.COMMAND_STATUS: self.status,
            m_common.COMMAND_SETUP: self.setup,
            m_common.COMMAND_START: self.start_proxy,
            m_common.COMMAND_STOP: self.stop,
        }

    def run(self):
        self.running = True
        self.state = GPS_OFFLINE
        while self.running:
            # grab EACH set of gpsd info to clear the buffer
            self.gpsd.next()
            # update information
            if self.gpsd.fix.mode == m_gpsd.MODE_3D:
                self.state = GPS_ONLINE
                self.latitude = self.gpsd.fix.latitude
                self.longitude = self.gpsd.fix.longitude
                self.altitude = self.gpsd.fix.altitude
                self.speed = self.gpsd.fix.speed
                self.error_estimate_longitude = self.gpsd.fix.epx
                self.error_estimate_latitude = self.gpsd.fix.epy
                self.error_estimate_altitude = self.gpsd.fix.epv
                self.satellites = self.gpsd.satellites

                dispatcher.send(
                    signal=m_common.LOCATION_UPDATE,
                    sender=self,
                    latitude=self.latitude,
                    longitude=self.longitude,
                    altitude=self.altitude,
                    speed=self.speed,
                    error_estimate_longitude=self.error_estimate_longitude,
                    error_estimate_latitude=self.error_estimate_latitude,
                    error_estimate_altitude=self.error_estimate_altitude
                )
            elif self.gpsd.fix.mode == m_gpsd.MODE_NO_FIX:
                self.state = GPS_OFFLINE
                self.satellites = self.gpsd.satellites

    def status(self):
        return {
            'latitude': self.latitude if self.latitude else COORDINATE_UNDEFINED,
            'longitude': self.longitude if self.longitude else COORDINATE_UNDEFINED,
            'altitude': self.altitude if self.altitude else COORDINATE_UNDEFINED,
            'error_estimate_latitude': self.error_estimate_latitude if self.error_estimate_latitude
            else ERROR_ESTIMATE_UNDEFINED,
            'error_estimate_longitude': self.error_estimate_longitude if self.error_estimate_longitude
            else ERROR_ESTIMATE_UNDEFINED,
            'error_estimate_altitude': self.error_estimate_altitude if self.error_estimate_altitude
            else ERROR_ESTIMATE_UNDEFINED
        }

    def setup(self, latitude, longitude):
        self.stop()
        self.state = GPS_OFFLINE
        self.latitude = latitude
        self.longitude = longitude

    def start_proxy(self):
        if self.running:
            raise m_common.LocationException(
                m_common.ERROR_COMMAND_FORBIDDEN.format(m_common.COMMAND_START, GPS_ROUTINE_ALREADY_RUNNING))
        else:
            threading.Thread.__init__(self)

    def stop(self):
        self.running = False
        self.join()

    @property
    def fix(self):
        return self.gpsd.fix

    @property
    def utc(self):
        return self.gpsd.utc

    def rest_get_command(self, command):
        # check that command exists
        if command not in GET_COMMANDS:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_COMMAND_UNKNOWN.format(command + '(GET)')
        # issue the command and return
        self.commands[command]()
        bottle.response.node_status = m_common.REST_REQUEST_FULFILLED
        return self.status()

    def rest_post_command(self, command):
        # check that command exists
        if command not in POST_COMMANDS:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_COMMAND_UNKNOWN.format(command + '(POST)')
        # node_load arguments
        arguments = {}
        for required_file_argument in REQUIRED_ARGUMENTS[command]['files']:
            arguments[required_file_argument] = bottle.request.files.get(required_file_argument)
        for required_form_argument in REQUIRED_ARGUMENTS[command]['forms']:
            arguments[required_form_argument] = bottle.request.forms.get(required_form_argument)
        # check that all arguments have been filled
        if any(value is None for argument, value in arguments.items()):
            missing_arguments = [argument for argument in arguments.keys() if arguments[argument] is None]
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_MISSING_ARGUMENT_IN_ARCHIVE.format(missing_arguments)
            # issue the command
        try:
            self.commands[command](**arguments)
            bottle.response.node_status = m_common.REST_REQUEST_FULFILLED
            # return the node node_status
            return self.status()
        except m_common.LocationException as e:
            bottle.response.node_status = m_common.REST_REQUEST_ERROR
            return m_common.ERROR_CONFIGURATION_FAIL.format(e.message)
<|endoftext|>