<commit_before>"""
Catalog module.
"""

from .providers import Provider

class AbstractCatalog(object):
    """
    Abstract object provides catalog.
    """

    def __init__(self, *used_providers):
        """
        Initializer.
        """
        self.__used_providers__ = set(used_providers)

    def __getattribute__(self, item):
        """
        Returns providers.

        :param item:
        :return:
        """
        attribute = super(AbstractCatalog, self).__getattribute__(item)
        if item in ('__used_providers__',):
            return attribute

        if attribute not in self.__used_providers__:
            raise AttributeError('Provider \'{}\' is not listed in '
                                 'dependencies'.format(item))
        return attribute

    @classmethod
    def __all_providers__(cls):
        """
        Returns set of all class providers.
        """
        providers = set()
        for attr_name in set(dir(cls)) - set(dir(AbstractCatalog)):
            provider = getattr(cls, attr_name)
            if not isinstance(provider, Provider):
                continue
            providers.add((attr_name, provider))
        return providers

    @classmethod
    def __override___(cls, overriding):
        """
        Overrides current catalog providers by overriding catalog providers.

        :param overriding: AbstractCatalog
        """
        overriden = overriding.__all_providers__() - cls.__all_providers__()
        for name, provider in overriden:
            overridden_provider = getattr(cls, name)
            overridden_provider.__override__(provider)

def overrides(catalog):
    """
    Catalog overriding decorator.

    :param catalog:
    :return:
    """
    def decorator(overriding_catalog):
        catalog.__override___(overriding_catalog)
        return overriding_catalog
    return decorator
<commit_msg>Adding possibility to get all providers by type from catalog<commit_after>"""
Catalog module.
"""

from .providers import Provider

class AbstractCatalog(object):
    """
    Abstract object provides catalog.
    """

    def __init__(self, *used_providers):
        """
        Initializer.
        """
        self.__used_providers__ = set(used_providers)

    def __getattribute__(self, item):
        """
        Returns providers.

        :param item:
        :return:
        """
        attribute = super(AbstractCatalog, self).__getattribute__(item)
        if item in ('__used_providers__',):
            return attribute

        if attribute not in self.__used_providers__:
            raise AttributeError('Provider \'{}\' is not listed in '
                                 'dependencies'.format(item))
        return attribute

    @classmethod
    def __all_providers__(cls, provider_type=Provider):
        """
        Returns set of all class providers.
        """
        providers = set()
        for attr_name in set(dir(cls)) - set(dir(AbstractCatalog)):
            provider = getattr(cls, attr_name)
            if not isinstance(provider, provider_type):
                continue
            providers.add((attr_name, provider))
        return providers

    @classmethod
    def __override___(cls, overriding):
        """
        Overrides current catalog providers by overriding catalog providers.

        :param overriding: AbstractCatalog
        """
        overriden = overriding.__all_providers__() - cls.__all_providers__()
        for name, provider in overriden:
            overridden_provider = getattr(cls, name)
            overridden_provider.__override__(provider)

def overrides(catalog):
    """
    Catalog overriding decorator.

    :param catalog:
    :return:
    """
    def decorator(overriding_catalog):
        catalog.__override___(overriding_catalog)
        return overriding_catalog
    return decorator
<|endoftext|>


<commit_before>import re
from collections import OrderedDict
from functools import lru_cache

import jsonref
import requests

class IdValue(str):
    """
    This is used to differentiate `id` values from other strings in `isinstance` checks.
    """
    def __init__(self, value):
        # Save the original value. For example, if the value is an integer, this attribute helps us to avoid coercing
        # the value to a string while merging.
        self.original_value = value
        str.__init__(value)

@lru_cache()
def get_latest_version():
    """
    Returns the tag of the latest version of OCDS.
    """
    return re.findall(r'\d+__\d+__\d+', requests.get('http://standard.open-contracting.org/schema/').text)[-1]

@lru_cache()
def get_latest_release_schema_url():
    """
    Returns the URL of the release schema in the latest version of OCDS.
    """
    return 'http://standard.open-contracting.org/schema/{}/release-schema.json'.format(get_latest_version())

def _get_types(prop):
    """
    Returns a property's `type` as a list.
    """
    if 'type' not in prop:
        return []
    if isinstance(prop['type'], str):
        return [prop['type']]
    return prop['type']

def _get_merge_rules(properties, path=None):
    """
    Yields merge rules as key-value pairs, in which the first element is a JSON path as a tuple, and the second element
    is a list of merge properties whose values are `true`.
    """
    if path is None:
        path = ()

    for key, value in properties.items():
        new_path = path + (key,)
        types = _get_types(value)

        # `omitWhenMerged` supersedes all other rules.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#omit-when-merged
        if value.get('omitWhenMerged'):
            yield (new_path, {'omitWhenMerged'})
        # `wholeListMerge` supersedes any nested rules.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        elif 'array' in types and value.get('wholeListMerge'):
            yield (new_path, {'wholeListMerge'})
        elif 'object' in types and 'properties' in value:
            yield from _get_merge_rules(value['properties'], path=new_path)
        elif 'array' in types and 'items' in value:
            item_types = _get_types(value['items'])
            # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
            if any(item_type != 'object' for item_type in item_types):
                yield (new_path, {'wholeListMerge'})
            elif 'object' in item_types and 'properties' in value['items']:
                # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
                if 'id' not in value['items']['properties']:
                    yield (new_path, {'wholeListMerge'})
                else:
                    yield from _get_merge_rules(value['items']['properties'], path=new_path)

        # `versionId` merely assists in identifying `id` fields that are not on objects in arrays.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#versioned-data

@lru_cache()
def _get_merge_rules_from_url_or_path(schema):
    if schema.startswith('http'):
        return jsonref.load_uri(schema)
    with open(schema) as f:
        return jsonref.load(f)

def get_merge_rules(schema=None):
    """
    Returns merge rules as key-value pairs, in which the key is a JSON path as a tuple, and the value is a list of
    merge properties whose values are `true`.
    """
    schema = schema or get_latest_release_schema_url()
    if isinstance(schema, dict):
        deref_schema = jsonref.JsonRef.replace_refs(schema)
    else:
        deref_schema = _get_merge_rules_from_url_or_path(schema)
    return dict(_get_merge_rules(deref_schema['properties']))

def flatten(obj, merge_rules=None, path=None, flattened=None):
    """
    Flattens a JSON object into key-value pairs, in which the key is the JSON path as a tuple. For example:

    {
        "a": "I am a",
        "b": ["A", "list"],
        "c": [
            {"ca": "I am ca"},
            {"cb": "I am cb"}
        ]
    }

    flattens to:

    {
        ('a',): 'I am a',
        ('b',): ['A', 'list'],
        ('c', 0, 'ca'): 'I am ca',
        ('c', 1, 'cb'): 'I am cb',
    }
    """
    if merge_rules is None:
        merge_rules = {}
    if path is None:
        path = ()
    if flattened is None:
        flattened = OrderedDict()

    if isinstance(obj, dict):
        iterable = obj.items()
        if not iterable:
            flattened[path] = {}
    else:
        iterable = enumerate(obj)
        if not iterable:
            flattened[path] = []

    for key, value in iterable:
        new_path = path + (key,)
        # Remove array indices to find the merge rule for this JSON path in the data.
        new_path_merge_rules = merge_rules.get(tuple(part for part in new_path if not isinstance(part, int)), [])

        if 'omitWhenMerged' in new_path_merge_rules:
            continue
        # If it is neither an object nor an array, if it is `wholeListMerge`, or if it is an array containing
        # non-objects (even if `wholeListMerge` is `false`), use the whole list merge strategy.
        # Note: Behavior is undefined and inconsistent if the array is not in the schema and contains objects in some
        # cases but not in others.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
        elif (not isinstance(value, (dict, list)) or 'wholeListMerge' in new_path_merge_rules or
                isinstance(value, list) and any(not isinstance(item, dict) for item in value)):
            flattened[new_path] = value
        # Recurse into objects, and arrays of objects that aren't `wholeListMerge`.
        else:
            flatten(value, merge_rules, new_path, flattened)

    return flattened

def unflatten(processed, merge_rules):
    """
    Unflattens a processed object into a JSON object.
    """
    unflattened = OrderedDict()

    for key in processed:
        current_node = unflattened
        for end, part in enumerate(key, 1):
            # If this is a path to an item of an array.
            # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#identifier-merge
            if isinstance(part, IdValue):
                # If the `id` of an object in the array matches, change into it.
                for node in current_node:
                    if node.get('id') == part.original_value:
                        current_node = node
                        break
                # Otherwise, append a new object, and change into it.
                else:
                    new_node = OrderedDict({'id': part.original_value})
                    current_node.append(new_node)
                    current_node = new_node
                continue

            # Otherwise, this is a path to a property of an object.
            node = current_node.get(part)

            # If this is a path to a node we visited before, change into it. If it's an `id` field, it's already been
            # set to its original value. However, if it sets `versionId`, pass-thru to set it to its versioned value.
            if node is not None:
                current_node = node
                continue

            # If this is a full path, copy the data.
            if len(key) == end:
                current_node[part] = processed[key]
                continue

            # If the path is to a new array, start a new array, and change into it.
            if isinstance(key[end], IdValue):
                new_node = []
            # If the path is to a new object, start a new object, and change into it.
            else:
                new_node = OrderedDict()
            current_node[part] = new_node
            current_node = new_node

    return unflattened

def process_flattened(flattened):
    """
    Replace numbers in JSON paths (representing positions in arrays) with special objects. This ensures that objects
    in arrays with different `id` values have different JSON paths – and makes it easy to identify such arrays.
    """
    # Keep arrays in order.
    processed = OrderedDict()

    for key in flattened:
        new_key = []
        for end, part in enumerate(key, 1):
            # If this is a path to an item in an array.
            if isinstance(part, int):
                # If it is an array of objects, get the `id` value to apply the identifier merge strategy.
                # http://standard.open-contracting.org/latest/en/schema/merging/#identifier-merge
                id_value = flattened.get(tuple(key[:end]) + ('id',))
                # If the object contain no top-level `id` value, we invent one.
                if id_value is None:
                    id_value = part
                part = IdValue(id_value)
            new_key.append(part)
        processed[tuple(new_key)] = flattened[key]

    return processed

def merge(releases, schema=None, merge_rules=None):
    """
    Merges a list of releases into a compiledRelease.
    """
    if not merge_rules:
        merge_rules = get_merge_rules(schema)

    merged = OrderedDict({('tag',): ['compiled']})
    for release in sorted(releases, key=lambda release: release['date']):
        release = release.copy()

        ocid = release['ocid']
        date = release['date']
        release.pop('tag', None)  # becomes ["compiled"]

        flat = flatten(release, merge_rules)
        processed = process_flattened(flat)

        # Add an `id` and `date`.
        merged[('id',)] = '{}-{}'.format(ocid, date)
        merged[('date',)] = date
        merged.update(processed)

    return unflatten(merged, merge_rules)

def merge_versioned(releases, schema=None, merge_rules=None):
    """
    Merges a list of releases into a versionedRelease.
    """
    if not merge_rules:
        merge_rules = get_merge_rules(schema)

    merged = OrderedDict()
    for release in sorted(releases, key=lambda release: release['date']):
        release = release.copy()

        # Don't version the OCID.
        ocid = release.pop('ocid')
        merged[('ocid',)] = ocid

        releaseID = release['id']
        date = release['date']
        tag = release.pop('tag', None)

        flat = flatten(release, merge_rules)
        processed = process_flattened(flat)

        for key, value in processed.items():
            # If value is unchanged, don't add to history.
            if key in merged and value == merged[key][-1]['value']:
                continue

            if key not in merged:
                merged[key] = []

            merged[key].append(OrderedDict([
                ('releaseID', releaseID),
                ('releaseDate', date),
                ('releaseTag', tag),
                ('value', value),
            ]))

    return unflatten(merged, merge_rules)
<commit_msg>Remove stale comment<commit_after>import re
from collections import OrderedDict
from functools import lru_cache

import jsonref
import requests

class IdValue(str):
    """
    This is used to differentiate `id` values from other strings in `isinstance` checks.
    """
    def __init__(self, value):
        # Save the original value. For example, if the value is an integer, this attribute helps us to avoid coercing
        # the value to a string while merging.
        self.original_value = value
        str.__init__(value)

@lru_cache()
def get_latest_version():
    """
    Returns the tag of the latest version of OCDS.
    """
    return re.findall(r'\d+__\d+__\d+', requests.get('http://standard.open-contracting.org/schema/').text)[-1]

@lru_cache()
def get_latest_release_schema_url():
    """
    Returns the URL of the release schema in the latest version of OCDS.
    """
    return 'http://standard.open-contracting.org/schema/{}/release-schema.json'.format(get_latest_version())

def _get_types(prop):
    """
    Returns a property's `type` as a list.
    """
    if 'type' not in prop:
        return []
    if isinstance(prop['type'], str):
        return [prop['type']]
    return prop['type']

def _get_merge_rules(properties, path=None):
    """
    Yields merge rules as key-value pairs, in which the first element is a JSON path as a tuple, and the second element
    is a list of merge properties whose values are `true`.
    """
    if path is None:
        path = ()

    for key, value in properties.items():
        new_path = path + (key,)
        types = _get_types(value)

        # `omitWhenMerged` supersedes all other rules.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#omit-when-merged
        if value.get('omitWhenMerged'):
            yield (new_path, {'omitWhenMerged'})
        # `wholeListMerge` supersedes any nested rules.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        elif 'array' in types and value.get('wholeListMerge'):
            yield (new_path, {'wholeListMerge'})
        elif 'object' in types and 'properties' in value:
            yield from _get_merge_rules(value['properties'], path=new_path)
        elif 'array' in types and 'items' in value:
            item_types = _get_types(value['items'])
            # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
            if any(item_type != 'object' for item_type in item_types):
                yield (new_path, {'wholeListMerge'})
            elif 'object' in item_types and 'properties' in value['items']:
                # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
                if 'id' not in value['items']['properties']:
                    yield (new_path, {'wholeListMerge'})
                else:
                    yield from _get_merge_rules(value['items']['properties'], path=new_path)

        # `versionId` merely assists in identifying `id` fields that are not on objects in arrays.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#versioned-data

@lru_cache()
def _get_merge_rules_from_url_or_path(schema):
    if schema.startswith('http'):
        return jsonref.load_uri(schema)
    with open(schema) as f:
        return jsonref.load(f)

def get_merge_rules(schema=None):
    """
    Returns merge rules as key-value pairs, in which the key is a JSON path as a tuple, and the value is a list of
    merge properties whose values are `true`.
    """
    schema = schema or get_latest_release_schema_url()
    if isinstance(schema, dict):
        deref_schema = jsonref.JsonRef.replace_refs(schema)
    else:
        deref_schema = _get_merge_rules_from_url_or_path(schema)
    return dict(_get_merge_rules(deref_schema['properties']))

def flatten(obj, merge_rules=None, path=None, flattened=None):
    """
    Flattens a JSON object into key-value pairs, in which the key is the JSON path as a tuple. For example:

    {
        "a": "I am a",
        "b": ["A", "list"],
        "c": [
            {"ca": "I am ca"},
            {"cb": "I am cb"}
        ]
    }

    flattens to:

    {
        ('a',): 'I am a',
        ('b',): ['A', 'list'],
        ('c', 0, 'ca'): 'I am ca',
        ('c', 1, 'cb'): 'I am cb',
    }
    """
    if merge_rules is None:
        merge_rules = {}
    if path is None:
        path = ()
    if flattened is None:
        flattened = OrderedDict()

    if isinstance(obj, dict):
        iterable = obj.items()
        if not iterable:
            flattened[path] = {}
    else:
        iterable = enumerate(obj)
        if not iterable:
            flattened[path] = []

    for key, value in iterable:
        new_path = path + (key,)
        # Remove array indices to find the merge rule for this JSON path in the data.
        new_path_merge_rules = merge_rules.get(tuple(part for part in new_path if not isinstance(part, int)), [])

        if 'omitWhenMerged' in new_path_merge_rules:
            continue
        # If it is neither an object nor an array, if it is `wholeListMerge`, or if it is an array containing
        # non-objects (even if `wholeListMerge` is `false`), use the whole list merge strategy.
        # Note: Behavior is undefined and inconsistent if the array is not in the schema and contains objects in some
        # cases but not in others.
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#whole-list-merge
        # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#objects
        elif (not isinstance(value, (dict, list)) or 'wholeListMerge' in new_path_merge_rules or
                isinstance(value, list) and any(not isinstance(item, dict) for item in value)):
            flattened[new_path] = value
        # Recurse into objects, and arrays of objects that aren't `wholeListMerge`.
        else:
            flatten(value, merge_rules, new_path, flattened)

    return flattened

def unflatten(processed, merge_rules):
    """
    Unflattens a processed object into a JSON object.
    """
    unflattened = OrderedDict()

    for key in processed:
        current_node = unflattened
        for end, part in enumerate(key, 1):
            # If this is a path to an item of an array.
            # See http://standard.open-contracting.org/1.1-dev/en/schema/merging/#identifier-merge
            if isinstance(part, IdValue):
                # If the `id` of an object in the array matches, change into it.
                for node in current_node:
                    if node.get('id') == part.original_value:
                        current_node = node
                        break
                # Otherwise, append a new object, and change into it.
                else:
                    new_node = OrderedDict({'id': part.original_value})
                    current_node.append(new_node)
                    current_node = new_node
                continue

            # Otherwise, this is a path to a property of an object.
            node = current_node.get(part)

            # If this is a path to a node we visited before, change into it. If it's an `id` field, it's already been
            # set to its original value.
            if node is not None:
                current_node = node
                continue

            # If this is a full path, copy the data.
            if len(key) == end:
                current_node[part] = processed[key]
                continue

            # If the path is to a new array, start a new array, and change into it.
            if isinstance(key[end], IdValue):
                new_node = []
            # If the path is to a new object, start a new object, and change into it.
            else:
                new_node = OrderedDict()

            current_node[part] = new_node
            current_node = new_node

    return unflattened

def process_flattened(flattened):
    """
    Replace numbers in JSON paths (representing positions in arrays) with special objects. This ensures that objects
    in arrays with different `id` values have different JSON paths – and makes it easy to identify such arrays.
    """
    # Keep arrays in order.
    processed = OrderedDict()

    for key in flattened:
        new_key = []
        for end, part in enumerate(key, 1):
            # If this is a path to an item in an array.
            if isinstance(part, int):
                # If it is an array of objects, get the `id` value to apply the identifier merge strategy.
                # http://standard.open-contracting.org/latest/en/schema/merging/#identifier-merge
                id_value = flattened.get(tuple(key[:end]) + ('id',))
                # If the object contain no top-level `id` value, we invent one.
                if id_value is None:
                    id_value = part
                part = IdValue(id_value)
            new_key.append(part)
        processed[tuple(new_key)] = flattened[key]

    return processed

def merge(releases, schema=None, merge_rules=None):
    """
    Merges a list of releases into a compiledRelease.
    """
    if not merge_rules:
        merge_rules = get_merge_rules(schema)

    merged = OrderedDict({('tag',): ['compiled']})
    for release in sorted(releases, key=lambda release: release['date']):
        release = release.copy()

        ocid = release['ocid']
        date = release['date']
        release.pop('tag', None)  # becomes ["compiled"]

        flat = flatten(release, merge_rules)
        processed = process_flattened(flat)

        # Add an `id` and `date`.
        merged[('id',)] = '{}-{}'.format(ocid, date)
        merged[('date',)] = date
        merged.update(processed)

    return unflatten(merged, merge_rules)

def merge_versioned(releases, schema=None, merge_rules=None):
    """
    Merges a list of releases into a versionedRelease.
    """
    if not merge_rules:
        merge_rules = get_merge_rules(schema)

    merged = OrderedDict()
    for release in sorted(releases, key=lambda release: release['date']):
        release = release.copy()

        # Don't version the OCID.
        ocid = release.pop('ocid')
        merged[('ocid',)] = ocid

        releaseID = release['id']
        date = release['date']
        tag = release.pop('tag', None)

        flat = flatten(release, merge_rules)
        processed = process_flattened(flat)

        for key, value in processed.items():
            # If value is unchanged, don't add to history.
            if key in merged and value == merged[key][-1]['value']:
                continue

            if key not in merged:
                merged[key] = []

            merged[key].append(OrderedDict([
                ('releaseID', releaseID),
                ('releaseDate', date),
                ('releaseTag', tag),
                ('value', value),
            ]))

    return unflatten(merged, merge_rules)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Oct2Py is a means to seamlessly call M-files and GNU Octave functions from Python.
It manages the Octave session for you, sharing data behind the scenes using
MAT files.  Usage is as simple as:

.. code-block:: python

    >>> import oct2py
    >>> oc = oct2py.Oct2Py()
    >>> x = oc.zeros(3,3)
    >>> print(x, x.dtype.str)  # doctest: +SKIP
    [[ 0.  0.  0.]
     [ 0.  0.  0.]
     [ 0.  0.  0.]] <f8

If you want to run legacy m-files, do not have MATLAB(TM), and do not fully
trust a code translator, this is your library.
"""

__title__ = 'oct2py'
__version__ = '3.9.0'
__author__ = 'Steven Silvester'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014-2016 Steven Silvester'
__all__ = ['Oct2Py', 'Oct2PyError', 'octave', 'Struct', 'demo', 'speed_check',
           'thread_check', '__version__', 'get_log']

import imp
import functools
import os
import ctypes

try:
    import thread
except ImportError:
    import _thread as thread

from .core import Oct2Py, Oct2PyError
from .utils import Struct, get_log
from .demo import demo
from .speed_check import speed_check
from .thread_check import thread_check

try:
    octave = Oct2Py()
except Oct2PyError as e:
    print(e)

def kill_octave():
    """Kill all octave instances (cross-platform).

    This will restart the "octave" instance.  If you have instantiated
    Any other Oct2Py objects, you must restart them.
    """
    import os
    if os.name == 'nt':
        os.system('taskkill /im octave /f')
    else:
        os.system('killall -9 octave')
        os.system('killall -9 octave-cli')
    octave.restart()

# clean up namespace
del functools, imp, os, ctypes, thread
<commit_msg>Bump version<commit_after># -*- coding: utf-8 -*-
"""
Oct2Py is a means to seamlessly call M-files and GNU Octave functions from Python.
It manages the Octave session for you, sharing data behind the scenes using
MAT files.  Usage is as simple as:

.. code-block:: python

    >>> import oct2py
    >>> oc = oct2py.Oct2Py()
    >>> x = oc.zeros(3,3)
    >>> print(x, x.dtype.str)  # doctest: +SKIP
    [[ 0.  0.  0.]
     [ 0.  0.  0.]
     [ 0.  0.  0.]] <f8

If you want to run legacy m-files, do not have MATLAB(TM), and do not fully
trust a code translator, this is your library.
"""

__title__ = 'oct2py'
__version__ = '3.9.1'
__author__ = 'Steven Silvester'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014-2016 Steven Silvester'
__all__ = ['Oct2Py', 'Oct2PyError', 'octave', 'Struct', 'demo', 'speed_check',
           'thread_check', '__version__', 'get_log']

import imp
import functools
import os
import ctypes

try:
    import thread
except ImportError:
    import _thread as thread

from .core import Oct2Py, Oct2PyError
from .utils import Struct, get_log
from .demo import demo
from .speed_check import speed_check
from .thread_check import thread_check

try:
    octave = Oct2Py()
except Oct2PyError as e:
    print(e)

def kill_octave():
    """Kill all octave instances (cross-platform).

    This will restart the "octave" instance.  If you have instantiated
    Any other Oct2Py objects, you must restart them.
    """
    import os
    if os.name == 'nt':
        os.system('taskkill /im octave /f')
    else:
        os.system('killall -9 octave')
        os.system('killall -9 octave-cli')
    octave.restart()

# clean up namespace
del functools, imp, os, ctypes, thread
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Swagger Support
~~~~~~~~~~~~~~~

Built in support for generating a swagger spec, along with built in Swagger UI.

To enable add the :py:class:`SwaggerSpec` resource API into your API::

    >>> from odinweb import api
    >>> from odinweb.swagger import SwaggerSpec
    >>> my_api = api.ApiCollection(
    ...    SwaggerSpec("Title of my Swagger spec", enable_ui=True),
    ... )

"""
import collections
import os

# Imported for typing support
from typing import List, Dict, Any, Union, Tuple  # noqa

from odin.fields.virtual import VirtualField
from .data_structures import PathParam  # noqa

from odin import fields
from odin.utils import getmeta, lazy_property, force_tuple

from . import doc
from . import resources
from ._compat import binary_type
from .constants import HTTPStatus, Type
from .containers import ResourceApi, CODECS
from .data_structures import UrlPath, Param, HttpResponse, NoPath, DefaultResource
from .decorators import Operation
from .exceptions import HttpError
from .utils import dict_filter

SWAGGER_SPEC_TYPE_MAPPING = [
    (fields.IntegerField, Type.Long),
    (fields.FloatField, Type.Float),
    (fields.EmailField, Type.Email),
    (fields.StringField, Type.String),
    (fields.TimeField, Type.Time),
    (fields.DateField, Type.Date),
    (fields.DateTimeField, Type.DateTime),
    (fields.BooleanField, Type.Boolean),
]
"""
Mapping of fields to Swagger types.
"""

def map_field_to_type(field):
    # type: (Any) -> Type
    for field_type, type_ in SWAGGER_SPEC_TYPE_MAPPING:
        if isinstance(field, field_type):
            return type_

def resource_definition(resource):
    """
    Generate a `Swagger Definitions Object <http://swagger.io/specification/#definitionsObject>`_
    from a resource.

    """
    meta = getmeta(resource)

    definition = {
        'type': "object",
        'properties': {}
    }

    for field in meta.all_fields:
        field_definition = {}

        type_def = map_field_to_type(field)
        if type_def:
            field_definition['type'] = str(type_def)
            if type_def.format:
                field_definition['format'] = type_def.format

        if field.doc_text:
            field_definition['description'] = field.doc_text

        if isinstance(field, VirtualField) or field in meta.readonly_fields:
            field_definition['readOnly'] = True

        # Use getattr to support calculated fields
        if getattr(field, 'choices', None):
            field_definition['enum'] = [c[0] for c in field.choices]

        definition['properties'][field.name] = field_definition

    return definition

class SwaggerSpec(ResourceApi):
    """
    Resource API instance that generates a Swagger spec of the current API.
    """
    api_name = 'swagger'
    SWAGGER_TAG = 'swagger'
    tags = (SWAGGER_TAG, )

    static_path = os.path.join(os.path.dirname(__file__), 'static')

    def __init__(self, title, enabled=True, enable_ui=False, host=None, schemes=None):
        # type: (str, bool, bool, str, Union[str, Tuple[str]]) -> None
        # Register operations
        if enabled:
            self._operations.append(Operation(SwaggerSpec.get_swagger))
            if enable_ui:
                self._operations.append(Operation(SwaggerSpec.get_ui, UrlPath.parse('ui')))
                self._operations.append(Operation(SwaggerSpec.get_static, UrlPath.parse('ui/{file_name:String}')))

        super(SwaggerSpec, self).__init__()
        self.title = title
        self.enabled = enabled
        self.enable_ui = enabled and enable_ui
        self.host = host
        self.schemes = set(force_tuple(schemes or ()))

        self._ui_cache = None

    @lazy_property
    def cenancestor(self):
        """
        Last universal ancestor (or the top level of the API structure).
        """
        ancestor = parent = self.parent
        while parent:
            ancestor = parent
            parent = getattr(parent, 'parent', None)
        return ancestor

    @lazy_property
    def base_path(self):
        """
        Calculate the APIs base path
        """
        path = UrlPath()

        # Walk up the API to find the base object
        parent = self.parent
        while parent:
            path_prefix = getattr(parent, 'path_prefix', NoPath)
            path = path_prefix + path
            parent = getattr(parent, 'parent', None)

        return path

    @property
    def swagger_path(self):
        return self.base_path + 'swagger'

    @staticmethod
    def generate_parameters(path):
        # type: (UrlPath) -> List[Dict[str, Any]]
        return [Param.path(node.name, node.type).to_swagger() for node in path.path_nodes]

    @staticmethod
    def swagger_node_formatter(path_node):
        # type: (PathParam) -> str
        """
        Format a node for swagger spec (default formatter for the format method).
        """
        return "{{{}}}".format(path_node.name)

    def security_definitions(self):
        """
        Chance to generate security definitions.
        """
        return None

    def parse_operations(self):
        """
        Flatten routes into a path -> method -> route structure
        """
        resource_defs = {
            getmeta(resources.Error).resource_name: resource_definition(resources.Error),
            getmeta(resources.Listing).resource_name: resource_definition(resources.Listing),
        }

        paths = collections.OrderedDict()
        for path, operation in self.parent.op_paths():
            # Cut of first item (will be the parents path)
            path = '/' + path[1:]  # type: UrlPath

            # Filter out swagger endpoints
            if self.SWAGGER_TAG in operation.tags:
                continue

            # Add to resource definitions
            if operation.resource:
                resource_defs[getmeta(operation.resource).resource_name] = resource_definition(operation.resource)

            # Add any resource definitions from responses
            if operation.responses:
                for response in operation.responses:
                    if response.resource is not DefaultResource:
                        resource_defs[getmeta(response.resource).resource_name] = resource_definition(response.resource)

            # Add path parameters
            path_spec = paths.setdefault(path.format(self.swagger_node_formatter), {})

            # Add parameters
            parameters = self.generate_parameters(path)
            if parameters:
                path_spec['parameters'] = parameters

            # Add methods
            for method in operation.methods:
                path_spec[method.value.lower()] = operation.to_swagger()

        return paths, resource_defs

    @doc.response(HTTPStatus.OK, "Swagger JSON of this API")
    def get_swagger(self, request):
        """
        Generate this document.
        """
        api_base = self.parent
        paths, definitions = self.parse_operations()
        codecs = getattr(self.cenancestor, 'registered_codecs', CODECS)  # type: dict
        return dict_filter({
            'swagger': '2.0',
            'info': {
                'title': self.title,
                'version': str(getattr(api_base, 'version', 0))
            },
            'host': self.host or request.host,
            'schemes': list(self.schemes) or None,
            'basePath': str(self.base_path),
            'consumes': list(codecs.keys()),
            'produces': list(codecs.keys()),
            'paths': paths,
            'definitions': definitions,
            'securityDefinitions': self.security_definitions(),
        })

    def load_static(self, file_name):
        file_path = os.path.abspath(os.path.join(self.static_path, file_name))
        # This is a security check to ensure this is not abused to
        # read files outside of the static folder.
        if not file_path.startswith(self.static_path):
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        try:
            return open(file_path, 'rb').read()
        except IOError:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

    @doc.response(HTTPStatus.OK, "HTML content")
    @doc.produces('text/html')
    def get_ui(self, _):
        """
        Load the Swagger UI interface
        """
        if not self._ui_cache:
            content = self.load_static('ui.html')
            if isinstance(content, binary_type):
                content = content.decode('UTF-8')
            self._ui_cache = content.replace(u"{{SWAGGER_PATH}}", str(self.swagger_path))
        return HttpResponse(self._ui_cache, headers={
            'Content-Type': 'text/html'
        })

    @doc.response(HTTPStatus.OK, "HTML content")
    def get_static(self, _, file_name=None):
        """
        Get static content for UI.
        """
        content_type = {
            'ss': 'text/css',
            'js': 'application/javascript',
        }.get(file_name[-2:])
        if not content_type:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        return HttpResponse(self.load_static(file_name), headers={
            'Content-Type': content_type,
            'Content-Encoding': 'gzip',
            'Cache-Control': 'public, max-age=300',
        })
<commit_msg>Fix to not generate a resource if one isn't defined (common for no response requests like DELETE)<commit_after># -*- coding: utf-8 -*-
"""
Swagger Support
~~~~~~~~~~~~~~~

Built in support for generating a swagger spec, along with built in Swagger UI.

To enable add the :py:class:`SwaggerSpec` resource API into your API::

    >>> from odinweb import api
    >>> from odinweb.swagger import SwaggerSpec
    >>> my_api = api.ApiCollection(
    ...    SwaggerSpec("Title of my Swagger spec", enable_ui=True),
    ... )

"""
import collections
import os

# Imported for typing support
from typing import List, Dict, Any, Union, Tuple  # noqa

from odin.fields.virtual import VirtualField
from .data_structures import PathParam  # noqa

from odin import fields
from odin.utils import getmeta, lazy_property, force_tuple

from . import doc
from . import resources
from ._compat import binary_type
from .constants import HTTPStatus, Type
from .containers import ResourceApi, CODECS
from .data_structures import UrlPath, Param, HttpResponse, NoPath, DefaultResource
from .decorators import Operation
from .exceptions import HttpError
from .utils import dict_filter

SWAGGER_SPEC_TYPE_MAPPING = [
    (fields.IntegerField, Type.Long),
    (fields.FloatField, Type.Float),
    (fields.EmailField, Type.Email),
    (fields.StringField, Type.String),
    (fields.TimeField, Type.Time),
    (fields.DateField, Type.Date),
    (fields.DateTimeField, Type.DateTime),
    (fields.BooleanField, Type.Boolean),
]
"""
Mapping of fields to Swagger types.
"""

def map_field_to_type(field):
    # type: (Any) -> Type
    for field_type, type_ in SWAGGER_SPEC_TYPE_MAPPING:
        if isinstance(field, field_type):
            return type_

def resource_definition(resource):
    """
    Generate a `Swagger Definitions Object <http://swagger.io/specification/#definitionsObject>`_
    from a resource.

    """
    meta = getmeta(resource)

    definition = {
        'type': "object",
        'properties': {}
    }

    for field in meta.all_fields:
        field_definition = {}

        type_def = map_field_to_type(field)
        if type_def:
            field_definition['type'] = str(type_def)
            if type_def.format:
                field_definition['format'] = type_def.format

        if field.doc_text:
            field_definition['description'] = field.doc_text

        if isinstance(field, VirtualField) or field in meta.readonly_fields:
            field_definition['readOnly'] = True

        # Use getattr to support calculated fields
        if getattr(field, 'choices', None):
            field_definition['enum'] = [c[0] for c in field.choices]

        definition['properties'][field.name] = field_definition

    return definition

class SwaggerSpec(ResourceApi):
    """
    Resource API instance that generates a Swagger spec of the current API.
    """
    api_name = 'swagger'
    SWAGGER_TAG = 'swagger'
    tags = (SWAGGER_TAG, )

    static_path = os.path.join(os.path.dirname(__file__), 'static')

    def __init__(self, title, enabled=True, enable_ui=False, host=None, schemes=None):
        # type: (str, bool, bool, str, Union[str, Tuple[str]]) -> None
        # Register operations
        if enabled:
            self._operations.append(Operation(SwaggerSpec.get_swagger))
            if enable_ui:
                self._operations.append(Operation(SwaggerSpec.get_ui, UrlPath.parse('ui')))
                self._operations.append(Operation(SwaggerSpec.get_static, UrlPath.parse('ui/{file_name:String}')))

        super(SwaggerSpec, self).__init__()
        self.title = title
        self.enabled = enabled
        self.enable_ui = enabled and enable_ui
        self.host = host
        self.schemes = set(force_tuple(schemes or ()))

        self._ui_cache = None

    @lazy_property
    def cenancestor(self):
        """
        Last universal ancestor (or the top level of the API structure).
        """
        ancestor = parent = self.parent
        while parent:
            ancestor = parent
            parent = getattr(parent, 'parent', None)
        return ancestor

    @lazy_property
    def base_path(self):
        """
        Calculate the APIs base path
        """
        path = UrlPath()

        # Walk up the API to find the base object
        parent = self.parent
        while parent:
            path_prefix = getattr(parent, 'path_prefix', NoPath)
            path = path_prefix + path
            parent = getattr(parent, 'parent', None)

        return path

    @property
    def swagger_path(self):
        return self.base_path + 'swagger'

    @staticmethod
    def generate_parameters(path):
        # type: (UrlPath) -> List[Dict[str, Any]]
        return [Param.path(node.name, node.type).to_swagger() for node in path.path_nodes]

    @staticmethod
    def swagger_node_formatter(path_node):
        # type: (PathParam) -> str
        """
        Format a node for swagger spec (default formatter for the format method).
        """
        return "{{{}}}".format(path_node.name)

    def security_definitions(self):
        """
        Chance to generate security definitions.
        """
        return None

    def parse_operations(self):
        """
        Flatten routes into a path -> method -> route structure
        """
        resource_defs = {
            getmeta(resources.Error).resource_name: resource_definition(resources.Error),
            getmeta(resources.Listing).resource_name: resource_definition(resources.Listing),
        }

        paths = collections.OrderedDict()
        for path, operation in self.parent.op_paths():
            # Cut of first item (will be the parents path)
            path = '/' + path[1:]  # type: UrlPath

            # Filter out swagger endpoints
            if self.SWAGGER_TAG in operation.tags:
                continue

            # Add to resource definitions
            if operation.resource:
                resource_defs[getmeta(operation.resource).resource_name] = resource_definition(operation.resource)

            # Add any resource definitions from responses
            if operation.responses:
                for response in operation.responses:
                    resource = response.resource
                    # Ensure we have a resource
                    if resource and resource is not DefaultResource:
                        resource_name = getmeta(resource).resource_name
                        # Don't generate a resource definition if one has already been created.
                        if resource_name not in resource_defs:
                            resource_defs[resource_name] = resource_definition(resource)

            # Add path parameters
            path_spec = paths.setdefault(path.format(self.swagger_node_formatter), {})

            # Add parameters
            parameters = self.generate_parameters(path)
            if parameters:
                path_spec['parameters'] = parameters

            # Add methods
            for method in operation.methods:
                path_spec[method.value.lower()] = operation.to_swagger()

        return paths, resource_defs

    @doc.response(HTTPStatus.OK, "Swagger JSON of this API")
    def get_swagger(self, request):
        """
        Generate this document.
        """
        api_base = self.parent
        paths, definitions = self.parse_operations()
        codecs = getattr(self.cenancestor, 'registered_codecs', CODECS)  # type: dict
        return dict_filter({
            'swagger': '2.0',
            'info': {
                'title': self.title,
                'version': str(getattr(api_base, 'version', 0))
            },
            'host': self.host or request.host,
            'schemes': list(self.schemes) or None,
            'basePath': str(self.base_path),
            'consumes': list(codecs.keys()),
            'produces': list(codecs.keys()),
            'paths': paths,
            'definitions': definitions,
            'securityDefinitions': self.security_definitions(),
        })

    def load_static(self, file_name):
        file_path = os.path.abspath(os.path.join(self.static_path, file_name))
        # This is a security check to ensure this is not abused to
        # read files outside of the static folder.
        if not file_path.startswith(self.static_path):
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        try:
            return open(file_path, 'rb').read()
        except IOError:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

    @doc.response(HTTPStatus.OK, "HTML content")
    @doc.produces('text/html')
    def get_ui(self, _):
        """
        Load the Swagger UI interface
        """
        if not self._ui_cache:
            content = self.load_static('ui.html')
            if isinstance(content, binary_type):
                content = content.decode('UTF-8')
            self._ui_cache = content.replace(u"{{SWAGGER_PATH}}", str(self.swagger_path))
        return HttpResponse(self._ui_cache, headers={
            'Content-Type': 'text/html'
        })

    @doc.response(HTTPStatus.OK, "HTML content")
    def get_static(self, _, file_name=None):
        """
        Get static content for UI.
        """
        content_type = {
            'ss': 'text/css',
            'js': 'application/javascript',
        }.get(file_name[-2:])
        if not content_type:
            raise HttpError(HTTPStatus.NOT_FOUND, 42)

        return HttpResponse(self.load_static(file_name), headers={
            'Content-Type': content_type,
            'Content-Encoding': 'gzip',
            'Cache-Control': 'public, max-age=300',
        })
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os.path
import subprocess

import github3
import unidiff

from inlineplz.interfaces.base import InterfaceBase

class GitHubInterface(InterfaceBase):
    def __init__(self, owner, repo, pr, token, url=None):
        if not url:
            self.gh = github3.GitHub(token=token)
        else:
            self.gh = github3.GitHubEnterprise(url, token=token)
        self.pull_request = self.gh.pull_request(owner, repo, pr)
        self.sha = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD']
        ).strip().decode('utf-8')
        self.diff = self.pull_request.diff()

    def post_messages(self, messages):
        for msg in messages:
            if not msg.content:
                continue
            msg_position = self.position(msg)
            if msg_position:
                if not self.is_duplicate(msg, msg_position):
                    self.pull_request.create_review_comment(
                        msg.content,
                        self.sha,
                        msg.path,
                        msg_position
                    )

    def is_duplicate(self, message, position):
        for comment in self.pull_request.review_comments():
            if (comment.position == position and
                    comment.path == message.path and
                    comment.body.strip() == message.content.strip()):
                return True
        return False

    def position(self, message):
        """Calculate position within the PR, which is not the line number"""
        patch = unidiff.PatchSet(self.diff.decode('utf-8').split('\n'))
        for patched_file in patch:
            if os.path.normpath(patched_file.target_file) == os.path.normpath('b/' + message.path):
                offset = 1
                for hunk_no, hunk in enumerate(patched_file):
                    for position, hunk_line in enumerate(hunk):
                        if '+' not in hunk_line.line_type:
                            continue
                        if hunk_line.target_line_no == message.line_number:
                            return position + offset
                    offset += len(hunk) + 1
<commit_msg>shell out to git diff instead of using github's diff<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os.path
import subprocess

import github3
import unidiff

from inlineplz.interfaces.base import InterfaceBase

class GitHubInterface(InterfaceBase):
    def __init__(self, owner, repo, pr, token, url=None):
        if not url:
            self.gh = github3.GitHub(token=token)
        else:
            self.gh = github3.GitHubEnterprise(url, token=token)
        self.pull_request = self.gh.pull_request(owner, repo, pr)
        self.sha = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD']
        ).strip().decode('utf-8')
        # diff with rename recognition
        self.diff = subprocess.check_output(
            ['git', 'diff', '-M', 'master..' + self.sha]
        ).strip().decode('utf-8')

    def post_messages(self, messages):
        for msg in messages:
            if not msg.content:
                continue
            msg_position = self.position(msg)
            if msg_position:
                if not self.is_duplicate(msg, msg_position):
                    self.pull_request.create_review_comment(
                        msg.content,
                        self.sha,
                        msg.path,
                        msg_position
                    )

    def is_duplicate(self, message, position):
        for comment in self.pull_request.review_comments():
            if (comment.position == position and
                    comment.path == message.path and
                    comment.body.strip() == message.content.strip()):
                return True
        return False

    def position(self, message):
        """Calculate position within the PR, which is not the line number"""
        patch = unidiff.PatchSet(self.diff.split('\n'))
        for patched_file in patch:
            if os.path.normpath(patched_file.target_file) == os.path.normpath('b/' + message.path):
                offset = 1
                for hunk_no, hunk in enumerate(patched_file):
                    for position, hunk_line in enumerate(hunk):
                        if '+' not in hunk_line.line_type:
                            continue
                        if hunk_line.target_line_no == message.line_number:
                            return position + offset
                    offset += len(hunk) + 1
<|endoftext|>


<commit_before>#
#  Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, Intel Corporation
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions
#  are met:
#
#      * Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#
#      * Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in
#        the documentation and/or other materials provided with the
#        distribution.
#
#      * Neither the name of Intel Corporation nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# Helper functions useful for both tests and experiments

import sys
import os
import socket
import subprocess
import io
import yaml
import shlex

import geopmpy.launcher

def detect_launcher():
    """
    Heuristic to determine the resource manager used on the system.
    Returns name of resource manager or launcher, otherwise a
    LookupError is raised.
    """
    # Try the environment
    result = os.environ.get('GEOPM_LAUNCHER', None)
    if not result:
        # Check for known host names
        slurm_hosts = ['mr-fusion', 'mcfly']
        alps_hosts = ['theta']
        hostname = socket.gethostname()
        if any(hostname.startswith(word) for word in slurm_hosts):
            result = 'srun'
        elif any(hostname.startswith(word) for word in alps_hosts):
            result = 'aprun'
    if not result:
        try:
            exec_str = 'srun --version'
            subprocess.check_call(exec_str, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE, shell=True)
            result = 'srun'
        except subprocess.CalledProcessError:
            pass
    if not result:
        try:
            exec_str = 'aprun --version'
            subprocess.check_call(exec_str, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE, shell=True)
            result = 'aprun'
        except subprocess.CalledProcessError:
            pass
    if not result:
        raise LookupError('Unable to determine resource manager')
    return result

def allocation_node_test(test_exec, stdout, stderr):
    argv = shlex.split(test_exec)
    launcher = detect_launcher()
    argv.insert(1, launcher)
    if launcher == 'aprun':
        argv.insert(2, '-q')  # Use quiet flag with aprun to suppress end of job info string
    argv.insert(2, '--geopm-ctl-disable')
    launcher = geopmpy.launcher.Factory().create(argv, num_rank=1, num_node=1,
                                                 job_name="geopm_allocation_test", quiet=True)
    launcher.run(stdout, stderr)

def geopmwrite(write_str):
    test_exec = "dummy -- geopmwrite " + write_str
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue().splitlines()
    last_line = None
    if len(output) > 0:
        last_line = output[-1]
    return last_line

def geopmread(read_str):
    test_exec = "dummy -- geopmread " + read_str
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    last_line = output.splitlines()[-1]

    if last_line.startswith('0x'):
        result = int(last_line)
    else:
        try:
            result = float(last_line)
        except ValueError:
            result = yaml.safe_load(output).values()[0]
    return result

def geopmread_domain():
    test_exec = "dummy -- geopmread -d"
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    result = {}
    for line in output.strip().split('\n'):
        domain, count = tuple(line.split())
        result[domain] = int(count)
    return result

def get_node_memory_info():
    test_exec = 'dummy -- cat /proc/meminfo'
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    result = {}
    for line in output.strip().split('\n'):
        if 'MemTotal' in line:
            key, total, units = tuple(line.split())
            if units != 'kB':
                raise RuntimeError("Unknown how to convert units: {}".format(units))
            # Convert to bytes.  Note: in spite of 'kB' label, meminfo
            # displays kibibytes
            result['MemTotal'] = float(total) * 1024
            break
    return result
<commit_msg>Fix broken geopmread helper<commit_after>#
#  Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, Intel Corporation
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions
#  are met:
#
#      * Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#
#      * Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in
#        the documentation and/or other materials provided with the
#        distribution.
#
#      * Neither the name of Intel Corporation nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# Helper functions useful for both tests and experiments

import sys
import os
import socket
import subprocess
import io
import yaml
import shlex

import geopmpy.launcher

def detect_launcher():
    """
    Heuristic to determine the resource manager used on the system.
    Returns name of resource manager or launcher, otherwise a
    LookupError is raised.
    """
    # Try the environment
    result = os.environ.get('GEOPM_LAUNCHER', None)
    if not result:
        # Check for known host names
        slurm_hosts = ['mr-fusion', 'mcfly']
        alps_hosts = ['theta']
        hostname = socket.gethostname()
        if any(hostname.startswith(word) for word in slurm_hosts):
            result = 'srun'
        elif any(hostname.startswith(word) for word in alps_hosts):
            result = 'aprun'
    if not result:
        try:
            exec_str = 'srun --version'
            subprocess.check_call(exec_str, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE, shell=True)
            result = 'srun'
        except subprocess.CalledProcessError:
            pass
    if not result:
        try:
            exec_str = 'aprun --version'
            subprocess.check_call(exec_str, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE, shell=True)
            result = 'aprun'
        except subprocess.CalledProcessError:
            pass
    if not result:
        raise LookupError('Unable to determine resource manager')
    return result

def allocation_node_test(test_exec, stdout, stderr):
    argv = shlex.split(test_exec)
    launcher = detect_launcher()
    argv.insert(1, launcher)
    if launcher == 'aprun':
        argv.insert(2, '-q')  # Use quiet flag with aprun to suppress end of job info string
    argv.insert(2, '--geopm-ctl-disable')
    launcher = geopmpy.launcher.Factory().create(argv, num_rank=1, num_node=1,
                                                 job_name="geopm_allocation_test", quiet=True)
    launcher.run(stdout, stderr)

def geopmwrite(write_str):
    test_exec = "dummy -- geopmwrite " + write_str
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue().splitlines()
    last_line = None
    if len(output) > 0:
        last_line = output[-1]
    return last_line

def geopmread(read_str):
    test_exec = "dummy -- geopmread " + read_str
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    last_line = output.splitlines()[-1]

    if last_line.startswith('0x'):
        result = int(last_line)
    else:
        try:
            result = float(last_line)
        except ValueError:
            result = list(yaml.safe_load(output).values())[0]
    return result

def geopmread_domain():
    test_exec = "dummy -- geopmread -d"
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    result = {}
    for line in output.strip().split('\n'):
        domain, count = tuple(line.split())
        result[domain] = int(count)
    return result

def get_node_memory_info():
    test_exec = 'dummy -- cat /proc/meminfo'
    stdout = io.StringIO()
    stderr = io.StringIO()
    try:
        allocation_node_test(test_exec, stdout, stderr)
    except subprocess.CalledProcessError as err:
        sys.stderr.write(stderr.getvalue())
        raise err
    output = stdout.getvalue()
    result = {}
    for line in output.strip().split('\n'):
        if 'MemTotal' in line:
            key, total, units = tuple(line.split())
            if units != 'kB':
                raise RuntimeError("Unknown how to convert units: {}".format(units))
            # Convert to bytes.  Note: in spite of 'kB' label, meminfo
            # displays kibibytes
            result['MemTotal'] = float(total) * 1024
            break
    return result
<|endoftext|>


<commit_before>__author__ = 'loopingz'

# Based on https://code.google.com/p/winsys/source/browse/trunk/random/file_handles.py code

import os, sys
from ctypes import *
from ctypes.wintypes import *
import Queue
import re
import struct
import threading

import ntsecuritycon
import pywintypes
import win32api
import win32con
import win32event
import win32file
import win32security
import winerror
from nxdrive.logging_config import get_logger
from time import sleep

UCHAR = c_ubyte
PVOID = c_void_p
ntdll = windll.ntdll

SystemHandleInformation = 16
STATUS_INFO_LENGTH_MISMATCH = 0xC0000004
STATUS_BUFFER_OVERFLOW = 0x80000005L
STATUS_INVALID_HANDLE = 0xC0000008L
STATUS_BUFFER_TOO_SMALL = 0xC0000023L
STATUS_SUCCESS = 0

CURRENT_PROCESS = win32api.GetCurrentProcess ()

threads = 0
log = get_logger(__name__)

class x_file_handles(Exception):
    pass

def signed_to_unsigned (signed):
    unsigned, = struct.unpack ("L", struct.pack ("l", signed))
    return unsigned

class SYSTEM_HANDLE_TABLE_ENTRY_INFO(Structure):
    _fields_ = [
        ("UniqueProcessId", USHORT),
        ("CreatorBackTraceIndex", USHORT),
        ("ObjectTypeIndex", UCHAR),
        ("HandleAttributes", UCHAR),
        ("HandleValue", USHORT),
        ("Object", PVOID),
        ("GrantedAccess", ULONG),
    ]

class SYSTEM_HANDLE_INFORMATION(Structure):
    _fields_ = [
        ("NumberOfHandles", ULONG),
        ("Handles", SYSTEM_HANDLE_TABLE_ENTRY_INFO * 1),
    ]

class LSA_UNICODE_STRING(Structure):
    _fields_ = [
        ("Length", USHORT),
        ("MaximumLength", USHORT),
        ("Buffer", LPWSTR),
    ]

class PUBLIC_OBJECT_TYPE_INFORMATION(Structure):
    _fields_ = [
        ("Name", LSA_UNICODE_STRING),
        ("Reserved", ULONG * 22),
    ]

class OBJECT_NAME_INFORMATION(Structure):
    _fields_ = [
        ("Name", LSA_UNICODE_STRING),
    ]

class IO_STATUS_BLOCK_UNION(Union):
    _fields_ = [
        ("Status", LONG),
        ("Pointer", PVOID),
    ]

class IO_STATUS_BLOCK(Structure):
    _anonymous_ = ("u",)
    _fields_ = [
        ("u", IO_STATUS_BLOCK_UNION),
        ("Information", POINTER(ULONG)),
    ]

class FILE_NAME_INFORMATION(Structure):
    filename_size = 4096
    _fields_ = [
        ("FilenameLength", ULONG),
        ("FileName", WCHAR * filename_size),
    ]

class GetObjectInfoThread(threading.Thread):
    def __init__(self, pid, handles, filter_type):
        super(GetObjectInfoThread, self).__init__(target=self._get_object_info, args=(self, pid, handles, filter_type))
        self._pid = pid
        self._handles = handles
        self._filter_type = filter_type
        self._result = []

    def get_pid(self):
        return self._pid

    @staticmethod
    def get_type_info(handle):
        public_object_type_information = PUBLIC_OBJECT_TYPE_INFORMATION ()
        size = DWORD(sizeof(public_object_type_information))
        while True:
            result = signed_to_unsigned(
                ntdll.NtQueryObject(
                handle, 2, byref(public_object_type_information), size, None
                )
            )
            if result == STATUS_SUCCESS:
                return public_object_type_information.Name.Buffer
            elif result == STATUS_INFO_LENGTH_MISMATCH:
                size = DWORD(size.value * 4)
                resize(public_object_type_information, size.value)
            elif result == STATUS_INVALID_HANDLE:
                return None
            elif result == 0xc0000005:
                # Access denied
                # Windows 10 result
                return None
            else:
                raise x_file_handles ("NtQueryObject.2", hex (result))

    @staticmethod
    def get_name_info(handle):
        object_name_information = OBJECT_NAME_INFORMATION ()
        size = DWORD (sizeof (object_name_information))
        while True:
            info = ntdll.NtQueryObject (
                    handle, 1, byref (object_name_information), size, None
                )
            result = signed_to_unsigned (
                info
            )
            if result == STATUS_SUCCESS:
                return object_name_information.Name.Buffer
            elif result in (STATUS_BUFFER_OVERFLOW, STATUS_BUFFER_TOO_SMALL, STATUS_INFO_LENGTH_MISMATCH):
                size = DWORD(size.value * 4)
                resize(object_name_information, size.value)
            else:
                return None

    @staticmethod
    def duplicate_handle(hProcess, handle):
        try:
            handle = win32api.DuplicateHandle(hProcess, handle, CURRENT_PROCESS,
                                                   0, 0, win32con.DUPLICATE_SAME_ACCESS)
        except win32api.error, (errno, errctx, errmsg):
            if errno in (
                winerror.ERROR_ACCESS_DENIED,
                winerror.ERROR_INVALID_PARAMETER,
                winerror.ERROR_INVALID_HANDLE,
                winerror.ERROR_NOT_SUPPORTED
            ):
                log.trace("Cant duplicate handle: %d errno: %d", handle, errno)
                return None
            else:
                raise
        if handle is not None:
            return int(handle)
        return None

    @staticmethod
    def get_object_info(pid, process_handle, handle, filter_type):
        #handle = GetObjectInfoThread.duplicate_handle(process_handle, handle)
        type = GetObjectInfoThread.get_type_info(handle)
        if filter_type is not None and filter_type != type and type is not None:
            return None
        name = GetObjectInfoThread.get_name_info(handle)
        if name is None:
            return None
        return (pid, type, name, handle)

    @staticmethod
    def _get_object_info(obj, pid, handles, filter_type):
        try:
            process_handler = win32api.OpenProcess (win32con.PROCESS_DUP_HANDLE, 0, pid)
        except win32api.error, (errno, errctx, errmsg):
            return
        for handle in handles:
            try:
                hDuplicate = win32api.DuplicateHandle (process_handler, handle, CURRENT_PROCESS,
                                                       0, 0, win32con.DUPLICATE_SAME_ACCESS)
            except win32api.error, (errno, errctx, errmsg):
                if errno in (
                    winerror.ERROR_ACCESS_DENIED,
                    winerror.ERROR_INVALID_PARAMETER,
                    winerror.ERROR_INVALID_HANDLE,
                    winerror.ERROR_NOT_SUPPORTED
                ):
                    log.trace("Cant duplicate handle: %d errno: %d", handle, errno)
                    return None
                else:
                    raise
            if hDuplicate is None:
                continue
            handle = int(hDuplicate)
            resource = GetObjectInfoThread.get_object_info(pid, process_handler, handle, filter_type)
            win32api.CloseHandle(hDuplicate)
            if resource is not None:
                obj._result.append(resource)
        win32api.CloseHandle(process_handler)

    def get_results(self):
        return self._result

class WindowsProcessFileHandlerSniffer():

    def __init__(self):
        self.DEVICE_DRIVES = dict()
        self._running = False
        self._threads = []
        self._pid_blacklist = dict()
        self._pid_blacklist[os.getpid()] = True
        # Get WIndows Drive
        for d in "abcdefghijklmnopqrstuvwxyz":
            try:
                device = win32file.QueryDosDevice(d + ":").strip("\x00").lower()
                self.DEVICE_DRIVES[device] = d.upper() + ":"
            except win32file.error, (errno, errctx, errmsg):
                if errno == 2:
                    pass
                else:
                    raise

    def get_process_handle(self, hProcess, handle):
        try:
            return win32api.DuplicateHandle (hProcess, handle, CURRENT_PROCESS,
                                                   0, 0, win32con.DUPLICATE_SAME_ACCESS)
        except win32api.error, (errno, errctx, errmsg):
            if errno in (
                winerror.ERROR_ACCESS_DENIED,
                winerror.ERROR_INVALID_PARAMETER,
                winerror.ERROR_INVALID_HANDLE,
                winerror.ERROR_NOT_SUPPORTED
            ):
                log.trace("Cant duplicate handle: %d errno: %d", handle, errno)
                return None
            else:
                raise

    def get_handles(self):
        system_handle_information = SYSTEM_HANDLE_INFORMATION ()
        size = DWORD (sizeof (system_handle_information))
        while True:
            result = ntdll.NtQuerySystemInformation (
                SystemHandleInformation,
                byref (system_handle_information),
                size,
                byref (size)
            )
            result = signed_to_unsigned (result)
            if result == STATUS_SUCCESS:
                break
            elif result == STATUS_INFO_LENGTH_MISMATCH:
                size = DWORD (size.value * 4)
                resize (system_handle_information, size.value)
            else:
                raise x_file_handles ("NtQuerySystemInformation", hex (result))

        result = dict()
        pHandles = cast (
            system_handle_information.Handles,
            POINTER (SYSTEM_HANDLE_TABLE_ENTRY_INFO * system_handle_information.NumberOfHandles)
        )
        for handle in pHandles.contents:
            if handle.UniqueProcessId not in result:
                result[handle.UniqueProcessId] = dict()
                result[handle.UniqueProcessId]["handles"] = []
                result[handle.UniqueProcessId]["risky_handles"] = []
            if handle.GrantedAccess == 0x1a0089 or handle.GrantedAccess == 0x1a019f or handle.GrantedAccess == 0x12019f\
                     or handle.GrantedAccess == 0x120189 or handle.GrantedAccess == 0x01f01ff\
                     or handle.GrantedAccess == 0x100081:
                # Some process like ssh-agent or chrome make NtQueryObject hang when queried
                result[handle.UniqueProcessId]["risky_handles"].append(handle.HandleValue)
                continue
            result[handle.UniqueProcessId]["handles"].append((handle.HandleValue, handle.GrantedAccess))
        return result

    def get_main_open_files(self, pids=None, filter_type='File'):
        handles = self.get_handles()
        info_threads = []
        # Remove stopped pids from blacklist
        remove_pids = []
        for pid in self._pid_blacklist:
            if not pid in handles:
                remove_pids.append(pid)
        for pid in remove_pids:
            del self._pid_blacklist[pid]

        # Go through handles ( stored by pid )
        for pid in handles:
            if pid in self._pid_blacklist:
                continue
            try:
                process_handler = win32api.OpenProcess (win32con.PROCESS_DUP_HANDLE, 0, pid)
            except win32api.error, (errno, errctx, errmsg):
                continue
            for handle_obj in handles[pid]['handles']:
                handle = handle_obj[0]
                hDuplicate = self.get_process_handle(process_handler, handle)
                if hDuplicate is None:
                    continue
                handle = int(hDuplicate)
                resource = GetObjectInfoThread.get_object_info(pid, process_handler, handle, filter_type)
                win32api.CloseHandle(hDuplicate)
                if resource is not None:
                    yield resource
            win32api.CloseHandle(process_handler)
            if len(handles[pid]['risky_handles']) > 0 and True:
                # As they can hang the kernel we need to launch another thread for those one
                thread = GetObjectInfoThread(pid, handles[pid]['risky_handles'], filter_type)
                # To avoid crash at the end
                thread.setDaemon(True)
                thread.start()
                info_threads.append(thread)
        # Wait 2s
        sleep(2)
        # Go through all the remaining thread
        for thread in info_threads:
            if thread.is_alive():
                # Should not be blacklist the pid
                self._pid_blacklist[thread.get_pid()] = True
                log.trace('Blacklisting process %d because of stuck thread', thread.get_pid())
            else:
                for result in thread.get_results():
                    yield result
        return

    def filepath_from_devicepath(self, devicepath):
        if devicepath is None: return None
        devicepath_lower = devicepath.lower()
        for device, drive in self.DEVICE_DRIVES.items():
            if devicepath_lower.startswith(device):
                return drive + devicepath[len(device):]
        return devicepath

    def get_open_files(self, pids=None):
        if self._running:
            raise Exception("Can't multithread open_files")
        se_debug = win32security.LookupPrivilegeValue (u"", win32security.SE_DEBUG_NAME)
        hToken = win32security.OpenProcessToken (
            CURRENT_PROCESS,
            ntsecuritycon.MAXIMUM_ALLOWED
        )
        try:
            self._running = True
            for file in self.get_main_open_files(pids):
                path = self.filepath_from_devicepath(file[2])
                if path is None or not os.path.exists(path) or os.path.isdir(path):
                    continue
                yield (file[0], path, file[1], file[3])
        except x_file_handles, (context, errno):
            log.trace("Error in %r - with errno : %d", context, errno)
        finally:
            self._running = False

<commit_msg>NXDRIVE-568: Use threads for all pids<commit_after>__author__ = 'loopingz'

# Based on https://code.google.com/p/winsys/source/browse/trunk/random/file_handles.py code

import os, sys
from ctypes import *
from ctypes.wintypes import *
import Queue
import re
import struct
import threading

import ntsecuritycon
import pywintypes
import win32api
import win32con
import win32event
import win32file
import win32security
import winerror
from nxdrive.logging_config import get_logger
from time import sleep

UCHAR = c_ubyte
PVOID = c_void_p
ntdll = windll.ntdll

SystemHandleInformation = 16
STATUS_INFO_LENGTH_MISMATCH = 0xC0000004
STATUS_BUFFER_OVERFLOW = 0x80000005L
STATUS_INVALID_HANDLE = 0xC0000008L
STATUS_BUFFER_TOO_SMALL = 0xC0000023L
STATUS_SUCCESS = 0

CURRENT_PROCESS = win32api.GetCurrentProcess ()

threads = 0
log = get_logger(__name__)

class x_file_handles(Exception):
    pass

def signed_to_unsigned (signed):
    unsigned, = struct.unpack ("L", struct.pack ("l", signed))
    return unsigned

class SYSTEM_HANDLE_TABLE_ENTRY_INFO(Structure):
    _fields_ = [
        ("UniqueProcessId", USHORT),
        ("CreatorBackTraceIndex", USHORT),
        ("ObjectTypeIndex", UCHAR),
        ("HandleAttributes", UCHAR),
        ("HandleValue", USHORT),
        ("Object", PVOID),
        ("GrantedAccess", ULONG),
    ]

class SYSTEM_HANDLE_INFORMATION(Structure):
    _fields_ = [
        ("NumberOfHandles", ULONG),
        ("Handles", SYSTEM_HANDLE_TABLE_ENTRY_INFO * 1),
    ]

class LSA_UNICODE_STRING(Structure):
    _fields_ = [
        ("Length", USHORT),
        ("MaximumLength", USHORT),
        ("Buffer", LPWSTR),
    ]

class PUBLIC_OBJECT_TYPE_INFORMATION(Structure):
    _fields_ = [
        ("Name", LSA_UNICODE_STRING),
        ("Reserved", ULONG * 22),
    ]

class OBJECT_NAME_INFORMATION(Structure):
    _fields_ = [
        ("Name", LSA_UNICODE_STRING),
    ]

class IO_STATUS_BLOCK_UNION(Union):
    _fields_ = [
        ("Status", LONG),
        ("Pointer", PVOID),
    ]

class IO_STATUS_BLOCK(Structure):
    _anonymous_ = ("u",)
    _fields_ = [
        ("u", IO_STATUS_BLOCK_UNION),
        ("Information", POINTER(ULONG)),
    ]

class FILE_NAME_INFORMATION(Structure):
    filename_size = 4096
    _fields_ = [
        ("FilenameLength", ULONG),
        ("FileName", WCHAR * filename_size),
    ]

class GetObjectInfoThread(threading.Thread):
    def __init__(self, pid, handles, filter_type):
        super(GetObjectInfoThread, self).__init__(target=self._get_object_info, args=(self, pid, handles, filter_type))
        self._pid = pid
        self._handles = handles
        self._filter_type = filter_type
        self._result = []
        self.lastGrantedAccess = None

    def get_pid(self):
        return self._pid

    @staticmethod
    def get_type_info(handle):
        public_object_type_information = PUBLIC_OBJECT_TYPE_INFORMATION ()
        size = DWORD(sizeof(public_object_type_information))
        while True:
            result = signed_to_unsigned(
                ntdll.NtQueryObject(
                handle, 2, byref(public_object_type_information), size, None
                )
            )
            if result == STATUS_SUCCESS:
                return public_object_type_information.Name.Buffer
            elif result == STATUS_INFO_LENGTH_MISMATCH:
                size = DWORD(size.value * 4)
                resize(public_object_type_information, size.value)
            elif result == STATUS_INVALID_HANDLE:
                return None
            elif result == 0xc0000005:
                # Access denied
                # Windows 10 result
                return None
            else:
                raise x_file_handles ("NtQueryObject.2", hex (result))

    @staticmethod
    def get_name_info(handle):
        object_name_information = OBJECT_NAME_INFORMATION ()
        size = DWORD (sizeof (object_name_information))
        while True:
            info = ntdll.NtQueryObject (
                    handle, 1, byref (object_name_information), size, None
                )
            result = signed_to_unsigned (
                info
            )
            if result == STATUS_SUCCESS:
                return object_name_information.Name.Buffer
            elif result in (STATUS_BUFFER_OVERFLOW, STATUS_BUFFER_TOO_SMALL, STATUS_INFO_LENGTH_MISMATCH):
                size = DWORD(size.value * 4)
                resize(object_name_information, size.value)
            else:
                return None

    @staticmethod
    def get_object_info(pid, process_handle, handle, filter_type):
        type = GetObjectInfoThread.get_type_info(handle)
        if filter_type is not None and filter_type != type and type is not None:
            return None
        name = GetObjectInfoThread.get_name_info(handle)
        if name is None:
            return None
        return (pid, type, name, handle)

    @staticmethod
    def _get_object_info(obj, pid, handles, filter_type):
        try:
            process_handler = win32api.OpenProcess (win32con.PROCESS_DUP_HANDLE, 0, pid)
        except win32api.error, (errno, errctx, errmsg):
            return
        for handle_obj in handles:
            obj.lastGrantedAccess = handle_obj[1]
            handle = handle_obj[0]
            hDuplicate = WindowsProcessFileHandlerSniffer.get_process_handle(process_handler, handle)
            if hDuplicate is None:
                continue
            handle = int(hDuplicate)
            resource = GetObjectInfoThread.get_object_info(pid, process_handler, handle, filter_type)
            win32api.CloseHandle(hDuplicate)
            if resource is not None:
                obj._result.append(resource)
        win32api.CloseHandle(process_handler)

    def get_results(self):
        return self._result

class WindowsProcessFileHandlerSniffer():

    def __init__(self):
        self.DEVICE_DRIVES = dict()
        self._running = False
        self._threads = []
        self._pid_blacklist = dict()
        self._pid_blacklist[os.getpid()] = True
        # Get WIndows Drive
        for d in "abcdefghijklmnopqrstuvwxyz":
            try:
                device = win32file.QueryDosDevice(d + ":").strip("\x00").lower()
                self.DEVICE_DRIVES[device] = d.upper() + ":"
            except win32file.error, (errno, errctx, errmsg):
                if errno == 2:
                    pass
                else:
                    raise

    @staticmethod
    def get_process_handle(hProcess, handle):
        try:
            return win32api.DuplicateHandle (hProcess, handle, CURRENT_PROCESS,
                                                   0, 0, win32con.DUPLICATE_SAME_ACCESS)
        except win32api.error, (errno, errctx, errmsg):
            if errno in (
                winerror.ERROR_ACCESS_DENIED,
                winerror.ERROR_INVALID_PARAMETER,
                winerror.ERROR_INVALID_HANDLE,
                winerror.ERROR_NOT_SUPPORTED
            ):
                log.trace("Cant duplicate handle: %d errno: %d", handle, errno)
                return None
            else:
                raise

    def get_handles(self):
        system_handle_information = SYSTEM_HANDLE_INFORMATION ()
        size = DWORD (sizeof (system_handle_information))
        while True:
            result = ntdll.NtQuerySystemInformation (
                SystemHandleInformation,
                byref (system_handle_information),
                size,
                byref (size)
            )
            result = signed_to_unsigned (result)
            if result == STATUS_SUCCESS:
                break
            elif result == STATUS_INFO_LENGTH_MISMATCH:
                size = DWORD (size.value * 4)
                resize (system_handle_information, size.value)
            else:
                raise x_file_handles ("NtQuerySystemInformation", hex (result))

        result = dict()
        pHandles = cast (
            system_handle_information.Handles,
            POINTER (SYSTEM_HANDLE_TABLE_ENTRY_INFO * system_handle_information.NumberOfHandles)
        )
        for handle in pHandles.contents:
            if handle.UniqueProcessId not in result:
                result[handle.UniqueProcessId] = dict()
                result[handle.UniqueProcessId]["handles"] = []
                result[handle.UniqueProcessId]["risky_handles"] = []

            # Useless for now but keep for later use in case
            if handle.GrantedAccess == 0x1a0089 or handle.GrantedAccess == 0x1a019f or handle.GrantedAccess == 0x12019f\
                     or handle.GrantedAccess == 0x120189 or handle.GrantedAccess == 0x01f01ff\
                     or handle.GrantedAccess == 0x100081:
                # Some process like ssh-agent or chrome make NtQueryObject hang when queried
                result[handle.UniqueProcessId]["risky_handles"].append((handle.HandleValue, handle.GrantedAccess))
                continue

            result[handle.UniqueProcessId]["risky_handles"].append((handle.HandleValue, handle.GrantedAccess))
        return result

    def get_main_open_files(self, pids=None, filter_type='File'):
        handles = self.get_handles()
        info_threads = []
        # Remove stopped pids from blacklist
        remove_pids = []
        for pid in self._pid_blacklist:
            if not pid in handles:
                remove_pids.append(pid)
        for pid in remove_pids:
            del self._pid_blacklist[pid]

        # Go through handles ( stored by pid )
        for pid in handles:
            if pid in self._pid_blacklist:
                continue
            try:
                process_handler = win32api.OpenProcess (win32con.PROCESS_DUP_HANDLE, 0, pid)
            except win32api.error, (errno, errctx, errmsg):
                continue
            for handle_obj in handles[pid]['handles']:
                handle = handle_obj[0]
                hDuplicate = WindowsProcessFileHandlerSniffer.get_process_handle(process_handler, handle)
                if hDuplicate is None:
                    continue
                handle = int(hDuplicate)
                resource = GetObjectInfoThread.get_object_info(pid, process_handler, handle, filter_type)
                win32api.CloseHandle(hDuplicate)
                if resource is not None:
                    yield resource
            win32api.CloseHandle(process_handler)
            if len(handles[pid]['risky_handles']) > 0:
                # As they can hang the kernel we need to launch another thread for those one
                thread = GetObjectInfoThread(pid, handles[pid]['risky_handles'], filter_type)
                # To avoid crash at the end
                thread.setDaemon(True)
                thread.start()
                info_threads.append(thread)
        # Wait 2s
        sleep(2)
        # Go through all the remaining thread
        for thread in info_threads:
            if thread.is_alive():
                # Should not be blacklist the pid
                self._pid_blacklist[thread.get_pid()] = True
                log.trace('Blacklisting process %d because of stuck thread (GrantedAccess: %d)',
                            thread.get_pid(), thread.lastGrantedAccess)
            else:
                for result in thread.get_results():
                    yield result
        return

    def filepath_from_devicepath(self, devicepath):
        if devicepath is None: return None
        devicepath_lower = devicepath.lower()
        for device, drive in self.DEVICE_DRIVES.items():
            if devicepath_lower.startswith(device):
                return drive + devicepath[len(device):]
        return devicepath

    def get_open_files(self, pids=None):
        if self._running:
            raise Exception("Can't multithread open_files")
        se_debug = win32security.LookupPrivilegeValue (u"", win32security.SE_DEBUG_NAME)
        hToken = win32security.OpenProcessToken (
            CURRENT_PROCESS,
            ntsecuritycon.MAXIMUM_ALLOWED
        )
        try:
            self._running = True
            for file in self.get_main_open_files(pids):
                path = self.filepath_from_devicepath(file[2])
                if path is None or not os.path.exists(path) or os.path.isdir(path):
                    continue
                yield (file[0], path, file[1], file[3])
        except x_file_handles, (context, errno):
            log.trace("Error in %r - with errno : %d", context, errno)
        finally:
            self._running = False
<|endoftext|>


<commit_before>import json
import os

from backend.api import serializers
from backend.cases.models import (
    Case,
    Candidate,
    Nodule,
    CaseSerializer
)
from backend.images.models import ImageSeries
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.core.serializers.json import DjangoJSONEncoder
from django.shortcuts import get_object_or_404
from rest_framework import renderers
from rest_framework import viewsets
from rest_framework.decorators import api_view
from rest_framework.decorators import renderer_classes
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

class CaseViewSet(viewsets.ModelViewSet):
    queryset = Case.objects.all()
    serializer_class = serializers.CaseSerializer

class CandidateViewSet(viewsets.ModelViewSet):
    queryset = Candidate.objects.all()
    serializer_class = serializers.CandidateSerializer

class NoduleViewSet(viewsets.ModelViewSet):
    queryset = Nodule.objects.all()
    serializer_class = serializers.NoduleSerializer

class ImageSeriesViewSet(viewsets.ModelViewSet):
    queryset = ImageSeries.objects.all()
    serializer_class = serializers.ImageSeriesSerializer

class ImageAvailableApiView(APIView):
    """
    View list of images from dataset directory
    """

    def __init__(self, *args, **kwargs):
        super(ImageAvailableApiView, self).__init__(**kwargs)
        self.fss = FileSystemStorage(settings.DATASOURCE_DIR)

    def walk(self, location, dir_name='root'):
        """
        Recursively walkthrough directories and files
        """
        list_dirs = self.fss.listdir(location)
        tree = {
            'name': dir_name,
            'children': [],
        }
        tree['children'] = sorted(list_dirs[1])
        for dirname in sorted(list_dirs[0]):
            tree['children'].append(self.walk(os.path.join(location, dirname), dirname))
        return tree

    def get(self, request):
        """
        Return a sorted(by name) list of files and folders
        in dataset

        Format::

            {'directories': [
                {
                    'name': directory_name1,
                    'children': [
                        file_name1,
                        file_name2,
                        {
                            'name': 'nested_dir_1',
                            'children': [
                                'file_name_1',
                                'file_name_2',
                                ....
                            ]
                        }
                        ... ]
                }, ... ]
            }
        """
        tree = self.walk(settings.DATASOURCE_DIR)
        return Response({'directories': tree})

@api_view(['GET'])
def candidate_mark(request, candidate_id):
    return Response({'response': "Candidate {} was marked".format(candidate_id)})

@api_view(['GET'])
def candidate_dismiss(request, candidate_id):
    return Response({'response': "Candidate {} was dismissed".format(candidate_id)})

class JsonHtmlRenderer(renderers.BaseRenderer):
    media_type = 'text/html'
    format = 'html'

    def render(self, data, media_type=None, renderer_context=None):
        return "
{}
".format(json.dumps(data, indent=4, sort_keys=True, cls=DjangoJSONEncoder))

@api_view(['GET'])
# Render .json and .html requests
@renderer_classes((JSONRenderer, JsonHtmlRenderer))
def case_report(request, case_id, format=None):
    case = get_object_or_404(Case, pk=case_id)

    return Response(CaseSerializer(case).data)
<commit_msg>Fix whitespace so flake8 and CI tests pass<commit_after>import json
import os

from backend.api import serializers
from backend.cases.models import (
    Case,
    Candidate,
    Nodule,
    CaseSerializer
)
from backend.images.models import ImageSeries
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.core.serializers.json import DjangoJSONEncoder
from django.shortcuts import get_object_or_404
from rest_framework import renderers
from rest_framework import viewsets
from rest_framework.decorators import api_view
from rest_framework.decorators import renderer_classes
from rest_framework.renderers import JSONRenderer
from rest_framework.response import Response
from rest_framework.views import APIView

class CaseViewSet(viewsets.ModelViewSet):
    queryset = Case.objects.all()
    serializer_class = serializers.CaseSerializer

class CandidateViewSet(viewsets.ModelViewSet):
    queryset = Candidate.objects.all()
    serializer_class = serializers.CandidateSerializer

class NoduleViewSet(viewsets.ModelViewSet):
    queryset = Nodule.objects.all()
    serializer_class = serializers.NoduleSerializer

class ImageSeriesViewSet(viewsets.ModelViewSet):
    queryset = ImageSeries.objects.all()
    serializer_class = serializers.ImageSeriesSerializer

class ImageAvailableApiView(APIView):
    """
    View list of images from dataset directory
    """

    def __init__(self, *args, **kwargs):
        super(ImageAvailableApiView, self).__init__(**kwargs)
        self.fss = FileSystemStorage(settings.DATASOURCE_DIR)

    def walk(self, location, dir_name='root'):
        """
        Recursively walkthrough directories and files
        """
        list_dirs = self.fss.listdir(location)
        tree = {
            'name': dir_name,
            'children': [],
        }
        tree['children'] = sorted(list_dirs[1])
        for dirname in sorted(list_dirs[0]):
            tree['children'].append(self.walk(os.path.join(location, dirname), dirname))
        return tree

    def get(self, request):
        """
        Return a sorted(by name) list of files and folders
        in dataset

        Format::

            {'directories': [
                {
                    'name': directory_name1,
                    'children': [
                        file_name1,
                        file_name2,
                        {
                            'name': 'nested_dir_1',
                            'children': [
                                'file_name_1',
                                'file_name_2',
                                ....
                            ]
                        }
                        ... ]
                }, ... ]
            }
        """
        tree = self.walk(settings.DATASOURCE_DIR)
        return Response({'directories': tree})

@api_view(['GET'])
def candidate_mark(request, candidate_id):
    return Response({'response': "Candidate {} was marked".format(candidate_id)})

@api_view(['GET'])
def candidate_dismiss(request, candidate_id):
    return Response({'response': "Candidate {} was dismissed".format(candidate_id)})

class JsonHtmlRenderer(renderers.BaseRenderer):
    media_type = 'text/html'
    format = 'html'

    def render(self, data, media_type=None, renderer_context=None):
        return "
{}
".format(json.dumps(data, indent=4, sort_keys=True, cls=DjangoJSONEncoder))

@api_view(['GET'])
# Render .json and .html requests
@renderer_classes((JSONRenderer, JsonHtmlRenderer))
def case_report(request, case_id, format=None):
    case = get_object_or_404(Case, pk=case_id)

    return Response(CaseSerializer(case).data)
<|endoftext|>