<commit_before>from __future__ import absolute_import, division, print_function
from textwrap import dedent, wrap
from collections import OrderedDict

import numpy as np
import six

# Parameter arguments that are listed first in the geom and
# stat class signatures

common_geom_params = ['mapping', 'data', 'stat', 'position',
                      'na_rm', 'inherit_aes', 'show_legend']
common_geom_param_values = {'mapping': None, 'data': None,
                            'inherit_aes': True, 'show_legend': None}

common_stat_params = ['mapping', 'data', 'geom', 'position', 'na_rm']
common_stat_param_values = common_geom_param_values

# Templates for docstrings

GEOM_SIGNATURE_TPL = """
.. rubric:: Usage

::

    {signature}

Only the ``mapping`` and ``data`` can be positional, the rest must
be keyword arguments. ``**kwargs`` can be aesthetics (or parameters)
used by the ``stat``.
"""

AESTHETICS_TPL = """
.. rubric:: Aesthetics

{aesthetics_table}

The **bold** aesthetics are required.
"""

STAT_SIGNATURE_TPL = """
.. rubric:: Usage

::

    {signature}

Only the ``mapping`` and ``data`` can be positional, the rest must
be keyword arguments. ``**kwargs`` can be aesthetics (or parameters)
used by the ``geom``.
"""

common_params_doc = {
    'mapping': """\
Aesthetic mappings created with :meth:`~plotnine.aes`. If specified and \
:py:`inherit.aes=True`, it is combined with the default mapping for the plot. \
You must supply mapping if there is no plot mapping.""",

    'data': """\
The data to be displayed in this layer. If :py:`None`, the data from \
from the :py:`ggplot()` call is used. If specified, it overrides the \
data from the :py:`ggplot()` call.""",

    'stat': """\
The statistical transformation to use on the data for this layer. \
If it is a string, it must be the registered and known to Plotnine.""",

    'position': """\
Position adjustment. If it is a string, it must be registered and \
known to Plotnine.""",

    'na_rm': """\
If :py:`False`, removes missing values with a warning. If :py:`True` \
silently removes missing values.""",

    'inherit_aes': """\
If :py:`False`, overrides the default aesthetics.""",

    'show_legend': """\
Whether this layer should be included in the legends. :py:`None` the \
default, includes any aesthetics that are mapped. If a :class:`bool`, \
:py:`False` never \ includes and :py:`True` always includes. A \
:class:`dict` can be used to *exclude* specific aesthetis of the layer \
from showing in the legend. e.g :py:`show_legend={'color': False}`, \
any other aesthetic are included by default."""
}

GEOM_PARAMS_TPL = """\
mapping: aes, optional
    {mapping}
data: dataframe, optional
    {data}
stat: str or stat, optional (default: {default_stat})
    {stat}
position: str or position, optional (default: {default_position})
    {position}
na_rm: bool, optional (default: {default_na_rm})
    {na_rm}
inherit_aes: bool, optional (default: {default_inherit_aes})
    {inherit_aes}
show_legend: bool or dict, optional (default: None)
    {show_legend}
"""

STAT_PARAMS_TPL = """\
mapping: aes, optional
    {mapping}
data: dataframe, optional
    {data}
geom: str or stat, optional (default: {default_geom})
    {stat}
position: str or position, optional (default: {default_position})
    {position}
na_rm: bool, optional (default: {default_na_rm})
    {na_rm}
"""

def dict_to_table(header, contents):
    """
    Convert dict to table

    Parameters
    ----------
    header : tuple
        Table header. Should have a length of 2.
    contents : dict
        The key becomes column 1 of table and the
        value becomes column 2 of table.
    """
    def to_text(row):
        name, value = row
        m = max_col1_size + 1 - len(name)
        spacing = ' ' * m

        return ''.join([name, spacing, value])

    thead = tuple(str(col) for col in header)
    rows = []
    for name, value in contents.items():
        # code highlighting
        if value != '':
            if isinstance(value, six.string_types):
                value = "'{}'".format(value)
            value = ':py:`{}`'.format(value)
        rows.append((name, value))

    n = np.max([len(header[0])] +
               [len(col1) for col1, _ in rows])
    hborder = tuple('='*n for col in header)
    rows = [hborder, thead, hborder] + rows + [hborder]
    max_col1_size = np.max([len(col1) for col1, _ in rows])
    table = '\n'.join([to_text(row) for row in rows])
    return table

def make_signature(name, params, common_params, common_param_values):
    """
    Create a signature for a geom or stat

    Gets the DEFAULT_PARAMS (params) and creates are comma
    separated list of the `name=value` pairs. The common_params
    come first in the list, and they get take their values from
    either the params-dict or the common_geom_param_values-dict.
    """
    tokens = []
    seen = set()

    def tokens_append(key, value):
        if isinstance(value, six.string_types):
            value = "'{}'".format(value)
        tokens.append('{}={}'.format(key, value))

    # preferred params come first
    for key in common_params:
        seen.add(key)
        try:
            value = params[key]
        except KeyError:
            value = common_param_values[key]
        tokens_append(key, value)

    # other params (these are the geom/stat specific parameters
    for key in (set(params) - seen):
        tokens_append(key, params[key])

    # name, 1 opening bracket, 4 spaces in SIGNATURE_TPL
    s1 = name + '('
    s2 = ', '.join(tokens) + ', **kwargs)'
    line_width = 78 - len(s1)
    indent_spaces = ' ' * (len(s1) + 4)
    s2_lines = wrap(s2, width=line_width, subsequent_indent=indent_spaces)
    return s1 + '\n'.join(s2_lines)

def get_geom_documentation(geom):
    """
    Create a structured documentation for the geom
    """
    # usage
    signature = make_signature(geom.__name__,
                               geom.DEFAULT_PARAMS,
                               common_geom_params,
                               common_geom_param_values)
    usage = GEOM_SIGNATURE_TPL.format(signature=signature)

    # common_parameters
    d = geom.DEFAULT_PARAMS
    common_parameters = GEOM_PARAMS_TPL.format(
        default_stat=d['stat'],
        default_position=d['position'],
        default_na_rm=d['na_rm'],
        default_inherit_aes=d.get('inherit_aes', True),
        **common_params_doc)

    # aesthetics
    contents = OrderedDict(('**{}**'.format(ae), '')
                           for ae in sorted(geom.REQUIRED_AES))
    if geom.DEFAULT_AES:
        d = geom.DEFAULT_AES.copy()
        d['group'] = ''  # All geoms understand the group aesthetic
        contents.update(sorted(d.items()))
    table = dict_to_table(('Aesthetic', 'Default value'),
                          contents)
    aesthetics = AESTHETICS_TPL.format(aesthetics_table=table)

    return usage, common_parameters, aesthetics

def get_stat_documentation(stat):
    """
    Create a structured documentation for the stat
    """
    # usage:
    signature = make_signature(stat.__name__,
                               stat.DEFAULT_PARAMS,
                               common_stat_params,
                               common_stat_param_values)
    usage = STAT_SIGNATURE_TPL.format(signature=signature)

    # common_parameters
    d = stat.DEFAULT_PARAMS
    common_parameters = STAT_PARAMS_TPL.format(
            default_geom=d['geom'],
            default_position=d['position'],
            default_na_rm=d['na_rm'],
            **common_params_doc)

    # aesthetics
    contents = OrderedDict(('**{}**'.format(ae), '')
                           for ae in sorted(stat.REQUIRED_AES))
    contents.update(sorted(stat.DEFAULT_AES.items()))
    table = dict_to_table(('Aesthetic', 'Default value'),
                          contents)
    aesthetics = AESTHETICS_TPL.format(aesthetics_table=table)

    return usage, common_parameters, aesthetics

def document(klass):
    """
    Decorator to document a class

    It replaces `{usage}`, `{common_parameters}` and
    `{aesthetics}` with generated documentation.
    """
    docstring = klass.__doc__
    if docstring is None:
        return klass

    # Dedented so that it lineups (in sphinx) with the part
    # generated parts when put together
    docstring = dedent(docstring)

    if klass.__name__.startswith('geom'):
        func = get_geom_documentation
    elif klass.__name__.startswith('stat'):
        func = get_stat_documentation
    else:
        return klass

    usage, common_parameters, aesthetics = func(klass)
    docstring = docstring.replace('{usage}', usage)
    docstring = docstring.replace('{common_parameters}',
                                  common_parameters)
    docstring = docstring.replace('{aesthetics}', aesthetics)

    klass.__doc__ = docstring
    return klass
<commit_msg>DOC: Formatting of the function signatures<commit_after>from __future__ import absolute_import, division, print_function
from textwrap import dedent, wrap
from collections import OrderedDict

import numpy as np
import six

# Parameter arguments that are listed first in the geom and
# stat class signatures

common_geom_params = ['mapping', 'data', 'stat', 'position',
                      'na_rm', 'inherit_aes', 'show_legend']
common_geom_param_values = {'mapping': None, 'data': None,
                            'inherit_aes': True, 'show_legend': None}

common_stat_params = ['mapping', 'data', 'geom', 'position', 'na_rm']
common_stat_param_values = common_geom_param_values

# Templates for docstrings

GEOM_SIGNATURE_TPL = """
.. rubric:: Usage

::

    {signature}

Only the ``mapping`` and ``data`` can be positional, the rest must
be keyword arguments. ``**kwargs`` can be aesthetics (or parameters)
used by the ``stat``.
"""

AESTHETICS_TPL = """
.. rubric:: Aesthetics

{aesthetics_table}

The **bold** aesthetics are required.
"""

STAT_SIGNATURE_TPL = """
.. rubric:: Usage

::

    {signature}

Only the ``mapping`` and ``data`` can be positional, the rest must
be keyword arguments. ``**kwargs`` can be aesthetics (or parameters)
used by the ``geom``.
"""

common_params_doc = {
    'mapping': """\
Aesthetic mappings created with :meth:`~plotnine.aes`. If specified and \
:py:`inherit.aes=True`, it is combined with the default mapping for the plot. \
You must supply mapping if there is no plot mapping.""",

    'data': """\
The data to be displayed in this layer. If :py:`None`, the data from \
from the :py:`ggplot()` call is used. If specified, it overrides the \
data from the :py:`ggplot()` call.""",

    'stat': """\
The statistical transformation to use on the data for this layer. \
If it is a string, it must be the registered and known to Plotnine.""",

    'position': """\
Position adjustment. If it is a string, it must be registered and \
known to Plotnine.""",

    'na_rm': """\
If :py:`False`, removes missing values with a warning. If :py:`True` \
silently removes missing values.""",

    'inherit_aes': """\
If :py:`False`, overrides the default aesthetics.""",

    'show_legend': """\
Whether this layer should be included in the legends. :py:`None` the \
default, includes any aesthetics that are mapped. If a :class:`bool`, \
:py:`False` never \ includes and :py:`True` always includes. A \
:class:`dict` can be used to *exclude* specific aesthetis of the layer \
from showing in the legend. e.g :py:`show_legend={'color': False}`, \
any other aesthetic are included by default."""
}

GEOM_PARAMS_TPL = """\
mapping: aes, optional
    {mapping}
data: dataframe, optional
    {data}
stat: str or stat, optional (default: {default_stat})
    {stat}
position: str or position, optional (default: {default_position})
    {position}
na_rm: bool, optional (default: {default_na_rm})
    {na_rm}
inherit_aes: bool, optional (default: {default_inherit_aes})
    {inherit_aes}
show_legend: bool or dict, optional (default: None)
    {show_legend}
"""

STAT_PARAMS_TPL = """\
mapping: aes, optional
    {mapping}
data: dataframe, optional
    {data}
geom: str or stat, optional (default: {default_geom})
    {stat}
position: str or position, optional (default: {default_position})
    {position}
na_rm: bool, optional (default: {default_na_rm})
    {na_rm}
"""

def dict_to_table(header, contents):
    """
    Convert dict to table

    Parameters
    ----------
    header : tuple
        Table header. Should have a length of 2.
    contents : dict
        The key becomes column 1 of table and the
        value becomes column 2 of table.
    """
    def to_text(row):
        name, value = row
        m = max_col1_size + 1 - len(name)
        spacing = ' ' * m

        return ''.join([name, spacing, value])

    thead = tuple(str(col) for col in header)
    rows = []
    for name, value in contents.items():
        # code highlighting
        if value != '':
            if isinstance(value, six.string_types):
                value = "'{}'".format(value)
            value = ':py:`{}`'.format(value)
        rows.append((name, value))

    n = np.max([len(header[0])] +
               [len(col1) for col1, _ in rows])
    hborder = tuple('='*n for col in header)
    rows = [hborder, thead, hborder] + rows + [hborder]
    max_col1_size = np.max([len(col1) for col1, _ in rows])
    table = '\n'.join([to_text(row) for row in rows])
    return table

def make_signature(name, params, common_params, common_param_values):
    """
    Create a signature for a geom or stat

    Gets the DEFAULT_PARAMS (params) and creates are comma
    separated list of the `name=value` pairs. The common_params
    come first in the list, and they get take their values from
    either the params-dict or the common_geom_param_values-dict.
    """
    tokens = []
    seen = set()

    def tokens_append(key, value):
        if isinstance(value, six.string_types):
            value = "'{}'".format(value)
        tokens.append('{}={}'.format(key, value))

    # preferred params come first
    for key in common_params:
        seen.add(key)
        try:
            value = params[key]
        except KeyError:
            value = common_param_values[key]
        tokens_append(key, value)

    # other params (these are the geom/stat specific parameters
    for key in (set(params) - seen):
        tokens_append(key, params[key])

    # name, 1 opening bracket, 4 spaces in SIGNATURE_TPL
    s1 = name + '('
    s2 = ', '.join(tokens) + ', **kwargs)'
    line_width = 78 - len(s1)
    indent_spaces = ' ' * (len(s1) + 4)
    newline_and_space = '\n' + indent_spaces
    s2_lines = wrap(s2, width=line_width)
    return s1 + newline_and_space.join(s2_lines)

def get_geom_documentation(geom):
    """
    Create a structured documentation for the geom
    """
    # usage
    signature = make_signature(geom.__name__,
                               geom.DEFAULT_PARAMS,
                               common_geom_params,
                               common_geom_param_values)
    usage = GEOM_SIGNATURE_TPL.format(signature=signature)

    # common_parameters
    d = geom.DEFAULT_PARAMS
    common_parameters = GEOM_PARAMS_TPL.format(
        default_stat=d['stat'],
        default_position=d['position'],
        default_na_rm=d['na_rm'],
        default_inherit_aes=d.get('inherit_aes', True),
        **common_params_doc)

    # aesthetics
    contents = OrderedDict(('**{}**'.format(ae), '')
                           for ae in sorted(geom.REQUIRED_AES))
    if geom.DEFAULT_AES:
        d = geom.DEFAULT_AES.copy()
        d['group'] = ''  # All geoms understand the group aesthetic
        contents.update(sorted(d.items()))
    table = dict_to_table(('Aesthetic', 'Default value'),
                          contents)
    aesthetics = AESTHETICS_TPL.format(aesthetics_table=table)

    return usage, common_parameters, aesthetics

def get_stat_documentation(stat):
    """
    Create a structured documentation for the stat
    """
    # usage:
    signature = make_signature(stat.__name__,
                               stat.DEFAULT_PARAMS,
                               common_stat_params,
                               common_stat_param_values)
    usage = STAT_SIGNATURE_TPL.format(signature=signature)

    # common_parameters
    d = stat.DEFAULT_PARAMS
    common_parameters = STAT_PARAMS_TPL.format(
            default_geom=d['geom'],
            default_position=d['position'],
            default_na_rm=d['na_rm'],
            **common_params_doc)

    # aesthetics
    contents = OrderedDict(('**{}**'.format(ae), '')
                           for ae in sorted(stat.REQUIRED_AES))
    contents.update(sorted(stat.DEFAULT_AES.items()))
    table = dict_to_table(('Aesthetic', 'Default value'),
                          contents)
    aesthetics = AESTHETICS_TPL.format(aesthetics_table=table)

    return usage, common_parameters, aesthetics

def document(klass):
    """
    Decorator to document a class

    It replaces `{usage}`, `{common_parameters}` and
    `{aesthetics}` with generated documentation.
    """
    docstring = klass.__doc__
    if docstring is None:
        return klass

    # Dedented so that it lineups (in sphinx) with the part
    # generated parts when put together
    docstring = dedent(docstring)

    if klass.__name__.startswith('geom'):
        func = get_geom_documentation
    elif klass.__name__.startswith('stat'):
        func = get_stat_documentation
    else:
        return klass

    usage, common_parameters, aesthetics = func(klass)
    docstring = docstring.replace('{usage}', usage)
    docstring = docstring.replace('{common_parameters}',
                                  common_parameters)
    docstring = docstring.replace('{aesthetics}', aesthetics)

    klass.__doc__ = docstring
    return klass
<|endoftext|>


<commit_before>import json

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp = self.get_object(obs_to_add)
            if object_misp:
                self.misp_event.add_object(object_misp)
            print(self.misp_event)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def get_object(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(**self.__get_attribute(obj_to_add))
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')
            domain_ip_object.add_attribute(**self.attribute)
            return domain_ip_object

    def __get_attribute(self, obj_yeti):
        typ_attribute = self.misp_mapping[obj_yeti['type']]
        attr_misp = {'value': obj_yeti['value']}
        if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst':
            attr_misp['object_relation'].update({'type': 'text',
                                                 'object_relation': 'ip'})
        elif 'domain' == typ_attribute:
            attr_misp.update({'type': 'domain',
                              'object_relation': 'domain'})
        elif 'hostname' == typ_attribute:
            attr_misp.update({'type': 'domain',
                              'object_relation': 'domain'})
        print('Attribute %s' % attr_misp)
        return attr_misp

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return yeti_client.get_result()
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<commit_msg>Update yeti.py<commit_after>import json

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            for n in neighboors['objs']:
                yield n

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add in self.get_neighboors(obs['id']):
            object_misp = self.get_object(obs_to_add)
            if object_misp:
                self.misp_event.add_object(object_misp)
            print(self.misp_event)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def get_object(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(**self.attribute)
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')

            return domain_ip_object

    def __get_relation(self, obj_yeti):
        typ_attribute = self.misp_mapping[obj_yeti['type']]
        attr_misp = {'value': obj_yeti['value']}
        if typ_attribute == 'ip-src' or typ_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == typ_attribute:
            return 'domain'
        elif 'hostname' == typ_attribute:
            return 'domain'

        print('Attribute %s' % attr_misp)
        return attr_misp

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return yeti_client.get_result()
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<|endoftext|>


<commit_before>__author__ = 'luigolas'

import numpy as np
import cv2
from package.utilities import safe_ln, FileNotFoundError
# from package.app import DB

CS_IIP = 1
CS_BGR = 2
CS_HSV = 3
CS_YCrCb = 4

colorspace_name = ["", "IIP", "BGR", "HSV", "YCrCb"]

iipA = np.asarray([[27.07439, -0.2280783, -1.806681],
                   [-5.646736, -7.722125, 12.86503],
                   [-4.163133, -4.579428, -4.576049]])

iipB = np.asarray([[0.9465229, 0.2946927, -0.1313419],
                   [-0.1179179, 0.9929960, 0.007371554],
                   [0.09230461, -0.04645794, 0.9946464]])

# iip_min = -576.559
# iip_min = -622.28125
# iip_max = 306.672
iip_min = -73.670
iip_max = 140.333

bgr2iipClip = 3  # Minimal value for RGB image before converting to IIP

hsvmax = [180, 256, 256]

class Image(np.ndarray):
    """
    Image class, basically an ndarray with imgname and colorspace parameters
    --To make it work with multiprocessing Queue it needs to be pickeable (reduce and getstate)
    ----More info: http://mail.scipy.org/pipermail/numpy-discussion/2007-April/027193.html

    :param imgname:
    :param colorspace:
    :return:
    :raise FileNotFoundError:
    """

    def __new__(cls, ndarray, colorspace=CS_BGR, imgname=None):
        if not isinstance(ndarray, np.ndarray):
            raise ValueError('Argument must be a valid numpy.ndarray')
        return ndarray.view(cls)

    def __init__(self, src, colorspace=CS_BGR, imgname=None):
        if not imgname and isinstance(src, Image):
            imgname = src.imgname
        self.imgname = imgname
        self.colorspace = colorspace
        # This _method should NOT affect DB

    def __deepcopy__(self, memo):
        # Allowing deepcopy of Images
        result = Image(self.copy(), self.colorspace, self.imgname)
        memo[id(self)] = result
        return result

    def __copy__(self):
        # Allowing copy of Images
        result = Image(self.copy(), self.colorspace, self.imgname)
        return result

    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.colorspace = getattr(obj, 'colorspace', CS_BGR)

    def __reduce__(self):
        # Enables pickling, needed for multiprocessing
        object_state = list(np.ndarray.__reduce__(self))
        subclass_state = (self.imgname, self.colorspace)
        object_state[2] = (object_state[2], subclass_state)
        return tuple(object_state)

    def __setstate__(self, state):
        # Enables pickling, needed for multiprocessing
        nd_state, own_state = state
        np.ndarray.__setstate__(self, nd_state)

        imgname, colorspace = own_state
        self.imgname, self.colorspace = imgname, colorspace

    @classmethod
    def from_filename(cls, imgname, colorspace=CS_BGR):
        """

        :param imgname:
        :param colorspace:
        :return: :raise FileNotFoundError:
        """
        img = cv2.imread(imgname)
        if img is None:
            raise FileNotFoundError("Image file not found at specified path")
        img = Image(img)
        img.colorspace = CS_BGR
        img.imgname = imgname
        if colorspace != CS_BGR:
            img = img.to_color_space(colorspace)
        return img

    def to_color_space(self, colorspace, normed=False):
        """

        :param colorspace:
        :return:
        """
        # Attempt to load from DB
        # try:
        #     if DB:
        #         img = DB[self.dbname(colorspace)]
        #         img = Image(img, colorspace)
        #         img.imgname = self.imgname
        #         return img
        # except FileNotFoundError:
        #     #Not in DataBase, continue calculating
        #     pass
        img = None
        if self.colorspace == CS_BGR:
            if colorspace == CS_IIP:
                img = self._bgr2iip()
            elif colorspace == CS_HSV:
                img = self._bgr2hsv(normed)
            elif colorspace == CS_YCrCb:
                img = self._bgr2YCrCb()
        elif self.colorspace == CS_YCrCb:
            if colorspace == CS_BGR:
                img = self._YCrCb2bgr()
        elif self.colorspace == CS_HSV:
            if colorspace == CS_BGR:
                img = self._hsv2bgr()
        # img.imgname = self.imgname
        #Save in Connection
        # if DB:
        #     DB[img.dbname()] = img
        return img

    def dbname(self):
        return abs(hash(self.tostring()))

    def _bgr2iip(self):
        # Convert to CV_32F3 , floating point in range 0.0 , 1.0
        # imgf32 = np.float32(self)
        # imgf32 = imgf32*1.0/255

        # Clip values to min value
        src = self.clip(bgr2iipClip)

        # src_xyz = cv2.cvtColor(imgf32, cv2.COLOR_BGR2XYZ)
        src_xyz = cv2.cvtColor(src, cv2.COLOR_BGR2XYZ)

        # img_iip = np.empty_like(src_xyz, np.float32)

        # http://stackoverflow.com/a/25922418/3337586
        img_iip = np.einsum('ij,klj->kli', iipB, src_xyz)
        img_iip = safe_ln(img_iip)
        img_iip = np.einsum('ij,klj->kli', iipA, img_iip)
        img_iip = img_iip.astype(np.float32)
        # for row_index, row in enumerate(src_xyz):
        #     for columm_index, element in enumerate(row):
        #         element = np.dot(iipB, element)
        #         element = safe_ln(element)  # element = np.log(element)
        #         element = np.dot(iipA, element)
        #         # element = (element - iip_min)/(iip_max - iip_min)  # Normalized to 0.0 ; 1.0
        #         # element = np.around(element, decimals=6)  # Remove some "extreme" precision
        #         img_iip[row_index][columm_index] = element
        return Image(img_iip, CS_IIP, self.imgname)

    def _bgr2hsv(self, normed=False):
        if normed:
            # Convert to CV_32F3 , floating point in range 0.0 , 1.0
            imgf32 = np.float32(self)
            imgf32 = imgf32 * 1.0 / 255
            img = cv2.cvtColor(imgf32, cv2.COLOR_BGR2HSV)
            img[:, :, 0] /= 360.
        else:
            img = cv2.cvtColor(self, cv2.COLOR_BGR2HSV)

        img = Image(img, CS_HSV, self.imgname)
        return img

    def _hsv2bgr(self, ):
        img = cv2.cvtColor(self, cv2.COLOR_HSV2BGR)
        img = Image(img, CS_BGR, self.imgname)
        return img

    def _bgr2YCrCb(self):
        img = cv2.cvtColor(self, cv2.COLOR_BGR2YCrCb)
        img = Image(img, CS_YCrCb, self.imgname)
        return img

    def _YCrCb2bgr(self):
        img = cv2.cvtColor(self, cv2.COLOR_YCrCb2BGR)
        img = Image(img, CS_BGR, self.imgname)
        return img
<commit_msg>Remove dbname<commit_after>__author__ = 'luigolas'

import numpy as np
import cv2
from package.utilities import safe_ln, FileNotFoundError
# from package.app import DB

CS_IIP = 1
CS_BGR = 2
CS_HSV = 3
CS_YCrCb = 4

colorspace_name = ["", "IIP", "BGR", "HSV", "YCrCb"]

iipA = np.asarray([[27.07439, -0.2280783, -1.806681],
                   [-5.646736, -7.722125, 12.86503],
                   [-4.163133, -4.579428, -4.576049]])

iipB = np.asarray([[0.9465229, 0.2946927, -0.1313419],
                   [-0.1179179, 0.9929960, 0.007371554],
                   [0.09230461, -0.04645794, 0.9946464]])

# iip_min = -576.559
# iip_min = -622.28125
# iip_max = 306.672
iip_min = -73.670
iip_max = 140.333

bgr2iipClip = 3  # Minimal value for RGB image before converting to IIP

hsvmax = [180, 256, 256]

class Image(np.ndarray):
    """
    Image class, basically an ndarray with imgname and colorspace parameters
    --To make it work with multiprocessing Queue it needs to be pickeable (reduce and getstate)
    ----More info: http://mail.scipy.org/pipermail/numpy-discussion/2007-April/027193.html

    :param imgname:
    :param colorspace:
    :return:
    :raise FileNotFoundError:
    """

    def __new__(cls, ndarray, colorspace=CS_BGR, imgname=None):
        if not isinstance(ndarray, np.ndarray):
            raise ValueError('Argument must be a valid numpy.ndarray')
        return ndarray.view(cls)

    def __init__(self, src, colorspace=CS_BGR, imgname=None):
        if not imgname and isinstance(src, Image):
            imgname = src.imgname
        self.imgname = imgname
        self.colorspace = colorspace
        # This _method should NOT affect DB

    def __deepcopy__(self, memo):
        # Allowing deepcopy of Images
        result = Image(self.copy(), self.colorspace, self.imgname)
        memo[id(self)] = result
        return result

    def __copy__(self):
        # Allowing copy of Images
        result = Image(self.copy(), self.colorspace, self.imgname)
        return result

    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.colorspace = getattr(obj, 'colorspace', CS_BGR)

    def __reduce__(self):
        # Enables pickling, needed for multiprocessing
        object_state = list(np.ndarray.__reduce__(self))
        subclass_state = (self.imgname, self.colorspace)
        object_state[2] = (object_state[2], subclass_state)
        return tuple(object_state)

    def __setstate__(self, state):
        # Enables pickling, needed for multiprocessing
        nd_state, own_state = state
        np.ndarray.__setstate__(self, nd_state)

        imgname, colorspace = own_state
        self.imgname, self.colorspace = imgname, colorspace

    @classmethod
    def from_filename(cls, imgname, colorspace=CS_BGR):
        """

        :param imgname:
        :param colorspace:
        :return: :raise FileNotFoundError:
        """
        img = cv2.imread(imgname)
        if img is None:
            raise FileNotFoundError("Image file not found at specified path")
        img = Image(img)
        img.colorspace = CS_BGR
        img.imgname = imgname
        if colorspace != CS_BGR:
            img = img.to_color_space(colorspace)
        return img

    def to_color_space(self, colorspace, normed=False):
        """

        :param colorspace:
        :return:
        """
        img = None
        if self.colorspace == CS_BGR:
            if colorspace == CS_IIP:
                img = self._bgr2iip()
            elif colorspace == CS_HSV:
                img = self._bgr2hsv(normed)
            elif colorspace == CS_YCrCb:
                img = self._bgr2YCrCb()
        elif self.colorspace == CS_YCrCb:
            if colorspace == CS_BGR:
                img = self._YCrCb2bgr()
        elif self.colorspace == CS_HSV:
            if colorspace == CS_BGR:
                img = self._hsv2bgr()
        return img

    def _bgr2iip(self):
        # Convert to CV_32F3 , floating point in range 0.0 , 1.0
        # imgf32 = np.float32(self)
        # imgf32 = imgf32*1.0/255

        # Clip values to min value
        src = self.clip(bgr2iipClip)

        # src_xyz = cv2.cvtColor(imgf32, cv2.COLOR_BGR2XYZ)
        src_xyz = cv2.cvtColor(src, cv2.COLOR_BGR2XYZ)

        # img_iip = np.empty_like(src_xyz, np.float32)

        # http://stackoverflow.com/a/25922418/3337586
        img_iip = np.einsum('ij,klj->kli', iipB, src_xyz)
        img_iip = safe_ln(img_iip)
        img_iip = np.einsum('ij,klj->kli', iipA, img_iip)
        img_iip = img_iip.astype(np.float32)
        # for row_index, row in enumerate(src_xyz):
        #     for columm_index, element in enumerate(row):
        #         element = np.dot(iipB, element)
        #         element = safe_ln(element)  # element = np.log(element)
        #         element = np.dot(iipA, element)
        #         # element = (element - iip_min)/(iip_max - iip_min)  # Normalized to 0.0 ; 1.0
        #         # element = np.around(element, decimals=6)  # Remove some "extreme" precision
        #         img_iip[row_index][columm_index] = element
        return Image(img_iip, CS_IIP, self.imgname)

    def _bgr2hsv(self, normed=False):
        if normed:
            # Convert to CV_32F3 , floating point in range 0.0 , 1.0
            imgf32 = np.float32(self)
            imgf32 = imgf32 * 1.0 / 255
            img = cv2.cvtColor(imgf32, cv2.COLOR_BGR2HSV)
            img[:, :, 0] /= 360.
        else:
            img = cv2.cvtColor(self, cv2.COLOR_BGR2HSV)

        img = Image(img, CS_HSV, self.imgname)
        return img

    def _hsv2bgr(self, ):
        img = cv2.cvtColor(self, cv2.COLOR_HSV2BGR)
        img = Image(img, CS_BGR, self.imgname)
        return img

    def _bgr2YCrCb(self):
        img = cv2.cvtColor(self, cv2.COLOR_BGR2YCrCb)
        img = Image(img, CS_YCrCb, self.imgname)
        return img

    def _YCrCb2bgr(self):
        img = cv2.cvtColor(self, cv2.COLOR_YCrCb2BGR)
        img = Image(img, CS_BGR, self.imgname)
        return img
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import logging
import os
import subprocess

import warnings
warnings.filterwarnings("ignore")

import datetime
import matplotlib
matplotlib.use('Agg')
import matplotlib.colorbar as colorbar
import matplotlib.colors as color
import matplotlib.pyplot as plt
import numpy as np
from scipy import ndimage

logger = logging.getLogger("sct.{}".format(__file__))

class QcImage(object):
    """
    Class used to create a .png file from a 2d image produced by the class "Slice"
    """
    _labels_regions = {'PONS': 50, 'MO': 51,
                       'C1': 1, 'C2': 2, 'C3': 3, 'C4': 4, 'C5': 5, 'C6': 6, 'C7': 7,
                       'T1': 8, 'T2': 9, 'T3': 10, 'T4': 11, 'T5': 12, 'T6': 13, 'T7': 14, 'T8': 15, 'T9': 16,
                       'T10': 17, 'T11': 18, 'T12': 19,
                       'L1': 20, 'L2': 21, 'L3': 22, 'L4': 23, 'L5': 24,
                       'S1': 25, 'S2': 26, 'S3': 27, 'S4': 28, 'S5': 29,
                       'Co': 30}
    _labels_color = ["#04663c", "#ff0000", "#50ff30",
                     "#ed1339", "#ffffff", "#e002e8",
                     "#ffee00", "#00c7ff", "#199f26",
                     "#563691", "#848545", "#ce2fe1",
                     "#2142a6", "#3edd76", "#c4c253",
                     "#e8618a", "#3128a3", "#1a41db",
                     "#939e41", "#3bec02", "#1c2c79",
                     "#18584e", "#b49992", "#e9e73a",
                     "#3b0e6e", "#6e856f", "#637394",
                     "#36e05b", "#530a1f", "#8179c4",
                     "#e1320c", "#52a4df", "#000ab5",
                     "#4a4242", "#0b53a5", "#b49c19",
                     "#50e7a9", "#bf5a42", "#fa8d8e",
                     "#83839a", "#320fef", "#82ffbf",
                     "#360ee7", "#551960", "#11371e",
                     "#e900c3", "#a21360", "#58a601",
                     "#811c90", "#235acf", "#49395d",
                     "#9f89b0", "#e08e08", "#3d2b54",
                     "#7d0434", "#fb1849", "#14aab4",
                     "#a22abd", "#d58240", "#ac2aff"]
    _seg_colormap = plt.cm.autumn

    def __init__(self, qc_report, interpolation, action_list):
        """

        Parameters
        ----------
        qc_report : QcReport
            The QC report object
        interpolation : str
            Type of interpolation used in matplotlib
        action_list : list of functions
            List of functions that generates a specific type of images
        """
        self.qc_report = qc_report
        self.interpolation = interpolation
        self.action_list = action_list

    """
    action_list contain the list of images that has to be generated.
    It can be seen as "figures" of matplotlib to be shown
    Ex: if 'colorbar' is in the list, the msct_qc process will generate a color bar in the "img" folder
    """

    def listed_seg(self, mask):
        img = np.rint(np.ma.masked_where(mask < 1, mask))
        fig = plt.imshow(img,
                         cmap=color.ListedColormap(self._labels_color),
                         norm=color.Normalize(vmin=0, vmax=len(self._labels_color)),
                         interpolation=self.interpolation,
                         alpha=1,
                         aspect=float(self.aspect_mask))
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def no_seg_seg(self, mask):
        values = np.ma.masked_equal(np.rint(mask), 0)
        fig = plt.imshow(values,
                         cmap=plt.cm.gray,
                         interpolation=self.interpolation,
                         aspect=self.aspect_mask)
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def sequential_seg(self, mask):
        values = np.ma.masked_equal(np.rint(mask), 0)
        fig = plt.imshow(values,
                         cmap=self._seg_colormap,
                         interpolation=self.interpolation,
                         aspect=self.aspect_mask)
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def label_vertebrae(self, mask):
        self.listed_seg(mask)
        ax = plt.gca()
        a = [0.0]
        data = mask
        for index, val in np.ndenumerate(data):
            if val not in a:
                a.append(val)
                index = int(val)
                if index in self._labels_regions.values():
                    color = self._labels_color[index]
                    x, y = ndimage.measurements.center_of_mass(np.where(data == val, data, 0))
                    label = self._labels_regions.keys()[list(self._labels_regions.values()).index(index)]
                    ax.text(y, x, label, color='black', weight='heavy', clip_on=True)
                    ax.text(y, x, label, color=color, clip_on=True)

    def colorbar(self):
        fig = plt.figure(figsize=(9, 1.5))
        ax = fig.add_axes([0.05, 0.80, 0.9, 0.15])
        colorbar.ColorbarBase(ax, cmap=self._seg_colormap, orientation='horizontal')
        return '{}_colorbar'.format(self.qc_report.img_base_name)

    def __call__(self, func):
        """wrapped function (f).

        In this case, it is the "mosaic" or "single" methods of the class "Slice"

        Parameters
        ----------
        func : function
            The wrapped function
        """

        def wrapped_f(sct_slice, *args):
            """

            Parameters
            ----------
            sct_slice : spinalcordtoolbox.report.slice:Slice
            args : list

            Returns
            -------

            """
            self.qc_report.slice_name = sct_slice.get_name()
            aspect_img, self.aspect_mask = sct_slice.aspect()
            self.qc_report.make_content_path()
            logger.info('QC: %s with %s slice', func.__name__, sct_slice.get_name())

            img, mask = func(sct_slice, *args)

            plt.figure(1)
            fig = plt.imshow(img, cmap=plt.cm.gray, interpolation=self.interpolation, aspect=float(aspect_img))
            fig.axes.get_xaxis().set_visible(False)
            fig.axes.get_yaxis().set_visible(False)
            self._save(self.qc_report.qc_params.abs_bkg_img_path())

            for action in self.action_list:
                logger.debug('Action List %s', action.__name__)
                plt.clf()
                plt.figure(1)
                action(self, mask)
                self._save(self.qc_report.qc_params.abs_overlay_img_path())
            plt.close()

            self.qc_report.update_description_file(img.shape)

        return wrapped_f

    def _save(self, img_path, format='png', bbox_inches='tight', pad_inches=0.00):
        """ Save the current figure into an image.

        Parameters
        ----------
        img_path : str
            path of the folder where the image is saved
        format : str
            image format
        bbox_inches : str
        pad_inches : float

        """
        logger.debug('Save image %s', img_path)
        plt.savefig(img_path,
                    format=format,
                    bbox_inches=bbox_inches,
                    pad_inches=pad_inches,
                    transparent=True,
                    dpi=600)

class Params(object):
    """Parses and stores the variables that will included into the QC details

    We derive the value of the contrast and subject name from the `input_file` path,
    by splitting it into `[subject]/[contrast]/input_file`
    """

    def __init__(self, input_file, command, args, orientation, dest_folder):
        """
        Parameters
        ----------
        input_file : str
            the input nifti file name
        command : str
            the command name
        args : str
            the command's arguments
        orientation : str
            The anatomical orientation
        dest_folder : str
            The absolute path of the QC root
        """
        abs_input_path = os.path.dirname(os.path.abspath(input_file))
        abs_input_path, contrast = os.path.split(abs_input_path)
        _, subject = os.path.split(abs_input_path)

        self.subject = subject
        self.contrast = contrast
        self.command = command
        self.args = args
        self.orientation = orientation
        self.root_folder = dest_folder
        self.mod_date = datetime.datetime.strftime(datetime.datetime.now(), '%Y_%m_%d_%H%M%S')

        self.qc_results = os.path.join(dest_folder, 'qc_results.json')
        self.bkg_img_path = os.path.join(subject, contrast, command, self.mod_date, 'bkg_img.png')
        self.overlay_img_path = os.path.join(subject, contrast, command, self.mod_date, 'overlay_img.png')

    def abs_bkg_img_path(self):
        return os.path.join(self.root_folder, self.bkg_img_path)

    def abs_overlay_img_path(self):
        return os.path.join(self.root_folder, self.overlay_img_path)

class QcReport(object):
    """This class generates the quality control report.

    It will also setup the folder structure so the report generator only needs to fetch the appropriate files.
    """

    def __init__(self, qc_params, usage):
        """
        Parameters
        ----------
        tool_name : str
            name of the sct tool being used. Is used to name the image file.
        qc_params : Params
            arguments of the "-param-qc" option in Terminal
        cmd_args : list of str
            the commands of the process being used to generate the images
        usage : str
             description of the process
        """
        self.tool_name = qc_params.command
        self.slice_name = qc_params.orientation
        self.qc_params = qc_params
        self.usage = usage

        import spinalcordtoolbox
        pardir = os.path.dirname(os.path.dirname(spinalcordtoolbox.__file__))
        self.assets_folder = os.path.join(pardir, 'assets')

        self.img_base_name = 'bkg_img'
        self.description_base_name = "qc_results"

    def make_content_path(self):
        """Creates the whole directory to contain the QC report

        :return: return "root folder of the report" and the "furthest folder path" containing the images
        """
        # make a new or update Qc directory
        target_img_folder = os.path.dirname(self.qc_params.abs_bkg_img_path())

        try:
            os.makedirs(target_img_folder)
        except OSError as err:
            if not os.path.isdir(target_img_folder):
                raise err

    def update_description_file(self, dimension):
        """Creates the description file with a JSON structure

        Parameters
        ----------
        dimension : (int, int)
            The dimension of the image frame
        """
        # get path of the toolbox
        try:
            cmd = 'git rev-parse --short HEAD'.split()
            git_hash = subprocess.check_output(cmd).strip('"')
        except (subprocess.CalledProcessError, OSError):
            git_hash = "N/A"

        output = {
            'command': self.qc_params.command,
            'version': git_hash,
            'args': ' '.join(self.qc_params.args),
            'subject': self.qc_params.subject,
            'contrast': self.qc_params.contrast,
            'orientation': self.qc_params.orientation,
            'background_img': self.qc_params.bkg_img_path,
            'overlay_img': self.qc_params.overlay_img_path,
            'dimension': '%dx%d' % dimension,
            'moddate': datetime.datetime.now().isoformat(' ')
        }
        logger.debug('Description file: %s', self.qc_params.qc_results)
        results = []
        if os.path.isfile(self.qc_params.qc_results):
            results = json.load(open(self.qc_params.qc_results, 'r'))
        results.append(output)
        json.dump(results, open(self.qc_params.qc_results, "w"), indent=2)
<commit_msg>Removed the version from the list of data collected for the QC reports (#1512)<commit_after># -*- coding: utf-8 -*-
import json
import logging
import os

import warnings
warnings.filterwarnings("ignore")

import datetime
import matplotlib
matplotlib.use('Agg')
import matplotlib.colorbar as colorbar
import matplotlib.colors as color
import matplotlib.pyplot as plt
import numpy as np
from scipy import ndimage

logger = logging.getLogger("sct.{}".format(__file__))

class QcImage(object):
    """
    Class used to create a .png file from a 2d image produced by the class "Slice"
    """
    _labels_regions = {'PONS': 50, 'MO': 51,
                       'C1': 1, 'C2': 2, 'C3': 3, 'C4': 4, 'C5': 5, 'C6': 6, 'C7': 7,
                       'T1': 8, 'T2': 9, 'T3': 10, 'T4': 11, 'T5': 12, 'T6': 13, 'T7': 14, 'T8': 15, 'T9': 16,
                       'T10': 17, 'T11': 18, 'T12': 19,
                       'L1': 20, 'L2': 21, 'L3': 22, 'L4': 23, 'L5': 24,
                       'S1': 25, 'S2': 26, 'S3': 27, 'S4': 28, 'S5': 29,
                       'Co': 30}
    _labels_color = ["#04663c", "#ff0000", "#50ff30",
                     "#ed1339", "#ffffff", "#e002e8",
                     "#ffee00", "#00c7ff", "#199f26",
                     "#563691", "#848545", "#ce2fe1",
                     "#2142a6", "#3edd76", "#c4c253",
                     "#e8618a", "#3128a3", "#1a41db",
                     "#939e41", "#3bec02", "#1c2c79",
                     "#18584e", "#b49992", "#e9e73a",
                     "#3b0e6e", "#6e856f", "#637394",
                     "#36e05b", "#530a1f", "#8179c4",
                     "#e1320c", "#52a4df", "#000ab5",
                     "#4a4242", "#0b53a5", "#b49c19",
                     "#50e7a9", "#bf5a42", "#fa8d8e",
                     "#83839a", "#320fef", "#82ffbf",
                     "#360ee7", "#551960", "#11371e",
                     "#e900c3", "#a21360", "#58a601",
                     "#811c90", "#235acf", "#49395d",
                     "#9f89b0", "#e08e08", "#3d2b54",
                     "#7d0434", "#fb1849", "#14aab4",
                     "#a22abd", "#d58240", "#ac2aff"]
    _seg_colormap = plt.cm.autumn

    def __init__(self, qc_report, interpolation, action_list):
        """

        Parameters
        ----------
        qc_report : QcReport
            The QC report object
        interpolation : str
            Type of interpolation used in matplotlib
        action_list : list of functions
            List of functions that generates a specific type of images
        """
        self.qc_report = qc_report
        self.interpolation = interpolation
        self.action_list = action_list

    """
    action_list contain the list of images that has to be generated.
    It can be seen as "figures" of matplotlib to be shown
    Ex: if 'colorbar' is in the list, the msct_qc process will generate a color bar in the "img" folder
    """

    def listed_seg(self, mask):
        img = np.rint(np.ma.masked_where(mask < 1, mask))
        fig = plt.imshow(img,
                         cmap=color.ListedColormap(self._labels_color),
                         norm=color.Normalize(vmin=0, vmax=len(self._labels_color)),
                         interpolation=self.interpolation,
                         alpha=1,
                         aspect=float(self.aspect_mask))
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def no_seg_seg(self, mask):
        values = np.ma.masked_equal(np.rint(mask), 0)
        fig = plt.imshow(values,
                         cmap=plt.cm.gray,
                         interpolation=self.interpolation,
                         aspect=self.aspect_mask)
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def sequential_seg(self, mask):
        values = np.ma.masked_equal(np.rint(mask), 0)
        fig = plt.imshow(values,
                         cmap=self._seg_colormap,
                         interpolation=self.interpolation,
                         aspect=self.aspect_mask)
        fig.axes.get_xaxis().set_visible(False)
        fig.axes.get_yaxis().set_visible(False)

    def label_vertebrae(self, mask):
        self.listed_seg(mask)
        ax = plt.gca()
        a = [0.0]
        data = mask
        for index, val in np.ndenumerate(data):
            if val not in a:
                a.append(val)
                index = int(val)
                if index in self._labels_regions.values():
                    color = self._labels_color[index]
                    x, y = ndimage.measurements.center_of_mass(np.where(data == val, data, 0))
                    label = self._labels_regions.keys()[list(self._labels_regions.values()).index(index)]
                    ax.text(y, x, label, color='black', weight='heavy', clip_on=True)
                    ax.text(y, x, label, color=color, clip_on=True)

    def colorbar(self):
        fig = plt.figure(figsize=(9, 1.5))
        ax = fig.add_axes([0.05, 0.80, 0.9, 0.15])
        colorbar.ColorbarBase(ax, cmap=self._seg_colormap, orientation='horizontal')
        return '{}_colorbar'.format(self.qc_report.img_base_name)

    def __call__(self, func):
        """wrapped function (f).

        In this case, it is the "mosaic" or "single" methods of the class "Slice"

        Parameters
        ----------
        func : function
            The wrapped function
        """

        def wrapped_f(sct_slice, *args):
            """

            Parameters
            ----------
            sct_slice : spinalcordtoolbox.report.slice:Slice
            args : list

            Returns
            -------

            """
            self.qc_report.slice_name = sct_slice.get_name()
            aspect_img, self.aspect_mask = sct_slice.aspect()
            self.qc_report.make_content_path()
            logger.info('QC: %s with %s slice', func.__name__, sct_slice.get_name())

            img, mask = func(sct_slice, *args)

            plt.figure(1)
            fig = plt.imshow(img, cmap=plt.cm.gray, interpolation=self.interpolation, aspect=float(aspect_img))
            fig.axes.get_xaxis().set_visible(False)
            fig.axes.get_yaxis().set_visible(False)
            self._save(self.qc_report.qc_params.abs_bkg_img_path())

            for action in self.action_list:
                logger.debug('Action List %s', action.__name__)
                plt.clf()
                plt.figure(1)
                action(self, mask)
                self._save(self.qc_report.qc_params.abs_overlay_img_path())
            plt.close()

            self.qc_report.update_description_file(img.shape)

        return wrapped_f

    def _save(self, img_path, format='png', bbox_inches='tight', pad_inches=0.00):
        """ Save the current figure into an image.

        Parameters
        ----------
        img_path : str
            path of the folder where the image is saved
        format : str
            image format
        bbox_inches : str
        pad_inches : float

        """
        logger.debug('Save image %s', img_path)
        plt.savefig(img_path,
                    format=format,
                    bbox_inches=bbox_inches,
                    pad_inches=pad_inches,
                    transparent=True,
                    dpi=600)

class Params(object):
    """Parses and stores the variables that will included into the QC details

    We derive the value of the contrast and subject name from the `input_file` path,
    by splitting it into `[subject]/[contrast]/input_file`
    """

    def __init__(self, input_file, command, args, orientation, dest_folder):
        """
        Parameters
        ----------
        input_file : str
            the input nifti file name
        command : str
            the command name
        args : str
            the command's arguments
        orientation : str
            The anatomical orientation
        dest_folder : str
            The absolute path of the QC root
        """
        abs_input_path = os.path.dirname(os.path.abspath(input_file))
        abs_input_path, contrast = os.path.split(abs_input_path)
        _, subject = os.path.split(abs_input_path)

        self.subject = subject
        self.contrast = contrast
        self.command = command
        self.args = args
        self.orientation = orientation
        self.root_folder = dest_folder
        self.mod_date = datetime.datetime.strftime(datetime.datetime.now(), '%Y_%m_%d_%H%M%S')

        self.qc_results = os.path.join(dest_folder, 'qc_results.json')
        self.bkg_img_path = os.path.join(subject, contrast, command, self.mod_date, 'bkg_img.png')
        self.overlay_img_path = os.path.join(subject, contrast, command, self.mod_date, 'overlay_img.png')

    def abs_bkg_img_path(self):
        return os.path.join(self.root_folder, self.bkg_img_path)

    def abs_overlay_img_path(self):
        return os.path.join(self.root_folder, self.overlay_img_path)

class QcReport(object):
    """This class generates the quality control report.

    It will also setup the folder structure so the report generator only needs to fetch the appropriate files.
    """

    def __init__(self, qc_params, usage):
        """
        Parameters
        ----------
        tool_name : str
            name of the sct tool being used. Is used to name the image file.
        qc_params : Params
            arguments of the "-param-qc" option in Terminal
        cmd_args : list of str
            the commands of the process being used to generate the images
        usage : str
             description of the process
        """
        self.tool_name = qc_params.command
        self.slice_name = qc_params.orientation
        self.qc_params = qc_params
        self.usage = usage

        import spinalcordtoolbox
        pardir = os.path.dirname(os.path.dirname(spinalcordtoolbox.__file__))
        self.assets_folder = os.path.join(pardir, 'assets')

        self.img_base_name = 'bkg_img'
        self.description_base_name = "qc_results"

    def make_content_path(self):
        """Creates the whole directory to contain the QC report

        :return: return "root folder of the report" and the "furthest folder path" containing the images
        """
        # make a new or update Qc directory
        target_img_folder = os.path.dirname(self.qc_params.abs_bkg_img_path())

        try:
            os.makedirs(target_img_folder)
        except OSError as err:
            if not os.path.isdir(target_img_folder):
                raise err

    def update_description_file(self, dimension):
        """Creates the description file with a JSON structure

        Parameters
        ----------
        dimension : (int, int)
            The dimension of the image frame
        """
        # get path of the toolbox

        output = {
            'command': self.qc_params.command,
            'args': ' '.join(self.qc_params.args),
            'subject': self.qc_params.subject,
            'contrast': self.qc_params.contrast,
            'orientation': self.qc_params.orientation,
            'background_img': self.qc_params.bkg_img_path,
            'overlay_img': self.qc_params.overlay_img_path,
            'dimension': '%dx%d' % dimension,
            'moddate': datetime.datetime.now().isoformat(' ')
        }
        logger.debug('Description file: %s', self.qc_params.qc_results)
        results = []
        if os.path.isfile(self.qc_params.qc_results):
            results = json.load(open(self.qc_params.qc_results, 'r'))
        results.append(output)
        json.dump(results, open(self.qc_params.qc_results, "w"), indent=2)
<|endoftext|>


<commit_before>import importlib
import json

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.db.models import Q, Count
from django.http import HttpResponseRedirect, HttpResponse, HttpResponseForbidden
from django.shortcuts import get_object_or_404, render, redirect
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt

from grid.models import Grid
from homepage.models import Dpotw, Gotw
from package.forms import PackageForm, PackageExampleForm, DocumentationForm
from package.models import Category, Package, PackageExample
from package.repos import get_all_repos

from .utils import quote_plus

def repo_data_for_js():
    repos = [handler.serialize() for handler in get_all_repos()]
    return json.dumps(repos)

def get_form_class(form_name):
    bits = form_name.split('.')
    form_module_name = '.'.join(bits[:-1])
    form_module = importlib.import_module(form_module_name)
    form_name = bits[-1]
    return getattr(form_module, form_name)

@login_required
def add_package(request, template_name="package/package_form.html"):

    if not request.user.profile.can_add_package:
        return HttpResponseForbidden("permission denied")

    new_package = Package()
    form = PackageForm(request.POST or None, instance=new_package)

    if form.is_valid():
        new_package = form.save()
        new_package.created_by = request.user
        new_package.last_modified_by = request.user
        new_package.save()
        #new_package.fetch_metadata()
        #new_package.fetch_commits()

        return HttpResponseRedirect(reverse("package", kwargs={"slug": new_package.slug}))

    return render(request, template_name, {
        "form": form,
        "repo_data": repo_data_for_js(),
        "action": "add",
        })

@login_required
def edit_package(request, slug, template_name="package/package_form.html"):

    if not request.user.profile.can_edit_package:
        return HttpResponseForbidden("permission denied")

    package = get_object_or_404(Package, slug=slug)
    form = PackageForm(request.POST or None, instance=package)

    if form.is_valid():
        modified_package = form.save()
        modified_package.last_modified_by = request.user
        modified_package.save()
        messages.add_message(request, messages.INFO, 'Package updated successfully')
        return HttpResponseRedirect(reverse("package", kwargs={"slug": modified_package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package,
        "repo_data": repo_data_for_js(),
        "action": "edit",
        })

@login_required
def update_package(request, slug):

    package = get_object_or_404(Package, slug=slug)
    package.fetch_metadata()
    package.fetch_commits()
    package.last_fetched = timezone.now()
    messages.add_message(request, messages.INFO, 'Package updated successfully')

    return HttpResponseRedirect(reverse("package", kwargs={"slug": package.slug}))

@login_required
def add_example(request, slug, template_name="package/add_example.html"):

    package = get_object_or_404(Package, slug=slug)
    new_package_example = PackageExample()
    form = PackageExampleForm(request.POST or None, instance=new_package_example)

    if form.is_valid():
        package_example = PackageExample(package=package,
                title=request.POST["title"],
                url=request.POST["url"])
        package_example.save()
        return HttpResponseRedirect(reverse("package", kwargs={"slug": package_example.package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package
        })

@login_required
def edit_example(request, slug, id, template_name="package/edit_example.html"):

    package_example = get_object_or_404(PackageExample, id=id)
    form = PackageExampleForm(request.POST or None, instance=package_example)

    if form.is_valid():
        form.save()
        return HttpResponseRedirect(reverse("package", kwargs={"slug": package_example.package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package_example.package
        })

def package_autocomplete(request):
    """
    Provides Package matching based on matches of the beginning
    """
    titles = []
    q = request.GET.get("q", "")
    if q:
        titles = (x.title for x in Package.objects.filter(title__istartswith=q))

    response = HttpResponse("\n".join(titles))

    setattr(response, "djangologging.suppress_output", True)
    return response

def category(request, slug, template_name="package/category.html"):
    category = get_object_or_404(Category, slug=slug)
    packages = category.package_set.select_related().annotate(usage_count=Count("usage")).order_by("-repo_watchers", "title")
    return render(request, template_name, {
        "category": category,
        "packages": packages,
        }
    )

def ajax_package_list(request, template_name="package/ajax_package_list.html"):
    q = request.GET.get("q", "")
    packages = []
    if q:
        _dash = "%s-%s" % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        _space = "%s %s" % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        _underscore = '%s_%s' % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        packages = Package.objects.filter(
                        Q(title__istartswith=q) |
                        Q(title__istartswith=_dash) |
                        Q(title__istartswith=_space) |
                        Q(title__istartswith=_underscore)
                    )

    packages_already_added_list = []
    grid_slug = request.GET.get("grid", "")
    if packages and grid_slug:
        grids = Grid.objects.filter(slug=grid_slug)
        if grids:
            grid = grids[0]
            packages_already_added_list = [x['slug'] for x in grid.packages.all().values('slug')]
            new_packages = tuple(packages.exclude(slug__in=packages_already_added_list))[:20]
            number_of_packages = len(new_packages)
            if number_of_packages < 20:
                try:
                    old_packages = packages.filter(slug__in=packages_already_added_list)[:20 - number_of_packages]
                except AssertionError:
                    old_packages = None

                if old_packages:
                    old_packages = tuple(old_packages)
                    packages = new_packages + old_packages
            else:
                packages = new_packages

    return render(request, template_name, {
        "packages": packages,
        'packages_already_added_list': packages_already_added_list,
        }
    )

@login_required
def usage(request, slug, action):
    success = False
    package = get_object_or_404(Package, slug=slug)

    # Update the current user's usage of the given package as specified by the
    # request.
    if package.usage.filter(username=request.user.username):
        if action.lower() == 'add':
            # The user is already using the package
            success = True
            change = 0
        else:
            # If the action was not add and the user has already specified
            # they are a use the package then remove their usage.
            package.usage.remove(request.user)
            success = True
            change = -1
    else:
        if action.lower() == 'lower':
            # The user is not using the package
            success = True
            change = 0
        else:
            # If the action was not lower and the user is not already using
            # the package then add their usage.
            package.usage.add(request.user)
            success = True
            change = 1

    # Invalidate the cache of this users's used_packages_list.
    if change == 1 or change == -1:
        cache_key = "sitewide_used_packages_list_%s" % request.user.pk
        cache.delete(cache_key)
        package.grid_clear_detail_template_cache()

    # Return an ajax-appropriate response if necessary
    if request.is_ajax():
        response = {'success': success}
        if success:
            response['change'] = change

        return HttpResponse(json.dumps(response))

    # Intelligently determine the URL to redirect the user to based on the
    # available information.
    next = request.GET.get('next') or request.META.get("HTTP_REFERER") or reverse("package", kwargs={"slug": package.slug})
    return HttpResponseRedirect(next)

def python3_list(request, template_name="package/python3_list.html"):
    packages = Package.objects.filter(version__supports_python3=True).distinct()
    packages = packages.order_by("-pypi_downloads", "-repo_watchers", "title")

    if request.GET.get('sort') and request.GET.get('sort') not in "category, category_id, commit, commit_list, created, created_by, created_by_id, documentation_url, dpotw, grid, gridpackage, id, last_fetched, last_modified_by, last_modified_by_id, modified, packageexample, participants, pypi_downloads, pypi_url, repo_description, repo_forks, repo_url, repo_watchers, slug, title, usage, version".split(','):
        # Some people have cached older versions of this view
        request.GET = request.GET.copy()
        del request.GET['sort']

    return render(
        request,
        template_name, {
            "packages": packages
        }
    )

def package_list(request, template_name="package/package_list.html"):

    categories = []
    for category in Category.objects.annotate(package_count=Count("package")):
        element = {
            "title": category.title,
            "description": category.description,
            "count": category.package_count,
            "slug": category.slug,
            "title_plural": category.title_plural,
            "show_pypi": category.show_pypi,
            "packages": category.package_set.annotate(usage_count=Count("usage")).order_by("-pypi_downloads", "-repo_watchers", "title")[:9]
        }
        categories.append(element)

    return render(
        request,
        template_name, {
            "categories": categories,
            "dpotw": Dpotw.objects.get_current(),
            "gotw": Gotw.objects.get_current(),
        }
    )

def package_detail(request, slug, template_name="package/package.html"):

    package = get_object_or_404(Package, slug=slug)
    no_development = package.no_development
    try:
        if package.category == Category.objects.get(slug='projects'):
            # projects get a bye because they are a website
            pypi_ancient = False
            pypi_no_release = False
        else:
            pypi_ancient = package.pypi_ancient
            pypi_no_release = package.pypi_ancient is None
        warnings = no_development or pypi_ancient or pypi_no_release
    except Category.DoesNotExist:
        pypi_ancient = False
        pypi_no_release = False
        warnings = no_development

    if request.GET.get("message"):
        messages.add_message(request, messages.INFO, request.GET.get("message"))

    return render(request, template_name,
            dict(
                package=package,
                pypi_ancient=pypi_ancient,
                no_development=no_development,
                pypi_no_release=pypi_no_release,
                warnings=warnings,
                latest_version=package.last_released(),
                repo=package.repo
            )
        )

def int_or_0(value):
    try:
        return int(value)
    except ValueError:
        return 0

@login_required
def post_data(request, slug):
    # if request.method == "POST":
        # try:
        #     # TODO Do this this with a form, really. Duh!
        #     package.repo_watchers = int_or_0(request.POST.get("repo_watchers"))
        #     package.repo_forks = int_or_0(request.POST.get("repo_forks"))
        #     package.repo_description = request.POST.get("repo_description")
        #     package.participants = request.POST.get('contributors')
        #     package.fetch_commits()  # also saves
        # except Exception as e:
        #     print e
    package = get_object_or_404(Package, slug=slug)
    package.fetch_pypi_data()
    package.repo.fetch_metadata(package)
    package.repo.fetch_commits(package)
    package.last_fetched = timezone.now()
    package.save()
    return HttpResponseRedirect(reverse("package", kwargs={"slug": package.slug}))

@login_required
def edit_documentation(request, slug, template_name="package/documentation_form.html"):
    package = get_object_or_404(Package, slug=slug)
    form = DocumentationForm(request.POST or None, instance=package)
    if form.is_valid():
        form.save()
        messages.add_message(request, messages.INFO, 'Package documentation updated successfully')
        return redirect(package)
    return render(request, template_name,
            dict(
                package=package,
                form=form
            )
        )

@csrf_exempt
def github_webhook(request):
    if request.method == "POST":
        data = json.loads(request.POST['payload'])

        # Webhook Test
        if "zen" in data:
            return HttpResponse(data['hook_id'])

        repo_url = data['repository']['url']

        # service test
        if repo_url == "http://github.com/mojombo/grit":
            return HttpResponse("Service Test pass")

        package = get_object_or_404(Package, repo_url=repo_url)
        package.repo.fetch_commits(package)
        package.last_fetched = timezone.now()
        package.save()
    return HttpResponse()
<commit_msg>Another python3 list fix<commit_after>import importlib
import json

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.db.models import Q, Count
from django.http import HttpResponseRedirect, HttpResponse, HttpResponseForbidden
from django.shortcuts import get_object_or_404, render, redirect
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt

from grid.models import Grid
from homepage.models import Dpotw, Gotw
from package.forms import PackageForm, PackageExampleForm, DocumentationForm
from package.models import Category, Package, PackageExample
from package.repos import get_all_repos

from .utils import quote_plus

def repo_data_for_js():
    repos = [handler.serialize() for handler in get_all_repos()]
    return json.dumps(repos)

def get_form_class(form_name):
    bits = form_name.split('.')
    form_module_name = '.'.join(bits[:-1])
    form_module = importlib.import_module(form_module_name)
    form_name = bits[-1]
    return getattr(form_module, form_name)

@login_required
def add_package(request, template_name="package/package_form.html"):

    if not request.user.profile.can_add_package:
        return HttpResponseForbidden("permission denied")

    new_package = Package()
    form = PackageForm(request.POST or None, instance=new_package)

    if form.is_valid():
        new_package = form.save()
        new_package.created_by = request.user
        new_package.last_modified_by = request.user
        new_package.save()
        #new_package.fetch_metadata()
        #new_package.fetch_commits()

        return HttpResponseRedirect(reverse("package", kwargs={"slug": new_package.slug}))

    return render(request, template_name, {
        "form": form,
        "repo_data": repo_data_for_js(),
        "action": "add",
        })

@login_required
def edit_package(request, slug, template_name="package/package_form.html"):

    if not request.user.profile.can_edit_package:
        return HttpResponseForbidden("permission denied")

    package = get_object_or_404(Package, slug=slug)
    form = PackageForm(request.POST or None, instance=package)

    if form.is_valid():
        modified_package = form.save()
        modified_package.last_modified_by = request.user
        modified_package.save()
        messages.add_message(request, messages.INFO, 'Package updated successfully')
        return HttpResponseRedirect(reverse("package", kwargs={"slug": modified_package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package,
        "repo_data": repo_data_for_js(),
        "action": "edit",
        })

@login_required
def update_package(request, slug):

    package = get_object_or_404(Package, slug=slug)
    package.fetch_metadata()
    package.fetch_commits()
    package.last_fetched = timezone.now()
    messages.add_message(request, messages.INFO, 'Package updated successfully')

    return HttpResponseRedirect(reverse("package", kwargs={"slug": package.slug}))

@login_required
def add_example(request, slug, template_name="package/add_example.html"):

    package = get_object_or_404(Package, slug=slug)
    new_package_example = PackageExample()
    form = PackageExampleForm(request.POST or None, instance=new_package_example)

    if form.is_valid():
        package_example = PackageExample(package=package,
                title=request.POST["title"],
                url=request.POST["url"])
        package_example.save()
        return HttpResponseRedirect(reverse("package", kwargs={"slug": package_example.package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package
        })

@login_required
def edit_example(request, slug, id, template_name="package/edit_example.html"):

    package_example = get_object_or_404(PackageExample, id=id)
    form = PackageExampleForm(request.POST or None, instance=package_example)

    if form.is_valid():
        form.save()
        return HttpResponseRedirect(reverse("package", kwargs={"slug": package_example.package.slug}))

    return render(request, template_name, {
        "form": form,
        "package": package_example.package
        })

def package_autocomplete(request):
    """
    Provides Package matching based on matches of the beginning
    """
    titles = []
    q = request.GET.get("q", "")
    if q:
        titles = (x.title for x in Package.objects.filter(title__istartswith=q))

    response = HttpResponse("\n".join(titles))

    setattr(response, "djangologging.suppress_output", True)
    return response

def category(request, slug, template_name="package/category.html"):
    category = get_object_or_404(Category, slug=slug)
    packages = category.package_set.select_related().annotate(usage_count=Count("usage")).order_by("-repo_watchers", "title")
    return render(request, template_name, {
        "category": category,
        "packages": packages,
        }
    )

def ajax_package_list(request, template_name="package/ajax_package_list.html"):
    q = request.GET.get("q", "")
    packages = []
    if q:
        _dash = "%s-%s" % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        _space = "%s %s" % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        _underscore = '%s_%s' % (settings.PACKAGINATOR_SEARCH_PREFIX, q)
        packages = Package.objects.filter(
                        Q(title__istartswith=q) |
                        Q(title__istartswith=_dash) |
                        Q(title__istartswith=_space) |
                        Q(title__istartswith=_underscore)
                    )

    packages_already_added_list = []
    grid_slug = request.GET.get("grid", "")
    if packages and grid_slug:
        grids = Grid.objects.filter(slug=grid_slug)
        if grids:
            grid = grids[0]
            packages_already_added_list = [x['slug'] for x in grid.packages.all().values('slug')]
            new_packages = tuple(packages.exclude(slug__in=packages_already_added_list))[:20]
            number_of_packages = len(new_packages)
            if number_of_packages < 20:
                try:
                    old_packages = packages.filter(slug__in=packages_already_added_list)[:20 - number_of_packages]
                except AssertionError:
                    old_packages = None

                if old_packages:
                    old_packages = tuple(old_packages)
                    packages = new_packages + old_packages
            else:
                packages = new_packages

    return render(request, template_name, {
        "packages": packages,
        'packages_already_added_list': packages_already_added_list,
        }
    )

@login_required
def usage(request, slug, action):
    success = False
    package = get_object_or_404(Package, slug=slug)

    # Update the current user's usage of the given package as specified by the
    # request.
    if package.usage.filter(username=request.user.username):
        if action.lower() == 'add':
            # The user is already using the package
            success = True
            change = 0
        else:
            # If the action was not add and the user has already specified
            # they are a use the package then remove their usage.
            package.usage.remove(request.user)
            success = True
            change = -1
    else:
        if action.lower() == 'lower':
            # The user is not using the package
            success = True
            change = 0
        else:
            # If the action was not lower and the user is not already using
            # the package then add their usage.
            package.usage.add(request.user)
            success = True
            change = 1

    # Invalidate the cache of this users's used_packages_list.
    if change == 1 or change == -1:
        cache_key = "sitewide_used_packages_list_%s" % request.user.pk
        cache.delete(cache_key)
        package.grid_clear_detail_template_cache()

    # Return an ajax-appropriate response if necessary
    if request.is_ajax():
        response = {'success': success}
        if success:
            response['change'] = change

        return HttpResponse(json.dumps(response))

    # Intelligently determine the URL to redirect the user to based on the
    # available information.
    next = request.GET.get('next') or request.META.get("HTTP_REFERER") or reverse("package", kwargs={"slug": package.slug})
    return HttpResponseRedirect(next)

def python3_list(request, template_name="package/python3_list.html"):
    packages = Package.objects.filter(version__supports_python3=True).distinct()
    packages = packages.order_by("-pypi_downloads", "-repo_watchers", "title")

    values = "category, category_id, commit, commit_list, created, created_by, created_by_id, documentation_url, dpotw, grid, gridpackage, id, last_fetched, last_modified_by, last_modified_by_id, modified, packageexample, participants, pypi_downloads, pypi_url, repo_description, repo_forks, repo_url, repo_watchers, slug, title, usage, version".split(',')
    values = [x.strip() for x in values]
    if request.GET.get('sort') and request.GET.get('sort') not in values:
        # Some people have cached older versions of this view
        request.GET = request.GET.copy()
        del request.GET['sort']

    return render(
        request,
        template_name, {
            "packages": packages
        }
    )

def package_list(request, template_name="package/package_list.html"):

    categories = []
    for category in Category.objects.annotate(package_count=Count("package")):
        element = {
            "title": category.title,
            "description": category.description,
            "count": category.package_count,
            "slug": category.slug,
            "title_plural": category.title_plural,
            "show_pypi": category.show_pypi,
            "packages": category.package_set.annotate(usage_count=Count("usage")).order_by("-pypi_downloads", "-repo_watchers", "title")[:9]
        }
        categories.append(element)

    return render(
        request,
        template_name, {
            "categories": categories,
            "dpotw": Dpotw.objects.get_current(),
            "gotw": Gotw.objects.get_current(),
        }
    )

def package_detail(request, slug, template_name="package/package.html"):

    package = get_object_or_404(Package, slug=slug)
    no_development = package.no_development
    try:
        if package.category == Category.objects.get(slug='projects'):
            # projects get a bye because they are a website
            pypi_ancient = False
            pypi_no_release = False
        else:
            pypi_ancient = package.pypi_ancient
            pypi_no_release = package.pypi_ancient is None
        warnings = no_development or pypi_ancient or pypi_no_release
    except Category.DoesNotExist:
        pypi_ancient = False
        pypi_no_release = False
        warnings = no_development

    if request.GET.get("message"):
        messages.add_message(request, messages.INFO, request.GET.get("message"))

    return render(request, template_name,
            dict(
                package=package,
                pypi_ancient=pypi_ancient,
                no_development=no_development,
                pypi_no_release=pypi_no_release,
                warnings=warnings,
                latest_version=package.last_released(),
                repo=package.repo
            )
        )

def int_or_0(value):
    try:
        return int(value)
    except ValueError:
        return 0

@login_required
def post_data(request, slug):
    # if request.method == "POST":
        # try:
        #     # TODO Do this this with a form, really. Duh!
        #     package.repo_watchers = int_or_0(request.POST.get("repo_watchers"))
        #     package.repo_forks = int_or_0(request.POST.get("repo_forks"))
        #     package.repo_description = request.POST.get("repo_description")
        #     package.participants = request.POST.get('contributors')
        #     package.fetch_commits()  # also saves
        # except Exception as e:
        #     print e
    package = get_object_or_404(Package, slug=slug)
    package.fetch_pypi_data()
    package.repo.fetch_metadata(package)
    package.repo.fetch_commits(package)
    package.last_fetched = timezone.now()
    package.save()
    return HttpResponseRedirect(reverse("package", kwargs={"slug": package.slug}))

@login_required
def edit_documentation(request, slug, template_name="package/documentation_form.html"):
    package = get_object_or_404(Package, slug=slug)
    form = DocumentationForm(request.POST or None, instance=package)
    if form.is_valid():
        form.save()
        messages.add_message(request, messages.INFO, 'Package documentation updated successfully')
        return redirect(package)
    return render(request, template_name,
            dict(
                package=package,
                form=form
            )
        )

@csrf_exempt
def github_webhook(request):
    if request.method == "POST":
        data = json.loads(request.POST['payload'])

        # Webhook Test
        if "zen" in data:
            return HttpResponse(data['hook_id'])

        repo_url = data['repository']['url']

        # service test
        if repo_url == "http://github.com/mojombo/grit":
            return HttpResponse("Service Test pass")

        package = get_object_or_404(Package, repo_url=repo_url)
        package.repo.fetch_commits(package)
        package.last_fetched = timezone.now()
        package.save()
    return HttpResponse()
<|endoftext|>