<commit_before># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from argparse import ArgumentParser
import logging
import distutils.util

from cytomine.cytomine import Cytomine, _cytomine_parameter_name_synonyms
from cytomine.models.project import Project
from cytomine.models.software import Software, Job, JobParameter, SoftwareParameterCollection
from cytomine.models.user import User

__author__ = "Begon Jean-Michel <jm.begon@gmail.com>"
__contributors = ["Mormont Romain <romainmormont@gmail.com", "Rubens Ulysse <urubens@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

def _inferred_number_type(v):
    """Return the inferred type for the given string. The string must contain either an interger or a float.
    """
    try:
        return int(v)
    except ValueError:
        return float(v)

def _convert_type(_type):
    # Not trying too hard for the finding types (list considered as string and number as floats)
    # Just want a first validation (when possible) with argparse.
    # Advanced checking is advised in actual job implementation.
    return {
        "Number": _inferred_number_type,
        "String": str,
        "Boolean": _to_bool,
        "Domain": int,
        "List": str,
        "ListDomain": str,
        "Date": str
    }[_type]

def _to_bool(v):
    """
    Convert the value to boolean.

    Parameters
    ----------
    v: object
        A object to convert to a boolean value.

    Returns
    -------
    b: bool
        The boolean value
    """
    if isinstance(v, str):
        return bool(distutils.util.strtobool(v))
    else:
        return bool(v)

def _software_params_to_argparse(parameters):
    """
    Converts a SoftwareParameterCollection into an ArgumentParser object.

    Parameters
    ----------
    parameters: SoftwareParameterCollection
        The software parameters
    Returns
    -------
    argparse: ArgumentParser
        An initialized argument parser
    """
    # Check software parameters
    argparse = ArgumentParser()
    for parameter in parameters:
        python_type = _convert_type(parameter.type)
        arg_desc = {"dest": parameter.name, "required": parameter.required, "type": python_type, "help": ""}  # TODO add help
        if parameter.type == "Boolean":
            default = _to_bool(parameter.defaultParamValue)
            arg_desc["nargs"] = "?"
            arg_desc["const"] = not default
            arg_desc["default"] = default
        else:
            arg_desc["default"] = None if parameter.defaultParamValue is None else python_type(parameter.defaultParamValue)
        argparse.add_argument(*_cytomine_parameter_name_synonyms(parameter.name), **arg_desc)
    return argparse

class CytomineJob(Cytomine):
    """
    ===========
    CytomineJob
    ===========
    A :class:`CytomineJob` represents a job in the cytomine context.
    This class does nothing by itself. It is just supposed to be
    inherited from or incorporated in any class representing
    a software/job registered in the Cytomine server

    Usage
    -----
    Either use the :meth:`connect`/meth:`close`methods or use it with a
    with statement:

    with CytomineJob(...) as job:
         do_your_stuff()

    Constructor parameters
    ----------------------
    software_id : int
        The identifier of the software on the Cytomine server
    project_id : int
        The identifier of the project to process on the Cytomine server
    parameters: Namespace
        A namespace mapping job parameters with their values (must not contain any other key than
        the parameters names)
    """
    def __init__(self, host, public_key, private_key, software_id, project_id, parameters=None, **kwargs):
        super(CytomineJob, self).__init__(host, public_key, private_key, **kwargs)
        self._job = None
        self._project = Project().fetch(project_id)
        self._software = Software().fetch(software_id)
        self._job_done = False
        self._parameters = parameters

    @classmethod
    def from_cli(cls, argv, **kwargs):
        # Parse CytomineJob constructor parameters
        argparse = cls._add_cytomine_cli_args(ArgumentParser())
        argparse.add_argument(*_cytomine_parameter_name_synonyms("software_id"),
                              dest="software_id", type=int, help="The Cytomine software id.", required=True)
        argparse.add_argument(*_cytomine_parameter_name_synonyms("project_id"),
                              dest="project_id", type=int, help="The Cytomine project id.", required=True)
        base_params, _ = argparse.parse_known_args(args=argv)

        log_level = base_params.verbose
        if base_params.log_level is not None:
            log_level = logging.getLevelName(base_params.log_level)

        cytomine_job = CytomineJob(
            host=base_params.host,
            public_key=base_params.public_key,
            private_key=base_params.private_key,
            software_id=base_params.software_id,
            project_id=base_params.project_id,
            parameters=None,
            verbose=log_level,
            **kwargs
        )

        # Parse and set job parameters
        params_collection = SoftwareParameterCollection(
            filters={"software": cytomine_job.software.id},
            withSetByServer=True
        ).fetch()
        cytomine_job.parameters, _ = _software_params_to_argparse(params_collection).parse_known_args(argv)
        return cytomine_job

    @property
    def job(self):
        """Return the job model

        Return
        ------
        job: cytomine.Job
            The job model
        """
        return self._job

    @property
    def project(self):
        """
        Protected method

        Return
        ------
        project_id : int
            The id of the project
        """
        return self._project

    @property
    def software(self):
        """
        Protected method

        Return
        ------
        software_id : int
            The id of the software
        """
        return self._software

    @property
    def parameters(self):
        """
        Return
        ------
        parameters : dict
            The id of the software
        """
        return self._parameters

    @parameters.setter
    def parameters(self, parameters):
        """
        Protected method.

        Parameters
        ----------
        parameters: dict
            Dictionnary mapping parameters name with their values.
        """
        self._parameters = parameters

    def done(self, status=True):
        """
        Indicates whether the job is finished or not

        Parameters
        ----------
        status : bool
            Whether the process is finished
        """
        self._job_done = status

    def is_done(self):
        """
        Return
        ------
        job_status : bool
            Whether the process is finished
        """
        return self._job_done

    def start(self):
        """
        Connect to the Cytomine server and switch to job connection
        Incurs dataflows
        """

        run_by_ui = False
        if not self.current_user.algo:
            # If user connects as a human (CLI execution)
            self._job = Job(self._project.id, self._software.id).save()
            user_job = User().fetch(self._job.userJob)
            self.set_credentials(user_job.publicKey, user_job.privateKey)
        else:
            # If the user executes the job through the Cytomine interface
            self._job = Job().fetch(self.current_user.job)
            run_by_ui = True

        # set job state to RUNNING
        self._job.status = Job.RUNNING
        self._job.update()

        # add software parameters
        if not run_by_ui and self._parameters is not None:
            parameters = vars(self._parameters)
            for software_param in self._software.parameters:
                name = software_param["name"]
                if name in parameters:
                    value = parameters[name]
                else:
                    value = software_param["defaultParamValue"]

                JobParameter(self._job.id, software_param["id"], value).save()

    def close(self, value):
        """
        Notify the Cytomine server of the job's end
        Incurs a dataflows
        """
        if value is None:
            status = Job.TERMINATED
            status_comment = "Job successfully terminated"
        else:
            status = Job.FAILED
            status_comment = str(value)[:255]

        self._job.status = status
        self._job.statusComment = status_comment
        self._job.update()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.close(value)
        return False

    def logger(self, start=0, end=100, update_period=None):
        """Return a logger for the current job."""
        return CytomineJobLogger(self, progress_start=start, progress_end=end, update_period=update_period)

    def monitor(self, iterable, start=0, end=100, update_period=None, prefix=""):
        """Return a monitor for the current job"""
        return self.logger(start=start, end=end, update_period=update_period).monitor(iterable, prefix=prefix)

class CytomineJobLogger(object):
    def __init__(self, cytomine_job: CytomineJob, progress_start=0, progress_end=100, update_period=None):
        """A logger serves as intermediary between the job implementation and the job status update requests.

        Parameters
        ----------
        cytomine_job: CytomineJob
            The job
        progress_start: float (range: [0.0, 100.0[)
            Progress at which the logger should start
        progress_end: float (range: ]0.0, 100.0])
            Progress at which the logger should stop
        update_period: int, float
            The number of iteration to wait before actually updating the status. For for updating at each iteration
        """
        self._cytomine_job = cytomine_job
        self._start = progress_start
        self._end = progress_end
        self._update_period = update_period

    def update(self, statusComment, current, total, status=Job.RUNNING):
        """
        Parameters
        ----------
        statusComment: str
            Job status update message
        current: int (range: [0, total[)
            Current iteration
        total: int
            Total number of iteration
        status: int
            Status of the job to send with the update
        """
        period = self._get_period(total)
        if period is not None and current % period != 0:
            return
        relative_progress = self._relative_progress(current / float(total))
        self._cytomine_job.job.update(progress=relative_progress, statusComment=statusComment, status=status)

    def _get_period(self, n_iter):
        """Return integer period given a maximum number of iteration """
        if self._update_period is None:
            return None
        if isinstance(self._update_period, float):
            return max(int(self._update_period * n_iter), 1)
        return self._update_period

    def _relative_progress(self, ratio):
        return int(self._start + (self._end - self._start) * ratio)

    def logger(self, progress_start, progress_end, update_period=None):
        """Return a logger that updates progress in a subrange of the current logger's range."""
        return CytomineJobLogger(
            self._cytomine_job,
            progress_start=self._relative_progress(progress_start / 100.),
            progress_end=self._relative_progress(progress_end / 100.),
            update_period=update_period
        )

    def monitor(self, iterable, prefix=""):
        """Return a monitor for the given iterable using this logger"""
        return CytomineJobProgressMonitor(self, iterable, comment_prefix=prefix)

class CytomineJobProgressMonitor(object):
    def __init__(self, cytomine_logger, iterable, comment_prefix=None):
        """
        cytomine_logger: CytomineJobLogger
            A logger
        iterable: iterable
            The iterable to iterate on
        comment_prefix: str
            A prefix for the status comment
        """
        self._cytomine_logger = cytomine_logger
        self._iterable = list(iterable)
        self._comment_prefix = comment_prefix

    def __iter__(self):
        for i, v in enumerate(self._iterable):
            self._cytomine_logger.update(
                "{} ({}/{}).".format(self._comment_prefix, i + 1, len(self)),
                current=i, total=len(self)
            )
            yield v

    def __len__(self):
        return len(list(self._iterable))

<commit_msg>simplify parameters names<commit_after># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from argparse import ArgumentParser
import logging
import distutils.util

from cytomine.cytomine import Cytomine, _cytomine_parameter_name_synonyms
from cytomine.models.project import Project
from cytomine.models.software import Software, Job, JobParameter, SoftwareParameterCollection
from cytomine.models.user import User

__author__ = "Begon Jean-Michel <jm.begon@gmail.com>"
__contributors = ["Mormont Romain <romainmormont@gmail.com", "Rubens Ulysse <urubens@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

def _inferred_number_type(v):
    """Return the inferred type for the given string. The string must contain either an interger or a float.
    """
    try:
        return int(v)
    except ValueError:
        return float(v)

def _convert_type(_type):
    # Not trying too hard for the finding types (list considered as string and number as floats)
    # Just want a first validation (when possible) with argparse.
    # Advanced checking is advised in actual job implementation.
    return {
        "Number": _inferred_number_type,
        "String": str,
        "Boolean": _to_bool,
        "Domain": int,
        "List": str,
        "ListDomain": str,
        "Date": str
    }[_type]

def _to_bool(v):
    """
    Convert the value to boolean.

    Parameters
    ----------
    v: object
        A object to convert to a boolean value.

    Returns
    -------
    b: bool
        The boolean value
    """
    if isinstance(v, str):
        return bool(distutils.util.strtobool(v))
    else:
        return bool(v)

def _software_params_to_argparse(parameters):
    """
    Converts a SoftwareParameterCollection into an ArgumentParser object.

    Parameters
    ----------
    parameters: SoftwareParameterCollection
        The software parameters
    Returns
    -------
    argparse: ArgumentParser
        An initialized argument parser
    """
    # Check software parameters
    argparse = ArgumentParser()
    for parameter in parameters:
        python_type = _convert_type(parameter.type)
        arg_desc = {"dest": parameter.name, "required": parameter.required, "type": python_type, "help": ""}  # TODO add help
        if parameter.type == "Boolean":
            default = _to_bool(parameter.defaultParamValue)
            arg_desc["nargs"] = "?"
            arg_desc["const"] = not default
            arg_desc["default"] = default
        else:
            arg_desc["default"] = None if parameter.defaultParamValue is None else python_type(parameter.defaultParamValue)
        argparse.add_argument(*_cytomine_parameter_name_synonyms(parameter.name), **arg_desc)
    return argparse

class CytomineJob(Cytomine):
    """
    ===========
    CytomineJob
    ===========
    A :class:`CytomineJob` represents a job in the cytomine context.
    This class does nothing by itself. It is just supposed to be
    inherited from or incorporated in any class representing
    a software/job registered in the Cytomine server

    Usage
    -----
    Either use the :meth:`connect`/meth:`close`methods or use it with a
    with statement:

    with CytomineJob(...) as job:
         do_your_stuff()

    Constructor parameters
    ----------------------
    software_id : int
        The identifier of the software on the Cytomine server
    project_id : int
        The identifier of the project to process on the Cytomine server
    parameters: Namespace
        A namespace mapping job parameters with their values (must not contain any other key than
        the parameters names)
    """
    def __init__(self, host, public_key, private_key, software_id, project_id, parameters=None, **kwargs):
        super(CytomineJob, self).__init__(host, public_key, private_key, **kwargs)
        self._job = None
        self._project = Project().fetch(project_id)
        self._software = Software().fetch(software_id)
        self._job_done = False
        self._parameters = parameters

    @classmethod
    def from_cli(cls, argv, **kwargs):
        # Parse CytomineJob constructor parameters
        argparse = cls._add_cytomine_cli_args(ArgumentParser())
        argparse.add_argument(*_cytomine_parameter_name_synonyms("software_id"),
                              dest="software_id", type=int, help="The Cytomine software id.", required=True)
        argparse.add_argument(*_cytomine_parameter_name_synonyms("project_id"),
                              dest="project_id", type=int, help="The Cytomine project id.", required=True)
        base_params, _ = argparse.parse_known_args(args=argv)

        log_level = base_params.verbose
        if base_params.log_level is not None:
            log_level = logging.getLevelName(base_params.log_level)

        cytomine_job = CytomineJob(
            host=base_params.host,
            public_key=base_params.public_key,
            private_key=base_params.private_key,
            software_id=base_params.software_id,
            project_id=base_params.project_id,
            parameters=None,
            verbose=log_level,
            **kwargs
        )

        # Parse and set job parameters
        params_collection = SoftwareParameterCollection(
            filters={"software": cytomine_job.software.id},
            withSetByServer=True
        ).fetch()
        cytomine_job.parameters, _ = _software_params_to_argparse(params_collection).parse_known_args(argv)
        return cytomine_job

    @property
    def job(self):
        """Return the job model

        Return
        ------
        job: cytomine.Job
            The job model
        """
        return self._job

    @property
    def project(self):
        """
        Protected method

        Return
        ------
        project_id : int
            The id of the project
        """
        return self._project

    @property
    def software(self):
        """
        Protected method

        Return
        ------
        software_id : int
            The id of the software
        """
        return self._software

    @property
    def parameters(self):
        """
        Return
        ------
        parameters : dict
            The id of the software
        """
        return self._parameters

    @parameters.setter
    def parameters(self, parameters):
        """
        Protected method.

        Parameters
        ----------
        parameters: dict
            Dictionnary mapping parameters name with their values.
        """
        self._parameters = parameters

    def done(self, status=True):
        """
        Indicates whether the job is finished or not

        Parameters
        ----------
        status : bool
            Whether the process is finished
        """
        self._job_done = status

    def is_done(self):
        """
        Return
        ------
        job_status : bool
            Whether the process is finished
        """
        return self._job_done

    def start(self):
        """
        Connect to the Cytomine server and switch to job connection
        Incurs dataflows
        """

        run_by_ui = False
        if not self.current_user.algo:
            # If user connects as a human (CLI execution)
            self._job = Job(self._project.id, self._software.id).save()
            user_job = User().fetch(self._job.userJob)
            self.set_credentials(user_job.publicKey, user_job.privateKey)
        else:
            # If the user executes the job through the Cytomine interface
            self._job = Job().fetch(self.current_user.job)
            run_by_ui = True

        # set job state to RUNNING
        self._job.status = Job.RUNNING
        self._job.update()

        # add software parameters
        if not run_by_ui and self._parameters is not None:
            parameters = vars(self._parameters)
            for software_param in self._software.parameters:
                name = software_param["name"]
                if name in parameters:
                    value = parameters[name]
                else:
                    value = software_param["defaultParamValue"]

                JobParameter(self._job.id, software_param["id"], value).save()

    def close(self, value):
        """
        Notify the Cytomine server of the job's end
        Incurs a dataflows
        """
        if value is None:
            status = Job.TERMINATED
            status_comment = "Job successfully terminated"
        else:
            status = Job.FAILED
            status_comment = str(value)[:255]

        self._job.status = status
        self._job.statusComment = status_comment
        self._job.update()

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, traceback):
        self.close(value)
        return False

    def logger(self, start=0, end=100, period=None):
        """Return a logger for the current job."""
        return CytomineJobLogger(self, start=start, end=end, period=period)

    def monitor(self, iterable, start=0, end=100, period=None, prefix=""):
        """Return a monitor for the current job"""
        return self.logger(start=start, end=end, period=period).monitor(iterable, prefix=prefix)

class CytomineJobLogger(object):
    def __init__(self, cytomine_job: CytomineJob, start=0, end=100, period=None):
        """A logger serves as intermediary between the job implementation and the job status update requests.

        Parameters
        ----------
        cytomine_job: CytomineJob
            The job
        start: float (range: [0.0, 100.0[)
            Progress at which the logger should start
        end: float (range: ]0.0, 100.0])
            Progress at which the logger should stop
        period: int, float
            The number of iteration to wait before actually updating the status. Also supports
            frequencies (float values).
        """
        self._cytomine_job = cytomine_job
        self._start = start
        self._end = end
        self._update_period = period

    def update(self, statusComment, current, total, status=Job.RUNNING):
        """
        Parameters
        ----------
        statusComment: str
            Job status update message
        current: int (range: [0, total[)
            Current iteration
        total: int
            Total number of iteration
        status: int
            Status of the job to send with the update
        """
        period = self._get_period(total)
        if period is not None and current % period != 0:
            return
        relative_progress = self._relative_progress(current / float(total))
        self._cytomine_job.job.update(progress=relative_progress, statusComment=statusComment, status=status)

    def _get_period(self, n_iter):
        """Return integer period given a maximum number of iteration """
        if self._update_period is None:
            return None
        if isinstance(self._update_period, float):
            return max(int(self._update_period * n_iter), 1)
        return self._update_period

    def _relative_progress(self, ratio):
        return int(self._start + (self._end - self._start) * ratio)

    def logger(self, progress_start, progress_end, update_period=None):
        """Return a logger that updates progress in a subrange of the current logger's range."""
        return CytomineJobLogger(
            self._cytomine_job,
            start=self._relative_progress(progress_start / 100.),
            end=self._relative_progress(progress_end / 100.),
            period=update_period
        )

    def monitor(self, iterable, prefix=""):
        """Return a monitor for the given iterable using this logger"""
        return CytomineJobProgressMonitor(self, iterable, comment_prefix=prefix)

class CytomineJobProgressMonitor(object):
    def __init__(self, cytomine_logger, iterable, comment_prefix=None):
        """
        cytomine_logger: CytomineJobLogger
            A logger
        iterable: iterable
            The iterable to iterate on
        comment_prefix: str
            A prefix for the status comment
        """
        self._cytomine_logger = cytomine_logger
        self._iterable = list(iterable)
        self._comment_prefix = comment_prefix

    def __iter__(self):
        for i, v in enumerate(self._iterable):
            self._cytomine_logger.update(
                "{} ({}/{}).".format(self._comment_prefix, i + 1, len(self)),
                current=i, total=len(self)
            )
            yield v

    def __len__(self):
        return len(list(self._iterable))

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Analyze docstrings to detect errors.

If no argument is provided, it does a quick check of docstrings and returns
a csv with all API functions and results of basic checks.

If a function or method is provided in the form "pandas.function",
"pandas.module.class.method", etc. a list of all errors in the docstring for
the specified function or method.

Usage::
    $ ./validate_docstrings.py
    $ ./validate_docstrings.py pandas.DataFrame.head
"""
import os
import sys
import csv
import re
import functools
import collections
import argparse
import contextlib
import pydoc
import inspect
import importlib
import doctest
try:
    from io import StringIO
except ImportError:
    from cStringIO import StringIO
import numpy

BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(0, os.path.join(BASE_PATH))
import pandas

sys.path.insert(1, os.path.join(BASE_PATH, 'doc', 'sphinxext'))
from numpydoc.docscrape import NumpyDocString

PRIVATE_CLASSES = ['NDFrame', 'IndexOpsMixin']

def _load_obj(obj_name):
    for maxsplit in range(1, obj_name.count('.') + 1):
        # TODO when py3 only replace by: module, *func_parts = ...
        func_name_split = obj_name.rsplit('.', maxsplit=maxsplit)
        module = func_name_split[0]
        func_parts = func_name_split[1:]
        try:
            obj = importlib.import_module(module)
        except ImportError:
            pass
        else:
            continue

    if 'module' not in locals():
        raise ImportError('No module can be imported '
                          'from "{}"'.format(obj_name))

    for part in func_parts:
        obj = getattr(obj, part)
    return obj

def _to_original_callable(obj):
    while True:
        if inspect.isfunction(obj) or inspect.isclass(obj):
            f = inspect.getfile(obj)
            if f.startswith('<') and f.endswith('>'):
                return None
            return obj
        if inspect.ismethod(obj):
            obj = obj.__func__
        elif isinstance(obj, functools.partial):
            obj = obj.func
        elif isinstance(obj, property):
            obj = obj.fget
        else:
            return None

def _output_header(title, width=80, char='#'):
    full_line = char * width
    side_len = (width - len(title) - 2) // 2
    adj = '' if len(title) % 2 == 0 else ' '
    title_line = '{side} {title}{adj} {side}'.format(side=char * side_len,
                                                     title=title,
                                                     adj=adj)

    return '\n{full_line}\n{title_line}\n{full_line}\n\n'.format(
        full_line=full_line, title_line=title_line)

class Docstring:
    def __init__(self, method_name, method_obj):
        self.method_name = method_name
        self.method_obj = method_obj
        self.raw_doc = method_obj.__doc__ or ''
        self.clean_doc = pydoc.getdoc(self.method_obj)
        self.doc = NumpyDocString(self.clean_doc)

    def __len__(self):
        return len(self.raw_doc)

    @property
    def is_function_or_method(self):
        return inspect.isfunction(self.method_obj)

    @property
    def source_file_name(self):
        fname = inspect.getsourcefile(self.method_obj)
        if fname:
            fname = os.path.relpath(fname, BASE_PATH)
            return fname

    @property
    def source_file_def_line(self):
        try:
            return inspect.getsourcelines(self.method_obj)[-1]
        except OSError:
            pass

    @property
    def github_url(self):
        url = 'https://github.com/pandas-dev/pandas/blob/master/'
        url += '{}#L{}'.format(self.source_file_name,
                               self.source_file_def_line)
        return url

    @property
    def start_blank_lines(self):
        i = None
        if self.raw_doc:
            for i, row in enumerate(self.raw_doc.split('\n')):
                if row.strip():
                    break
        return i

    @property
    def end_blank_lines(self):
        i = None
        if self.raw_doc:
            for i, row in enumerate(reversed(self.raw_doc.split('\n'))):
                if row.strip():
                    break
        return i

    @property
    def double_blank_lines(self):
        prev = True
        for row in self.raw_doc.split('\n'):
            if not prev and not row.strip():
                return True
            prev = row.strip()
        return False

    @property
    def summary(self):
        if not self.doc['Extended Summary'] and len(self.doc['Summary']) > 1:
            return ''
        return ' '.join(self.doc['Summary'])

    @property
    def extended_summary(self):
        if not self.doc['Extended Summary'] and len(self.doc['Summary']) > 1:
            return ' '.join(self.doc['Summary'])
        return ' '.join(self.doc['Extended Summary'])

    @property
    def needs_summary(self):
        return not (bool(self.summary) and bool(self.extended_summary))

    @property
    def doc_parameters(self):
        return collections.OrderedDict((name, (type_, ''.join(desc)))
                                       for name, type_, desc
                                       in self.doc['Parameters'])

    @property
    def signature_parameters(self):
        if (inspect.isclass(self.method_obj)
                and self.method_name.split('.')[-1] in {'dt', 'str', 'cat'}):
            # accessor classes have a signature, but don't want to show this
            return tuple()
        try:
            signature = inspect.signature(self.method_obj)
        except (TypeError, ValueError):
            # Some objects, mainly in C extensions do not support introspection
            # of the signature
            return tuple()
        params = tuple(signature.parameters.keys())
        if params and params[0] in ('self', 'cls'):
            return params[1:]
        return params

    @property
    def parameter_mismatches(self):
        errs = []
        signature_params = self.signature_parameters
        doc_params = tuple(self.doc_parameters)
        missing = set(signature_params) - set(doc_params)
        if missing:
            errs.append('Parameters {!r} not documented'.format(missing))
        extra = set(doc_params) - set(signature_params)
        if extra:
            errs.append('Unknown parameters {!r}'.format(extra))
        if (not missing and not extra and signature_params != doc_params
                and not (not signature_params and not doc_params)):
            errs.append('Wrong parameters order. ' +
                        'Actual: {!r}. '.format(signature_params) +
                        'Documented: {!r}'.format(doc_params))

        return errs

    @property
    def correct_parameters(self):
        return not bool(self.parameter_mismatches)

    def parameter_type(self, param):
        return self.doc_parameters[param][0]

    def parameter_desc(self, param):
        return self.doc_parameters[param][1]

    @property
    def see_also(self):
        return collections.OrderedDict((name, ''.join(desc))
                                       for name, desc, _
                                       in self.doc['See Also'])

    @property
    def examples(self):
        return self.doc['Examples']

    @property
    def returns(self):
        return self.doc['Returns']

    @property
    def first_line_ends_in_dot(self):
        if self.doc:
            return self.doc.split('\n')[0][-1] == '.'

    @property
    def deprecated(self):
        pattern = re.compile('.. deprecated:: ')
        return (self.method_name.startswith('pandas.Panel') or
                bool(pattern.search(self.summary)) or
                bool(pattern.search(self.extended_summary)))

    @property
    def mentioned_private_classes(self):
        return [klass for klass in PRIVATE_CLASSES if klass in self.raw_doc]

    @property
    def examples_errors(self):
        flags = doctest.NORMALIZE_WHITESPACE | doctest.IGNORE_EXCEPTION_DETAIL
        finder = doctest.DocTestFinder()
        runner = doctest.DocTestRunner(optionflags=flags)
        context = {'np': numpy, 'pd': pandas}
        error_msgs = ''
        for test in finder.find(self.raw_doc, self.method_name, globs=context):
            f = StringIO()
            with contextlib.redirect_stdout(f):
                runner.run(test)
            error_msgs += f.getvalue()
        return error_msgs

def get_api_items():
    api_fname = os.path.join(BASE_PATH, 'doc', 'source', 'api.rst')

    previous_line = current_section = current_subsection = ''
    position = None
    with open(api_fname) as f:
        for line in f:
            line = line.strip()
            if len(line) == len(previous_line):
                if set(line) == set('-'):
                    current_section = previous_line
                    continue
                if set(line) == set('~'):
                    current_subsection = previous_line
                    continue

            if line.startswith('.. currentmodule::'):
                current_module = line.replace('.. currentmodule::', '').strip()
                continue

            if line == '.. autosummary::':
                position = 'autosummary'
                continue

            if position == 'autosummary':
                if line == '':
                    position = 'items'
                    continue

            if position == 'items':
                if line == '':
                    position = None
                    continue
                item = line.strip()
                func = importlib.import_module(current_module)
                for part in item.split('.'):
                    func = getattr(func, part)

                yield ('.'.join([current_module, item]), func,
                       current_section, current_subsection)

            previous_line = line

def _csv_row(func_name, func_obj, section, subsection, in_api, seen={}):
    obj_type = type(func_obj).__name__
    original_callable = _to_original_callable(func_obj)
    if original_callable is None:
        return [func_name, obj_type] + [''] * 12, ''
    else:
        doc = Docstring(func_name, original_callable)
        key = doc.source_file_name, doc.source_file_def_line
        shared_code = seen.get(key, '')
        return [func_name,
                obj_type,
                in_api,
                int(doc.deprecated),
                section,
                subsection,
                doc.source_file_name,
                doc.source_file_def_line,
                doc.github_url,
                int(bool(doc.summary)),
                int(bool(doc.extended_summary)),
                int(doc.correct_parameters),
                int(bool(doc.examples)),
                shared_code], key

def validate_all():
    writer = csv.writer(sys.stdout)
    cols = ('Function or method',
            'Type',
            'In API doc',
            'Is deprecated',
            'Section',
            'Subsection',
            'File',
            'Code line',
            'GitHub link',
            'Has summary',
            'Has extended summary',
            'Parameters ok',
            'Has examples',
            'Shared code with')
    writer.writerow(cols)
    seen = {}
    api_items = list(get_api_items())
    for func_name, func, section, subsection in api_items:
        row, key = _csv_row(func_name, func, section, subsection,
                            in_api=1, seen=seen)
        seen[key] = func_name
        writer.writerow(row)

    api_item_names = set(list(zip(*api_items))[0])
    for class_ in (pandas.Series, pandas.DataFrame, pandas.Panel):
        for member in inspect.getmembers(class_):
            func_name = 'pandas.{}.{}'.format(class_.__name__, member[0])
            if (not member[0].startswith('_') and
                    func_name not in api_item_names):
                func = _load_obj(func_name)
                row, key = _csv_row(func_name, func, section='', subsection='',
                                    in_api=0)
                writer.writerow(row)

    return 0

def validate_one(func_name):
    func_obj = _load_obj(func_name)
    doc = Docstring(func_name, func_obj)

    sys.stderr.write(_output_header('Docstring ({})'.format(func_name)))
    sys.stderr.write('{}\n'.format(doc.clean_doc))

    errs = []
    if doc.start_blank_lines != 1:
        errs.append('Docstring text (summary) should start in the line '
                    'immediately after the opening quotes (not in the same '
                    'line, or leaving a blank line in between)')
    if doc.end_blank_lines != 1:
        errs.append('Closing quotes should be placed in the line after '
                    'the last text in the docstring (do not close the '
                    'quotes in the same line as the text, or leave a '
                    'blank line between the last text and the quotes)')
    if doc.double_blank_lines:
        errs.append('Use only one blank line to separate sections or '
                    'paragraphs')

    if not doc.summary:
        errs.append('No summary found (a short summary in a single line '
                    'should be present at the beginning of the docstring)')
    else:
        if not doc.summary[0].isupper():
            errs.append('Summary does not start with capital')
        if doc.summary[-1] != '.':
            errs.append('Summary does not end with dot')
        if (doc.is_function_or_method and
                doc.summary.split(' ')[0][-1] == 's'):
            errs.append('Summary must start with infinitive verb, '
                        'not third person (e.g. use "Generate" instead of '
                        '"Generates")')
    if not doc.extended_summary:
        errs.append('No extended summary found')

    param_errs = doc.parameter_mismatches
    for param in doc.doc_parameters:
        if not doc.parameter_type(param):
            param_errs.append('Parameter "{}" has no type'.format(param))
        else:
            if doc.parameter_type(param)[-1] == '.':
                param_errs.append('Parameter "{}" type '
                                  'should not finish with "."'.format(param))

        if not doc.parameter_desc(param):
            param_errs.append('Parameter "{}" '
                              'has no description'.format(param))
        else:
            if not doc.parameter_desc(param)[0].isupper():
                param_errs.append('Parameter "{}" description '
                                  'should start with '
                                  'capital letter'.format(param))
            if doc.parameter_desc(param)[-1] != '.':
                param_errs.append('Parameter "{}" description '
                                  'should finish with "."'.format(param))
    if param_errs:
        errs.append('Errors in parameters section')
        for param_err in param_errs:
            errs.append('\t{}'.format(param_err))

    if not doc.returns:
        errs.append('No returns section found')

    mentioned_errs = doc.mentioned_private_classes
    if mentioned_errs:
        errs.append('Private classes ({}) should not be mentioned in public '
                    'docstring.'.format(mentioned_errs))

    if not doc.see_also:
        errs.append('See Also section not found')
    else:
        for rel_name, rel_desc in doc.see_also.items():
            if not rel_desc:
                errs.append('Missing description for '
                            'See Also "{}" reference'.format(rel_name))
    examples_errs = ''
    if not doc.examples:
        errs.append('No examples section found')
    else:
        examples_errs = doc.examples_errors
        if examples_errs:
            errs.append('Examples do not pass tests')

    sys.stderr.write(_output_header('Validation'))
    if errs:
        sys.stderr.write('Errors found:\n')
        for err in errs:
            sys.stderr.write('\t{}\n'.format(err))
    else:
        sys.stderr.write('Docstring for "{}" correct. :)\n'.format(func_name))

    if examples_errs:
        sys.stderr.write(_output_header('Doctests'))
        sys.stderr.write(examples_errs)

    return len(errs)

def main(function):
    if function is None:
        return validate_all()
    else:
        return validate_one(function)

if __name__ == '__main__':
    argparser = argparse.ArgumentParser(
        description='validate pandas docstrings')
    argparser.add_argument('function',
                           nargs='?',
                           default=None,
                           help=('function or method to validate '
                                 '(e.g. pandas.DataFrame.head) '
                                 'if not provided, all docstrings '
                                 'are validated'))
    args = argparser.parse_args()
    sys.exit(main(args.function))
<commit_msg>DOC: Validation script checks for tabs (#20112)<commit_after>#!/usr/bin/env python
"""
Analyze docstrings to detect errors.

If no argument is provided, it does a quick check of docstrings and returns
a csv with all API functions and results of basic checks.

If a function or method is provided in the form "pandas.function",
"pandas.module.class.method", etc. a list of all errors in the docstring for
the specified function or method.

Usage::
    $ ./validate_docstrings.py
    $ ./validate_docstrings.py pandas.DataFrame.head
"""
import os
import sys
import csv
import re
import functools
import collections
import argparse
import contextlib
import pydoc
import inspect
import importlib
import doctest
try:
    from io import StringIO
except ImportError:
    from cStringIO import StringIO
import numpy

BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(0, os.path.join(BASE_PATH))
import pandas

sys.path.insert(1, os.path.join(BASE_PATH, 'doc', 'sphinxext'))
from numpydoc.docscrape import NumpyDocString

PRIVATE_CLASSES = ['NDFrame', 'IndexOpsMixin']

def _load_obj(obj_name):
    for maxsplit in range(1, obj_name.count('.') + 1):
        # TODO when py3 only replace by: module, *func_parts = ...
        func_name_split = obj_name.rsplit('.', maxsplit=maxsplit)
        module = func_name_split[0]
        func_parts = func_name_split[1:]
        try:
            obj = importlib.import_module(module)
        except ImportError:
            pass
        else:
            continue

    if 'module' not in locals():
        raise ImportError('No module can be imported '
                          'from "{}"'.format(obj_name))

    for part in func_parts:
        obj = getattr(obj, part)
    return obj

def _to_original_callable(obj):
    while True:
        if inspect.isfunction(obj) or inspect.isclass(obj):
            f = inspect.getfile(obj)
            if f.startswith('<') and f.endswith('>'):
                return None
            return obj
        if inspect.ismethod(obj):
            obj = obj.__func__
        elif isinstance(obj, functools.partial):
            obj = obj.func
        elif isinstance(obj, property):
            obj = obj.fget
        else:
            return None

def _output_header(title, width=80, char='#'):
    full_line = char * width
    side_len = (width - len(title) - 2) // 2
    adj = '' if len(title) % 2 == 0 else ' '
    title_line = '{side} {title}{adj} {side}'.format(side=char * side_len,
                                                     title=title,
                                                     adj=adj)

    return '\n{full_line}\n{title_line}\n{full_line}\n\n'.format(
        full_line=full_line, title_line=title_line)

class Docstring:
    def __init__(self, method_name, method_obj):
        self.method_name = method_name
        self.method_obj = method_obj
        self.raw_doc = method_obj.__doc__ or ''
        self.clean_doc = pydoc.getdoc(self.method_obj)
        self.doc = NumpyDocString(self.clean_doc)

    def __len__(self):
        return len(self.raw_doc)

    @property
    def is_function_or_method(self):
        return inspect.isfunction(self.method_obj)

    @property
    def source_file_name(self):
        fname = inspect.getsourcefile(self.method_obj)
        if fname:
            fname = os.path.relpath(fname, BASE_PATH)
            return fname

    @property
    def source_file_def_line(self):
        try:
            return inspect.getsourcelines(self.method_obj)[-1]
        except OSError:
            pass

    @property
    def github_url(self):
        url = 'https://github.com/pandas-dev/pandas/blob/master/'
        url += '{}#L{}'.format(self.source_file_name,
                               self.source_file_def_line)
        return url

    @property
    def start_blank_lines(self):
        i = None
        if self.raw_doc:
            for i, row in enumerate(self.raw_doc.split('\n')):
                if row.strip():
                    break
        return i

    @property
    def end_blank_lines(self):
        i = None
        if self.raw_doc:
            for i, row in enumerate(reversed(self.raw_doc.split('\n'))):
                if row.strip():
                    break
        return i

    @property
    def double_blank_lines(self):
        prev = True
        for row in self.raw_doc.split('\n'):
            if not prev and not row.strip():
                return True
            prev = row.strip()
        return False

    @property
    def summary(self):
        if not self.doc['Extended Summary'] and len(self.doc['Summary']) > 1:
            return ''
        return ' '.join(self.doc['Summary'])

    @property
    def extended_summary(self):
        if not self.doc['Extended Summary'] and len(self.doc['Summary']) > 1:
            return ' '.join(self.doc['Summary'])
        return ' '.join(self.doc['Extended Summary'])

    @property
    def needs_summary(self):
        return not (bool(self.summary) and bool(self.extended_summary))

    @property
    def doc_parameters(self):
        return collections.OrderedDict((name, (type_, ''.join(desc)))
                                       for name, type_, desc
                                       in self.doc['Parameters'])

    @property
    def signature_parameters(self):
        if (inspect.isclass(self.method_obj)
                and self.method_name.split('.')[-1] in {'dt', 'str', 'cat'}):
            # accessor classes have a signature, but don't want to show this
            return tuple()
        try:
            signature = inspect.signature(self.method_obj)
        except (TypeError, ValueError):
            # Some objects, mainly in C extensions do not support introspection
            # of the signature
            return tuple()
        params = tuple(signature.parameters.keys())
        if params and params[0] in ('self', 'cls'):
            return params[1:]
        return params

    @property
    def parameter_mismatches(self):
        errs = []
        signature_params = self.signature_parameters
        doc_params = tuple(self.doc_parameters)
        missing = set(signature_params) - set(doc_params)
        if missing:
            errs.append('Parameters {!r} not documented'.format(missing))
        extra = set(doc_params) - set(signature_params)
        if extra:
            errs.append('Unknown parameters {!r}'.format(extra))
        if (not missing and not extra and signature_params != doc_params
                and not (not signature_params and not doc_params)):
            errs.append('Wrong parameters order. ' +
                        'Actual: {!r}. '.format(signature_params) +
                        'Documented: {!r}'.format(doc_params))

        return errs

    @property
    def correct_parameters(self):
        return not bool(self.parameter_mismatches)

    def parameter_type(self, param):
        return self.doc_parameters[param][0]

    def parameter_desc(self, param):
        return self.doc_parameters[param][1]

    @property
    def see_also(self):
        return collections.OrderedDict((name, ''.join(desc))
                                       for name, desc, _
                                       in self.doc['See Also'])

    @property
    def examples(self):
        return self.doc['Examples']

    @property
    def returns(self):
        return self.doc['Returns']

    @property
    def first_line_ends_in_dot(self):
        if self.doc:
            return self.doc.split('\n')[0][-1] == '.'

    @property
    def deprecated(self):
        pattern = re.compile('.. deprecated:: ')
        return (self.method_name.startswith('pandas.Panel') or
                bool(pattern.search(self.summary)) or
                bool(pattern.search(self.extended_summary)))

    @property
    def mentioned_private_classes(self):
        return [klass for klass in PRIVATE_CLASSES if klass in self.raw_doc]

    @property
    def examples_errors(self):
        flags = doctest.NORMALIZE_WHITESPACE | doctest.IGNORE_EXCEPTION_DETAIL
        finder = doctest.DocTestFinder()
        runner = doctest.DocTestRunner(optionflags=flags)
        context = {'np': numpy, 'pd': pandas}
        error_msgs = ''
        for test in finder.find(self.raw_doc, self.method_name, globs=context):
            f = StringIO()
            with contextlib.redirect_stdout(f):
                runner.run(test)
            error_msgs += f.getvalue()
        return error_msgs

def get_api_items():
    api_fname = os.path.join(BASE_PATH, 'doc', 'source', 'api.rst')

    previous_line = current_section = current_subsection = ''
    position = None
    with open(api_fname) as f:
        for line in f:
            line = line.strip()
            if len(line) == len(previous_line):
                if set(line) == set('-'):
                    current_section = previous_line
                    continue
                if set(line) == set('~'):
                    current_subsection = previous_line
                    continue

            if line.startswith('.. currentmodule::'):
                current_module = line.replace('.. currentmodule::', '').strip()
                continue

            if line == '.. autosummary::':
                position = 'autosummary'
                continue

            if position == 'autosummary':
                if line == '':
                    position = 'items'
                    continue

            if position == 'items':
                if line == '':
                    position = None
                    continue
                item = line.strip()
                func = importlib.import_module(current_module)
                for part in item.split('.'):
                    func = getattr(func, part)

                yield ('.'.join([current_module, item]), func,
                       current_section, current_subsection)

            previous_line = line

def _csv_row(func_name, func_obj, section, subsection, in_api, seen={}):
    obj_type = type(func_obj).__name__
    original_callable = _to_original_callable(func_obj)
    if original_callable is None:
        return [func_name, obj_type] + [''] * 12, ''
    else:
        doc = Docstring(func_name, original_callable)
        key = doc.source_file_name, doc.source_file_def_line
        shared_code = seen.get(key, '')
        return [func_name,
                obj_type,
                in_api,
                int(doc.deprecated),
                section,
                subsection,
                doc.source_file_name,
                doc.source_file_def_line,
                doc.github_url,
                int(bool(doc.summary)),
                int(bool(doc.extended_summary)),
                int(doc.correct_parameters),
                int(bool(doc.examples)),
                shared_code], key

def validate_all():
    writer = csv.writer(sys.stdout)
    cols = ('Function or method',
            'Type',
            'In API doc',
            'Is deprecated',
            'Section',
            'Subsection',
            'File',
            'Code line',
            'GitHub link',
            'Has summary',
            'Has extended summary',
            'Parameters ok',
            'Has examples',
            'Shared code with')
    writer.writerow(cols)
    seen = {}
    api_items = list(get_api_items())
    for func_name, func, section, subsection in api_items:
        row, key = _csv_row(func_name, func, section, subsection,
                            in_api=1, seen=seen)
        seen[key] = func_name
        writer.writerow(row)

    api_item_names = set(list(zip(*api_items))[0])
    for class_ in (pandas.Series, pandas.DataFrame, pandas.Panel):
        for member in inspect.getmembers(class_):
            func_name = 'pandas.{}.{}'.format(class_.__name__, member[0])
            if (not member[0].startswith('_') and
                    func_name not in api_item_names):
                func = _load_obj(func_name)
                row, key = _csv_row(func_name, func, section='', subsection='',
                                    in_api=0)
                writer.writerow(row)

    return 0

def validate_one(func_name):
    func_obj = _load_obj(func_name)
    doc = Docstring(func_name, func_obj)

    sys.stderr.write(_output_header('Docstring ({})'.format(func_name)))
    sys.stderr.write('{}\n'.format(doc.clean_doc))

    errs = []
    if doc.start_blank_lines != 1:
        errs.append('Docstring text (summary) should start in the line '
                    'immediately after the opening quotes (not in the same '
                    'line, or leaving a blank line in between)')
    if doc.end_blank_lines != 1:
        errs.append('Closing quotes should be placed in the line after '
                    'the last text in the docstring (do not close the '
                    'quotes in the same line as the text, or leave a '
                    'blank line between the last text and the quotes)')
    if doc.double_blank_lines:
        errs.append('Use only one blank line to separate sections or '
                    'paragraphs')

    if not doc.summary:
        errs.append('No summary found (a short summary in a single line '
                    'should be present at the beginning of the docstring)')
    else:
        if not doc.summary[0].isupper():
            errs.append('Summary does not start with capital')
        if doc.summary[-1] != '.':
            errs.append('Summary does not end with dot')
        if (doc.is_function_or_method and
                doc.summary.split(' ')[0][-1] == 's'):
            errs.append('Summary must start with infinitive verb, '
                        'not third person (e.g. use "Generate" instead of '
                        '"Generates")')
    if not doc.extended_summary:
        errs.append('No extended summary found')

    param_errs = doc.parameter_mismatches
    for param in doc.doc_parameters:
        if not doc.parameter_type(param):
            param_errs.append('Parameter "{}" has no type'.format(param))
        else:
            if doc.parameter_type(param)[-1] == '.':
                param_errs.append('Parameter "{}" type '
                                  'should not finish with "."'.format(param))

        if not doc.parameter_desc(param):
            param_errs.append('Parameter "{}" '
                              'has no description'.format(param))
        else:
            if not doc.parameter_desc(param)[0].isupper():
                param_errs.append('Parameter "{}" description '
                                  'should start with '
                                  'capital letter'.format(param))
            if doc.parameter_desc(param)[-1] != '.':
                param_errs.append('Parameter "{}" description '
                                  'should finish with "."'.format(param))
    if param_errs:
        errs.append('Errors in parameters section')
        for param_err in param_errs:
            errs.append('\t{}'.format(param_err))

    if not doc.returns:
        errs.append('No returns section found')

    mentioned_errs = doc.mentioned_private_classes
    if mentioned_errs:
        errs.append('Private classes ({}) should not be mentioned in public '
                    'docstring.'.format(mentioned_errs))

    if not doc.see_also:
        errs.append('See Also section not found')
    else:
        for rel_name, rel_desc in doc.see_also.items():
            if not rel_desc:
                errs.append('Missing description for '
                            'See Also "{}" reference'.format(rel_name))

    for line in doc.raw_doc.splitlines():
        if re.match("^ *\t", line):
            errs.append('Tabs found at the start of line "{}", '
                        'please use whitespace only'.format(line.lstrip()))

    examples_errs = ''
    if not doc.examples:
        errs.append('No examples section found')
    else:
        examples_errs = doc.examples_errors
        if examples_errs:
            errs.append('Examples do not pass tests')

    sys.stderr.write(_output_header('Validation'))
    if errs:
        sys.stderr.write('Errors found:\n')
        for err in errs:
            sys.stderr.write('\t{}\n'.format(err))
    else:
        sys.stderr.write('Docstring for "{}" correct. :)\n'.format(func_name))

    if examples_errs:
        sys.stderr.write(_output_header('Doctests'))
        sys.stderr.write(examples_errs)

    return len(errs)

def main(function):
    if function is None:
        return validate_all()
    else:
        return validate_one(function)

if __name__ == '__main__':
    argparser = argparse.ArgumentParser(
        description='validate pandas docstrings')
    argparser.add_argument('function',
                           nargs='?',
                           default=None,
                           help=('function or method to validate '
                                 '(e.g. pandas.DataFrame.head) '
                                 'if not provided, all docstrings '
                                 'are validated'))
    args = argparser.parse_args()
    sys.exit(main(args.function))
<|endoftext|>


<commit_before>import os
import shutil
import pkg_resources
import pickle
import dreambeam
from antpat import radfarfield
from antpat.io.NECread import readNECout_tvecfuns
from antpat.reps.hamaker import convDPE2LOFARcc, convLOFARcc2DPE

class FeedWiz(object):

    def __init__(self):
        feed_paths = self._get_feed_paths()
        self.feedplugins = {}
        for feed_path in feed_paths:
            feedmodpath = os.path.join(feed_path, '_feeds.py')
            if os.path.exists(feedmodpath):
                feed_category = os.path.basename(feed_path)
                self.feedplugins[feed_category] = feed_path

    def _get_feed_paths(self):
        resource_path = '/'.join(('configs', 'feed_paths.txt'))
        rootpath = os.path.dirname(os.path.dirname(dreambeam.__file__))
        feed_paths_file = pkg_resources.resource_filename(dreambeam.__name__,
                                                          resource_path)
        feed_paths = []
        with open(feed_paths_file) as fp:
            lines = fp.readlines()
            for line in lines:
                line = line.rstrip()
                if not (line.startswith('#') or line == ''):
                    feed_path = os.path.join(rootpath, line)
                    feed_paths.append(feed_path)
        return feed_paths

    def __getitem__(self, feed_category):
        try:
            feed_path = self.feedplugins[feed_category]
        except KeyError:
            return None
        feedplugin = FeedPlugin(feed_path)
        return feedplugin

    def list_names(self):
        names = []
        for feed_plugin in self.feed_plugins:
            names.extend(feed_plugin.get_bands())
        return names

    def list_models4receptor(self, feed_name):
        models = []
        for feed_plugin in self.feed_plugins:
            models.extend(feed_plugin.list_models4receptor(feed_name))
        return models

class FeedPlugin(object):
    dpesuffix = '.dpe.pkl'
    _bndversep = '-'  # Band Version Separator
    _mdlversep = '-'  # Model Version Separator
    SHAREDIR = 'share'  # Dir for native telescope project data.
    DATADIR = 'data'    # Dir for telescope data for RIME level work.

    def __init__(self, path2plugin):
        self.path_ = path2plugin

    def prep_dpefiles(self):
        coeffilemetas = self._search_coeff_files()
        if coeffilemetas != []:
            for coeffilemeta in coeffilemetas:
                self.get_stndpolel(coeffilemeta['band'],
                                   coeffilemeta['modeltype'],
                                   coeffilemeta['version'])

    def find_necoutfiles(self):
        necfiles = []
        necoutdir = os.path.join(self.path_, self.SHAREDIR)
        ls = os.listdir(necoutdir)
        for lsentry in ls:
            if lsentry.endswith('.out'):
                necfiles.append(lsentry)
        return necfiles

    def _parsefilename(self, filename):
        filebase, fileext = filename.split('.', 1)
        band, version, = filebase.split(self._bndversep, 1)
        return band, version

    def _convNEC2antpat(self, necoutfile, scalefac):
        print("Generating antpat from necfile: {}".format(necoutfile))
        necoutfile = os.path.join(self.path_, self.SHAREDIR, necoutfile)
        tvf = readNECout_tvecfuns(necoutfile)
        tvf.scale(scalefac)
        antpat = radfarfield.RadFarField(tvf)
        return antpat

    def compute_ccfile(self, necoutfile, scalefac, freq_center, freq_range,
                       kord, tord, ford, channels):
        """Create a HamakerArts coefficients file from a NEC out file
        and save it in a .cc file.
        """
        antpat = self._convNEC2antpat(necoutfile, scalefac)
        band, haversion = self._parsefilename(necoutfile)
        # freq_center should be close to maximum gain frequency,
        # while freq_range should avoid going outside the frequencies
        # of NEC4 file which is 0 to 100e6 Hz (also avoid edges).
        print("Generating coefficients files...")
        artsdata, filename = convDPE2LOFARcc(antpat, freq_center, freq_range,
                                             HAcoefband=band,
                                             HAcoefversion=haversion,
                                             kord=kord, tord=tord, ford=ford,
                                             channels=channels)
        srcpath = os.path.abspath(filename)
        destfilepath = os.path.join(self.path_, self.SHAREDIR, filename)
        shutil.move(srcpath, destfilepath)

    def _search_coeff_files(self):
        coeffilemetas = []
        modelspecdir = os.path.join(self.path_, self.SHAREDIR)
        ls = os.listdir(modelspecdir)
        for lsentry in ls:
            if lsentry.endswith('.cc'):
                modeltype = 'Hamaker'
                ccfilename = lsentry.rstrip('.cc')
                version, band = ccfilename.split('Coeff')
                coeffilemetas.append({'band': band, 'modeltype': modeltype,
                                      'version': version})
        return coeffilemetas

    def get_bands(self):
        coeffilemetas = self._search_coeff_files()
        bands = []
        for coeffilemeta in coeffilemetas:
            bands.append(coeffilemeta['band'])
        return bands

    def list_models4band(self, band):
        coeffilemetas = self._search_coeff_files()
        models = []
        for coeffilemeta in coeffilemetas:
            if coeffilemeta['band'] == band:
                modelstring = '{}{}{}'.format(coeffilemeta['modeltype'],
                                              self._mdlversep,
                                              coeffilemeta['version'])
                models.append(modelstring)
        return models

    def list_feednames(self):
        coeffilemetas = self._search_coeff_files()
        feednames = []
        for coeffilemeta in coeffilemetas:
            feedname = self._mdlversep.join([coeffilemeta['band'],
                                             coeffilemeta['modeltype'],
                                             coeffilemeta['version']])
            feednames.append(feedname)
        return feednames

    def list_versions4bandmodels(self, band, modeltype):
        coeffilemetas = self._search_coeff_files()
        versions = []
        for coeffilemeta in coeffilemetas:
            if coeffilemeta['band'] == band and coeffilemeta['modeltype'] == modeltype:
                version = coeffilemeta['version']
                versions.append(version)
        return versions

    def _get_ccfile(self, band, haversion):
        inpfile = haversion+"Coeff"+band+".cc"
        return inpfile

    def _get_dpfile(self, band, version):
        outfile = band+self._bndversep+version+self.dpesuffix
        return outfile

    def list_dpes(self):
        dpes = []
        dpedir = os.path.join(self.path_, self.DATADIR)
        ls = os.listdir(dpedir)
        for lsentry in ls:
            if lsentry.endswith(self.dpesuffix):
                modeltype = 'Hamaker'  # FIXME:
                dpefilename = lsentry.rstrip(self.dpesuffix)
                band, version = dpefilename.split(self._bndversep, 1)
                dpes.append({'band': band, 'modeltype': modeltype,
                             'version': version})
        return dpes

    def load_dpolel(self, band, modelstring):
        modeltype, version = modelstring.split(self._mdlversep, 1)
        if version is None:
            versions = self.list_versions4bandmodels(band, modeltype)
            version = versions[0]
        if modeltype == 'Hamaker':
            dpfile = self._get_dpfile(band, version)
            if os.path.exists(dpfile):
                dpolel = pickle.load(open(dpfile, 'rb'))
            else:
                dpolel = self._gen_dpolel_hamaker(band, version)
        return dpolel

    def _gen_dpolel_hamaker(self, band, haversion):
        """Generate dualPolElem for the given band.
        Reads the 'lofar_elem_resp' packages c++ header files of
        Hamaker-Arts coefficients and writes pickled instances of DualPolElem
        class.
        Also adds nominal LOFAR frequency channels."""
        inpfile = self._get_ccfile(band, haversion)
        outfile = self._get_dpfile(band, haversion)
        inppath = os.path.join(self.path_, self.SHAREDIR, inpfile)
        outpath = os.path.join(self.path_, self.DATADIR, outfile)
        stndpolel = convLOFARcc2DPE(inppath, outpath)
        return stndpolel
<commit_msg>Create DATADIR for feed plugin when initializing FeedPlugin().<commit_after>import os
import shutil
import pkg_resources
import pickle
import dreambeam
from antpat import radfarfield
from antpat.io.NECread import readNECout_tvecfuns
from antpat.reps.hamaker import convDPE2LOFARcc, convLOFARcc2DPE

class FeedWiz(object):

    def __init__(self):
        feed_paths = self._get_feed_paths()
        self.feedplugins = {}
        for feed_path in feed_paths:
            feedmodpath = os.path.join(feed_path, '_feeds.py')
            if os.path.exists(feedmodpath):
                feed_category = os.path.basename(feed_path)
                self.feedplugins[feed_category] = feed_path

    def _get_feed_paths(self):
        resource_path = '/'.join(('configs', 'feed_paths.txt'))
        rootpath = os.path.dirname(os.path.dirname(dreambeam.__file__))
        feed_paths_file = pkg_resources.resource_filename(dreambeam.__name__,
                                                          resource_path)
        feed_paths = []
        with open(feed_paths_file) as fp:
            lines = fp.readlines()
            for line in lines:
                line = line.rstrip()
                if not (line.startswith('#') or line == ''):
                    feed_path = os.path.join(rootpath, line)
                    feed_paths.append(feed_path)
        return feed_paths

    def __getitem__(self, feed_category):
        try:
            feed_path = self.feedplugins[feed_category]
        except KeyError:
            return None
        feedplugin = FeedPlugin(feed_path)
        return feedplugin

    def list_names(self):
        names = []
        for feed_plugin in self.feed_plugins:
            names.extend(feed_plugin.get_bands())
        return names

    def list_models4receptor(self, feed_name):
        models = []
        for feed_plugin in self.feed_plugins:
            models.extend(feed_plugin.list_models4receptor(feed_name))
        return models

class FeedPlugin(object):
    dpesuffix = '.dpe.pkl'
    _bndversep = '-'  # Band Version Separator
    _mdlversep = '-'  # Model Version Separator
    SHAREDIR = 'share'  # Dir for native telescope project data.
    DATADIR = 'data'    # Dir for telescope data for RIME level work.

    def __init__(self, path2plugin):
        self.path_ = path2plugin
        # Make sure DATADIR path exists:
        datadirpath = os.path.join(self.path_, self.DATADIR)
        if not os.path.exists(datadirpath):
            os.mkdir(datadirpath)

    def prep_dpefiles(self):
        coeffilemetas = self._search_coeff_files()
        if coeffilemetas != []:
            for coeffilemeta in coeffilemetas:
                self.get_stndpolel(coeffilemeta['band'],
                                   coeffilemeta['modeltype'],
                                   coeffilemeta['version'])

    def find_necoutfiles(self):
        necfiles = []
        necoutdir = os.path.join(self.path_, self.SHAREDIR)
        ls = os.listdir(necoutdir)
        for lsentry in ls:
            if lsentry.endswith('.out'):
                necfiles.append(lsentry)
        return necfiles

    def _parsefilename(self, filename):
        filebase, fileext = filename.split('.', 1)
        band, version, = filebase.split(self._bndversep, 1)
        return band, version

    def _convNEC2antpat(self, necoutfile, scalefac):
        print("Generating antpat from necfile: {}".format(necoutfile))
        necoutfile = os.path.join(self.path_, self.SHAREDIR, necoutfile)
        tvf = readNECout_tvecfuns(necoutfile)
        tvf.scale(scalefac)
        antpat = radfarfield.RadFarField(tvf)
        return antpat

    def compute_ccfile(self, necoutfile, scalefac, freq_center, freq_range,
                       kord, tord, ford, channels):
        """Create a HamakerArts coefficients file from a NEC out file
        and save it in a .cc file.
        """
        antpat = self._convNEC2antpat(necoutfile, scalefac)
        band, haversion = self._parsefilename(necoutfile)
        # freq_center should be close to maximum gain frequency,
        # while freq_range should avoid going outside the frequencies
        # of NEC4 file which is 0 to 100e6 Hz (also avoid edges).
        print("Generating coefficients files...")
        artsdata, filename = convDPE2LOFARcc(antpat, freq_center, freq_range,
                                             HAcoefband=band,
                                             HAcoefversion=haversion,
                                             kord=kord, tord=tord, ford=ford,
                                             channels=channels)
        srcpath = os.path.abspath(filename)
        destfilepath = os.path.join(self.path_, self.SHAREDIR, filename)
        shutil.move(srcpath, destfilepath)

    def _search_coeff_files(self):
        coeffilemetas = []
        modelspecdir = os.path.join(self.path_, self.SHAREDIR)
        ls = os.listdir(modelspecdir)
        for lsentry in ls:
            if lsentry.endswith('.cc'):
                modeltype = 'Hamaker'
                ccfilename = lsentry.rstrip('.cc')
                version, band = ccfilename.split('Coeff')
                coeffilemetas.append({'band': band, 'modeltype': modeltype,
                                      'version': version})
        return coeffilemetas

    def get_bands(self):
        coeffilemetas = self._search_coeff_files()
        bands = []
        for coeffilemeta in coeffilemetas:
            bands.append(coeffilemeta['band'])
        return bands

    def list_models4band(self, band):
        coeffilemetas = self._search_coeff_files()
        models = []
        for coeffilemeta in coeffilemetas:
            if coeffilemeta['band'] == band:
                modelstring = '{}{}{}'.format(coeffilemeta['modeltype'],
                                              self._mdlversep,
                                              coeffilemeta['version'])
                models.append(modelstring)
        return models

    def list_feednames(self):
        coeffilemetas = self._search_coeff_files()
        feednames = []
        for coeffilemeta in coeffilemetas:
            feedname = self._mdlversep.join([coeffilemeta['band'],
                                             coeffilemeta['modeltype'],
                                             coeffilemeta['version']])
            feednames.append(feedname)
        return feednames

    def list_versions4bandmodels(self, band, modeltype):
        coeffilemetas = self._search_coeff_files()
        versions = []
        for coeffilemeta in coeffilemetas:
            if coeffilemeta['band'] == band and coeffilemeta['modeltype'] == modeltype:
                version = coeffilemeta['version']
                versions.append(version)
        return versions

    def _get_ccfile(self, band, haversion):
        inpfile = haversion+"Coeff"+band+".cc"
        return inpfile

    def _get_dpfile(self, band, version):
        outfile = band+self._bndversep+version+self.dpesuffix
        return outfile

    def list_dpes(self):
        dpes = []
        dpedir = os.path.join(self.path_, self.DATADIR)
        ls = os.listdir(dpedir)
        for lsentry in ls:
            if lsentry.endswith(self.dpesuffix):
                modeltype = 'Hamaker'  # FIXME:
                dpefilename = lsentry.rstrip(self.dpesuffix)
                band, version = dpefilename.split(self._bndversep, 1)
                dpes.append({'band': band, 'modeltype': modeltype,
                             'version': version})
        return dpes

    def load_dpolel(self, band, modelstring):
        modeltype, version = modelstring.split(self._mdlversep, 1)
        if version is None:
            versions = self.list_versions4bandmodels(band, modeltype)
            version = versions[0]
        if modeltype == 'Hamaker':
            dpfile = self._get_dpfile(band, version)
            if os.path.exists(dpfile):
                dpolel = pickle.load(open(dpfile, 'rb'))
            else:
                dpolel = self._gen_dpolel_hamaker(band, version)
        return dpolel

    def _gen_dpolel_hamaker(self, band, haversion):
        """Generate dualPolElem for the given band.
        Reads the 'lofar_elem_resp' packages c++ header files of
        Hamaker-Arts coefficients and writes pickled instances of DualPolElem
        class.
        Also adds nominal LOFAR frequency channels."""
        inpfile = self._get_ccfile(band, haversion)
        outfile = self._get_dpfile(band, haversion)
        inppath = os.path.join(self.path_, self.SHAREDIR, inpfile)
        outpath = os.path.join(self.path_, self.DATADIR, outfile)
        stndpolel = convLOFARcc2DPE(inppath, outpath)
        return stndpolel
<|endoftext|>