<commit_before>#! /usr/bin/env python3

# Copyright(c) 2017 Intel Corporation.
# License: MIT See LICENSE file in root directory.
# NPS

# Object detector using SSD Mobile Net

from mvnc import mvncapi as mvnc
import numpy as numpy
import cv2
import queue
import threading

class SsdMobileNetProcessor:

    # Tiny Yolo assumes input images are these dimensions.
    SSDMN_NETWORK_IMAGE_WIDTH = 300
    SSDMN_NETWORK_IMAGE_HEIGHT = 300

    def __init__(self, network_graph_filename: str, ncs_device: mvnc.Device,
                 inital_box_prob_thresh: float, classification_mask:list=None):
        """Initializes an instance of the class

        :param network_graph_filename: is the path and filename to the graph
               file that was created by the ncsdk compiler
        :param ncs_device: is an open ncs device object to use for inferences for this graph file
        :param inital_box_prob_thresh: the initial box probablity threshold. between 0.0 and 1.0
        :param classification_mask: a list of 0 or 1 values, one for each classification label in the
        _classification_mask list.  if the value is 0 then the corresponding classification won't be reported.
        :return : None
        """

        # Load graph from disk and allocate graph.
        try:
            with open(network_graph_filename, mode='rb') as graph_file:
                graph_in_memory = graph_file.read()
            self._graph = mvnc.Graph("SSD MobileNet Graph")
            self._fifo_in, self._fifo_out = self._graph.allocate_with_fifos(ncs_device, graph_in_memory)

        except:
            print('\n\n')
            print('Error - could not load tiny yolo graph file: ' + network_graph_filename)
            print('\n\n')
            raise

        self._classification_labels=SsdMobileNetProcessor.get_classification_labels()

        self._box_probability_threshold = inital_box_prob_thresh
        self._classification_mask=classification_mask
        if (self._classification_mask is None):
            # if no mask passed then create one to accept all classifications
            self._classification_mask = [1, 1, 1, 1, 1, 1, 1,
                                         1, 1, 1, 1, 1, 1, 1,
                                         1, 1, 1, 1, 1, 1, 1]

        self._end_flag = True

    def cleanup(self):
        """Call once when done with the instance of the class

        :return: None
        """

        self._drain_queues()
        self._fifo_in.destroy()
        self._fifo_out.destroy()
        self._graph.destroy()

    @staticmethod
    def get_classification_labels():
        """get a list of the classifications that are supported by this neural network

        :return: the list of the classification strings
        """
        ret_labels = list(['background',
          'aeroplane', 'bicycle', 'bird', 'boat',
          'bottle', 'bus', 'car', 'cat', 'chair',
          'cow', 'dining table', 'dog', 'horse',
          'motorbike', 'person', 'potted plant',
          'sheep', 'sofa', 'train', 'tvmonitor'])
        return ret_labels

    def start_aysnc_inference(self, input_image:numpy.ndarray):
        """Start an asynchronous inference.  When its complete it will go to the output FIFO queue which
           can be read using the get_async_inference_result() method
           If there is no room on the input queue this function will block indefinitely until there is room,
           when there is room, it will queue the inference and return immediately

        :param input_image: he image on which to run the inference.
             it can be any size but is assumed to be opencv standard format of BGRBGRBGR...
        :return: None
        """

        # resize image to network width and height
        # then convert to float32, normalize (divide by 255),
        # and finally convert to float16 to pass to LoadTensor as input
        # for an inference
        # this returns a new image so the input_image is unchanged
        inference_image = cv2.resize(input_image,
                                 (SsdMobileNetProcessor.SSDMN_NETWORK_IMAGE_WIDTH,
                                  SsdMobileNetProcessor.SSDMN_NETWORK_IMAGE_HEIGHT),
                                 cv2.INTER_LINEAR)

        # modify inference_image for network input
        inference_image = inference_image - 127.5
        inference_image = inference_image * 0.007843

        # Load tensor and get result.  This executes the inference on the NCS
        self._graph.queue_inference_with_fifo_elem(self._fifo_in, self._fifo_out, inference_image.astype(numpy.float32), input_image)

        return

    # Reads the next available object from the output FIFO queue.
    # If there is nothing on the output FIFO, this fuction will block indefinitiley
    # until there is.
    # Returns tuple of the filtered results along with the original input image
    # the filtered results is a list of lists. each of the inner lists represent one found object and contain
    # the following 6 values:
    #    string that is network classification ie 'cat', or 'chair' etc
    #    float value for box X pixel location of upper left within source image
    #    float value for box Y pixel location of upper left within source image
    #    float value for box X pixel location of lower right within source image
    #    float value for box Y pixel location of lower right within source image
    #    float value that is the probability for the network classification 0.0 - 1.0 inclusive.
    def get_async_inference_result(self):
        """Reads the next available object from the output FIFO queue.  If there is nothing on the output FIFO,
        this fuction will block indefinitiley until there is.

        :return: tuple of the filtered results along with the original input image
        the filtered results is a list of lists. each of the inner lists represent one found object and contain
        the following 6 values:
           string that is network classification ie 'cat', or 'chair' etc
           float value for box X pixel location of upper left within source image
          float value for box Y pixel location of upper left within source image
          float value for box X pixel location of lower right within source image
          float value for box Y pixel location of lower right within source image
          float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """

        # get the result from the queue
        output, input_image = self._fifo_out.read_elem()

        # save original width and height
        input_image_width = input_image.shape[1]
        input_image_height = input_image.shape[0]

        # filter out all the objects/boxes that don't meet thresholds
        return self._filter_objects(output, input_image_width, input_image_height), input_image

    # get the number of elemets in the input queue
    def is_input_queue_empty(self):
        """ determines if the input queue for this instance is empty

        :return: True if input queue is empty or False if not.
        """
        count = self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL)
        return (count == 0)

    def _drain_queues(self):
        """clears everything from the input and output queues. call this to clear both input and output
        queues after no longer putting work into the input queues  (calling start_async_inference)

        :return: None.
        """

        while (self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL) != 0):
            # at least one item to process in the input queue, read one from output queue
            # and then loop back around
            print("input FIFO has at least one item")
            self._fifo_out.read_elem()

        while (self._fifo_out.get_option(mvnc.FifoOption.RO_READ_FILL_LEVEL) != 0):
            # output FIFO not empty so keep reading from it until it is
            print("output FIFO has at least one item")
            self._fifo_out.read_elem()

        print ("Done Draining queues")
        print ("Input FIFO fill level: " + str(self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL)))
        print ("Output FIFO fill level: " + str(self._fifo_out.get_option(mvnc.FifoOption.RO_READ_FILL_LEVEL)))
        return

    def do_sync_inference(self, input_image:numpy.ndarray):
        """Do a single inference synchronously.
        Don't mix this with calls to get_async_inference_result, Use one or the other.  It is assumed
        that the input queue is empty when this is called which will be the case if this isn't mixed
        with calls to get_async_inference_result.

        :param input_image: the image on which to run the inference it can be any size.
        :return: filtered results which is a list of lists. Each of the inner lists represent one
        found object and contain the following 6 values:
            string that is network classification ie 'cat', or 'chair' etc
            float value for box X pixel location of upper left within source image
            float value for box Y pixel location of upper left within source image
            float value for box X pixel location of lower right within source image
            float value for box Y pixel location of lower right within source image
            float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """
        self.start_aysnc_inference(input_image)
        filtered_objects, original_image = self.get_async_inference_result()

        return filtered_objects

    def get_box_probability_threshold(self):
        """Determine the current box probabilty threshold for this instance.  It will be between 0.0 and 1.0.
        A higher number means less boxes will be returned.

        :return: the box probability threshold currently in place for this instance.
        """
        return self._box_probability_threshold

    def set_box_probability_threshold(self, value):
        """set the box probability threshold.

        :param value: the new box probability threshold value, it must be between 0.0 and 1.0.
        lower values will allow less certain boxes in the inferences
        which will result in more boxes per image.  Higher values will
        filter out less certain boxes and result in fewer boxes per
        inference.
        :return: None
        """
        self._box_probability_threshold = value

    def _filter_objects(self, inference_result:numpy.ndarray, input_image_width:int, input_image_height:int):
        """Interpret the output from a single inference of the neural network
        and filter out objects/boxes with probabilities under the box probability threshold

        :param inference_result: the array of floats returned from the NCAPI as float32.
        :param input_image_width: width of the original input image, used to determine size of boxes
        :param input_image_height: height of original input image used to determine size of boxes
        :return: list of lists. each of the inner lists represent one found object
        that match the filter criteria.  The each contain the following 6 values:
        string that is network classification ie 'cat', or 'chair' etc
            float value for box X pixel location of upper left within source image
            float value for box Y pixel location of upper left within source image
            float value for box X pixel location of lower right within source image
            float value for box Y pixel location of lower right within source image
            float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """

        # the inference result is in this format:
        #   a.	First value holds the number of valid detections = num_valid.
        #   b.	The next 6 values are unused.
        #   c.	The next (7 * num_valid) values contain the valid detections data
        #       Each group of 7 values will describe an object/box These 7 values in order.
        #       The values are:
        #         0: image_id (always 0)
        #         1: class_id (this is an index into labels)
        #         2: score (this is the probability for the class)
        #         3: box left location within image as number between 0.0 and 1.0
        #         4: box top location within image as number between 0.0 and 1.0
        #         5: box right location within image as number between 0.0 and 1.0
        #         6: box bottom location within image as number between 0.0 and 1.0

        # number of boxes returned
        num_valid_boxes = int(inference_result[0])

        classes_boxes_and_probs = []
        for box_index in range(num_valid_boxes):
                base_index = 7+ box_index * 7
                if (inference_result[base_index + 2] < self._box_probability_threshold):
                    # probability/confidence is too low for this box, omit it.
                    continue
                if (self._classification_mask[int(inference_result[base_index + 1])] != 1):
                    # masking off these types of objects
                    continue

                if (not numpy.isfinite(inference_result[base_index]) or
                        not numpy.isfinite(inference_result[base_index + 1]) or
                        not numpy.isfinite(inference_result[base_index + 2]) or
                        not numpy.isfinite(inference_result[base_index + 3]) or
                        not numpy.isfinite(inference_result[base_index + 4]) or
                        not numpy.isfinite(inference_result[base_index + 5]) or
                        not numpy.isfinite(inference_result[base_index + 6])):
                    # boxes with non finite (inf, nan, etc) numbers must be ignored
                    continue

                x1 = max(int(inference_result[base_index + 3] * input_image_width), 0)
                y1 = max(int(inference_result[base_index + 4] * input_image_height), 0)
                x2 = min(int(inference_result[base_index + 5] * input_image_width), input_image_width-1)
                y2 = min(int(inference_result[base_index + 6] * input_image_height), input_image_height-1)

                classes_boxes_and_probs.append([self._classification_labels[int(inference_result[base_index + 1])], # label
                                                x1, y1, # upper left in source image
                                                x2, y2, # lower right in source image
                                                inference_result[base_index + 2] # confidence
                                                ])
        return classes_boxes_and_probs

<commit_msg>fixed tiny yolo references<commit_after>#! /usr/bin/env python3

# Copyright(c) 2017 Intel Corporation.
# License: MIT See LICENSE file in root directory.
# NPS

# Object detector using SSD Mobile Net

from mvnc import mvncapi as mvnc
import numpy as numpy
import cv2
import queue
import threading

class SsdMobileNetProcessor:

    # Neural network assumes input images are these dimensions.
    SSDMN_NETWORK_IMAGE_WIDTH = 300
    SSDMN_NETWORK_IMAGE_HEIGHT = 300

    def __init__(self, network_graph_filename: str, ncs_device: mvnc.Device,
                 inital_box_prob_thresh: float, classification_mask:list=None):
        """Initializes an instance of the class

        :param network_graph_filename: is the path and filename to the graph
               file that was created by the ncsdk compiler
        :param ncs_device: is an open ncs device object to use for inferences for this graph file
        :param inital_box_prob_thresh: the initial box probablity threshold. between 0.0 and 1.0
        :param classification_mask: a list of 0 or 1 values, one for each classification label in the
        _classification_mask list.  if the value is 0 then the corresponding classification won't be reported.
        :return : None
        """

        # Load graph from disk and allocate graph.
        try:
            with open(network_graph_filename, mode='rb') as graph_file:
                graph_in_memory = graph_file.read()
            self._graph = mvnc.Graph("SSD MobileNet Graph")
            self._fifo_in, self._fifo_out = self._graph.allocate_with_fifos(ncs_device, graph_in_memory)

        except:
            print('\n\n')
            print('Error - could not load neural network graph file: ' + network_graph_filename)
            print('\n\n')
            raise

        self._classification_labels=SsdMobileNetProcessor.get_classification_labels()

        self._box_probability_threshold = inital_box_prob_thresh
        self._classification_mask=classification_mask
        if (self._classification_mask is None):
            # if no mask passed then create one to accept all classifications
            self._classification_mask = [1, 1, 1, 1, 1, 1, 1,
                                         1, 1, 1, 1, 1, 1, 1,
                                         1, 1, 1, 1, 1, 1, 1]

        self._end_flag = True

    def cleanup(self):
        """Call once when done with the instance of the class

        :return: None
        """

        self._drain_queues()
        self._fifo_in.destroy()
        self._fifo_out.destroy()
        self._graph.destroy()

    @staticmethod
    def get_classification_labels():
        """get a list of the classifications that are supported by this neural network

        :return: the list of the classification strings
        """
        ret_labels = list(['background',
          'aeroplane', 'bicycle', 'bird', 'boat',
          'bottle', 'bus', 'car', 'cat', 'chair',
          'cow', 'dining table', 'dog', 'horse',
          'motorbike', 'person', 'potted plant',
          'sheep', 'sofa', 'train', 'tvmonitor'])
        return ret_labels

    def start_aysnc_inference(self, input_image:numpy.ndarray):
        """Start an asynchronous inference.  When its complete it will go to the output FIFO queue which
           can be read using the get_async_inference_result() method
           If there is no room on the input queue this function will block indefinitely until there is room,
           when there is room, it will queue the inference and return immediately

        :param input_image: he image on which to run the inference.
             it can be any size but is assumed to be opencv standard format of BGRBGRBGR...
        :return: None
        """

        # resize image to network width and height
        # then convert to float32, normalize (divide by 255),
        # and finally convert to float16 to pass to LoadTensor as input
        # for an inference
        # this returns a new image so the input_image is unchanged
        inference_image = cv2.resize(input_image,
                                 (SsdMobileNetProcessor.SSDMN_NETWORK_IMAGE_WIDTH,
                                  SsdMobileNetProcessor.SSDMN_NETWORK_IMAGE_HEIGHT),
                                 cv2.INTER_LINEAR)

        # modify inference_image for network input
        inference_image = inference_image - 127.5
        inference_image = inference_image * 0.007843

        # Load tensor and get result.  This executes the inference on the NCS
        self._graph.queue_inference_with_fifo_elem(self._fifo_in, self._fifo_out, inference_image.astype(numpy.float32), input_image)

        return

    # Reads the next available object from the output FIFO queue.
    # If there is nothing on the output FIFO, this fuction will block indefinitiley
    # until there is.
    # Returns tuple of the filtered results along with the original input image
    # the filtered results is a list of lists. each of the inner lists represent one found object and contain
    # the following 6 values:
    #    string that is network classification ie 'cat', or 'chair' etc
    #    float value for box X pixel location of upper left within source image
    #    float value for box Y pixel location of upper left within source image
    #    float value for box X pixel location of lower right within source image
    #    float value for box Y pixel location of lower right within source image
    #    float value that is the probability for the network classification 0.0 - 1.0 inclusive.
    def get_async_inference_result(self):
        """Reads the next available object from the output FIFO queue.  If there is nothing on the output FIFO,
        this fuction will block indefinitiley until there is.

        :return: tuple of the filtered results along with the original input image
        the filtered results is a list of lists. each of the inner lists represent one found object and contain
        the following 6 values:
           string that is network classification ie 'cat', or 'chair' etc
           float value for box X pixel location of upper left within source image
          float value for box Y pixel location of upper left within source image
          float value for box X pixel location of lower right within source image
          float value for box Y pixel location of lower right within source image
          float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """

        # get the result from the queue
        output, input_image = self._fifo_out.read_elem()

        # save original width and height
        input_image_width = input_image.shape[1]
        input_image_height = input_image.shape[0]

        # filter out all the objects/boxes that don't meet thresholds
        return self._filter_objects(output, input_image_width, input_image_height), input_image

    # get the number of elemets in the input queue
    def is_input_queue_empty(self):
        """ determines if the input queue for this instance is empty

        :return: True if input queue is empty or False if not.
        """
        count = self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL)
        return (count == 0)

    def _drain_queues(self):
        """clears everything from the input and output queues. call this to clear both input and output
        queues after no longer putting work into the input queues  (calling start_async_inference)

        :return: None.
        """

        while (self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL) != 0):
            # at least one item to process in the input queue, read one from output queue
            # and then loop back around
            print("input FIFO has at least one item")
            self._fifo_out.read_elem()

        while (self._fifo_out.get_option(mvnc.FifoOption.RO_READ_FILL_LEVEL) != 0):
            # output FIFO not empty so keep reading from it until it is
            print("output FIFO has at least one item")
            self._fifo_out.read_elem()

        print ("Done Draining queues")
        print ("Input FIFO fill level: " + str(self._fifo_in.get_option(mvnc.FifoOption.RO_WRITE_FILL_LEVEL)))
        print ("Output FIFO fill level: " + str(self._fifo_out.get_option(mvnc.FifoOption.RO_READ_FILL_LEVEL)))
        return

    def do_sync_inference(self, input_image:numpy.ndarray):
        """Do a single inference synchronously.
        Don't mix this with calls to get_async_inference_result, Use one or the other.  It is assumed
        that the input queue is empty when this is called which will be the case if this isn't mixed
        with calls to get_async_inference_result.

        :param input_image: the image on which to run the inference it can be any size.
        :return: filtered results which is a list of lists. Each of the inner lists represent one
        found object and contain the following 6 values:
            string that is network classification ie 'cat', or 'chair' etc
            float value for box X pixel location of upper left within source image
            float value for box Y pixel location of upper left within source image
            float value for box X pixel location of lower right within source image
            float value for box Y pixel location of lower right within source image
            float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """
        self.start_aysnc_inference(input_image)
        filtered_objects, original_image = self.get_async_inference_result()

        return filtered_objects

    def get_box_probability_threshold(self):
        """Determine the current box probabilty threshold for this instance.  It will be between 0.0 and 1.0.
        A higher number means less boxes will be returned.

        :return: the box probability threshold currently in place for this instance.
        """
        return self._box_probability_threshold

    def set_box_probability_threshold(self, value):
        """set the box probability threshold.

        :param value: the new box probability threshold value, it must be between 0.0 and 1.0.
        lower values will allow less certain boxes in the inferences
        which will result in more boxes per image.  Higher values will
        filter out less certain boxes and result in fewer boxes per
        inference.
        :return: None
        """
        self._box_probability_threshold = value

    def _filter_objects(self, inference_result:numpy.ndarray, input_image_width:int, input_image_height:int):
        """Interpret the output from a single inference of the neural network
        and filter out objects/boxes with probabilities under the box probability threshold

        :param inference_result: the array of floats returned from the NCAPI as float32.
        :param input_image_width: width of the original input image, used to determine size of boxes
        :param input_image_height: height of original input image used to determine size of boxes
        :return: list of lists. each of the inner lists represent one found object
        that match the filter criteria.  The each contain the following 6 values:
        string that is network classification ie 'cat', or 'chair' etc
            float value for box X pixel location of upper left within source image
            float value for box Y pixel location of upper left within source image
            float value for box X pixel location of lower right within source image
            float value for box Y pixel location of lower right within source image
            float value that is the probability for the network classification 0.0 - 1.0 inclusive.
        """

        # the inference result is in this format:
        #   a.	First value holds the number of valid detections = num_valid.
        #   b.	The next 6 values are unused.
        #   c.	The next (7 * num_valid) values contain the valid detections data
        #       Each group of 7 values will describe an object/box These 7 values in order.
        #       The values are:
        #         0: image_id (always 0)
        #         1: class_id (this is an index into labels)
        #         2: score (this is the probability for the class)
        #         3: box left location within image as number between 0.0 and 1.0
        #         4: box top location within image as number between 0.0 and 1.0
        #         5: box right location within image as number between 0.0 and 1.0
        #         6: box bottom location within image as number between 0.0 and 1.0

        # number of boxes returned
        num_valid_boxes = int(inference_result[0])

        classes_boxes_and_probs = []
        for box_index in range(num_valid_boxes):
                base_index = 7+ box_index * 7
                if (inference_result[base_index + 2] < self._box_probability_threshold):
                    # probability/confidence is too low for this box, omit it.
                    continue
                if (self._classification_mask[int(inference_result[base_index + 1])] != 1):
                    # masking off these types of objects
                    continue

                if (not numpy.isfinite(inference_result[base_index]) or
                        not numpy.isfinite(inference_result[base_index + 1]) or
                        not numpy.isfinite(inference_result[base_index + 2]) or
                        not numpy.isfinite(inference_result[base_index + 3]) or
                        not numpy.isfinite(inference_result[base_index + 4]) or
                        not numpy.isfinite(inference_result[base_index + 5]) or
                        not numpy.isfinite(inference_result[base_index + 6])):
                    # boxes with non finite (inf, nan, etc) numbers must be ignored
                    continue

                x1 = max(int(inference_result[base_index + 3] * input_image_width), 0)
                y1 = max(int(inference_result[base_index + 4] * input_image_height), 0)
                x2 = min(int(inference_result[base_index + 5] * input_image_width), input_image_width-1)
                y2 = min(int(inference_result[base_index + 6] * input_image_height), input_image_height-1)

                classes_boxes_and_probs.append([self._classification_labels[int(inference_result[base_index + 1])], # label
                                                x1, y1, # upper left in source image
                                                x2, y2, # lower right in source image
                                                inference_result[base_index + 2] # confidence
                                                ])
        return classes_boxes_and_probs

<|endoftext|>


<commit_before># Copyright (c) 2013 Rackspace
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Determines operating system version and os depended commands.
"""
from trove.guestagent.common.operating_system import get_os

OS = get_os()
REDIS_CONFIG = '/etc/redis/redis.conf'
REDIS_CONF_DIR = '/etc/redis'
REDIS_INIT = '/etc/init/redis-server.conf'
REDIS_CLI = '/usr/bin/redis-cli'
REDIS_BIN = '/usr/bin/redis-server'
REDIS_CMD_ENABLE = 'update-rc.d redis-server enable'
REDIS_CMD_DISABLE = 'update-rc.d redis-server disable'
REDIS_CMD_START = 'service redis-server start || /bin/true'
REDIS_CMD_STOP = 'service redis-server stop || /bin/true'
REDIS_PACKAGE = 'redis-server'
SERVICE_CANDIDATES = ['redis-server']

if OS is 'redhat':
    REDIS_BIN = '/usr/libexec/redis-server'
    REDIS_CMD_ENABLE = 'chkconfig redis-server on'
    REDIS_CMD_DISABLE = 'chkconfig redis-server off'
    REDIS_CMD_START = 'service redis-server start'
    REDIS_CMD_STOP = 'service redis-server stop'
<commit_msg>Update redis system.py paths for current RHEL/CentOS/Fedora<commit_after># Copyright (c) 2013 Rackspace
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Determines operating system version and OS dependent commands.
"""
from trove.guestagent.common.operating_system import get_os

OS = get_os()
REDIS_CONFIG = '/etc/redis/redis.conf'
REDIS_CONF_DIR = '/etc/redis'
REDIS_INIT = '/etc/init/redis-server.conf'
REDIS_CLI = '/usr/bin/redis-cli'
REDIS_BIN = '/usr/bin/redis-server'
REDIS_CMD_ENABLE = 'update-rc.d redis-server enable'
REDIS_CMD_DISABLE = 'update-rc.d redis-server disable'
REDIS_CMD_START = 'service redis-server start || /bin/true'
REDIS_CMD_STOP = 'service redis-server stop || /bin/true'
REDIS_PACKAGE = 'redis-server'
SERVICE_CANDIDATES = ['redis-server']

if OS is 'redhat':
    REDIS_CONFIG = '/etc/redis.conf'
    REDIS_CMD_ENABLE = 'systemctl enable redis'
    REDIS_CMD_DISABLE = 'systemctl disable redis'
    REDIS_CMD_START = 'systemctl start redis'
    REDIS_CMD_STOP = 'systemctl stop redis'
    REDIS_PACKAGE = 'redis'
    SERVICE_CANDIDATES = ['redis']
<|endoftext|>


<commit_before>import avro.schema
import os
import json
import schema_salad.validate as validate
import copy
import yaml
import copy
import logging
import pprint
from aslist import aslist
import schema_salad.schema
import urlparse
import pprint
from pkg_resources import resource_stream
import stat
from builder import Builder
import tempfile
import glob
from errors import WorkflowException
from pathmapper import abspath
from rdflib import URIRef
from rdflib.namespace import RDFS

_logger = logging.getLogger("cwltool")

supportedProcessRequirements = ["DockerRequirement",
                                "ExpressionEngineRequirement",
                                "SchemaDefRequirement",
                                "EnvVarRequirement",
                                "CreateFileRequirement",
                                "ScatterFeatureRequirement",
                                "SubworkflowFeatureRequirement",
                                "MultipleInputFeatureRequirement",
                                "FormatOntologyRequirement"]

def get_schema():
    f = resource_stream(__name__, 'schemas/draft-3/cwl-avro.yml')
    j = yaml.load(f)
    j["name"] = "https://w3id.org/cwl/cwl"
    return schema_salad.schema.load_schema(j)

def get_feature(self, feature):
    for t in reversed(self.requirements):
        if t["class"] == feature:
            return (t, True)
    for t in reversed(self.hints):
        if t["class"] == feature:
            return (t, False)
    return (None, None)

def shortname(inputid):
    (_, d) = urlparse.urldefrag(inputid)
    return d.split("/")[-1].split(".")[-1]

class StdFsAccess(object):
    def __init__(self, basedir):
        self.basedir = basedir

    def _abs(self, p):
        return abspath(p, self.basedir)

    def glob(self, pattern):
        return glob.glob(self._abs(pattern))

    def open(self, fn, mode):
        return open(self._abs(fn), mode)

    def exists(self, fn):
        return os.path.exists(self._abs(fn))

def checkRequirements(rec, supportedProcessRequirements):
    if isinstance(rec, dict):
        if "requirements" in rec:
            for r in rec["requirements"]:
                if r["class"] not in supportedProcessRequirements:
                    raise Exception("Unsupported requirement %s" % r["class"])
        if "scatter" in rec:
            if isinstance(rec["scatter"], list) and rec["scatter"] > 1:
                raise Exception("Unsupported complex scatter type '%s'" % rec.get("scatterMethod"))
        for d in rec:
            checkRequirements(rec[d], supportedProcessRequirements)
    if isinstance(rec, list):
        for d in rec:
            checkRequirements(d, supportedProcessRequirements)

def adjustFiles(rec, op):
    """Apply a mapping function to each File path in the object `rec`."""

    if isinstance(rec, dict):
        if rec.get("class") == "File":
            rec["path"] = op(rec["path"])
        for d in rec:
            adjustFiles(rec[d], op)
    if isinstance(rec, list):
        for d in rec:
            adjustFiles(d, op)

def formatSubclassOf(fmt, cls, ontology):
    if ontology is None:
        return False

    for s,p,o in ontology.triples( (fmt, RDFS.subClassOf, None) ):
        if o == cls:
            return True

    for s,p,o in ontology.triples( (fmt, RDFS.subClassOf, None) ):
        if formatSubclassOf(o, cls, ontology):
            return True

    return False

def checkFormat(actualFile, inputFormats, requirements, ontology):
    for af in aslist(actualFile):
        if "format" not in af:
            raise validate.ValidationException("Missing required 'format' for File %s" % af)
        for inpf in aslist(inputFormats):
            if af["format"] == inpf or formatSubclassOf(inpf, af["format"], ontology):
                return
        raise validate.ValidationException("Incompatible file format %s required format(s) %s" % (af["format"], inputFormats))

class Process(object):
    def __init__(self, toolpath_object, **kwargs):
        (_, self.names, _) = get_schema()
        self.tool = toolpath_object
        self.requirements = kwargs.get("requirements", []) + self.tool.get("requirements", [])
        self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])

        self.validate_hints(self.tool.get("hints", []), strict=kwargs.get("strict"))

        self.schemaDefs = {}

        sd, _ = self.get_requirement("SchemaDefRequirement")

        if sd:
            sdtypes = sd["types"]
            av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in sdtypes}, set())
            for i in av:
                self.schemaDefs[i["name"]] = i
            avro.schema.make_avsc_object(av, self.names)

        # Build record schema from inputs
        self.inputs_record_schema = {"name": "input_record_schema", "type": "record", "fields": []}
        self.outputs_record_schema = {"name": "outputs_record_schema", "type": "record", "fields": []}

        for key in ("inputs", "outputs"):
            for i in self.tool[key]:
                c = copy.copy(i)
                doc_url, _ = urlparse.urldefrag(c['id'])
                c["name"] = shortname(c["id"])
                del c["id"]

                if "type" not in c:
                    raise validate.ValidationException("Missing `type` in parameter `%s`" % c["name"])

                if "default" in c and "null" not in aslist(c["type"]):
                    c["type"] = ["null"] + aslist(c["type"])
                else:
                    c["type"] = c["type"]

                if key == "inputs":
                    self.inputs_record_schema["fields"].append(c)
                elif key == "outputs":
                    self.outputs_record_schema["fields"].append(c)

        try:
            self.inputs_record_schema = schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set())
            avro.schema.make_avsc_object(self.inputs_record_schema, self.names)
        except avro.schema.SchemaParseException as e:
            raise validate.ValidationException("Got error `%s` while prcoessing inputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.inputs_record_schema, indent=4)))

        try:
            self.outputs_record_schema = schema_salad.schema.make_valid_avro(self.outputs_record_schema, {}, set())
            avro.schema.make_avsc_object(self.outputs_record_schema, self.names)
        except avro.schema.SchemaParseException as e:
            raise validate.ValidationException("Got error `%s` while prcoessing outputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4)))

    def _init_job(self, joborder, input_basedir, **kwargs):
        builder = Builder()
        builder.job = copy.deepcopy(joborder)

        for i in self.tool["inputs"]:
            d = shortname(i["id"])
            if d not in builder.job and "default" in i:
                builder.job[d] = i["default"]

        for r in self.requirements:
            if r["class"] not in supportedProcessRequirements:
                raise WorkflowException("Unsupported process requirement %s" % (r["class"]))

        # Validate job order
        try:
            validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job)
        except validate.ValidationException as e:
            raise WorkflowException("Error validating input record, " + str(e))

        builder.files = []
        builder.bindings = []
        builder.schemaDefs = self.schemaDefs
        builder.names = self.names
        builder.requirements = self.requirements

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            builder.outdir = kwargs.get("docker_outdir") or "/tmp/job_output"
            builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp/job_tmp"
        else:
            builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()

        builder.fs_access = kwargs.get("fs_access") or StdFsAccess(input_basedir)

        formatReq, _ = self.get_requirement("FormatOntologyRequirement")
        ontology = None
        if formatReq:
            import rdflib
            ontology = rdflib.Graph()
            for a in aslist(formatReq.get("formatOntologies", [])):
                ontology.parse(a)

        for i in self.tool["inputs"]:
            d = shortname(i["id"])
            if d in builder.job and i.get("format"):
                checkFormat(builder.job[d], builder.do_eval(i["format"]), self.requirements, ontology)

        builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))

        return builder

    def validate_hints(self, hints, strict):
        for r in hints:
            try:
                if self.names.get_name(r["class"], "") is not None:
                    validate.validate_ex(self.names.get_name(r["class"], ""), r, strict=strict)
                else:
                    _logger.info(validate.ValidationException("Unknown hint %s" % (r["class"])))
            except validate.ValidationException as v:
                raise validate.ValidationException("Validating hint `%s`: %s" % (r["class"], str(v)))

    def get_requirement(self, feature):
        return get_feature(self, feature)

def empty_subtree(dirpath):
    # Test if a directory tree contains any files (does not count empty
    # subdirectories)
    for d in os.listdir(dirpath):
        d = os.path.join(dirpath, d)
        if stat.S_ISDIR(os.stat(d).st_mode):
            if empty_subtree(d) is False:
                return False
        else:
            return False
    return True
<commit_msg>Fixed file format reasoning.<commit_after>import avro.schema
import os
import json
import schema_salad.validate as validate
import copy
import yaml
import copy
import logging
import pprint
from aslist import aslist
import schema_salad.schema
import urlparse
import pprint
from pkg_resources import resource_stream
import stat
from builder import Builder
import tempfile
import glob
from errors import WorkflowException
from pathmapper import abspath
from rdflib import URIRef
from rdflib.namespace import RDFS

_logger = logging.getLogger("cwltool")

supportedProcessRequirements = ["DockerRequirement",
                                "ExpressionEngineRequirement",
                                "SchemaDefRequirement",
                                "EnvVarRequirement",
                                "CreateFileRequirement",
                                "ScatterFeatureRequirement",
                                "SubworkflowFeatureRequirement",
                                "MultipleInputFeatureRequirement",
                                "FormatOntologyRequirement"]

def get_schema():
    f = resource_stream(__name__, 'schemas/draft-3/cwl-avro.yml')
    j = yaml.load(f)
    j["name"] = "https://w3id.org/cwl/cwl"
    return schema_salad.schema.load_schema(j)

def get_feature(self, feature):
    for t in reversed(self.requirements):
        if t["class"] == feature:
            return (t, True)
    for t in reversed(self.hints):
        if t["class"] == feature:
            return (t, False)
    return (None, None)

def shortname(inputid):
    (_, d) = urlparse.urldefrag(inputid)
    return d.split("/")[-1].split(".")[-1]

class StdFsAccess(object):
    def __init__(self, basedir):
        self.basedir = basedir

    def _abs(self, p):
        return abspath(p, self.basedir)

    def glob(self, pattern):
        return glob.glob(self._abs(pattern))

    def open(self, fn, mode):
        return open(self._abs(fn), mode)

    def exists(self, fn):
        return os.path.exists(self._abs(fn))

def checkRequirements(rec, supportedProcessRequirements):
    if isinstance(rec, dict):
        if "requirements" in rec:
            for r in rec["requirements"]:
                if r["class"] not in supportedProcessRequirements:
                    raise Exception("Unsupported requirement %s" % r["class"])
        if "scatter" in rec:
            if isinstance(rec["scatter"], list) and rec["scatter"] > 1:
                raise Exception("Unsupported complex scatter type '%s'" % rec.get("scatterMethod"))
        for d in rec:
            checkRequirements(rec[d], supportedProcessRequirements)
    if isinstance(rec, list):
        for d in rec:
            checkRequirements(d, supportedProcessRequirements)

def adjustFiles(rec, op):
    """Apply a mapping function to each File path in the object `rec`."""

    if isinstance(rec, dict):
        if rec.get("class") == "File":
            rec["path"] = op(rec["path"])
        for d in rec:
            adjustFiles(rec[d], op)
    if isinstance(rec, list):
        for d in rec:
            adjustFiles(d, op)

def formatSubclassOf(fmt, cls, ontology):
    if ontology is None:
        return False

    fmt = URIRef(fmt)

    for s,p,o in ontology.triples( (fmt, RDFS.subClassOf, None) ):
        print s,p,o
        if o == URIRef(cls):
            return True

    for s,p,o in ontology.triples( (fmt, RDFS.subClassOf, None) ):
        if formatSubclassOf(o, cls, ontology):
            return True

    return False

def checkFormat(actualFile, inputFormats, requirements, ontology):
    for af in aslist(actualFile):
        if "format" not in af:
            raise validate.ValidationException("Missing required 'format' for File %s" % af)
        for inpf in aslist(inputFormats):
            if af["format"] == inpf or formatSubclassOf(af["format"], inpf, ontology):
                return
        raise validate.ValidationException("Incompatible file format %s required format(s) %s" % (af["format"], inputFormats))

class Process(object):
    def __init__(self, toolpath_object, **kwargs):
        (_, self.names, _) = get_schema()
        self.tool = toolpath_object
        self.requirements = kwargs.get("requirements", []) + self.tool.get("requirements", [])
        self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])

        self.validate_hints(self.tool.get("hints", []), strict=kwargs.get("strict"))

        self.schemaDefs = {}

        sd, _ = self.get_requirement("SchemaDefRequirement")

        if sd:
            sdtypes = sd["types"]
            av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in sdtypes}, set())
            for i in av:
                self.schemaDefs[i["name"]] = i
            avro.schema.make_avsc_object(av, self.names)

        # Build record schema from inputs
        self.inputs_record_schema = {"name": "input_record_schema", "type": "record", "fields": []}
        self.outputs_record_schema = {"name": "outputs_record_schema", "type": "record", "fields": []}

        for key in ("inputs", "outputs"):
            for i in self.tool[key]:
                c = copy.copy(i)
                doc_url, _ = urlparse.urldefrag(c['id'])
                c["name"] = shortname(c["id"])
                del c["id"]

                if "type" not in c:
                    raise validate.ValidationException("Missing `type` in parameter `%s`" % c["name"])

                if "default" in c and "null" not in aslist(c["type"]):
                    c["type"] = ["null"] + aslist(c["type"])
                else:
                    c["type"] = c["type"]

                if key == "inputs":
                    self.inputs_record_schema["fields"].append(c)
                elif key == "outputs":
                    self.outputs_record_schema["fields"].append(c)

        try:
            self.inputs_record_schema = schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set())
            avro.schema.make_avsc_object(self.inputs_record_schema, self.names)
        except avro.schema.SchemaParseException as e:
            raise validate.ValidationException("Got error `%s` while prcoessing inputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.inputs_record_schema, indent=4)))

        try:
            self.outputs_record_schema = schema_salad.schema.make_valid_avro(self.outputs_record_schema, {}, set())
            avro.schema.make_avsc_object(self.outputs_record_schema, self.names)
        except avro.schema.SchemaParseException as e:
            raise validate.ValidationException("Got error `%s` while prcoessing outputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4)))

    def _init_job(self, joborder, input_basedir, **kwargs):
        builder = Builder()
        builder.job = copy.deepcopy(joborder)

        for i in self.tool["inputs"]:
            d = shortname(i["id"])
            if d not in builder.job and "default" in i:
                builder.job[d] = i["default"]

        for r in self.requirements:
            if r["class"] not in supportedProcessRequirements:
                raise WorkflowException("Unsupported process requirement %s" % (r["class"]))

        # Validate job order
        try:
            validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job)
        except validate.ValidationException as e:
            raise WorkflowException("Error validating input record, " + str(e))

        builder.files = []
        builder.bindings = []
        builder.schemaDefs = self.schemaDefs
        builder.names = self.names
        builder.requirements = self.requirements

        dockerReq, _ = self.get_requirement("DockerRequirement")
        if dockerReq and kwargs.get("use_container"):
            builder.outdir = kwargs.get("docker_outdir") or "/tmp/job_output"
            builder.tmpdir = kwargs.get("docker_tmpdir") or "/tmp/job_tmp"
        else:
            builder.outdir = kwargs.get("outdir") or tempfile.mkdtemp()
            builder.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp()

        builder.fs_access = kwargs.get("fs_access") or StdFsAccess(input_basedir)

        formatReq, _ = self.get_requirement("FormatOntologyRequirement")
        ontology = None
        if formatReq:
            import rdflib
            ontology = rdflib.Graph()
            for a in aslist(formatReq.get("formatOntologies", [])):
                ontology.parse(a)

        for i in self.tool["inputs"]:
            d = shortname(i["id"])
            if d in builder.job and i.get("format"):
                checkFormat(builder.job[d], builder.do_eval(i["format"]), self.requirements, ontology)

        builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))

        return builder

    def validate_hints(self, hints, strict):
        for r in hints:
            try:
                if self.names.get_name(r["class"], "") is not None:
                    validate.validate_ex(self.names.get_name(r["class"], ""), r, strict=strict)
                else:
                    _logger.info(validate.ValidationException("Unknown hint %s" % (r["class"])))
            except validate.ValidationException as v:
                raise validate.ValidationException("Validating hint `%s`: %s" % (r["class"], str(v)))

    def get_requirement(self, feature):
        return get_feature(self, feature)

def empty_subtree(dirpath):
    # Test if a directory tree contains any files (does not count empty
    # subdirectories)
    for d in os.listdir(dirpath):
        d = os.path.join(dirpath, d)
        if stat.S_ISDIR(os.stat(d).st_mode):
            if empty_subtree(d) is False:
                return False
        else:
            return False
    return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
Tests for putting velocity differentials into SkyCoord objects.
"""

import pytest

import numpy as np

from ... import units as u
from ...tests.helper import assert_quantity_allclose
from .. import (SkyCoord, ICRS, SphericalRepresentation, SphericalDifferential,
                SphericalCosLatDifferential, CartesianRepresentation,
                CartesianDifferential, Galactic, PrecessedGeocentric)

try:
    import scipy
    HAS_SCIPY = True
except ImportError:
    HAS_SCIPY = False

def test_creation_frameobjs():
    i = ICRS(1*u.deg, 2*u.deg, pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr)
    sc = SkyCoord(i)

    for attrnm in ['ra', 'dec', 'pm_ra_cosdec', 'pm_dec']:
        assert_quantity_allclose(getattr(i, attrnm), getattr(sc, attrnm))

    sc_nod = SkyCoord(ICRS(1*u.deg, 2*u.deg))

    for attrnm in ['ra', 'dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_nod, attrnm))

def test_creation_attrs():
    sc1 = SkyCoord(1*u.deg, 2*u.deg,
                   pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   frame='fk5')
    assert_quantity_allclose(sc1.ra, 1*u.deg)
    assert_quantity_allclose(sc1.dec, 2*u.deg)
    assert_quantity_allclose(sc1.pm_ra_cosdec, .2*u.arcsec/u.kyr)
    assert_quantity_allclose(sc1.pm_dec, .1*u.arcsec/u.kyr)

    sc2 = SkyCoord(1*u.deg, 2*u.deg,
                   pm_ra=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   differential_cls=SphericalDifferential)
    assert_quantity_allclose(sc2.ra, 1*u.deg)
    assert_quantity_allclose(sc2.dec, 2*u.deg)
    assert_quantity_allclose(sc2.pm_ra, .2*u.arcsec/u.kyr)
    assert_quantity_allclose(sc2.pm_dec, .1*u.arcsec/u.kyr)

    sc3 = SkyCoord('1:2:3 4:5:6',
                   pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   unit=(u.hour, u.deg))

    assert_quantity_allclose(sc3.ra, 1*u.hourangle + 2*u.arcmin*15 + 3*u.arcsec*15)
    assert_quantity_allclose(sc3.dec, 4*u.deg + 5*u.arcmin + 6*u.arcsec)
    # might as well check with sillier units?
    assert_quantity_allclose(sc3.pm_ra_cosdec, 1.2776637006616473e-07 * u.arcmin / u.fortnight)
    assert_quantity_allclose(sc3.pm_dec, 6.388318503308237e-08 * u.arcmin / u.fortnight)

def test_creation_copy_basic():
    i = ICRS(1*u.deg, 2*u.deg, pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr)
    sc = SkyCoord(i)
    sc_cpy = SkyCoord(sc)

    for attrnm in ['ra', 'dec', 'pm_ra_cosdec', 'pm_dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_cpy, attrnm))

def test_creation_copy_rediff():
    sc = SkyCoord(1*u.deg, 2*u.deg,
                  pm_ra=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                  differential_cls=SphericalDifferential)

    sc_cpy = SkyCoord(sc)
    for attrnm in ['ra', 'dec', 'pm_ra', 'pm_dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_cpy, attrnm))

    sc_newdiff = SkyCoord(sc, differential_cls=SphericalCosLatDifferential)
    reprepr = sc.represent_as(SphericalRepresentation, SphericalCosLatDifferential)
    assert_quantity_allclose(sc_newdiff.pm_ra_cosdec,
                             reprepr.differentials['s'].d_lon_coslat)

def test_creation_cartesian():
    rep = CartesianRepresentation([10, 0., 0.]*u.pc)
    dif = CartesianDifferential([0, 100, 0.]*u.pc/u.Myr)
    rep = rep.with_differentials(dif)
    c = SkyCoord(rep)

    sdif = dif.represent_as(SphericalCosLatDifferential, rep)
    assert_quantity_allclose(c.pm_ra_cosdec, sdif.d_lon_coslat)

@pytest.mark.xfail  #TODO: FIX
def test_useful_error_missing():
    sc_nod = SkyCoord(ICRS(1*u.deg, 2*u.deg))
    try:
        sc_nod.l
    except AttributeError as e:
        # this is double-checking the *normal* behavior
        msg_l = e.args[0]

    try:
        sc_nod.pm_dec
    except AttributeError as e:
        msg_pm_dec = e.args[0]

    assert "has no attribute" in msg_l
    assert "has no attribute" in msg_pm_dec

# ----------------------Operations on SkyCoords w/ velocities-------------------

# define some fixtires to get baseline coordinates to try operations with
@pytest.fixture(scope="module", params=[(False, False),
                                        (True, False),
                                        (False, True),
                                        (True, True)])
def sc(request):
    incldist, inclrv = request.param

    args = [1*u.deg, 2*u.deg]
    kwargs = dict(pm_dec=1*u.mas/u.yr, pm_ra_cosdec=2*u.mas/u.yr)
    if incldist:
        kwargs['distance'] = 213.4*u.pc
    if inclrv:
        kwargs['radial_velocity'] = 61*u.km/u.s

    return SkyCoord(*args, **kwargs)

@pytest.fixture(scope="module")
def sc2():
    return SkyCoord(1*u.deg, 2*u.deg,
                pm_dec=1*u.mas/u.yr, pm_ra_cosdec=2*u.mas/u.yr)

@pytest.fixture(scope="module")
def scmany():
    return SkyCoord(ICRS(ra=[1]*100*u.deg, dec=[2]*100*u.deg,
                     pm_ra_cosdec=np.random.randn(100)*u.mas/u.yr,
                     pm_dec=np.random.randn(100)*u.mas/u.yr,))

def test_separation(sc, sc2):
    sc.separation(sc2)

def test_accessors(sc, scmany):
    sc.data.differentials['s']
    sc.spherical
    sc.galactic

    scmany[0]
    scmany.spherical
    scmany.galactic

def test_transforms(sc):
    trans = sc.transform_to('galactic')
    assert isinstance(trans.frame, Galactic)

@pytest.mark.skipif(str('not HAS_SCIPY'))
def test_matching(sc, scmany):
    # just check that it works and yields something
    idx, d2d, d3d = sc.match_to_catalog_sky(scmany)

def test_position_angle(sc, sc2):
    sc.position_angle(sc2)

@pytest.mark.xfail  #TODO: FIX
def test_constellations(sc):
    const = sc.get_constellation()
    assert const == 'Pisces'

@pytest.mark.xfail  #TODO: FIX
def test_skyoffset_frame(sc):
    sc.skyoffset_frame()
<commit_msg>Allow a bit looser but still useful error message<commit_after># -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

"""
Tests for putting velocity differentials into SkyCoord objects.
"""

import pytest

import numpy as np

from ... import units as u
from ...tests.helper import assert_quantity_allclose
from .. import (SkyCoord, ICRS, SphericalRepresentation, SphericalDifferential,
                SphericalCosLatDifferential, CartesianRepresentation,
                CartesianDifferential, Galactic, PrecessedGeocentric)

try:
    import scipy
    HAS_SCIPY = True
except ImportError:
    HAS_SCIPY = False

def test_creation_frameobjs():
    i = ICRS(1*u.deg, 2*u.deg, pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr)
    sc = SkyCoord(i)

    for attrnm in ['ra', 'dec', 'pm_ra_cosdec', 'pm_dec']:
        assert_quantity_allclose(getattr(i, attrnm), getattr(sc, attrnm))

    sc_nod = SkyCoord(ICRS(1*u.deg, 2*u.deg))

    for attrnm in ['ra', 'dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_nod, attrnm))

def test_creation_attrs():
    sc1 = SkyCoord(1*u.deg, 2*u.deg,
                   pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   frame='fk5')
    assert_quantity_allclose(sc1.ra, 1*u.deg)
    assert_quantity_allclose(sc1.dec, 2*u.deg)
    assert_quantity_allclose(sc1.pm_ra_cosdec, .2*u.arcsec/u.kyr)
    assert_quantity_allclose(sc1.pm_dec, .1*u.arcsec/u.kyr)

    sc2 = SkyCoord(1*u.deg, 2*u.deg,
                   pm_ra=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   differential_cls=SphericalDifferential)
    assert_quantity_allclose(sc2.ra, 1*u.deg)
    assert_quantity_allclose(sc2.dec, 2*u.deg)
    assert_quantity_allclose(sc2.pm_ra, .2*u.arcsec/u.kyr)
    assert_quantity_allclose(sc2.pm_dec, .1*u.arcsec/u.kyr)

    sc3 = SkyCoord('1:2:3 4:5:6',
                   pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                   unit=(u.hour, u.deg))

    assert_quantity_allclose(sc3.ra, 1*u.hourangle + 2*u.arcmin*15 + 3*u.arcsec*15)
    assert_quantity_allclose(sc3.dec, 4*u.deg + 5*u.arcmin + 6*u.arcsec)
    # might as well check with sillier units?
    assert_quantity_allclose(sc3.pm_ra_cosdec, 1.2776637006616473e-07 * u.arcmin / u.fortnight)
    assert_quantity_allclose(sc3.pm_dec, 6.388318503308237e-08 * u.arcmin / u.fortnight)

def test_creation_copy_basic():
    i = ICRS(1*u.deg, 2*u.deg, pm_ra_cosdec=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr)
    sc = SkyCoord(i)
    sc_cpy = SkyCoord(sc)

    for attrnm in ['ra', 'dec', 'pm_ra_cosdec', 'pm_dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_cpy, attrnm))

def test_creation_copy_rediff():
    sc = SkyCoord(1*u.deg, 2*u.deg,
                  pm_ra=.2*u.mas/u.yr, pm_dec=.1*u.mas/u.yr,
                  differential_cls=SphericalDifferential)

    sc_cpy = SkyCoord(sc)
    for attrnm in ['ra', 'dec', 'pm_ra', 'pm_dec']:
        assert_quantity_allclose(getattr(sc, attrnm), getattr(sc_cpy, attrnm))

    sc_newdiff = SkyCoord(sc, differential_cls=SphericalCosLatDifferential)
    reprepr = sc.represent_as(SphericalRepresentation, SphericalCosLatDifferential)
    assert_quantity_allclose(sc_newdiff.pm_ra_cosdec,
                             reprepr.differentials['s'].d_lon_coslat)

def test_creation_cartesian():
    rep = CartesianRepresentation([10, 0., 0.]*u.pc)
    dif = CartesianDifferential([0, 100, 0.]*u.pc/u.Myr)
    rep = rep.with_differentials(dif)
    c = SkyCoord(rep)

    sdif = dif.represent_as(SphericalCosLatDifferential, rep)
    assert_quantity_allclose(c.pm_ra_cosdec, sdif.d_lon_coslat)

def test_useful_error_missing():
    sc_nod = SkyCoord(ICRS(1*u.deg, 2*u.deg))
    try:
        sc_nod.l
    except AttributeError as e:
        # this is double-checking the *normal* behavior
        msg_l = e.args[0]

    try:
        sc_nod.pm_dec
    except Exception as e:
        msg_pm_dec = e.args[0]

    assert "has no attribute" in msg_l
    assert "has no associated differentials" in msg_pm_dec

# ----------------------Operations on SkyCoords w/ velocities-------------------

# define some fixtires to get baseline coordinates to try operations with
@pytest.fixture(scope="module", params=[(False, False),
                                        (True, False),
                                        (False, True),
                                        (True, True)])
def sc(request):
    incldist, inclrv = request.param

    args = [1*u.deg, 2*u.deg]
    kwargs = dict(pm_dec=1*u.mas/u.yr, pm_ra_cosdec=2*u.mas/u.yr)
    if incldist:
        kwargs['distance'] = 213.4*u.pc
    if inclrv:
        kwargs['radial_velocity'] = 61*u.km/u.s

    return SkyCoord(*args, **kwargs)

@pytest.fixture(scope="module")
def sc2():
    return SkyCoord(1*u.deg, 2*u.deg,
                pm_dec=1*u.mas/u.yr, pm_ra_cosdec=2*u.mas/u.yr)

@pytest.fixture(scope="module")
def scmany():
    return SkyCoord(ICRS(ra=[1]*100*u.deg, dec=[2]*100*u.deg,
                     pm_ra_cosdec=np.random.randn(100)*u.mas/u.yr,
                     pm_dec=np.random.randn(100)*u.mas/u.yr,))

def test_separation(sc, sc2):
    sc.separation(sc2)

def test_accessors(sc, scmany):
    sc.data.differentials['s']
    sc.spherical
    sc.galactic

    scmany[0]
    scmany.spherical
    scmany.galactic

def test_transforms(sc):
    trans = sc.transform_to('galactic')
    assert isinstance(trans.frame, Galactic)

@pytest.mark.skipif(str('not HAS_SCIPY'))
def test_matching(sc, scmany):
    # just check that it works and yields something
    idx, d2d, d3d = sc.match_to_catalog_sky(scmany)

def test_position_angle(sc, sc2):
    sc.position_angle(sc2)

@pytest.mark.xfail  #TODO: FIX
def test_constellations(sc):
    const = sc.get_constellation()
    assert const == 'Pisces'

@pytest.mark.xfail  #TODO: FIX
def test_skyoffset_frame(sc):
    sc.skyoffset_frame()
<|endoftext|>


<commit_before>"""
==========================================
Statistical functions (:mod:`scipy.stats`)
==========================================

.. module:: scipy.stats

This module contains a large number of probability distributions as
well as a growing library of statistical functions.

Each included distribution is an instance of the class rv_continous:
For each given name the following methods are available:

.. autosummary::
   :toctree: generated/

   rv_continuous
   rv_continuous.pdf
   rv_continuous.logpdf
   rv_continuous.cdf
   rv_continuous.logcdf
   rv_continuous.sf
   rv_continuous.logsf
   rv_continuous.ppf
   rv_continuous.isf
   rv_continuous.moment
   rv_continuous.stats
   rv_continuous.entropy
   rv_continuous.fit
   rv_continuous.expect

Calling the instance as a function returns a frozen pdf whose shape,
location, and scale parameters are fixed.

Similarly, each discrete distribution is an instance of the class
rv_discrete:

.. autosummary::
   :toctree: generated/

   rv_discrete
   rv_discrete.rvs
   rv_discrete.pmf
   rv_discrete.logpmf
   rv_discrete.cdf
   rv_discrete.logcdf
   rv_discrete.sf
   rv_discrete.logsf
   rv_discrete.ppf
   rv_discrete.isf
   rv_discrete.stats
   rv_discrete.moment
   rv_discrete.entropy
   rv_discrete.expect

Continuous distributions
========================

.. autosummary::
   :toctree: generated/

   alpha             -- Alpha
   anglit            -- Anglit
   arcsine           -- Arcsine
   beta              -- Beta
   betaprime         -- Beta Prime
   bradford          -- Bradford
   burr              -- Burr
   cauchy            -- Cauchy
   chi               -- Chi
   chi2              -- Chi-squared
   cosine            -- Cosine
   dgamma            -- Double Gamma
   dweibull          -- Double Weibull
   erlang            -- Erlang
   expon             -- Exponential
   exponweib         -- Exponentiated Weibull
   exponpow          -- Exponential Power
   f                 -- F (Snecdor F)
   fatiguelife       -- Fatigue Life (Birnbaum-Sanders)
   fisk              -- Fisk
   foldcauchy        -- Folded Cauchy
   foldnorm          -- Folded Normal
   frechet_r         -- Frechet Right Sided, Extreme Value Type II (Extreme LB) or weibull_min
   frechet_l         -- Frechet Left Sided, Weibull_max
   genlogistic       -- Generalized Logistic
   genpareto         -- Generalized Pareto
   genexpon          -- Generalized Exponential
   genextreme        -- Generalized Extreme Value
   gausshyper        -- Gauss Hypergeometric
   gamma             -- Gamma
   gengamma          -- Generalized gamma
   genhalflogistic   -- Generalized Half Logistic
   gilbrat           -- Gilbrat
   gompertz          -- Gompertz (Truncated Gumbel)
   gumbel_r          -- Right Sided Gumbel, Log-Weibull, Fisher-Tippett, Extreme Value Type I
   gumbel_l          -- Left Sided Gumbel, etc.
   halfcauchy        -- Half Cauchy
   halflogistic      -- Half Logistic
   halfnorm          -- Half Normal
   hypsecant         -- Hyperbolic Secant
   invgamma          -- Inverse Gamma
   invgauss          -- Inverse Gaussian
   invweibull        -- Inverse Weibull
   johnsonsb         -- Johnson SB
   johnsonsu         -- Johnson SU
   ksone             -- Kolmogorov-Smirnov one-sided (no stats)
   kstwobign         -- Kolmogorov-Smirnov two-sided test for Large N (no stats)
   laplace           -- Laplace
   logistic          -- Logistic
   loggamma          -- Log-Gamma
   loglaplace        -- Log-Laplace (Log Double Exponential)
   lognorm           -- Log-Normal
   lomax             -- Lomax (Pareto of the second kind)
   maxwell           -- Maxwell
   mielke            -- Mielke's Beta-Kappa
   nakagami          -- Nakagami
   ncx2              -- Non-central chi-squared
   ncf               -- Non-central F
   nct               -- Non-central Student's T
   norm              -- Normal (Gaussian)
   pareto            -- Pareto
   pearson3          -- Pearson type III
   powerlaw          -- Power-function
   powerlognorm      -- Power log normal
   powernorm         -- Power normal
   rdist             -- R-distribution
   reciprocal        -- Reciprocal
   rayleigh          -- Rayleigh
   rice              -- Rice
   recipinvgauss     -- Reciprocal Inverse Gaussian
   semicircular      -- Semicircular
   t                 -- Student's T
   triang            -- Triangular
   truncexpon        -- Truncated Exponential
   truncnorm         -- Truncated Normal
   tukeylambda       -- Tukey-Lambda
   uniform           -- Uniform
   vonmises          -- Von-Mises (Circular)
   wald              -- Wald
   weibull_min       -- Minimum Weibull (see Frechet)
   weibull_max       -- Maximum Weibull (see Frechet)
   wrapcauchy        -- Wrapped Cauchy

Multivariate distributions
==========================

.. autosummary::
   :toctree: generated/

   multivariate_normal   -- Multivariate normal distribution

Discrete distributions
======================

.. autosummary::
   :toctree: generated/

   bernoulli         -- Bernoulli
   binom             -- Binomial
   boltzmann         -- Boltzmann (Truncated Discrete Exponential)
   dlaplace          -- Discrete Laplacian
   geom              -- Geometric
   hypergeom         -- Hypergeometric
   logser            -- Logarithmic (Log-Series, Series)
   nbinom            -- Negative Binomial
   planck            -- Planck (Discrete Exponential)
   poisson           -- Poisson
   randint           -- Discrete Uniform
   skellam           -- Skellam
   zipf              -- Zipf

Statistical functions
=====================

Several of these functions have a similar version in scipy.stats.mstats
which work for masked arrays.

.. autosummary::
   :toctree: generated/

   describe          -- Descriptive statistics
   gmean             -- Geometric mean
   hmean             -- Harmonic mean
   kurtosis          -- Fisher or Pearson kurtosis
   kurtosistest      --
   mode              -- Modal value
   moment            -- Central moment
   normaltest        --
   skew              -- Skewness
   skewtest          --
   tmean             -- Truncated arithmetic mean
   tvar              -- Truncated variance
   tmin              --
   tmax              --
   tstd              --
   tsem              --
   nanmean           -- Mean, ignoring NaN values
   nanstd            -- Standard deviation, ignoring NaN values
   nanmedian         -- Median, ignoring NaN values
   variation         -- Coefficient of variation

.. autosummary::
   :toctree: generated/

   cumfreq           _
   histogram2        _
   histogram         _
   itemfreq          _
   percentileofscore _
   scoreatpercentile _
   relfreq           _

.. autosummary::
   :toctree: generated/

   binned_statistic     -- Compute a binned statistic for a set of data.
   binned_statistic_2d  -- Compute a 2-D binned statistic for a set of data.
   binned_statistic_dd  -- Compute a d-D binned statistic for a set of data.

.. autosummary::
   :toctree: generated/

   obrientransform
   signaltonoise
   bayes_mvs
   sem
   zmap
   zscore

.. autosummary::
   :toctree: generated/

   threshold
   trimboth
   trim1

.. autosummary::
   :toctree: generated/

   f_oneway
   pearsonr
   spearmanr
   pointbiserialr
   kendalltau
   linregress

.. autosummary::
   :toctree: generated/

   ttest_1samp
   ttest_ind
   ttest_rel
   kstest
   chisquare
   power_divergence
   ks_2samp
   mannwhitneyu
   tiecorrect
   rankdata
   ranksums
   wilcoxon
   kruskal
   friedmanchisquare

.. autosummary::
   :toctree: generated/

   ansari
   bartlett
   levene
   shapiro
   anderson
   binom_test
   fligner
   mood

.. autosummary::
   :toctree: generated/

   boxcox
   boxcox_normmax
   boxcox_llf

Contingency table functions
===========================

.. autosummary::
   :toctree: generated/

   chi2_contingency
   contingency.expected_freq
   contingency.margins
   fisher_exact

Plot-tests
==========

.. autosummary::
   :toctree: generated/

   ppcc_max
   ppcc_plot
   probplot
   boxcox_normplot

Masked statistics functions
===========================

.. toctree::

   stats.mstats

Univariate and multivariate kernel density estimation (:mod:`scipy.stats.kde`)
==============================================================================

.. autosummary::
   :toctree: generated/

   gaussian_kde

For many more stat related functions install the software R and the
interface package rpy.

"""
from __future__ import division, print_function, absolute_import

from .stats import *
from .distributions import *
from .rv import *
from .morestats import *
from ._binned_statistic import *
from .kde import gaussian_kde
from . import mstats
from .contingency import chi2_contingency
from ._multivariate import *

#remove vonmises_cython from __all__, I don't know why it is included
__all__ = [s for s in dir() if not (s.startswith('_') or s.endswith('cython'))]

from numpy.testing import Tester
test = Tester().test
<commit_msg>Add scipy.stats.sigmaclip to the html docs<commit_after>"""
==========================================
Statistical functions (:mod:`scipy.stats`)
==========================================

.. module:: scipy.stats

This module contains a large number of probability distributions as
well as a growing library of statistical functions.

Each included distribution is an instance of the class rv_continous:
For each given name the following methods are available:

.. autosummary::
   :toctree: generated/

   rv_continuous
   rv_continuous.pdf
   rv_continuous.logpdf
   rv_continuous.cdf
   rv_continuous.logcdf
   rv_continuous.sf
   rv_continuous.logsf
   rv_continuous.ppf
   rv_continuous.isf
   rv_continuous.moment
   rv_continuous.stats
   rv_continuous.entropy
   rv_continuous.fit
   rv_continuous.expect

Calling the instance as a function returns a frozen pdf whose shape,
location, and scale parameters are fixed.

Similarly, each discrete distribution is an instance of the class
rv_discrete:

.. autosummary::
   :toctree: generated/

   rv_discrete
   rv_discrete.rvs
   rv_discrete.pmf
   rv_discrete.logpmf
   rv_discrete.cdf
   rv_discrete.logcdf
   rv_discrete.sf
   rv_discrete.logsf
   rv_discrete.ppf
   rv_discrete.isf
   rv_discrete.stats
   rv_discrete.moment
   rv_discrete.entropy
   rv_discrete.expect

Continuous distributions
========================

.. autosummary::
   :toctree: generated/

   alpha             -- Alpha
   anglit            -- Anglit
   arcsine           -- Arcsine
   beta              -- Beta
   betaprime         -- Beta Prime
   bradford          -- Bradford
   burr              -- Burr
   cauchy            -- Cauchy
   chi               -- Chi
   chi2              -- Chi-squared
   cosine            -- Cosine
   dgamma            -- Double Gamma
   dweibull          -- Double Weibull
   erlang            -- Erlang
   expon             -- Exponential
   exponweib         -- Exponentiated Weibull
   exponpow          -- Exponential Power
   f                 -- F (Snecdor F)
   fatiguelife       -- Fatigue Life (Birnbaum-Sanders)
   fisk              -- Fisk
   foldcauchy        -- Folded Cauchy
   foldnorm          -- Folded Normal
   frechet_r         -- Frechet Right Sided, Extreme Value Type II (Extreme LB) or weibull_min
   frechet_l         -- Frechet Left Sided, Weibull_max
   genlogistic       -- Generalized Logistic
   genpareto         -- Generalized Pareto
   genexpon          -- Generalized Exponential
   genextreme        -- Generalized Extreme Value
   gausshyper        -- Gauss Hypergeometric
   gamma             -- Gamma
   gengamma          -- Generalized gamma
   genhalflogistic   -- Generalized Half Logistic
   gilbrat           -- Gilbrat
   gompertz          -- Gompertz (Truncated Gumbel)
   gumbel_r          -- Right Sided Gumbel, Log-Weibull, Fisher-Tippett, Extreme Value Type I
   gumbel_l          -- Left Sided Gumbel, etc.
   halfcauchy        -- Half Cauchy
   halflogistic      -- Half Logistic
   halfnorm          -- Half Normal
   hypsecant         -- Hyperbolic Secant
   invgamma          -- Inverse Gamma
   invgauss          -- Inverse Gaussian
   invweibull        -- Inverse Weibull
   johnsonsb         -- Johnson SB
   johnsonsu         -- Johnson SU
   ksone             -- Kolmogorov-Smirnov one-sided (no stats)
   kstwobign         -- Kolmogorov-Smirnov two-sided test for Large N (no stats)
   laplace           -- Laplace
   logistic          -- Logistic
   loggamma          -- Log-Gamma
   loglaplace        -- Log-Laplace (Log Double Exponential)
   lognorm           -- Log-Normal
   lomax             -- Lomax (Pareto of the second kind)
   maxwell           -- Maxwell
   mielke            -- Mielke's Beta-Kappa
   nakagami          -- Nakagami
   ncx2              -- Non-central chi-squared
   ncf               -- Non-central F
   nct               -- Non-central Student's T
   norm              -- Normal (Gaussian)
   pareto            -- Pareto
   pearson3          -- Pearson type III
   powerlaw          -- Power-function
   powerlognorm      -- Power log normal
   powernorm         -- Power normal
   rdist             -- R-distribution
   reciprocal        -- Reciprocal
   rayleigh          -- Rayleigh
   rice              -- Rice
   recipinvgauss     -- Reciprocal Inverse Gaussian
   semicircular      -- Semicircular
   t                 -- Student's T
   triang            -- Triangular
   truncexpon        -- Truncated Exponential
   truncnorm         -- Truncated Normal
   tukeylambda       -- Tukey-Lambda
   uniform           -- Uniform
   vonmises          -- Von-Mises (Circular)
   wald              -- Wald
   weibull_min       -- Minimum Weibull (see Frechet)
   weibull_max       -- Maximum Weibull (see Frechet)
   wrapcauchy        -- Wrapped Cauchy

Multivariate distributions
==========================

.. autosummary::
   :toctree: generated/

   multivariate_normal   -- Multivariate normal distribution

Discrete distributions
======================

.. autosummary::
   :toctree: generated/

   bernoulli         -- Bernoulli
   binom             -- Binomial
   boltzmann         -- Boltzmann (Truncated Discrete Exponential)
   dlaplace          -- Discrete Laplacian
   geom              -- Geometric
   hypergeom         -- Hypergeometric
   logser            -- Logarithmic (Log-Series, Series)
   nbinom            -- Negative Binomial
   planck            -- Planck (Discrete Exponential)
   poisson           -- Poisson
   randint           -- Discrete Uniform
   skellam           -- Skellam
   zipf              -- Zipf

Statistical functions
=====================

Several of these functions have a similar version in scipy.stats.mstats
which work for masked arrays.

.. autosummary::
   :toctree: generated/

   describe          -- Descriptive statistics
   gmean             -- Geometric mean
   hmean             -- Harmonic mean
   kurtosis          -- Fisher or Pearson kurtosis
   kurtosistest      --
   mode              -- Modal value
   moment            -- Central moment
   normaltest        --
   skew              -- Skewness
   skewtest          --
   tmean             -- Truncated arithmetic mean
   tvar              -- Truncated variance
   tmin              --
   tmax              --
   tstd              --
   tsem              --
   nanmean           -- Mean, ignoring NaN values
   nanstd            -- Standard deviation, ignoring NaN values
   nanmedian         -- Median, ignoring NaN values
   variation         -- Coefficient of variation

.. autosummary::
   :toctree: generated/

   cumfreq           _
   histogram2        _
   histogram         _
   itemfreq          _
   percentileofscore _
   scoreatpercentile _
   relfreq           _

.. autosummary::
   :toctree: generated/

   binned_statistic     -- Compute a binned statistic for a set of data.
   binned_statistic_2d  -- Compute a 2-D binned statistic for a set of data.
   binned_statistic_dd  -- Compute a d-D binned statistic for a set of data.

.. autosummary::
   :toctree: generated/

   obrientransform
   signaltonoise
   bayes_mvs
   sem
   zmap
   zscore

.. autosummary::
   :toctree: generated/

   sigmaclip
   threshold
   trimboth
   trim1

.. autosummary::
   :toctree: generated/

   f_oneway
   pearsonr
   spearmanr
   pointbiserialr
   kendalltau
   linregress

.. autosummary::
   :toctree: generated/

   ttest_1samp
   ttest_ind
   ttest_rel
   kstest
   chisquare
   power_divergence
   ks_2samp
   mannwhitneyu
   tiecorrect
   rankdata
   ranksums
   wilcoxon
   kruskal
   friedmanchisquare

.. autosummary::
   :toctree: generated/

   ansari
   bartlett
   levene
   shapiro
   anderson
   binom_test
   fligner
   mood

.. autosummary::
   :toctree: generated/

   boxcox
   boxcox_normmax
   boxcox_llf

Contingency table functions
===========================

.. autosummary::
   :toctree: generated/

   chi2_contingency
   contingency.expected_freq
   contingency.margins
   fisher_exact

Plot-tests
==========

.. autosummary::
   :toctree: generated/

   ppcc_max
   ppcc_plot
   probplot
   boxcox_normplot

Masked statistics functions
===========================

.. toctree::

   stats.mstats

Univariate and multivariate kernel density estimation (:mod:`scipy.stats.kde`)
==============================================================================

.. autosummary::
   :toctree: generated/

   gaussian_kde

For many more stat related functions install the software R and the
interface package rpy.

"""
from __future__ import division, print_function, absolute_import

from .stats import *
from .distributions import *
from .rv import *
from .morestats import *
from ._binned_statistic import *
from .kde import gaussian_kde
from . import mstats
from .contingency import chi2_contingency
from ._multivariate import *

#remove vonmises_cython from __all__, I don't know why it is included
__all__ = [s for s in dir() if not (s.startswith('_') or s.endswith('cython'))]

from numpy.testing import Tester
test = Tester().test
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#    Copyright (c) 2014 Serv. Tecnol. Avanzados (http://www.serviciosbaeza.com)
#                       Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
#    Copyright (c) 2015 Antiun Ingeniería S.L. (http://www.antiun.com)
#                       Antonio Espinosa <antonioea@antiun.com>
# © 2015 Antiun Ingeniería S.L. - Jairo Llopis
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from openerp import models, fields

class ResPartner(models.Model):
    _inherit = 'res.partner'

    department_id = fields.Many2one(
        "res.partner.department",
        "Department",
        oldname="department")

class ResPartnerDepartment(models.Model):
    _name = 'res.partner.department'
    _order = "parent_left"
    _parent_order = "name"
    _parent_store = True
    _description = "Department"

    name = fields.Char(required=True, translate=True)
    parent_id = fields.Many2one(
        "res.partner.department",
        "Parent department")
    child_ids = fields.One2many(
        "res.partner.department",
        "parent_id",
        "Child departments",
        oldname="children")
    parent_left = fields.Integer(index=True)
    parent_right = fields.Integer(index=True)
<commit_msg>Add ondelete option to parent_id, runbot fails to lack of attribute.<commit_after># -*- coding: utf-8 -*-
#    Copyright (c) 2014 Serv. Tecnol. Avanzados (http://www.serviciosbaeza.com)
#                       Pedro M. Baeza <pedro.baeza@serviciosbaeza.com>
#    Copyright (c) 2015 Antiun Ingeniería S.L. (http://www.antiun.com)
#                       Antonio Espinosa <antonioea@antiun.com>
# © 2015 Antiun Ingeniería S.L. - Jairo Llopis
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from openerp import models, fields

class ResPartner(models.Model):
    _inherit = 'res.partner'

    department_id = fields.Many2one(
        "res.partner.department",
        "Department",
        oldname="department")

class ResPartnerDepartment(models.Model):
    _name = 'res.partner.department'
    _order = "parent_left"
    _parent_order = "name"
    _parent_store = True
    _description = "Department"

    name = fields.Char(required=True, translate=True)
    parent_id = fields.Many2one(
        "res.partner.department",
        "Parent department",
        ondelete='restrict')
    child_ids = fields.One2many(
        "res.partner.department",
        "parent_id",
        "Child departments",
        oldname="children")
    parent_left = fields.Integer(index=True)
    parent_right = fields.Integer(index=True)
<|endoftext|>