<commit_before># (c) 2015 - Jaguar Land Rover.
#
# Mozilla Public License 2.0
#
# Python-based hmi PoC for Software Loading Manager
#

import gtk
import dbus
import dbus.service
from dbus.mainloop.glib import DBusGMainLoop

#
# hmi service
#
class HMIService(dbus.service.Object):
    def __init__(self):
        self.bus = dbus.SessionBus()
        hmi_bus_name = dbus.service.BusName('org.genivi.hmi',
                                            bus=self.bus)

        dbus.service.Object.__init__(self,
                                     hmi_bus_name,
                                     '/org/genivi/hmi')

    @dbus.service.method('org.genivi.hmi',
                         async_callbacks=('send_reply', 'send_error'))
    def update_notification(self,
                              package_id,
                              major,
                              minor,
                              patch,
                              command,
                              size,
                              description,
                              vendor,
                              target,
                              send_reply,
                              send_error):

        print "HMI: Got update_notification()"
        print "  ID:     {}".format(package_id)
        print "  ver:    {}.{}.{} ".format(major, minor, patch)
        print "  cmd:    {}".format(command)
        print "  size:   {}".format(size)
        print "  descr:  {}".format(description)
        print "  vendor: {}".format(vendor)
        print "  target: {}".format(target)
        print "---"

        #
        # Send back an async reply to the invoking software_loading_manager
        # so that we can continue with user interaction without
        # risking a DBUS timeout
        #
        send_reply(True)

        print
        print
        print "DIALOG:"
        print "DIALOG: Available update"
        print "DIALOG:   Description: {}".format(description)
        print "DIALOG:   Vendor:      {}".format(vendor)
        print "DIALOG:"
        resp = raw_input("DIALOG: Install? (yes/no): ")
        print
        #
        # Setup DBUS call to software loading manager.
        #
        slm_bus_name = dbus.service.BusName('org.genivi.software_loading_manager',
                                            bus=self.bus)
        slm_obj = self.bus.get_object(slm_bus_name.get_name(),
                                         "/org/genivi/software_loading_manager")

        package_confirmation = slm_obj.get_dbus_method("package_confirmation",
                                                       "org.genivi.software_loading_manager")

        if len(resp) == 0 or (resp[0] != 'y' and resp[0] != 'Y'):
            approved = False
        else:
            approved = True

        #
        # Call software_loading_manager.package_confirmation()
        # to inform it of user approval / decline.
        #
        package_confirmation(approved,
                             package_id,
                             major,
                             minor,
                             patch,
                             command,
                             size,
                             description,
                             vendor,
                             target)

        return None

    @dbus.service.method('org.genivi.hmi')
    def update_report(self,
                      update_id,
                      results):
        global active
        print "Update report"
        print "  ID:          {}".format(update_id)
        print "  results:"
        for result in results:
            print "    operation_id: {}".format(result['id'])
            print "    code:         {}".format(result['result_code'])
            print "    text:         {}".format(result['result_text'])
            print "  ---"
        print "---"
        active = False
        return None

print
print "HMI Simulator"
print "Please enter package installation approval when prompted"
print

DBusGMainLoop(set_as_default=True)
pkg_mgr = HMIService()

while True:
    gtk.main_iteration()
<commit_msg>Added sexy, ascii-graphics progress bars.<commit_after># (c) 2015 - Jaguar Land Rover.
#
# Mozilla Public License 2.0
#
# Python-based hmi PoC for Software Loading Manager
#

import gtk
import dbus
import dbus.service
from dbus.mainloop.glib import DBusGMainLoop
import time
import threading

class DisplayProgress(threading.Thread):

    def __init__(self, *arg):
        super(DisplayProgress, self).__init__(*arg)
        self.in_progress = True
        self.update_description = None
        self.update_start_time = None
        self.update_stop_time = None
        self.operation_hmi_message = None
        self.operation_start_time = None
        self.operation_stop_time = None
        print "DisplayProgress(): Called"

    def set_manifest(self, description, start_time, stop_time):
        self.update_description = description
        self.update_start_time = start_time
        self.update_stop_time = stop_time

    def set_operation(self, hmi_message, start_time, stop_time):
        self.operation_hmi_message = hmi_message
        self.operation_start_time = start_time
        self.operation_stop_time = stop_time

    def exit_thread(self):
        self.in_progress = False
        self.join()

    def run(self):
        while self.in_progress:
            if not self.update_description:
                time.sleep(0.5)
                continue

            time.sleep(0.1)

            ct = time.time()
            if self.update_stop_time >= ct:
                completion = (self.update_stop_time - ct) / (self.update_stop_time - self.update_start_time)
            else:
                completion = 0.0

            print "\033[HUpdate:    {}".format(self.update_description)

            print "{}{} ".format("+"*int(60.0 - completion*60), "-"*int(completion*60))

            # If no operation is in progress, clear sreen
            if not self.operation_start_time:
                continue

            if self.update_stop_time >= ct:
                completion = (self.operation_stop_time - ct) / (self.operation_stop_time - self.operation_start_time)
            else:
                completion = 0.0

            print "\nOperation: {}".format(self.operation_hmi_message)
            print "{}{} ".format("+"*int(60.0 - completion*60), "-"*int(completion*60))
            print "\033[J"

#
# hmi service
#
class HMIService(dbus.service.Object):
    def __init__(self):
        self.bus = dbus.SessionBus()
        hmi_bus_name = dbus.service.BusName('org.genivi.hmi',
                                            bus=self.bus)

        dbus.service.Object.__init__(self,
                                     hmi_bus_name,
                                     '/org/genivi/hmi')
        self.progress_thread = DisplayProgress()
        self.progress_thread.start()

    @dbus.service.method('org.genivi.hmi',
                         async_callbacks=('send_reply', 'send_error'))
    def update_notification(self,
                            update_id,
                            description,
                            send_reply,
                            send_error):

        print "HMI: Got update_notification()"
        print "  ID:            {}".format(update_id)
        print "  description:   {}".format(description)
        print "---"

        #
        # Send back an async reply to the invoking software_loading_manager
        # so that we can continue with user interaction without
        # risking a DBUS timeout
        #
        send_reply(True)

        print
        print
        print "DIALOG:"
        print "DIALOG: Available update"
        print "DIALOG:   Description: {}".format(description)
        print "DIALOG:   Vendor:      {}".format(vendor)
        print "DIALOG:"
        resp = raw_input("DIALOG: Install? (yes/no): ")
        print
        #
        # Setup DBUS call to software loading manager.
        #
        slm_bus_name = dbus.service.BusName('org.genivi.software_loading_manager',
                                            bus=self.bus)
        slm_obj = self.bus.get_object(slm_bus_name.get_name(),
                                         "/org/genivi/software_loading_manager")

        package_confirmation = slm_obj.get_dbus_method("package_confirmation",
                                                       "org.genivi.software_loading_manager")

        if len(resp) == 0 or (resp[0] != 'y' and resp[0] != 'Y'):
            approved = False
        else:
            approved = True

        #
        # Call software_loading_manager.package_confirmation()
        # to inform it of user approval / decline.
        #
        package_confirmation(approved, update_id)

        return None

    @dbus.service.method('org.genivi.hmi')
    def manifest_started(self,
                         update_id,
                         total_time_estimate,
                         description):
        print "\033[H\033[J"
        ct = time.time()
        self.progress_thread.set_manifest(description,
                                          ct,
                                          ct + float(total_time_estimate) / 1000.0)

        return None

    @dbus.service.method('org.genivi.hmi')
    def operation_started(self,
                          operation_id,
                          time_estimate,
                          hmi_message):
        ct = time.time()
        self.progress_thread.set_operation(hmi_message, ct, ct + float(time_estimate) / 1000.0)
        return None

    @dbus.service.method('org.genivi.hmi')
    def update_report(self,
                      update_id,
                      results):

        self.progress_thread.exit_thread()

        print "Update report"
        print "  ID:          {}".format(update_id)
        print "  results:"
        for result in results:
            print "    operation_id: {}".format(result['id'])
            print "    code:         {}".format(result['result_code'])
            print "    text:         {}".format(result['result_text'])
            print "  ---"
        print "---"
        return None

print
print "HMI Simulator"
print "Please enter package installation approval when prompted"
print

gtk.gdk.threads_init()
DBusGMainLoop(set_as_default=True)
pkg_mgr = HMIService()

while True:
    gtk.main_iteration()
<|endoftext|>


<commit_before>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import numpy as np
from federatedml.feature.instance import Instance
from federatedml.util import consts
from federatedml.statistic.statics import MultivariateStatisticalSummary

class LeastSquaredErrorLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        lse_loss = y.join(y_pred, lambda y, yp: ((y - yp) * (y - yp), 1))
        lse_sum, sample_num = lse_loss.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return lse_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        return 2 * (y_pred - y)

    @staticmethod
    def compute_hess(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            shape = (y - y_pred).shape
            return np.full(shape, 2)
        else:
            return 2

class LeastAbsoluteErrorLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        median = statistics.get_median()["label"]
        return y.mapValues(lambda x: np.asarray([median])), np.asarray([median])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        lae = y.join(y_pred, lambda y, yp: (np.abs(y - yp), 1))
        lae_sum, sample_num = lae.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return lae_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            diff = y_pred - y
            diff[diff > consts.FLOAT_ZERO] = 1
            diff[diff < consts.FLOAT_ZERO] = -1
            diff[np.abs(diff) <= consts.FLOAT_ZERO] = 0

            return diff
        else:
            diff = y_pred - y
            if diff > consts.FLOAT_ZERO:
                return 1
            elif diff < consts.FLOAT_ZERO:
                return -1
            else:
                return 0

    @staticmethod
    def compute_hess(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            shape = (y - y_pred).shape
            return np.full(shape, 1)
        else:
            return 1

class HuberLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, delta):
        if delta is None:
            self.delta = consts.FLOAT_ZERO
        else:
            self.delta = delta

        if np.abs(self.delta) < consts.FLOAT_ZERO:
            self.delta = consts.FLOAT_ZERO

    def compute_loss(self, y, y_pred):
        huber_loss = y.join(y_pred, lambda y, yp:
        (self.delta ** 2 * (np.sqrt(1 + ((yp - y) / self.delta) ** 2) - 1), 1))
        huber_sum, sample_num = huber_loss.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return huber_sum / sample_num

    @staticmethod
    def predict(value):
        return value

    def compute_grad(self, y, y_pred):
        diff = y_pred - y
        return diff / np.sqrt(1.0 + diff * diff / (self.delta ** 2))

    def compute_hess(self, y, y_pred):
        diff = y_pred - y
        return 1.0 / (1.0 + diff * diff / (self.delta ** 2)) ** 1.5

class FairLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, c):
        if c is None:
            self.c = const.FLOAT_ZERO
        else:
            self.c = c

        if np.abs(self.c) < consts.FLOAT_ZERO:
            self.c = consts.FLOAT_ZERO

    @staticmethod
    def predict(value):
        return value

    def compute_loss(self, y, y_pred):
        fair_loss = y.join(y_pred, lambda y, yp:
        (self.c * np.abs(yp - y) - self.c ** 2 * np.log(np.abs(yp - y) / self.c + 1), 1))
        fair_loss_sum, sample_num = fair_loss.reduce(
            lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return fair_loss_sum / sample_num

    def compute_grad(self, y, y_pred):
        diff = y_pred - y
        return self.c * diff / (np.abs(diff) + self.c)

    def compute_hess(self, y, y_pred):
        diff = y_pred - y
        return self.c ** 2 / (np.abs(diff) + self.c) ** 2

class LogCoshLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        log_cosh_loss = y.join(y_pred, lambda y, yp: (np.log(np.cosh(yp - y)), 1))
        log_cosh_sum, sample_num = log_cosh_loss.reduce(
            lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return log_cosh_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        return np.tanh(y_pred - y)

    @staticmethod
    def compute_hess(y, y_pred):
        return 1 - np.tanh(y_pred - y) ** 2

class TweedieLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, rho=None):
        if rho is None:
            self.rho = consts.FLOAT_ZERO
        else:
            self.rho = rho

    @staticmethod
    def predict(value):
        return value

    def compute_loss(self, y, y_pred):
        tweedie_loss = y.join(y_pred,
                              lambda y, yp:
                              (-y * np.exp(1 - self.rho) * np.log(max(yp, consts.FLOAT_ZERO)) / (1 - self.rho) +
                               np.exp(2 - self.rho) * np.log(max(consts.FLOAT_ZERO, yp)) / (2 - self.rho), 1))
        tweedie_loss_sum, sample_num = tweedie_loss.reduce(lambda tuple1, tuple2:
                                                           (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return tweedie_loss_sum / sample_num

    def compute_grad(self, y, y_pred):
        return -y * np.exp(1 - self.rho) * y_pred + np.exp(2 - self.rho) * y_pred

    def compute_hess(self, y, y_pred):
        return -y * (1 - self.rho) * np.exp(1 - self.rho) * y_pred + (2 - self.rho) * np.exp(2 - self.rho) * y_pred
<commit_msg>fix loss func<commit_after>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import numpy as np
from federatedml.feature.instance import Instance
from federatedml.util import consts
from federatedml.statistic.statics import MultivariateStatisticalSummary

class LeastSquaredErrorLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        lse_loss = y.join(y_pred, lambda y, yp: ((y - yp) * (y - yp), 1))
        lse_sum, sample_num = lse_loss.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return lse_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        return 2 * (y_pred - y)

    @staticmethod
    def compute_hess(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            shape = (y - y_pred).shape
            return np.full(shape, 2)
        else:
            return 2

class LeastAbsoluteErrorLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        median = statistics.get_median()["label"]
        return y.mapValues(lambda x: np.asarray([median])), np.asarray([median])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        lae = y.join(y_pred, lambda y, yp: (np.abs(y - yp), 1))
        lae_sum, sample_num = lae.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return lae_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            diff = y_pred - y
            diff[diff > consts.FLOAT_ZERO] = 1
            diff[diff < consts.FLOAT_ZERO] = -1
            diff[np.abs(diff) <= consts.FLOAT_ZERO] = 0

            return diff
        else:
            diff = y_pred - y
            if diff > consts.FLOAT_ZERO:
                return 1
            elif diff < consts.FLOAT_ZERO:
                return -1
            else:
                return 0

    @staticmethod
    def compute_hess(y, y_pred):
        if type(y).__name__ == "ndarray" or type(y_pred).__name__ == "ndarray":
            shape = (y - y_pred).shape
            return np.full(shape, 1)
        else:
            return 1

class HuberLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, delta):
        if delta is None:
            self.delta = consts.FLOAT_ZERO
        else:
            self.delta = delta

        if np.abs(self.delta) < consts.FLOAT_ZERO:
            self.delta = consts.FLOAT_ZERO

    def compute_loss(self, y, y_pred):
        huber_loss = y.join(y_pred, lambda y, yp:
        (self.delta ** 2 * (np.sqrt(1 + ((yp - y) / self.delta) ** 2) - 1), 1))
        huber_sum, sample_num = huber_loss.reduce(lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return huber_sum / sample_num

    @staticmethod
    def predict(value):
        return value

    def compute_grad(self, y, y_pred):
        diff = y_pred - y
        return diff / np.sqrt(1.0 + diff * diff / (self.delta ** 2))

    def compute_hess(self, y, y_pred):
        diff = y_pred - y
        return 1.0 / (1.0 + diff * diff / (self.delta ** 2)) ** 1.5

class FairLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, c):
        if c is None:
            self.c = const.FLOAT_ZERO
        else:
            self.c = c

        if np.abs(self.c) < consts.FLOAT_ZERO:
            self.c = consts.FLOAT_ZERO

    @staticmethod
    def predict(value):
        return value

    def compute_loss(self, y, y_pred):
        fair_loss = y.join(y_pred, lambda y, yp:
        (self.c * np.abs(yp - y) - self.c ** 2 * np.log(np.abs(yp - y) / self.c + 1), 1))
        fair_loss_sum, sample_num = fair_loss.reduce(
            lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return fair_loss_sum / sample_num

    def compute_grad(self, y, y_pred):
        diff = y_pred - y
        return self.c * diff / (np.abs(diff) + self.c)

    def compute_hess(self, y, y_pred):
        diff = y_pred - y
        return self.c ** 2 / (np.abs(diff) + self.c) ** 2

class LogCoshLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    @staticmethod
    def predict(value):
        return value

    @staticmethod
    def compute_loss(y, y_pred):
        log_cosh_loss = y.join(y_pred, lambda y, yp: (np.log(np.cosh(yp - y)), 1))
        log_cosh_sum, sample_num = log_cosh_loss.reduce(
            lambda tuple1, tuple2: (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return log_cosh_sum / sample_num

    @staticmethod
    def compute_grad(y, y_pred):
        return np.tanh(y_pred - y)

    @staticmethod
    def compute_hess(y, y_pred):
        return 1 - np.tanh(y_pred - y) ** 2

class TweedieLoss(object):
    @staticmethod
    def initialize(y):
        y_inst = y.mapValues(lambda label: Instance(features=np.asarray([label])))
        y_inst.schema = {"header": ["label"]}
        statistics = MultivariateStatisticalSummary(y_inst, -1)
        mean = statistics.get_mean()["label"]
        return y.mapValues(lambda x: np.asarray([mean])), np.asarray([mean])

    def __init__(self, rho=None):
        if rho is None:
            self.rho = consts.FLOAT_ZERO
        else:
            self.rho = rho

    @staticmethod
    def predict(value):
        return value

    def compute_loss(self, y, y_pred):
        tweedie_loss = y.join(y_pred,
                              lambda y, yp:
                              (-y * np.exp(1 - self.rho) * np.log(max(yp, consts.FLOAT_ZERO)) / (1 - self.rho) +
                               np.exp(2 - self.rho) * np.log(max(consts.FLOAT_ZERO, yp)) / (2 - self.rho), 1))
        tweedie_loss_sum, sample_num = tweedie_loss.reduce(lambda tuple1, tuple2:
                                                           (tuple1[0] + tuple2[0], tuple1[1] + tuple2[1]))
        return tweedie_loss_sum / sample_num

    def compute_grad(self, y, y_pred):
        return -y * np.exp(1 - self.rho) * y_pred + np.exp(2 - self.rho) * y_pred

    def compute_hess(self, y, y_pred):
        return -y * (1 - self.rho) * np.exp(1 - self.rho) * y_pred + (2 - self.rho) * np.exp(2 - self.rho) * y_pred
<|endoftext|>


<commit_before>import numpy as np
import matplotlib
import pytest

import marvin.utils.plot.map as mapplot

matplotlib_2 = pytest.mark.skipif(int(matplotlib.__version__.split('.')[0]) <= 1,
                                  reason='matplotlib-2.0 or higher required')

value = np.array([[1, -2, 3],
                  [-4, 5, 6],
                  [7, 8, 9]])

ivar = np.array([[4, 10, 1],
                 [0.01, 0.04, 0.0001],
                 [100000, 0, 1]])

mask_simple = np.array([[0, 1, 0],
                        [1, 0, 1],
                        [0, 1, 0]])

mask_binary = np.array([[0b000, 0b001, 0b010],
                        [0b011, 0b100, 0b101],
                        [0b110, 0b111, 0b000]])

mask_daplike = np.array([[0b0000000000000000000000000000000,
                          0b0000000000000000000000000000001,
                          0b0000000000000000000000000100000],
                         [0b0000000000000000000000001000001,
                          0b0000000000000000000000010000000,
                          0b1000000000000000000000000000001],
                         [0b0000000000000000000000000000000,
                          0b1000000000000000000000011100001,
                          0b0000000000000000000000000000000]])

nocov = np.array([[0, 1, 0],
                  [1, 0, 1],
                  [0, 1, 0]], dtype=bool)

bad_data = np.array([[0, 0, 1],
                     [1, 1, 1],
                     [0, 1, 0]])

# SNR = value * np.sqrt(ivar)
# [[  2.00e+00,  -6.32e+00,   3.00e+00],
#  [ -4.00e-01,   1.00e+00,   6.00e-02],
#  [  2.21e+03,   0.00e+00,   9.00e+00]]

snr_min_none = np.array([[0, 0, 0],
                         [0, 0, 0],
                         [0, 1, 0]])

snr_min_0 = np.array([[0, 0, 0],
                      [0, 0, 0],
                      [0, 1, 0]])

snr_min_1 = np.array([[0, 0, 0],
                      [1, 0, 1],
                      [0, 1, 0]])

snr_min_3 = np.array([[1, 0, 0],
                      [1, 1, 1],
                      [0, 1, 0]])

log_cb_true = np.array([[0, 1, 0],
                        [1, 0, 0],
                        [0, 0, 0]])

log_cb_false = np.zeros(value.shape, dtype=bool)

image_none_true = np.array([[0, 1, 1],
                            [1, 1, 1],
                            [0, 1, 0]])

image_3_true = np.array([[1, 1, 1],
                         [1, 1, 1],
                         [0, 1, 0]])

image_none_false = np.array([[0, 1, 1],
                             [1, 1, 1],
                             [0, 1, 0]])

image_3_false = np.array([[1, 1, 1],
                          [1, 1, 1],
                          [0, 1, 0]])

class TestMasks(object):

    @pytest.mark.parametrize('value, mask, expected', [(value, mask_simple, nocov),
                                                       (value, mask_binary, nocov),
                                                       (value, mask_daplike, nocov)])
    def test_no_coverage_mask(self, value, mask, expected):
        nocov = mapplot.no_coverage_mask(value, mask)
        assert np.all(nocov == expected)

    @pytest.mark.parametrize('value, mask, expected', [(value, mask_daplike, bad_data)])
    def test_bad_data_mask(self, value, mask, expected):
        bad_data = mapplot.bad_data_mask(value, mask)
        assert np.all(bad_data == expected)

    @pytest.mark.parametrize('value, ivar, snr_min, expected',
                             [(value, ivar, None, snr_min_none),
                              (value, ivar, 0, snr_min_0),
                              (value, ivar, 1, snr_min_1),
                              (value, ivar, 3, snr_min_3)])
    def test_low_snr_mask(self, value, ivar, snr_min, expected):
        low_snr = mapplot.low_snr_mask(value, ivar, snr_min)
        assert np.all(low_snr == expected)

    @pytest.mark.parametrize('value, log_cb, expected',
                             [(value, True, log_cb_true),
                              (value, False, log_cb_false)])
    def test_log_colorbar_mask(self, value, log_cb, expected):
        log_colorbar = mapplot.log_colorbar_mask(value, log_cb)
        assert np.all(log_colorbar == expected)

    @pytest.mark.parametrize('value, nocov, bad_data, snr_min, log_cb, expected_mask',
                             [(value, nocov, bad_data, snr_min_none, log_cb_true, image_none_true),
                              (value, nocov, bad_data, snr_min_3, log_cb_true, image_3_true),
                              (value, nocov, bad_data, snr_min_none, log_cb_false,
                               image_none_false),
                              (value, nocov, bad_data, snr_min_3, log_cb_false, image_3_false)])
    def test_make_image(self, value, nocov, bad_data, snr_min, log_cb, expected_mask):
        image = mapplot.make_image(value, nocov, bad_data, snr_min, log_cb)
        expected = np.ma.array(value, mask=expected_mask)
        assert np.all(image.data == expected.data)
        assert np.all(image.mask == expected.mask)

class TestMapPlot(object):

    @pytest.mark.parametrize('cube_size, sky_coords, expected',
                             [([36, 36], True, np.array([-18, 18, -18, 18])),
                              ([35, 35], True, np.array([-17.5, 17.5, -17.5, 17.5])),
                              ([36, 36], False, np.array([0, 35, 0, 35]))])
    def test_set_extent(self, cube_size, sky_coords, expected):
        extent = mapplot.set_extent(cube_size, sky_coords)
        assert np.all(extent == expected)

    @matplotlib_2
    def test_set_hatch_linewidth(self):
        __ = mapplot.set_patch_style([0, 1, 0, 1], facecolor='#A8A8A8')
        assert matplotlib.rcParams['hatch.linewidth'] == 0.5

    @matplotlib_2
    def test_set_hatch_color(self):
        __ = mapplot.set_patch_style([0, 1, 0, 1], facecolor='#A8A8A8')
        assert matplotlib.rcParams['hatch.color'] == 'w'

    def test_ax_facecolor(self):
        fig, ax = mapplot.ax_setup(sky_coords=True, fig=None, ax=None, facecolor='#A8A8A8')
        assert ax.get_facecolor() == (0.6588235294117647, 0.6588235294117647, 0.6588235294117647,
                                      1.0)

    @pytest.mark.parametrize('sky_coords, expected',
                             [(True, 'arcsec'),
                              (False, 'spaxel')])
    def test_ax_labels(self, sky_coords, expected):
        fig, ax = mapplot.ax_setup(sky_coords, fig=None, ax=None, facecolor='#A8A8A8')
        assert ax.get_xlabel() == expected
        assert ax.get_ylabel() == expected

    @pytest.mark.parametrize('title, property_name, channel, expected',
                             [('test-title', None, None, 'test-title'),
                              ('test-title', 'prop', 'channel', 'test-title'),
                              (None, 'prop', 'channel', 'prop channel'),
                              (None, 'emline_gflux', 'ha_6564', 'emline gflux ha 6564'),
                              (None, 'prop_flux', None, 'prop flux'),
                              (None, None, 'channel', 'channel'),
                              (None, None, None, '')])
    def test_set_title(self, title, property_name, channel, expected):
        title = mapplot.set_title(title, property_name, channel)
        assert title == expected

    @pytest.mark.parametrize('property_name, channel',
                             [('emline_gflux', 'ha_6564'),
                              ('emline_gflux', 'oiii_5008'),
                              ('stellar_vel', None),
                              ('stellar_sigma', None),
                              ('specindex', 'd4000')])
    def test_plot_dapmap(self, maps, property_name, channel):
        map_ = maps.getMap(property_name, channel=None)
        fig, ax = mapplot.plot(dapmap=map_)
        # fig, ax = mapplot.plot(value=np.array([[0, 1], [2, 3]]))  # dapmap=ha
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._axes.Axes)

    @pytest.mark.parametrize('mask', [None, mask_simple, mask_binary, mask_daplike])
    @pytest.mark.parametrize('ivar', [None, ivar])
    @pytest.mark.parametrize('value', [value])
    def test_plot_user_defined_map(self, value, ivar, mask):
        fig, ax = mapplot.plot(value=value, ivar=ivar, mask=mask)
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._axes.Axes)

# TODO add more asserts to test_plot
<commit_msg>adds map fixture<commit_after>import numpy as np
import matplotlib
import pytest

from marvin.tools.maps import Maps
import marvin.utils.plot.map as mapplot

matplotlib_2 = pytest.mark.skipif(int(matplotlib.__version__.split('.')[0]) <= 1,
                                  reason='matplotlib-2.0 or higher required')

value = np.array([[1, -2, 3],
                  [-4, 5, 6],
                  [7, 8, 9]])

ivar = np.array([[4, 10, 1],
                 [0.01, 0.04, 0.0001],
                 [100000, 0, 1]])

mask_simple = np.array([[0, 1, 0],
                        [1, 0, 1],
                        [0, 1, 0]])

mask_binary = np.array([[0b000, 0b001, 0b010],
                        [0b011, 0b100, 0b101],
                        [0b110, 0b111, 0b000]])

mask_daplike = np.array([[0b0000000000000000000000000000000,
                          0b0000000000000000000000000000001,
                          0b0000000000000000000000000100000],
                         [0b0000000000000000000000001000001,
                          0b0000000000000000000000010000000,
                          0b1000000000000000000000000000001],
                         [0b0000000000000000000000000000000,
                          0b1000000000000000000000011100001,
                          0b0000000000000000000000000000000]])

nocov = np.array([[0, 1, 0],
                  [1, 0, 1],
                  [0, 1, 0]], dtype=bool)

bad_data = np.array([[0, 0, 1],
                     [1, 1, 1],
                     [0, 1, 0]])

# SNR = value * np.sqrt(ivar)
# [[  2.00e+00,  -6.32e+00,   3.00e+00],
#  [ -4.00e-01,   1.00e+00,   6.00e-02],
#  [  2.21e+03,   0.00e+00,   9.00e+00]]

snr_min_none = np.array([[0, 0, 0],
                         [0, 0, 0],
                         [0, 1, 0]])

snr_min_0 = np.array([[0, 0, 0],
                      [0, 0, 0],
                      [0, 1, 0]])

snr_min_1 = np.array([[0, 0, 0],
                      [1, 0, 1],
                      [0, 1, 0]])

snr_min_3 = np.array([[1, 0, 0],
                      [1, 1, 1],
                      [0, 1, 0]])

log_cb_true = np.array([[0, 1, 0],
                        [1, 0, 0],
                        [0, 0, 0]])

log_cb_false = np.zeros(value.shape, dtype=bool)

image_none_true = np.array([[0, 1, 1],
                            [1, 1, 1],
                            [0, 1, 0]])

image_3_true = np.array([[1, 1, 1],
                         [1, 1, 1],
                         [0, 1, 0]])

image_none_false = np.array([[0, 1, 1],
                             [1, 1, 1],
                             [0, 1, 0]])

image_3_false = np.array([[1, 1, 1],
                          [1, 1, 1],
                          [0, 1, 0]])

@pytest.fixture(scope='module')
def maps(galaxy):
    yield Maps(plateifu=galaxy.plateifu)

class TestMasks(object):

    @pytest.mark.parametrize('value, mask, expected', [(value, mask_simple, nocov),
                                                       (value, mask_binary, nocov),
                                                       (value, mask_daplike, nocov)])
    def test_no_coverage_mask(self, value, mask, expected):
        nocov = mapplot.no_coverage_mask(value, mask)
        assert np.all(nocov == expected)

    @pytest.mark.parametrize('value, mask, expected', [(value, mask_daplike, bad_data)])
    def test_bad_data_mask(self, value, mask, expected):
        bad_data = mapplot.bad_data_mask(value, mask)
        assert np.all(bad_data == expected)

    @pytest.mark.parametrize('value, ivar, snr_min, expected',
                             [(value, ivar, None, snr_min_none),
                              (value, ivar, 0, snr_min_0),
                              (value, ivar, 1, snr_min_1),
                              (value, ivar, 3, snr_min_3),
                              (value, None, None, np.zeros(value.shape, dtype=bool)),
                              (value, None, 1, np.zeros(value.shape, dtype=bool))])
    def test_low_snr_mask(self, value, ivar, snr_min, expected):
        low_snr = mapplot.low_snr_mask(value, ivar, snr_min)
        assert np.all(low_snr == expected)

    @pytest.mark.parametrize('value, log_cb, expected',
                             [(value, True, log_cb_true),
                              (value, False, log_cb_false)])
    def test_log_colorbar_mask(self, value, log_cb, expected):
        log_colorbar = mapplot.log_colorbar_mask(value, log_cb)
        assert np.all(log_colorbar == expected)

    @pytest.mark.parametrize('value, nocov, bad_data, snr_min, log_cb, expected_mask',
                             [(value, nocov, bad_data, snr_min_none, log_cb_true, image_none_true),
                              (value, nocov, bad_data, snr_min_3, log_cb_true, image_3_true),
                              (value, nocov, bad_data, snr_min_none, log_cb_false,
                               image_none_false),
                              (value, nocov, bad_data, snr_min_3, log_cb_false, image_3_false)])
    def test_make_image(self, value, nocov, bad_data, snr_min, log_cb, expected_mask):
        image = mapplot.make_image(value, nocov, bad_data, snr_min, log_cb)
        expected = np.ma.array(value, mask=expected_mask)
        assert np.all(image.data == expected.data)
        assert np.all(image.mask == expected.mask)

class TestMapPlot(object):

    @pytest.mark.parametrize('cube_size, sky_coords, expected',
                             [([36, 36], True, np.array([-18, 18, -18, 18])),
                              ([35, 35], True, np.array([-17.5, 17.5, -17.5, 17.5])),
                              ([36, 36], False, np.array([0, 35, 0, 35]))])
    def test_set_extent(self, cube_size, sky_coords, expected):
        extent = mapplot.set_extent(cube_size, sky_coords)
        assert np.all(extent == expected)

    @matplotlib_2
    def test_set_hatch_linewidth(self):
        __ = mapplot.set_patch_style([0, 1, 0, 1], facecolor='#A8A8A8')
        assert matplotlib.rcParams['hatch.linewidth'] == 0.5

    @matplotlib_2
    def test_set_hatch_color(self):
        __ = mapplot.set_patch_style([0, 1, 0, 1], facecolor='#A8A8A8')
        assert matplotlib.rcParams['hatch.color'] == 'w'

    def test_ax_facecolor(self):
        fig, ax = mapplot.ax_setup(sky_coords=True, fig=None, ax=None, facecolor='#A8A8A8')
        assert ax.get_facecolor() == (0.6588235294117647, 0.6588235294117647, 0.6588235294117647,
                                      1.0)

    @pytest.mark.parametrize('sky_coords, expected',
                             [(True, 'arcsec'),
                              (False, 'spaxel')])
    def test_ax_labels(self, sky_coords, expected):
        fig, ax = mapplot.ax_setup(sky_coords, fig=None, ax=None, facecolor='#A8A8A8')
        assert ax.get_xlabel() == expected
        assert ax.get_ylabel() == expected

    @pytest.mark.parametrize('title, property_name, channel, expected',
                             [('test-title', None, None, 'test-title'),
                              ('test-title', 'prop', 'channel', 'test-title'),
                              (None, 'prop', 'channel', 'prop channel'),
                              (None, 'emline_gflux', 'ha_6564', 'emline gflux ha 6564'),
                              (None, 'prop_flux', None, 'prop flux'),
                              (None, None, 'channel', 'channel'),
                              (None, None, None, '')])
    def test_set_title(self, title, property_name, channel, expected):
        title = mapplot.set_title(title, property_name, channel)
        assert title == expected

    @pytest.mark.parametrize('property_name, channel',
                             [('emline_gflux', 'ha_6564'),
                              ('emline_gflux', 'oiii_5008'),
                              ('stellar_vel', None),
                              ('stellar_sigma', None),
                              ('specindex', 'd4000')])
    def test_plot_dapmap(self, maps, property_name, channel):
        map_ = maps.getMap(property_name, channel=channel)
        fig, ax = mapplot.plot(dapmap=map_)
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._axes.Axes)

    @pytest.mark.parametrize('mask', [None, mask_simple, mask_binary, mask_daplike])
    @pytest.mark.parametrize('ivar', [None, ivar])
    @pytest.mark.parametrize('value', [value])
    def test_plot_user_defined_map(self, value, ivar, mask):
        fig, ax = mapplot.plot(value=value, ivar=ivar, mask=mask)
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._axes.Axes)

# TODO add more asserts to test_plot
<|endoftext|>


<commit_before># Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import warnings
from functools import wraps
from typing import Any, Optional, Union

import torch

from pytorch_lightning import _logger as log

if torch.distributed.is_available():
    from torch.distributed import ReduceOp, group
else:
    class ReduceOp:
        SUM = None

    class group:
        WORLD = None

def rank_zero_only(fn):

    @wraps(fn)
    def wrapped_fn(*args, **kwargs):
        if rank_zero_only.rank == 0:
            return fn(*args, **kwargs)

    return wrapped_fn

# add the attribute to the function but don't overwrite in case Trainer has already set it
rank_zero_only.rank = getattr(rank_zero_only, 'rank', int(os.environ.get('LOCAL_RANK', 0)))

def _warn(*args, **kwargs):
    warnings.warn(*args, **kwargs)

def _info(*args, **kwargs):
    log.info(*args, **kwargs)

def _debug(*args, **kwargs):
    log.debug(*args, **kwargs)

rank_zero_debug = rank_zero_only(_debug)
rank_zero_info = rank_zero_only(_info)
rank_zero_warn = rank_zero_only(_warn)

def find_free_network_port() -> int:
    """
    Finds a free port on localhost.
    It is useful in single-node training when we don't want to connect to a real master node but
    have to set the `MASTER_PORT` environment variable.
    """
    import socket
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("", 0))
    s.listen(1)
    port = s.getsockname()[1]
    s.close()
    return port

def gather_all_tensors(result: Union[torch.Tensor], group: Optional[Any] = None):
    """
    Function to gather all tensors from several ddp processes onto a list that
    is broadcasted to all processes

    Args:
        result: the value to sync
        group: the process group to gather results from. Defaults to all processes (world)

    Return:
        gathered_result: list with size equal to the process group where
            gathered_result[i] corresponds to result tensor from process i
    """
    if group is None:
        group = torch.distributed.group.WORLD

    # convert tensors to contiguous format
    result = result.contiguous()

    world_size = torch.distributed.get_world_size(group)

    gathered_result = [torch.zeros_like(result) for _ in range(world_size)]

    # sync and broadcast all
    torch.distributed.barrier(group=group)
    torch.distributed.all_gather(gathered_result, result, group)

    return gathered_result

def sync_ddp_if_available(
    result: Union[torch.Tensor], group: Optional[Any] = None, reduce_op: Optional[Union[ReduceOp, str]] = None
) -> torch.Tensor:
    """
    Function to reduce a tensor across worker processes during distributed training
    Args:
        result: the value to sync and reduce (typically tensor or number)
        group: the process group to gather results from. Defaults to all processes (world)
        reduce_op: the reduction operation. Defaults to sum.
            Can also be a string of 'avg', 'mean' to calculate the mean during reduction.
    Return:
        reduced value
    """
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        return sync_ddp(result, group=group, reduce_op=reduce_op)
    return result

def sync_ddp(
    result: Union[torch.Tensor], group: Optional[Any] = None, reduce_op: Optional[Union[ReduceOp, str]] = None
) -> torch.Tensor:
    """
    Function to reduce the tensors from several ddp processes to one master process

    Args:
        result: the value to sync and reduce (typically tensor or number)
        group: the process group to gather results from. Defaults to all processes (world)
        reduce_op: the reduction operation. Defaults to sum.
            Can also be a string of 'avg', 'mean' to calculate the mean during reduction.

    Return:
        reduced value
    """
    divide_by_world_size = False

    if group is None:
        group = torch.distributed.group.WORLD

    op = reduce_op if isinstance(reduce_op, ReduceOp) else ReduceOp.SUM

    if isinstance(reduce_op, str) and reduce_op.lower() in ("avg", "mean"):
        divide_by_world_size = True

    # sync all processes before reduction
    torch.distributed.barrier(group=group)
    torch.distributed.all_reduce(result, op=op, group=group, async_op=False)

    if divide_by_world_size:
        result = result / torch.distributed.get_world_size(group)

    return result

class AllGatherGrad(torch.autograd.Function):
    @staticmethod
    def forward(ctx, tensor, group=group.WORLD):
        ctx.group = group

        gathered_tensor = [
            torch.zeros_like(tensor) for _ in range(torch.distributed.get_world_size())
        ]

        torch.distributed.all_gather(gathered_tensor, tensor, group=group)
        gathered_tensor = torch.stack(gathered_tensor, dim=0)

        return gathered_tensor

    @staticmethod
    def backward(ctx, *grad_output):
        grad_output = torch.cat(grad_output)

        torch.distributed.all_reduce(
            grad_output,
            op=torch.distributed.ReduceOp.SUM,
            async_op=False,
            group=ctx.group
        )

        return grad_output[torch.distributed.get_rank()]

def all_gather_ddp_if_available(
    tensor: Union[torch.Tensor], group: Optional[Any] = None, sync_grads: bool = False
) -> torch.Tensor:
    """
    Function to gather a tensor from several distributed processes

    Args:
        tensor: tensor of shape (batch, ...)
        group: the process group to gather results from. Defaults to all processes (world)
        sync_grads: flag that allows users to synchronize gradients for all_gather op

    Return:
        A tensor of shape (world_size, batch, ...)
    """
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        if sync_grads:
            return AllGatherGrad.apply(tensor, group)
        else:
            with torch.no_grad():
                return AllGatherGrad.apply(tensor, group)
    return tensor
<commit_msg>[bugfix] Group defaults to WORLD if None (#5125)<commit_after># Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import warnings
from functools import wraps
from typing import Any, Optional, Union

import torch

from pytorch_lightning import _logger as log

if torch.distributed.is_available():
    from torch.distributed import ReduceOp, group
else:
    class ReduceOp:
        SUM = None

    class group:
        WORLD = None

def rank_zero_only(fn):

    @wraps(fn)
    def wrapped_fn(*args, **kwargs):
        if rank_zero_only.rank == 0:
            return fn(*args, **kwargs)

    return wrapped_fn

# add the attribute to the function but don't overwrite in case Trainer has already set it
rank_zero_only.rank = getattr(rank_zero_only, 'rank', int(os.environ.get('LOCAL_RANK', 0)))

def _warn(*args, **kwargs):
    warnings.warn(*args, **kwargs)

def _info(*args, **kwargs):
    log.info(*args, **kwargs)

def _debug(*args, **kwargs):
    log.debug(*args, **kwargs)

rank_zero_debug = rank_zero_only(_debug)
rank_zero_info = rank_zero_only(_info)
rank_zero_warn = rank_zero_only(_warn)

def find_free_network_port() -> int:
    """
    Finds a free port on localhost.
    It is useful in single-node training when we don't want to connect to a real master node but
    have to set the `MASTER_PORT` environment variable.
    """
    import socket
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("", 0))
    s.listen(1)
    port = s.getsockname()[1]
    s.close()
    return port

def gather_all_tensors(result: Union[torch.Tensor], group: Optional[Any] = None):
    """
    Function to gather all tensors from several ddp processes onto a list that
    is broadcasted to all processes

    Args:
        result: the value to sync
        group: the process group to gather results from. Defaults to all processes (world)

    Return:
        gathered_result: list with size equal to the process group where
            gathered_result[i] corresponds to result tensor from process i
    """
    if group is None:
        group = torch.distributed.group.WORLD

    # convert tensors to contiguous format
    result = result.contiguous()

    world_size = torch.distributed.get_world_size(group)

    gathered_result = [torch.zeros_like(result) for _ in range(world_size)]

    # sync and broadcast all
    torch.distributed.barrier(group=group)
    torch.distributed.all_gather(gathered_result, result, group)

    return gathered_result

def sync_ddp_if_available(
    result: Union[torch.Tensor], group: Optional[Any] = None, reduce_op: Optional[Union[ReduceOp, str]] = None
) -> torch.Tensor:
    """
    Function to reduce a tensor across worker processes during distributed training
    Args:
        result: the value to sync and reduce (typically tensor or number)
        group: the process group to gather results from. Defaults to all processes (world)
        reduce_op: the reduction operation. Defaults to sum.
            Can also be a string of 'avg', 'mean' to calculate the mean during reduction.
    Return:
        reduced value
    """
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        return sync_ddp(result, group=group, reduce_op=reduce_op)
    return result

def sync_ddp(
    result: Union[torch.Tensor], group: Optional[Any] = None, reduce_op: Optional[Union[ReduceOp, str]] = None
) -> torch.Tensor:
    """
    Function to reduce the tensors from several ddp processes to one master process

    Args:
        result: the value to sync and reduce (typically tensor or number)
        group: the process group to gather results from. Defaults to all processes (world)
        reduce_op: the reduction operation. Defaults to sum.
            Can also be a string of 'avg', 'mean' to calculate the mean during reduction.

    Return:
        reduced value
    """
    divide_by_world_size = False

    if group is None:
        group = torch.distributed.group.WORLD

    op = reduce_op if isinstance(reduce_op, ReduceOp) else ReduceOp.SUM

    if isinstance(reduce_op, str) and reduce_op.lower() in ("avg", "mean"):
        divide_by_world_size = True

    # sync all processes before reduction
    torch.distributed.barrier(group=group)
    torch.distributed.all_reduce(result, op=op, group=group, async_op=False)

    if divide_by_world_size:
        result = result / torch.distributed.get_world_size(group)

    return result

class AllGatherGrad(torch.autograd.Function):
    @staticmethod
    def forward(ctx, tensor, group=group.WORLD):
        ctx.group = group

        gathered_tensor = [
            torch.zeros_like(tensor) for _ in range(torch.distributed.get_world_size())
        ]

        torch.distributed.all_gather(gathered_tensor, tensor, group=group)
        gathered_tensor = torch.stack(gathered_tensor, dim=0)

        return gathered_tensor

    @staticmethod
    def backward(ctx, *grad_output):
        grad_output = torch.cat(grad_output)

        torch.distributed.all_reduce(
            grad_output,
            op=torch.distributed.ReduceOp.SUM,
            async_op=False,
            group=ctx.group
        )

        return grad_output[torch.distributed.get_rank()]

def all_gather_ddp_if_available(
    tensor: Union[torch.Tensor], group: Optional[Any] = None, sync_grads: bool = False
) -> torch.Tensor:
    """
    Function to gather a tensor from several distributed processes

    Args:
        tensor: tensor of shape (batch, ...)
        group: the process group to gather results from. Defaults to all processes (world)
        sync_grads: flag that allows users to synchronize gradients for all_gather op

    Return:
        A tensor of shape (world_size, batch, ...)
    """
    group = group if group is not None else torch.distributed.group.WORLD
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        if sync_grads:
            return AllGatherGrad.apply(tensor, group)
        else:
            with torch.no_grad():
                return AllGatherGrad.apply(tensor, group)
    return tensor
<|endoftext|>


<commit_before>import os
import os.path
import filecmp
import pytest
from file_manip_toolkit.unfman import CPS2Format

TESTDIR = 'tests\\test_data\\unfman\\'
DEINTERLEAVE_FILE = 'cps2_test.13.15.17.19.combined'
INTERLEAVE_FILES = ['cps2_test.13', 'cps2_test.15', 'cps2_test.17', 'cps2_test.19']
CUSTOM_FILES = ['custom1.13', 'custom1.15', 'custom1.17', 'custom1.19']

# tests work for the most part
# but some thought should be given to refactoring them
# maybe

@pytest.fixture(params=[2, 4])
def numbytes(request):
    return request.param

@pytest.fixture(params=[2, 4])
def nsplits(request):
    return request.param

@pytest.fixture(params=['testdir\\', 'custom'])
def outputs(request):
    return request.param

@pytest.mark.parametrize('filepaths, outputs, expected', [
    ([DEINTERLEAVE_FILE], '', (4, INTERLEAVE_FILES)),
    ([DEINTERLEAVE_FILE], TESTDIR, (4, [''.join([TESTDIR, name]) for name in INTERLEAVE_FILES])),
    ([DEINTERLEAVE_FILE], ''.join([TESTDIR, 'custom1']), (4, [''.join([TESTDIR, name]) for name in CUSTOM_FILES])),
])
def test_format_savepaths_deinterleave(filepaths, outputs, expected):
    cps2 = CPS2Format.new(filepaths, outputs, False)
    results = cps2.format_savepaths()
    numfiles, filenames = expected
    assert len(results) == numfiles
    assert results == filenames

@pytest.mark.parametrize('filepaths, outputs, expected', [
    (INTERLEAVE_FILES, '', (1, [DEINTERLEAVE_FILE])),
    (INTERLEAVE_FILES, TESTDIR, (1, [''.join([TESTDIR, DEINTERLEAVE_FILE])])),
    (INTERLEAVE_FILES, ''.join([TESTDIR, 'custom1']), (1, [''.join([TESTDIR, 'custom1.combined'])])),
])
def test_format_savepaths_interleave(filepaths, outputs, expected):
    cps2 = CPS2Format.new(filepaths, outputs, False)
    results = cps2.format_savepaths()
    numfiles, filenames = expected
    assert len(results) == numfiles
    assert results == filenames

# deinterleave runs slower during testing - find out why?
# DOESNT WORK WITH empty case?'' not sure how to test
joined_deinterleave = ['\\'.join([TESTDIR, DEINTERLEAVE_FILE])]
joined_interleave = ['\\'.join([TESTDIR, name]) for name in INTERLEAVE_FILES]
@pytest.mark.skip
@pytest.mark.parametrize('test_data, expected', [
    (joined_deinterleave, 4),
    (joined_interleave, 1),
])
def test_run(test_data, expected, outputs, tmpdir):
    fns = tmpdir.mkdir(outputs)

    cps2 = CPS2Format.new(test_data, str(fns), False)
    cps2.run()

    assert len(tmpdir.join(outputs).listdir()) == expected

    for f in tmpdir.join(outputs).listdir():
        assert filecmp.cmp(f.strpath, os.path.join(TESTDIR, f.basename))
<commit_msg>os.path.join might be able to fix this<commit_after>import os
import os.path
import filecmp
import pytest
from file_manip_toolkit.unfman import CPS2Format

TESTDIR = 'tests\\test_data\\unfman\\'
DEINTERLEAVE_FILE = 'cps2_test.13.15.17.19.combined'
INTERLEAVE_FILES = ['cps2_test.13', 'cps2_test.15', 'cps2_test.17', 'cps2_test.19']
CUSTOM_FILES = ['custom1.13', 'custom1.15', 'custom1.17', 'custom1.19']

# tests work for the most part
# but some thought should be given to refactoring them
# maybe

@pytest.fixture(params=[2, 4])
def numbytes(request):
    return request.param

@pytest.fixture(params=[2, 4])
def nsplits(request):
    return request.param

@pytest.fixture(params=['testdir\\', 'custom'])
def outputs(request):
    return request.param

@pytest.mark.parametrize('filepaths, outputs, expected', [
    ([DEINTERLEAVE_FILE], '', (4, INTERLEAVE_FILES)),
    ([DEINTERLEAVE_FILE], TESTDIR, (4, [''.join([TESTDIR, name]) for name in INTERLEAVE_FILES])),
    ([DEINTERLEAVE_FILE], os.path.join(TESTDIR, 'custom1'), (4, [''.join([TESTDIR, name]) for name in CUSTOM_FILES])),
])
def test_format_savepaths_deinterleave(filepaths, outputs, expected):
    cps2 = CPS2Format.new(filepaths, outputs, False)
    results = cps2.format_savepaths()
    numfiles, filenames = expected
    assert len(results) == numfiles
    assert results == filenames

@pytest.mark.parametrize('filepaths, outputs, expected', [
    (INTERLEAVE_FILES, '', (1, [DEINTERLEAVE_FILE])),
    (INTERLEAVE_FILES, TESTDIR, (1, [''.join([TESTDIR, DEINTERLEAVE_FILE])])),
    (INTERLEAVE_FILES, os.path.join(TESTDIR, 'custom1'), (1, [''.join([TESTDIR, 'custom1.combined'])])),
])
def test_format_savepaths_interleave(filepaths, outputs, expected):
    cps2 = CPS2Format.new(filepaths, outputs, False)
    results = cps2.format_savepaths()
    numfiles, filenames = expected
    assert len(results) == numfiles
    assert results == filenames

# deinterleave runs slower during testing - find out why?
# DOESNT WORK WITH empty case?'' not sure how to test
joined_deinterleave = ['\\'.join([TESTDIR, DEINTERLEAVE_FILE])]
joined_interleave = ['\\'.join([TESTDIR, name]) for name in INTERLEAVE_FILES]
@pytest.mark.skip
@pytest.mark.parametrize('test_data, expected', [
    (joined_deinterleave, 4),
    (joined_interleave, 1),
])
def test_run(test_data, expected, outputs, tmpdir):
    fns = tmpdir.mkdir(outputs)

    cps2 = CPS2Format.new(test_data, str(fns), False)
    cps2.run()

    assert len(tmpdir.join(outputs).listdir()) == expected

    for f in tmpdir.join(outputs).listdir():
        assert filecmp.cmp(f.strpath, os.path.join(TESTDIR, f.basename))
<|endoftext|>


<commit_before>import numpy as np
import scipy as sp
import sys
import caffe
import os.path as opath
from multiprocessing import Pool
import argparse

def save_code_and_label(params):
    database_code = params['database_code']
    validation_code = params['validation_code']
    database_labels = params['database_labels']
    validation_labels = params['validation_labels']
    path = params['path']
    np.save(opath.join(path, "database_code.npy"), database_code)
    np.save(opath.join(path, "database_label.npy"), database_labels)
    np.save(opath.join(path, "validation_code.npy"), validation_code)
    np.save(opath.join(path, "validation_label.npy"), validation_labels)

def mean_average_precision(params):
    database_code = params['database_code']
    validation_code = params['validation_code']
    database_labels = params['database_labels']
    validation_labels = params['validation_labels']
    R = params['R']
    query_num = validation_code.shape[0]

    sim = np.dot(database_code, validation_code.T)
    ids = np.argsort(-sim, axis=0)
    APx = []
    ground_truth = np.sum(np.dot(database_labels, validation_labels.T), axis=0)
    validation_labels[validation_labels==0] = -1
    for i in range(query_num):
        label = validation_labels[i, :]
        if ground_truth[i] > 0:
            idx = ids[:, i]
            imatch = np.sum(database_labels[idx[0:R], :] == label, axis=1) > 0
            relevant_num = np.sum(imatch)
            Lx = np.cumsum(imatch)
            Px = Lx.astype(float) / np.arange(1, R+1, 1)
            if relevant_num != 0:
                APx.append(np.sum(Px * imatch) / relevant_num)
            else:
                APx.append(np.sum(Px * imatch) / 1)

    return np.mean(np.array(APx))

def get_codes_and_labels(params):
    device_id = int(params["gpu"])
    caffe.set_device(device_id)
    caffe.set_mode_gpu()
    model_file = params['model_file']
    pretrained_model = params['pretrained_model']
    dims = params['image_dims']
    scale = params['scale']
    database = open(params['database'], 'r').readlines()
    validation = open(params['validation'], 'r').readlines()
    batch_size = params['batch_size']

    if 'mean_file' in params:
        mean_file = params['mean_file']
        net = caffe.Classifier(model_file, pretrained_model, channel_swap=(2,1,0), image_dims=dims, mean=np.load(mean_file).mean(1).mean(1), raw_scale=scale)
    else:
        net = caffe.Classifier(model_file, pretrained_model, channel_swap=(2,1,0), image_dims=dims, raw_scale=scale)

    database_code = []
    validation_code = []
    database_labels = []
    validation_labels = []
    cur_pos = 0

    while 1:
        lines = database[cur_pos : cur_pos + batch_size]
        if len(lines) == 0:
            break;
        cur_pos = cur_pos + len(lines)
        images = [caffe.io.load_image(line.strip().split(" ")[0]) for line in lines]
        labels = [[int(i) for i in line.strip().split(" ")[1:]] for line in lines]
        codes = net.predict(images, oversample=False)
        #codes = net.predict(images)
        [database_code.append(c) for c in codes]
        [database_labels.append(l) for l in labels]

        print str(cur_pos) + "/" + str(len(database))
        if len(lines) < batch_size:
            break;

    cur_pos = 0
    while 1:
        lines = validation[cur_pos : cur_pos + batch_size]
        if len(lines) == 0:
            break;
        cur_pos = cur_pos + len(lines)
        images = [caffe.io.load_image(line.strip().split(" ")[0]) for line in lines]
        labels = [[int(i) for i in line.strip().split(" ")[1:]] for line in lines]

        codes = net.predict(images, oversample=False)
        #codes = net.predict(images)
        [validation_code.append(c) for c in codes]
        [validation_labels.append(l) for l in labels]

        print str(cur_pos) + "/" + str(len(validation))
        if len(lines) < batch_size:
            break;

    return dict(database_code=np.sign(np.array(database_code)), database_labels=np.array(database_labels), validation_code=np.sign(np.array(validation_code)), validation_labels=np.array(validation_labels))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Predict Hash Code.')
    parser.add_argument('--gpu', metavar='N', type=str, nargs='?', default="0",
                    help='running gpu id')
    parser.add_argument('--model_path', type=str, nargs="?",
                    help='the caffemodel path')
    parser.add_argument('--save_path', type=str, nargs='?',
                    help='the path to save predict code')
    parser.add_argument('--code_path', type=str, nargs='?', default="",
                    help='the predicted hash code path')

    args = parser.parse_args()

    if args.code_path:
        code_and_label = {}
        code_and_label["database_code"] = np.load(opath.join(args.code_path, "database_code.npy"))
        code_and_label['database_labels'] = np.load(opath.join(args.code_path,"database_label.npy"))
        code_and_label["validation_code"] = np.load(opath.join(args.code_path, "validation_code.npy"))
        code_and_label['validation_labels'] = np.load(opath.join(args.code_path, "validation_label.npy"))
    else:
        nthreads = 8
        params = []
        for i in range(nthreads):
            params.append(dict(model_file="./models/predict/deploy.prototxt",
                  pretrained_model=args.model_path[0],
                  image_dims=(256,256),
                  scale=255,
                  database="./data/nuswide_81/test/database" + str(i) + ".txt",
                  validation="./data/nuswide_81/test/test" + str(i) + ".txt",
                  batch_size=50,
                  mean_file="./python/caffe/imagenet/ilsvrc_2012_mean.npy",
                  gpu=args.gpu[0]))

        pool = Pool(nthreads)
        results = pool.map(get_codes_and_labels, params)
        print("start combine")
        code_and_label = results[0]
        for i in range(1, nthreads):
            code_and_label['database_code'] = np.vstack([code_and_label["database_code"], results[i]['database_code']])
            code_and_label['database_labels'] = np.vstack([code_and_label["database_labels"],results[i]['database_labels']])
            code_and_label['validation_code'] = np.vstack([code_and_label["validation_code"], results[i]['validation_code']])
            code_and_label['validation_labels'] = np.vstack([code_and_label["validation_labels"], results[i]['validation_labels']])

        print("saving ...")
        code_and_label['path'] = str(args.save_path[0])
        save_code_and_label(code_and_label)
        print("saving done")

print("start predict")
code_and_label['R'] = 5000
mAP = mean_average_precision(code_and_label)
print ("MAP: "+ str(mAP))
<commit_msg>Update predict_parallel.py<commit_after>import numpy as np
import scipy as sp
import sys
import caffe
import os.path as opath
from multiprocessing import Pool
import argparse

def save_code_and_label(params):
    database_code = params['database_code']
    validation_code = params['validation_code']
    database_labels = params['database_labels']
    validation_labels = params['validation_labels']
    path = params['path']
    np.save(opath.join(path, "database_code.npy"), database_code)
    np.save(opath.join(path, "database_label.npy"), database_labels)
    np.save(opath.join(path, "validation_code.npy"), validation_code)
    np.save(opath.join(path, "validation_label.npy"), validation_labels)

def mean_average_precision(params):
    database_code = params['database_code']
    validation_code = params['validation_code']
    database_labels = params['database_labels']
    validation_labels = params['validation_labels']
    R = params['R']
    query_num = validation_code.shape[0]

    sim = np.dot(database_code, validation_code.T)
    ids = np.argsort(-sim, axis=0)
    APx = []
    ground_truth = np.sum(np.dot(database_labels, validation_labels.T), axis=0)
    validation_labels[validation_labels==0] = -1
    for i in range(query_num):
        label = validation_labels[i, :]
        if ground_truth[i] > 0:
            idx = ids[:, i]
            imatch = np.sum(database_labels[idx[0:R], :] == label, axis=1) > 0
            relevant_num = np.sum(imatch)
            Lx = np.cumsum(imatch)
            Px = Lx.astype(float) / np.arange(1, R+1, 1)
            if relevant_num != 0:
                APx.append(np.sum(Px * imatch) / relevant_num)
            else:
                APx.append(np.sum(Px * imatch) / 1)

    return np.mean(np.array(APx))

def get_codes_and_labels(params):
    device_id = int(params["gpu"])
    caffe.set_device(device_id)
    caffe.set_mode_gpu()
    model_file = params['model_file']
    pretrained_model = params['pretrained_model']
    dims = params['image_dims']
    scale = params['scale']
    database = open(params['database'], 'r').readlines()
    validation = open(params['validation'], 'r').readlines()
    batch_size = params['batch_size']

    if 'mean_file' in params:
        mean_file = params['mean_file']
        net = caffe.Classifier(model_file, pretrained_model, channel_swap=(2,1,0), image_dims=dims, mean=np.load(mean_file).mean(1).mean(1), raw_scale=scale)
    else:
        net = caffe.Classifier(model_file, pretrained_model, channel_swap=(2,1,0), image_dims=dims, raw_scale=scale)

    database_code = []
    validation_code = []
    database_labels = []
    validation_labels = []
    cur_pos = 0

    while 1:
        lines = database[cur_pos : cur_pos + batch_size]
        if len(lines) == 0:
            break;
        cur_pos = cur_pos + len(lines)
        images = [caffe.io.load_image(line.strip().split(" ")[0]) for line in lines]
        labels = [[int(i) for i in line.strip().split(" ")[1:]] for line in lines]
        codes = net.predict(images, oversample=False)
        #codes = net.predict(images)
        [database_code.append(c) for c in codes]
        [database_labels.append(l) for l in labels]

        print str(cur_pos) + "/" + str(len(database))
        if len(lines) < batch_size:
            break;

    cur_pos = 0
    while 1:
        lines = validation[cur_pos : cur_pos + batch_size]
        if len(lines) == 0:
            break;
        cur_pos = cur_pos + len(lines)
        images = [caffe.io.load_image(line.strip().split(" ")[0]) for line in lines]
        labels = [[int(i) for i in line.strip().split(" ")[1:]] for line in lines]

        codes = net.predict(images, oversample=False)
        #codes = net.predict(images)
        [validation_code.append(c) for c in codes]
        [validation_labels.append(l) for l in labels]

        print str(cur_pos) + "/" + str(len(validation))
        if len(lines) < batch_size:
            break;

    return dict(database_code=np.sign(np.array(database_code)), database_labels=np.array(database_labels), validation_code=np.sign(np.array(validation_code)), validation_labels=np.array(validation_labels))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Predict Hash Code.')
    parser.add_argument('--gpu', metavar='N', type=int, nargs='?', default="0",
                    help='running gpu id')
    parser.add_argument('--model_path', type=str, nargs="?",
                    help='the caffemodel path')
    parser.add_argument('--save_path', type=str, nargs='?',
                    help='the path to save predict code')
    parser.add_argument('--code_path', type=str, nargs='?', default="",
                    help='the predicted hash code path')

    args = parser.parse_args()

    if args.code_path:
        code_and_label = {}
        code_and_label["database_code"] = np.load(opath.join(args.code_path, "database_code.npy"))
        code_and_label['database_labels'] = np.load(opath.join(args.code_path,"database_label.npy"))
        code_and_label["validation_code"] = np.load(opath.join(args.code_path, "validation_code.npy"))
        code_and_label['validation_labels'] = np.load(opath.join(args.code_path, "validation_label.npy"))
    else:
        nthreads = 8
        params = []
        for i in range(nthreads):
            params.append(dict(model_file="./models/predict/deploy.prototxt",
                  pretrained_model=args.model_path,
                  image_dims=(256,256),
                  scale=255,
                  database="./data/nuswide_81/test/database" + str(i) + ".txt",
                  validation="./data/nuswide_81/test/test" + str(i) + ".txt",
                  batch_size=50,
                  mean_file="./python/caffe/imagenet/ilsvrc_2012_mean.npy",
                  gpu=args.gpu))

        pool = Pool(nthreads)
        results = pool.map(get_codes_and_labels, params)
        print("start combine")
        code_and_label = results[0]
        for i in range(1, nthreads):
            code_and_label['database_code'] = np.vstack([code_and_label["database_code"], results[i]['database_code']])
            code_and_label['database_labels'] = np.vstack([code_and_label["database_labels"],results[i]['database_labels']])
            code_and_label['validation_code'] = np.vstack([code_and_label["validation_code"], results[i]['validation_code']])
            code_and_label['validation_labels'] = np.vstack([code_and_label["validation_labels"], results[i]['validation_labels']])

        print("saving ...")
        code_and_label['path'] = str(args.save_path)
        save_code_and_label(code_and_label)
        print("saving done")

print("start predict")
code_and_label['R'] = 5000
mAP = mean_average_precision(code_and_label)
print ("MAP: "+ str(mAP))
<|endoftext|>


<commit_before>from base64 import b64decode
import json
import logging
import sys
from redash.query_runner import *
from redash.utils import JSONEncoder

logger = logging.getLogger(__name__)

try:
    import gspread
    from oauth2client.client import SignedJwtAssertionCredentials
    from dateutil import parser
    enabled = True
except ImportError:
    enabled = False

def _load_key(filename):
    with open(filename, "rb") as f:
        return json.loads(f.read())

def _guess_type(value):
    if value == '':
        return TYPE_STRING
    try:
        val = int(value)
        return TYPE_INTEGER
    except ValueError:
        pass
    try:
        val = float(value)
        return TYPE_FLOAT
    except ValueError:
        pass
    if unicode(value).lower() in ('true', 'false'):
        return TYPE_BOOLEAN
    try:
        val = parser.parse(value)
        return TYPE_DATETIME
    except ValueError:
        pass
    return TYPE_STRING

def _value_eval_list(value):
    value_list = []
    for member in value:
        if member == '' or member is None:
            val = None
            value_list.append(val)
            continue
        try:
            val = int(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        try:
            val = float(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        if unicode(member).lower() in ('true', 'false'):
            if unicode(member).lower() == 'true':
                value_list.append(True)
            else:
                value_list.append(False)
            continue
        try:
            val = parser.parse(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        value_list.append(member)
    return value_list

HEADER_INDEX = 0

class WorksheetNotFoundError(Exception):
    def __init__(self, worksheet_num, worksheet_count):
        message = "Worksheet number {} not found. Spreadsheet has {} worksheets. Note that the worksheet count is zero based.".format(worksheet_num, worksheet_count)
        super(WorksheetNotFoundError, self).__init__(message)

def parse_worksheet(worksheet):
    if not worksheet:
        return {'columns': [], 'rows': []}

    column_names = []
    columns = []

    for j, column_name in enumerate(worksheet[HEADER_INDEX]):
        column_names.append(column_name)
        columns.append({
            'name': column_name,
            'friendly_name': column_name,
            'type': TYPE_STRING
        })

    if len(worksheet) > 1:
        for j, value in enumerate(worksheet[HEADER_INDEX+1]):
            columns[j]['type'] = _guess_type(value)

    rows = [dict(zip(column_names, _value_eval_list(row))) for row in worksheet[HEADER_INDEX + 1:]]
    data = {'columns': columns, 'rows': rows}

    return data

def parse_spreadsheet(spreadsheet, worksheet_num):
    worksheets = spreadsheet.worksheets()
    worksheet_count = len(worksheets)
    if worksheet_num >= worksheet_count:
        raise WorksheetNotFoundError(worksheet_num, worksheet_count)

    worksheet = worksheets[worksheet_num].get_all_values()

    return parse_worksheet(worksheet)

class GoogleSpreadsheet(BaseQueryRunner):
    @classmethod
    def annotate_query(cls):
        return False

    @classmethod
    def type(cls):
        return "google_spreadsheets"

    @classmethod
    def enabled(cls):
        return enabled

    @classmethod
    def configuration_schema(cls):
        return {
            'type': 'object',
            'properties': {
                'jsonKeyFile': {
                    "type": "string",
                    'title': 'JSON Key File'
                }
            },
            'required': ['jsonKeyFile'],
            'secret': ['jsonKeyFile']
        }

    def __init__(self, configuration):
        super(GoogleSpreadsheet, self).__init__(configuration)

    def _get_spreadsheet_service(self):
        scope = [
            'https://spreadsheets.google.com/feeds',
        ]

        key = json.loads(b64decode(self.configuration['jsonKeyFile']))
        credentials = SignedJwtAssertionCredentials(key['client_email'], key["private_key"], scope=scope)
        spreadsheetservice = gspread.authorize(credentials)
        return spreadsheetservice

    def run_query(self, query):
        logger.debug("Spreadsheet is about to execute query: %s", query)
        values = query.split("|")
        key = values[0] #key of the spreadsheet
        worksheet_num = 0 if len(values) != 2 else int(values[1])# if spreadsheet contains more than one worksheet - this is the number of it
        try:
            spreadsheet_service = self._get_spreadsheet_service()
            spreadsheet = spreadsheet_service.open_by_key(key)

            data = parse_spreadsheet(spreadsheet, worksheet_num)

            json_data = json.dumps(data, cls=JSONEncoder)
            error = None
        except gspread.SpreadsheetNotFound:
            error = "Spreadsheet ({}) not found. Make sure you used correct id.".format(key)
            json_data = None

        return json_data, error

register(GoogleSpreadsheet)
<commit_msg>Move dateutil import higher<commit_after>from base64 import b64decode
import json
import logging
from dateutil import parser
from redash.query_runner import *
from redash.utils import JSONEncoder

logger = logging.getLogger(__name__)

try:
    import gspread
    from oauth2client.client import SignedJwtAssertionCredentials
    enabled = True
except ImportError:
    enabled = False

def _load_key(filename):
    with open(filename, "rb") as f:
        return json.loads(f.read())

def _guess_type(value):
    if value == '':
        return TYPE_STRING
    try:
        val = int(value)
        return TYPE_INTEGER
    except ValueError:
        pass
    try:
        val = float(value)
        return TYPE_FLOAT
    except ValueError:
        pass
    if unicode(value).lower() in ('true', 'false'):
        return TYPE_BOOLEAN
    try:
        val = parser.parse(value)
        return TYPE_DATETIME
    except ValueError:
        pass
    return TYPE_STRING

def _value_eval_list(value):
    value_list = []
    for member in value:
        if member == '' or member is None:
            val = None
            value_list.append(val)
            continue
        try:
            val = int(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        try:
            val = float(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        if unicode(member).lower() in ('true', 'false'):
            if unicode(member).lower() == 'true':
                value_list.append(True)
            else:
                value_list.append(False)
            continue
        try:
            val = parser.parse(member)
            value_list.append(val)
            continue
        except ValueError:
            pass
        value_list.append(member)
    return value_list

HEADER_INDEX = 0

class WorksheetNotFoundError(Exception):
    def __init__(self, worksheet_num, worksheet_count):
        message = "Worksheet number {} not found. Spreadsheet has {} worksheets. Note that the worksheet count is zero based.".format(worksheet_num, worksheet_count)
        super(WorksheetNotFoundError, self).__init__(message)

def parse_worksheet(worksheet):
    if not worksheet:
        return {'columns': [], 'rows': []}

    column_names = []
    columns = []

    for j, column_name in enumerate(worksheet[HEADER_INDEX]):
        column_names.append(column_name)
        columns.append({
            'name': column_name,
            'friendly_name': column_name,
            'type': TYPE_STRING
        })

    if len(worksheet) > 1:
        for j, value in enumerate(worksheet[HEADER_INDEX+1]):
            columns[j]['type'] = _guess_type(value)

    rows = [dict(zip(column_names, _value_eval_list(row))) for row in worksheet[HEADER_INDEX + 1:]]
    data = {'columns': columns, 'rows': rows}

    return data

def parse_spreadsheet(spreadsheet, worksheet_num):
    worksheets = spreadsheet.worksheets()
    worksheet_count = len(worksheets)
    if worksheet_num >= worksheet_count:
        raise WorksheetNotFoundError(worksheet_num, worksheet_count)

    worksheet = worksheets[worksheet_num].get_all_values()

    return parse_worksheet(worksheet)

class GoogleSpreadsheet(BaseQueryRunner):
    @classmethod
    def annotate_query(cls):
        return False

    @classmethod
    def type(cls):
        return "google_spreadsheets"

    @classmethod
    def enabled(cls):
        return enabled

    @classmethod
    def configuration_schema(cls):
        return {
            'type': 'object',
            'properties': {
                'jsonKeyFile': {
                    "type": "string",
                    'title': 'JSON Key File'
                }
            },
            'required': ['jsonKeyFile'],
            'secret': ['jsonKeyFile']
        }

    def __init__(self, configuration):
        super(GoogleSpreadsheet, self).__init__(configuration)

    def _get_spreadsheet_service(self):
        scope = [
            'https://spreadsheets.google.com/feeds',
        ]

        key = json.loads(b64decode(self.configuration['jsonKeyFile']))
        credentials = SignedJwtAssertionCredentials(key['client_email'], key["private_key"], scope=scope)
        spreadsheetservice = gspread.authorize(credentials)
        return spreadsheetservice

    def run_query(self, query):
        logger.debug("Spreadsheet is about to execute query: %s", query)
        values = query.split("|")
        key = values[0] #key of the spreadsheet
        worksheet_num = 0 if len(values) != 2 else int(values[1])# if spreadsheet contains more than one worksheet - this is the number of it
        try:
            spreadsheet_service = self._get_spreadsheet_service()
            spreadsheet = spreadsheet_service.open_by_key(key)

            data = parse_spreadsheet(spreadsheet, worksheet_num)

            json_data = json.dumps(data, cls=JSONEncoder)
            error = None
        except gspread.SpreadsheetNotFound:
            error = "Spreadsheet ({}) not found. Make sure you used correct id.".format(key)
            json_data = None

        return json_data, error

register(GoogleSpreadsheet)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    FFmpeg

    Your entrypoint for every Task you want to do with FFmpeg

    :copyright: (c) 2011 by Mathias Koehler.
    :license: BSD, see LICENSE for more details.
"""

import os

from fcntl import fcntl, F_SETFL, F_GETFL
from select import select
from subprocess import Popen, PIPE, STDOUT
from itertools import chain

from .options import OptionStore, Option

class Input(OptionStore):

    def __init__(self, file, *args):
        self.file = file
        OptionStore.__init__(self, *args)

    def __iter__(self):
        return chain(OptionStore.__iter__(self), ['-i', self.file])

class Output(OptionStore):

    def __init__(self, file, *args):
        self.file = file
        OptionStore.__init__(self, *args)

    def overwrite(self):
        self.add_option('-y', None)
        return self

    def __iter__(self):
        return chain(OptionStore.__iter__(self), [self.file])

class FFmpeg(OptionStore):

    def __init__(self, binary="ffmpeg", *args):
        self.binary = binary
        OptionStore.__init__(self, *args)

    def run(self):
        self.pipe = Popen(self, executable=self.binary,
                     stderr=PIPE)
        fcntl(self.pipe.stderr.fileno(), F_SETFL,
              fcntl(self.pipe.stderr.fileno(), F_GETFL) | os.O_NONBLOCK)
        return self

    def wait_for_data(self):
        while True:
            ready = select([self.pipe.stderr.fileno()], [], [])[0]
            if ready:
                return True

    def add_option(self, key, value):
        self._list.insert(0, Options({key: value}))
<commit_msg>Change run to handle the unbuffered output<commit_after># -*- coding: utf-8 -*-
"""
    FFmpeg

    Your entrypoint for every Task you want to do with FFmpeg

    :copyright: (c) 2011 by Mathias Koehler.
    :license: BSD, see LICENSE for more details.
"""

import os

from fcntl import fcntl, F_SETFL, F_GETFL
from select import select
from subprocess import Popen, PIPE, STDOUT
from itertools import chain

from .options import OptionStore, Option

class Input(OptionStore):

    def __init__(self, file, *args):
        self.file = file
        OptionStore.__init__(self, *args)

    def __iter__(self):
        return chain(OptionStore.__iter__(self), ['-i', self.file])

class Output(OptionStore):

    def __init__(self, file, *args):
        self.file = file
        OptionStore.__init__(self, *args)

    def overwrite(self):
        self.add_option('-y', None)
        return self

    def __iter__(self):
        return chain(OptionStore.__iter__(self), [self.file])

class FFmpeg(OptionStore):

    def __init__(self, binary="ffmpeg", *args):
        self.binary = binary
        OptionStore.__init__(self, *args)

    def run(self):
        """Execute through subprocess the :attr:`binary` with all Options
        that are appended in this Store as arguments.
        """
        self.pipe = Popen(list(self), stderr=PIPE)
        fcntl(self.pipe.stderr.fileno(), F_SETFL,
              fcntl(self.pipe.stderr.fileno(), F_GETFL) | os.O_NONBLOCK)
        return self

    def wait_for_data(self):
        """ As long as the :attr:`binary` is executed through :meth:`run`
        we process a syscall to check if ffmpeg has written to stderr. If
        ffmpeg has written to stderr we return true. If the binary isn't
        running any more we return False.

        This is a helper class to deal with the unbuffered output from ffmpeg
        """
        while self.pipe.poll() is None:
            ready = select([self.pipe.stderr.fileno()], [], [])[0]
            if ready:
                return True
        return False

    def poll(self):
        """Check if :attr:`binary` is running"""
        return self.pipe.poll()

    def add_option(self, key, value):
        self._list.insert(0, Option({key: value}))

    def __iter__(self):
        return chain([self.binary], OptionStore.__iter__(self))
<|endoftext|>


<commit_before>import pytest
from pupa.scrape import Vote as ScrapeVote
from pupa.importers import VoteImporter, BillImporter
from opencivicdata.models import VoteEvent, Jurisdiction, Person, Organization, Bill

class DumbMockImporter(object):
    """ this is a mock importer that implements a resolve_json_id that is just a pass-through """

    def resolve_json_id(self, json_id):
        return json_id

@pytest.mark.django_db
def test_full_vote():
    j = Jurisdiction.objects.create(id='jid', division_id='did')
    session = j.legislative_sessions.create(name='1900')
    person = Person.objects.create(id='person-id', name='Adam Smith')
    org = Organization.objects.create(id='org-id', name='House', chamber='lower',
                                      classification='legislature')
    bill = Bill.objects.create(id='bill-id', identifier='HB 1', legislative_session=session,
                               from_organization=org)
    com = Organization.objects.create(id='com-id', name='Arbitrary Committee', parent=org)

    vote = ScrapeVote(legislative_session='1900', motion_text='passage', start_date='1900-04-01',
                      classification='passage:bill', result='pass', bill=bill.identifier)
    vote.set_count('yes', 20)
    vote.yes('John Smith')
    vote.no('Adam Smith')

    dmi = DumbMockImporter()
    bi = BillImporter('jid', dmi)

    VoteImporter('jid', dmi, dmi, bi).import_data([vote.as_dict()])

    assert VoteEvent.objects.count() == 1
    ve = VoteEvent.objects.get()
    assert ve.legislative_session_id == session.id
    assert ve.classification == ['passage:bill']
    assert ve.bill_id == bill.id
    count = ve.counts.get()
    assert count.option == 'yes'
    assert count.value == 20
    votes = list(ve.votes.all())
    assert len(votes) == 2
    for v in ve.votes.all():
        if v.voter_name == 'John Smith':
            assert v.option == 'yes'
        else:
            assert v.option == 'no'
<commit_msg>fix a test that relied on old name for vote.classification<commit_after>import pytest
from pupa.scrape import Vote as ScrapeVote
from pupa.importers import VoteImporter, BillImporter
from opencivicdata.models import VoteEvent, Jurisdiction, Person, Organization, Bill

class DumbMockImporter(object):
    """ this is a mock importer that implements a resolve_json_id that is just a pass-through """

    def resolve_json_id(self, json_id):
        return json_id

@pytest.mark.django_db
def test_full_vote():
    j = Jurisdiction.objects.create(id='jid', division_id='did')
    session = j.legislative_sessions.create(name='1900')
    person = Person.objects.create(id='person-id', name='Adam Smith')
    org = Organization.objects.create(id='org-id', name='House', chamber='lower',
                                      classification='legislature')
    bill = Bill.objects.create(id='bill-id', identifier='HB 1', legislative_session=session,
                               from_organization=org)
    com = Organization.objects.create(id='com-id', name='Arbitrary Committee', parent=org)

    vote = ScrapeVote(legislative_session='1900', motion_text='passage', start_date='1900-04-01',
                      classification='passage:bill', result='pass', bill=bill.identifier)
    vote.set_count('yes', 20)
    vote.yes('John Smith')
    vote.no('Adam Smith')

    dmi = DumbMockImporter()
    bi = BillImporter('jid', dmi)

    VoteImporter('jid', dmi, dmi, bi).import_data([vote.as_dict()])

    assert VoteEvent.objects.count() == 1
    ve = VoteEvent.objects.get()
    assert ve.legislative_session_id == session.id
    assert ve.motion_classification == ['passage:bill']
    assert ve.bill_id == bill.id
    count = ve.counts.get()
    assert count.option == 'yes'
    assert count.value == 20
    votes = list(ve.votes.all())
    assert len(votes) == 2
    for v in ve.votes.all():
        if v.voter_name == 'John Smith':
            assert v.option == 'yes'
        else:
            assert v.option == 'no'
<|endoftext|>