<commit_before>import uuid
from datetime import datetime

from cloudyvents.cyvents import CYvent
import em_core_findworkers
from epumgmt.api import RunVM
from epumgmt.main import em_args
from epumgmt.api.exceptions import IncompatibleEnvironment, ProgrammingError

def find_latest_status(p, c, m, run_name, cloudinitd, findworkersfirst=True):
    """Finds new workers, EPU controllers, and gathers all the information possible for status

    The information is all stored to the RunVM objects in persistence

    Return "allvms" for convenience, a list of RunVM instances
    """
    if findworkersfirst:
        try:
            em_core_findworkers.find_once(p, c, m, run_name)
        except IncompatibleEnvironment,e:
            c.log.error("Problem finding new workers: %s" % str(e))
    allvms = m.persistence.get_run_vms_or_none(run_name)
    if not allvms or len(allvms) == 0:
        raise IncompatibleEnvironment("Cannot find any VMs associated with run '%s'" % run_name)

    #_find_latest_turtle_status(c, m, run_name, cloudinitd, allvms)
    _find_latest_worker_status(c, m, run_name, cloudinitd, allvms)

    return allvms

def pretty_status(p, c, m, run_name, cloudinitd):
    """Log information about VM instances that are part of the run.

    To update or not is based on a given parameter to the program.

    If you are interested in using a more API-like method, see find_latest_status()
    """

    c.log.debug("Obtaining status")

    no_update = p.get_arg_or_none(em_args.STATUS_NOUPDATE)

    if no_update:
        c.log.debug("The %s flag is suppressing status update" % em_args.STATUS_NOUPDATE.long_syntax)
        allvms = m.persistence.get_run_vms_or_none(run_name)
    else:
        c.log.info("Getting the latest status information")
        allvms = find_latest_status(p, c, m, run_name, cloudinitd)

    c.log.info("\n%s" % _report(allvms))

# ----------------------------------------------------------------------------------------------------
# IMPL
# ----------------------------------------------------------------------------------------------------

def _find_latest_worker_status(c, m, run_name, cloudinitd, allvms):
    """Update what can be found for any nodes launched via EPU controllers
    """

    c.log.info("Updating worker status")

    # Print each new info item
    trace = False

    m.remote_svc_adapter.initialize(m, run_name, cloudinitd)
    if not m.remote_svc_adapter.is_channel_open():
        c.log.warn("Cannot get worker status: there is no channel open to the EPU controllers")
        return

    controller_map = m.remote_svc_adapter.controller_map(allvms)
    if not len(controller_map):
        c.log.warn("Cannot get worker status: there is a channel to the EPU controllers but no controllers are configured")
        return

    controllers = []
    for instanceid in controller_map.keys():
        controllers.extend(controller_map[instanceid])

    provisioner_vm = None
    for vm in allvms:
        if vm.service_type == "provisioner":
            provisioner_vm = vm
            break
    if not provisioner_vm:
        # This is an exception because it should be there especially if is_channel_open() passed above
        raise ProgrammingError("Cannot update worker status without provisioner channel into the system")

    controller_state_map = m.remote_svc_adapter.worker_state(controllers, provisioner_vm)

    # Generate "iaas_state" and "heartbeat_state" cloudyvents.  Discover worker 'parent' in the process.
    for controller in controllers:
        state = controller_state_map[controller]
        for wis in state.instances:
            vm = _get_vm_with_nodeid(wis.nodeid, allvms)
            if not vm:
                # Can't make a RunVM yet for this, unfortunately
                c.log.warn("Controller '%s' knows about worker we have no IaaS id for yet: %s" % (controller, wis.nodeid))
                continue

            newparent = False
            if not vm.parent:
                vm.parent = controller
                newparent = True
            elif vm.parent != controller:
                raise ProgrammingError("Previous RunVM had a different parent "
                        "'%s', new status query indicates parent is '%s'" % (vm.parent, controller))

            newevent = _get_events_from_wis(wis, vm, controller, trace, c)

            if newevent or newparent:
                m.persistence.store_run_vms(run_name, [vm])

    # Generate "de_state", "de_conf_report", and "last_queuelen_size" cloudyvents.
    for instanceid in controller_map.keys():
        vm = _get_vm_with_instanceid(instanceid, allvms)
        if not vm:
            raise ProgrammingError("instanceid in RunVMs that is not in controller_map?")

        any_newevent = False
        controllers = controller_map[instanceid]
        for controller in controllers:
            state = controller_state_map[controller]
            newevent = _get_events_from_controller_state(state, vm, controller, trace, c)
            if newevent:
                any_newevent = True

        if any_newevent:
            m.persistence.store_run_vms(run_name, [vm])

def _get_events_from_wis(wis, vm, controller, trace, c):
    """See if there is anything in the WorkerInstanceState, return True if events were added
    """
    newevent = False

    if wis.iaas_state:
        event = CYvent(controller,
                       "iaas_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(wis.iaas_state_time),
                       extra={"nodeid":wis.nodeid,
                              "state":wis.iaas_state,
                              "instanceid": vm.instanceid})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("iaas_state for %s: %s (from controller '%s')" % (vm.instanceid, wis.iaas_state, controller))

    if wis.heartbeat_state:
        event = CYvent(controller,
                       "heartbeat_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(wis.heartbeat_time),
                       extra={"nodeid":wis.nodeid,
                              "state":wis.heartbeat_state,
                              "instanceid": vm.instanceid})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("heartbeat_state for %s: %s (from controller '%s')" % (vm.instanceid, wis.heartbeat_state, controller))

    return newevent

def _get_events_from_controller_state(state, vm, controller, trace, c):
    """See if there is anything in the EPUControllerState, return True if any events were added
    """
    newevent = False

    if state.de_state:
        event = CYvent(controller,
                       "de_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.capture_time),
                       extra={"de_state": state.de_state})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("de_state for controller %s: %s" % (controller, state.de_state))

    if state.de_conf_report:
        event = CYvent(controller,
                       "de_conf_report",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.capture_time),
                       extra={"de_conf_report": state.de_conf_report})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("de_conf_report for controller %s: %s" % (controller, state.de_conf_report))

    if state.last_queuelen_size >= 0:
        event = CYvent(controller,
                       "last_queuelen_size",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.last_queuelen_time),
                       extra={"last_queuelen_size": state.last_queuelen_size})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("last_queuelen_size for controller %s: %s" % (controller, state.last_queuelen_size))

    return newevent

def _get_vm_with_nodeid(nodeid, allvms):
    """Return RunVM object for nodeid or None if not found
    """
    for vm in allvms:
        if vm.nodeid == nodeid:
            return vm
    return None

def _get_vm_with_instanceid(instanceid, allvms):
    """Return RunVM object for instanceid or None if not found
    """
    for vm in allvms:
        if vm.instanceid == instanceid:
            return vm
    return None

def _filter_out_workers(allvms):

    # We are currently in a transition state, the WORKER_SUFFIX idea is being obsoleted
    # by a direct "parent_epu_controller" instance variable in RunVM

    nonworkers = []
    for vm in allvms:
        if vm.parent:
            continue
        elif vm.service_type.endswith(RunVM.WORKER_SUFFIX):
            # For now, to be backwards compatible, WORKER_SUFFIX suffix signals a worker
            continue
        nonworkers.append(vm)
    return nonworkers

def _filter_out_services(allvms):

    # We are currently in a transition state, the WORKER_SUFFIX idea is being obsoleted
    # by a direct "parent_epu_controller" instance variable in RunVM

    workers = []
    for vm in allvms:
        if vm.parent:
            workers.append(vm)
        elif vm.service_type.endswith(RunVM.WORKER_SUFFIX):
            # For now, to be backwards compatible, WORKER_SUFFIX suffix signals a worker
            workers.append(vm)
    return workers

def _find_state_from_events(vm):

    if not vm:
        return None
    if not vm.events:
        return None
    latest = None
    for ev in vm.events:
        if ev.name == "iaas_state":
            if latest:
                if latest.timestamp < ev.timestamp:
                    latest = ev
            else:
                latest = ev

    if not latest:
        return None
    return latest.extra["state"]

def _get_vm_with_controller(controller, vm_list):
    for vm in vm_list:
        for ev in vm.events:
            if ev.source == controller:
                return vm
    return None

def _latest_controller_state(controller, vm):
    if not vm:
        return None, None
    if not vm.events:
        return None, None
    latest_destate = None
    for ev in vm.events:
        if ev.name == "de_state":
            if latest_destate:
                if latest_destate.timestamp < ev.timestamp:
                    latest_destate = ev
            else:
                latest_destate = ev
    latest_qlen = None
    for ev in vm.events:
        if ev.name == "last_queuelen_size":
            if latest_qlen:
                if latest_qlen.timestamp < ev.timestamp:
                    latest_qlen = ev
            else:
                latest_qlen = ev

    ret_state = latest_destate
    if latest_destate:
        if latest_destate.extra.has_key("de_state"):
            ret_state = latest_destate.extra["de_state"]
        if latest_destate.extra.has_key("state"):
            ret_state = latest_destate.extra["state"]

    ret_qlen = latest_qlen
    if latest_qlen:
        if latest_qlen.extra.has_key("last_queuelen_size"):
            ret_qlen = latest_qlen.extra["last_queuelen_size"]

    return ret_state, ret_qlen

# ----------------------------------------------------------------------------------------------------
# REPORT
# ----------------------------------------------------------------------------------------------------

def _report(allvms):
    txt = "\n------------\nBase System:\n------------\n\n"
    default_typetxt = "(unknown)"
    default_hostname = "(unknown)"

    services = _filter_out_workers(allvms)
    widest_service = _widest_type(services)
    if len(default_typetxt) > widest_service:
        widest_service = len(default_typetxt)
    default_typetxt = _pad_txt(default_typetxt, widest_service)

    for vm in services:
        typetxt = default_typetxt
        hostname = default_hostname
        if vm.service_type:
            typetxt = _pad_txt(vm.service_type, widest_service)
        if vm.hostname:
            hostname = vm.hostname
        vm_info = (typetxt, vm.instanceid, hostname)
        txt += "%s | %s | %s\n" % vm_info

    txt += "\n--------\nWorkers:\n--------\n\n"

    workers = _filter_out_services(allvms)

    default_status = "(unknown)"
    default_controller = "(unknown controller)"
    by_controller = {} # key: controller, value: list of vm_info tuples for it

    for vm in workers:

        hostname = default_hostname
        if vm.hostname:
            hostname = vm.hostname

        status = _find_state_from_events(vm)
        if not status:
            status = default_status

        controller = default_controller
        if vm.parent:
            controller = vm.parent

        vm_info = (status, vm.instanceid, hostname)
        if by_controller.has_key(controller):
            by_controller[controller].append(vm_info)
        else:
            by_controller[controller] = [vm_info]

    widest_status = len(default_status)
    for vm_info_list in by_controller.values():
        for vm_info in vm_info_list:
            if len(vm_info[0]) > widest_status:
                widest_status = len(vm_info[0])

    for controller in by_controller.keys():
        txt += "%s:\n" % controller

        vm = _get_vm_with_controller(controller, services)
        if vm:
            latest_destate, latest_qlen = _latest_controller_state(controller, vm)
            if latest_destate:
                txt += "  EPU state: %s" % latest_destate
            else:
                txt += "  EPU state: unknown"
            if latest_qlen is not None:
                txt += ", Queue length: %s" % latest_qlen

        txt += "\n  Workers:\n"
        for vm_info in by_controller[controller]:
            status = _pad_txt(vm_info[0], widest_status)
            txt += "    %s | %s | %s\n" % (status, vm_info[1], vm_info[2])
        txt += "\n"

    return txt

def _widest_type(run_vms):
    widest = 0
    for vm in run_vms:
        if vm.service_type:
            if len(vm.service_type) > widest:
                widest = len(vm.service_type)
    return widest

def _pad_txt(txt, widest):
    if len(txt) >= widest:
        return txt
    difference = widest - len(txt)
    while difference:
        txt += " "
        difference -= 1
    return txt
<commit_msg>Remove unused param from _latest_controller_state<commit_after>import uuid
from datetime import datetime

from cloudyvents.cyvents import CYvent
import em_core_findworkers
from epumgmt.api import RunVM
from epumgmt.main import em_args
from epumgmt.api.exceptions import IncompatibleEnvironment, ProgrammingError

def find_latest_status(p, c, m, run_name, cloudinitd, findworkersfirst=True):
    """Finds new workers, EPU controllers, and gathers all the information possible for status

    The information is all stored to the RunVM objects in persistence

    Return "allvms" for convenience, a list of RunVM instances
    """
    if findworkersfirst:
        try:
            em_core_findworkers.find_once(p, c, m, run_name)
        except IncompatibleEnvironment,e:
            c.log.error("Problem finding new workers: %s" % str(e))
    allvms = m.persistence.get_run_vms_or_none(run_name)
    if not allvms or len(allvms) == 0:
        raise IncompatibleEnvironment("Cannot find any VMs associated with run '%s'" % run_name)

    #_find_latest_turtle_status(c, m, run_name, cloudinitd, allvms)
    _find_latest_worker_status(c, m, run_name, cloudinitd, allvms)

    return allvms

def pretty_status(p, c, m, run_name, cloudinitd):
    """Log information about VM instances that are part of the run.

    To update or not is based on a given parameter to the program.

    If you are interested in using a more API-like method, see find_latest_status()
    """

    c.log.debug("Obtaining status")

    no_update = p.get_arg_or_none(em_args.STATUS_NOUPDATE)

    if no_update:
        c.log.debug("The %s flag is suppressing status update" % em_args.STATUS_NOUPDATE.long_syntax)
        allvms = m.persistence.get_run_vms_or_none(run_name)
    else:
        c.log.info("Getting the latest status information")
        allvms = find_latest_status(p, c, m, run_name, cloudinitd)

    c.log.info("\n%s" % _report(allvms))

# ----------------------------------------------------------------------------------------------------
# IMPL
# ----------------------------------------------------------------------------------------------------

def _find_latest_worker_status(c, m, run_name, cloudinitd, allvms):
    """Update what can be found for any nodes launched via EPU controllers
    """

    c.log.info("Updating worker status")

    # Print each new info item
    trace = False

    m.remote_svc_adapter.initialize(m, run_name, cloudinitd)
    if not m.remote_svc_adapter.is_channel_open():
        c.log.warn("Cannot get worker status: there is no channel open to the EPU controllers")
        return

    controller_map = m.remote_svc_adapter.controller_map(allvms)
    if not len(controller_map):
        c.log.warn("Cannot get worker status: there is a channel to the EPU controllers but no controllers are configured")
        return

    controllers = []
    for instanceid in controller_map.keys():
        controllers.extend(controller_map[instanceid])

    provisioner_vm = None
    for vm in allvms:
        if vm.service_type == "provisioner":
            provisioner_vm = vm
            break
    if not provisioner_vm:
        # This is an exception because it should be there especially if is_channel_open() passed above
        raise ProgrammingError("Cannot update worker status without provisioner channel into the system")

    controller_state_map = m.remote_svc_adapter.worker_state(controllers, provisioner_vm)

    # Generate "iaas_state" and "heartbeat_state" cloudyvents.  Discover worker 'parent' in the process.
    for controller in controllers:
        state = controller_state_map[controller]
        for wis in state.instances:
            vm = _get_vm_with_nodeid(wis.nodeid, allvms)
            if not vm:
                # Can't make a RunVM yet for this, unfortunately
                c.log.warn("Controller '%s' knows about worker we have no IaaS id for yet: %s" % (controller, wis.nodeid))
                continue

            newparent = False
            if not vm.parent:
                vm.parent = controller
                newparent = True
            elif vm.parent != controller:
                raise ProgrammingError("Previous RunVM had a different parent "
                        "'%s', new status query indicates parent is '%s'" % (vm.parent, controller))

            newevent = _get_events_from_wis(wis, vm, controller, trace, c)

            if newevent or newparent:
                m.persistence.store_run_vms(run_name, [vm])

    # Generate "de_state", "de_conf_report", and "last_queuelen_size" cloudyvents.
    for instanceid in controller_map.keys():
        vm = _get_vm_with_instanceid(instanceid, allvms)
        if not vm:
            raise ProgrammingError("instanceid in RunVMs that is not in controller_map?")

        any_newevent = False
        controllers = controller_map[instanceid]
        for controller in controllers:
            state = controller_state_map[controller]
            newevent = _get_events_from_controller_state(state, vm, controller, trace, c)
            if newevent:
                any_newevent = True

        if any_newevent:
            m.persistence.store_run_vms(run_name, [vm])

def _get_events_from_wis(wis, vm, controller, trace, c):
    """See if there is anything in the WorkerInstanceState, return True if events were added
    """
    newevent = False

    if wis.iaas_state:
        event = CYvent(controller,
                       "iaas_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(wis.iaas_state_time),
                       extra={"nodeid":wis.nodeid,
                              "state":wis.iaas_state,
                              "instanceid": vm.instanceid})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("iaas_state for %s: %s (from controller '%s')" % (vm.instanceid, wis.iaas_state, controller))

    if wis.heartbeat_state:
        event = CYvent(controller,
                       "heartbeat_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(wis.heartbeat_time),
                       extra={"nodeid":wis.nodeid,
                              "state":wis.heartbeat_state,
                              "instanceid": vm.instanceid})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("heartbeat_state for %s: %s (from controller '%s')" % (vm.instanceid, wis.heartbeat_state, controller))

    return newevent

def _get_events_from_controller_state(state, vm, controller, trace, c):
    """See if there is anything in the EPUControllerState, return True if any events were added
    """
    newevent = False

    if state.de_state:
        event = CYvent(controller,
                       "de_state",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.capture_time),
                       extra={"de_state": state.de_state})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("de_state for controller %s: %s" % (controller, state.de_state))

    if state.de_conf_report:
        event = CYvent(controller,
                       "de_conf_report",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.capture_time),
                       extra={"de_conf_report": state.de_conf_report})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("de_conf_report for controller %s: %s" % (controller, state.de_conf_report))

    if state.last_queuelen_size >= 0:
        event = CYvent(controller,
                       "last_queuelen_size",
                       str(uuid.uuid4()),
                       datetime.fromtimestamp(state.last_queuelen_time),
                       extra={"last_queuelen_size": state.last_queuelen_size})
        vm.events.append(event)
        newevent = True
        if trace:
            c.log.debug("last_queuelen_size for controller %s: %s" % (controller, state.last_queuelen_size))

    return newevent

def _get_vm_with_nodeid(nodeid, allvms):
    """Return RunVM object for nodeid or None if not found
    """
    for vm in allvms:
        if vm.nodeid == nodeid:
            return vm
    return None

def _get_vm_with_instanceid(instanceid, allvms):
    """Return RunVM object for instanceid or None if not found
    """
    for vm in allvms:
        if vm.instanceid == instanceid:
            return vm
    return None

def _filter_out_workers(allvms):

    # We are currently in a transition state, the WORKER_SUFFIX idea is being obsoleted
    # by a direct "parent_epu_controller" instance variable in RunVM

    nonworkers = []
    for vm in allvms:
        if vm.parent:
            continue
        elif vm.service_type.endswith(RunVM.WORKER_SUFFIX):
            # For now, to be backwards compatible, WORKER_SUFFIX suffix signals a worker
            continue
        nonworkers.append(vm)
    return nonworkers

def _filter_out_services(allvms):

    # We are currently in a transition state, the WORKER_SUFFIX idea is being obsoleted
    # by a direct "parent_epu_controller" instance variable in RunVM

    workers = []
    for vm in allvms:
        if vm.parent:
            workers.append(vm)
        elif vm.service_type.endswith(RunVM.WORKER_SUFFIX):
            # For now, to be backwards compatible, WORKER_SUFFIX suffix signals a worker
            workers.append(vm)
    return workers

def _find_state_from_events(vm):

    if not vm:
        return None
    if not vm.events:
        return None
    latest = None
    for ev in vm.events:
        if ev.name == "iaas_state":
            if latest:
                if latest.timestamp < ev.timestamp:
                    latest = ev
            else:
                latest = ev

    if not latest:
        return None
    return latest.extra["state"]

def _get_vm_with_controller(controller, vm_list):
    for vm in vm_list:
        for ev in vm.events:
            if ev.source == controller:
                return vm
    return None

def _latest_controller_state(vm):
    if not vm:
        return None, None
    if not vm.events:
        return None, None
    latest_destate = None
    for ev in vm.events:
        if ev.name == "de_state":
            if latest_destate:
                if latest_destate.timestamp < ev.timestamp:
                    latest_destate = ev
            else:
                latest_destate = ev
    latest_qlen = None
    for ev in vm.events:
        if ev.name == "last_queuelen_size":
            if latest_qlen:
                if latest_qlen.timestamp < ev.timestamp:
                    latest_qlen = ev
            else:
                latest_qlen = ev

    ret_state = latest_destate
    if latest_destate:
        if latest_destate.extra.has_key("de_state"):
            ret_state = latest_destate.extra["de_state"]
        if latest_destate.extra.has_key("state"):
            ret_state = latest_destate.extra["state"]

    ret_qlen = latest_qlen
    if latest_qlen:
        if latest_qlen.extra.has_key("last_queuelen_size"):
            ret_qlen = latest_qlen.extra["last_queuelen_size"]

    return ret_state, ret_qlen

# ----------------------------------------------------------------------------------------------------
# REPORT
# ----------------------------------------------------------------------------------------------------

def _report(allvms):
    txt = "\n------------\nBase System:\n------------\n\n"
    default_typetxt = "(unknown)"
    default_hostname = "(unknown)"

    services = _filter_out_workers(allvms)
    widest_service = _widest_type(services)
    if len(default_typetxt) > widest_service:
        widest_service = len(default_typetxt)
    default_typetxt = _pad_txt(default_typetxt, widest_service)

    for vm in services:
        typetxt = default_typetxt
        hostname = default_hostname
        if vm.service_type:
            typetxt = _pad_txt(vm.service_type, widest_service)
        if vm.hostname:
            hostname = vm.hostname
        vm_info = (typetxt, vm.instanceid, hostname)
        txt += "%s | %s | %s\n" % vm_info

    txt += "\n--------\nWorkers:\n--------\n\n"

    workers = _filter_out_services(allvms)

    default_status = "(unknown)"
    default_controller = "(unknown controller)"
    by_controller = {} # key: controller, value: list of vm_info tuples for it

    for vm in workers:

        hostname = default_hostname
        if vm.hostname:
            hostname = vm.hostname

        status = _find_state_from_events(vm)
        if not status:
            status = default_status

        controller = default_controller
        if vm.parent:
            controller = vm.parent

        vm_info = (status, vm.instanceid, hostname)
        if by_controller.has_key(controller):
            by_controller[controller].append(vm_info)
        else:
            by_controller[controller] = [vm_info]

    widest_status = len(default_status)
    for vm_info_list in by_controller.values():
        for vm_info in vm_info_list:
            if len(vm_info[0]) > widest_status:
                widest_status = len(vm_info[0])

    for controller in by_controller.keys():
        txt += "%s:\n" % controller

        vm = _get_vm_with_controller(controller, services)
        if vm:
            latest_destate, latest_qlen = _latest_controller_state(vm)
            if latest_destate:
                txt += "  EPU state: %s" % latest_destate
            else:
                txt += "  EPU state: unknown"
            if latest_qlen is not None:
                txt += ", Queue length: %s" % latest_qlen

        txt += "\n  Workers:\n"
        for vm_info in by_controller[controller]:
            status = _pad_txt(vm_info[0], widest_status)
            txt += "    %s | %s | %s\n" % (status, vm_info[1], vm_info[2])
        txt += "\n"

    return txt

def _widest_type(run_vms):
    widest = 0
    for vm in run_vms:
        if vm.service_type:
            if len(vm.service_type) > widest:
                widest = len(vm.service_type)
    return widest

def _pad_txt(txt, widest):
    if len(txt) >= widest:
        return txt
    difference = widest - len(txt)
    while difference:
        txt += " "
        difference -= 1
    return txt
<|endoftext|>


<commit_before>from pyqrllib.pyqrllib import bin2hstr

from qrl.core.State import State
from qrl.core.StateContainer import StateContainer
from qrl.core.misc import logger
from qrl.core.txs.Transaction import Transaction
from qrl.crypto.misc import sha256

class MultiSigVote(Transaction):
    """
    MultiSigSpend for the transaction of QRL from a multi sig wallet to another wallet.
    """

    def __init__(self, protobuf_transaction=None):
        super(MultiSigVote, self).__init__(protobuf_transaction)

    @property
    def shared_key(self):
        return self._data.multi_sig_vote.shared_key

    @property
    def unvote(self):
        return self._data.multi_sig_vote.unvote

    @property
    def prev_tx_hash(self):
        return self._data.multi_sig_vote.prev_tx_hash

    def set_prev_tx_hash(self, prev_tx_hash: bytes):
        self._data.multi_sig_vote.prev_tx_hash = prev_tx_hash

    def get_data_hash(self):
        tmp_tx_hash = (self.master_addr +
                       self.fee.to_bytes(8, byteorder='big', signed=False) +
                       self.shared_key +
                       self.unvote.to_bytes(1, byteorder='big', signed=False))

        return sha256(tmp_tx_hash)

    @staticmethod
    def create(shared_key: bytes,
               unvote: bool,
               fee: int,
               xmss_pk,
               master_addr: bytes = None):
        multi_sig_vote = MultiSigVote()

        if master_addr:
            multi_sig_vote._data.master_addr = master_addr

        multi_sig_vote._data.public_key = bytes(xmss_pk)

        multi_sig_vote._data.multi_sig_vote.shared_key = shared_key
        multi_sig_vote._data.multi_sig_vote.unvote = unvote

        multi_sig_vote._data.fee = int(fee)

        multi_sig_vote.validate_or_raise(verify_signature=False)

        return multi_sig_vote

    def _validate_custom(self):
        if self.fee < 0:
            logger.warning('MultiSigVote [%s] Invalid Fee = %d', bin2hstr(self.txhash), self.fee)
            return False

        return True

    def _validate_extended(self, state_container: StateContainer):
        if state_container.block_number < state_container.current_dev_config.hard_fork_heights[0]:
            logger.warning("[MultiSigVote] Hard Fork Feature not yet activated")
            return False

        addr_from_state = state_container.addresses_state[self.addr_from]
        vote_stats = state_container.votes_stats[self.shared_key]
        multi_sig_spend_tx = state_container.multi_sig_spend_txs[self.shared_key]
        block_number = state_container.block_number
        if vote_stats.executed:
            logger.warning("[MultiSigVote] Invalid Tx as MultiSigSpend has already been executed")
            return False

        if multi_sig_spend_tx is None:
            logger.warning("MultiSigSpend not found, Shared Key %s", bin2hstr(self.shared_key))
            return False

        if block_number > multi_sig_spend_tx.expiry_block_number:
            logger.warning("[MultiSigVote] Voted for expired Multi Sig Spend Txn")
            logger.warning("Expiry Block Number: %s, Current Block Number: %s",
                           multi_sig_spend_tx.expiry_block_number,
                           block_number)
            return False

        if self.addr_from not in vote_stats.signatories:
            logger.warning("Address not found in signatory list")
            logger.warning("Address %s Shared Key %s",
                           bin2hstr(self.addr_from),
                           bin2hstr(self.shared_key),
                           bin2hstr(vote_stats.multi_sig_address))
            return False

        index = vote_stats.get_address_index(self.addr_from)
        if vote_stats.unvotes[index] == self.unvote:
            logger.warning("[MultiSigVote] Invalid as Vote type already executed")
            logger.warning("Vote type %s", self.unvote)
            return False

        tx_balance = addr_from_state.balance

        if tx_balance < self.fee:
            logger.warning('[MultiSigVote] State validation failed for %s because: Insufficient funds',
                           bin2hstr(self.txhash))
            logger.warning('balance: %s, fee: %s', tx_balance, self.fee)
            return False

        return True

    def set_affected_address(self, addresses_set: set):
        super().set_affected_address(addresses_set)

    def apply(self,
              state: State,
              state_container: StateContainer) -> bool:
        address_state = state_container.addresses_state[self.addr_from]
        address_state.update_balance(state_container, self.fee, subtract=True)
        state_container.paginated_tx_hash.insert(address_state, self.txhash)
        vote_stats = state_container.votes_stats[self.shared_key]
        multi_sig_address = vote_stats.multi_sig_address

        weight, found = state_container.addresses_state[multi_sig_address].get_weight_by_signatory(self.addr_from)
        if not found:
            logger.info("[MultiSigVote] Address is not the signatory for the multi sig address")
            return False

        self.set_prev_tx_hash(vote_stats.get_vote_tx_hash_by_signatory_address(self.addr_from))

        if not vote_stats.apply_vote_stats(self, weight, state_container):
            logger.info("[MultiSigVote] Failed to apply vote_stats")
            return False

        return self._apply_state_changes_for_PK(state_container)

    def revert(self,
               state: State,
               state_container: StateContainer) -> bool:
        address_state = state_container.addresses_state[self.addr_from]
        address_state.update_balance(state_container, self.fee)
        state_container.paginated_tx_hash.remove(address_state, self.txhash)
        vote_stats = state_container.votes_stats[self.shared_key]
        multi_sig_address = vote_stats.multi_sig_address

        weight, found = state_container.addresses_state[multi_sig_address].get_weight_by_signatory(self.addr_from)
        if not found:
            logger.info("[MultiSigVote] Address is not the signatory for the multi sig address")
            return False

        if not vote_stats.revert_vote_stats(self, weight, state_container):
            logger.info("[MultiSigVote] Failed to revert vote_stats")
            return False

        return self._revert_state_changes_for_PK(state_container)
<commit_msg>Reject MultiSigVote in case of invalid shared_key<commit_after>from pyqrllib.pyqrllib import bin2hstr

from qrl.core.State import State
from qrl.core.StateContainer import StateContainer
from qrl.core.misc import logger
from qrl.core.txs.Transaction import Transaction
from qrl.crypto.misc import sha256

class MultiSigVote(Transaction):
    """
    MultiSigSpend for the transaction of QRL from a multi sig wallet to another wallet.
    """

    def __init__(self, protobuf_transaction=None):
        super(MultiSigVote, self).__init__(protobuf_transaction)

    @property
    def shared_key(self):
        return self._data.multi_sig_vote.shared_key

    @property
    def unvote(self):
        return self._data.multi_sig_vote.unvote

    @property
    def prev_tx_hash(self):
        return self._data.multi_sig_vote.prev_tx_hash

    def set_prev_tx_hash(self, prev_tx_hash: bytes):
        self._data.multi_sig_vote.prev_tx_hash = prev_tx_hash

    def get_data_hash(self):
        tmp_tx_hash = (self.master_addr +
                       self.fee.to_bytes(8, byteorder='big', signed=False) +
                       self.shared_key +
                       self.unvote.to_bytes(1, byteorder='big', signed=False))

        return sha256(tmp_tx_hash)

    @staticmethod
    def create(shared_key: bytes,
               unvote: bool,
               fee: int,
               xmss_pk,
               master_addr: bytes = None):
        multi_sig_vote = MultiSigVote()

        if master_addr:
            multi_sig_vote._data.master_addr = master_addr

        multi_sig_vote._data.public_key = bytes(xmss_pk)

        multi_sig_vote._data.multi_sig_vote.shared_key = shared_key
        multi_sig_vote._data.multi_sig_vote.unvote = unvote

        multi_sig_vote._data.fee = int(fee)

        multi_sig_vote.validate_or_raise(verify_signature=False)

        return multi_sig_vote

    def _validate_custom(self):
        if self.fee < 0:
            logger.warning('MultiSigVote [%s] Invalid Fee = %d', bin2hstr(self.txhash), self.fee)
            return False

        return True

    def _validate_extended(self, state_container: StateContainer):
        if state_container.block_number < state_container.current_dev_config.hard_fork_heights[0]:
            logger.warning("[MultiSigVote] Hard Fork Feature not yet activated")
            return False

        addr_from_state = state_container.addresses_state[self.addr_from]
        vote_stats = state_container.votes_stats[self.shared_key]
        if vote_stats is None:
            logger.warning("[MultiSigVote] Invalid Shared key %s", bin2hstr(self.shared_key))
            return False
        multi_sig_spend_tx = state_container.multi_sig_spend_txs[self.shared_key]
        block_number = state_container.block_number
        if vote_stats.executed:
            logger.warning("[MultiSigVote] Invalid Tx as MultiSigSpend has already been executed")
            return False

        if multi_sig_spend_tx is None:
            logger.warning("MultiSigSpend not found, Shared Key %s", bin2hstr(self.shared_key))
            return False

        if block_number > multi_sig_spend_tx.expiry_block_number:
            logger.warning("[MultiSigVote] Voted for expired Multi Sig Spend Txn")
            logger.warning("Expiry Block Number: %s, Current Block Number: %s",
                           multi_sig_spend_tx.expiry_block_number,
                           block_number)
            return False

        if self.addr_from not in vote_stats.signatories:
            logger.warning("Address not found in signatory list")
            logger.warning("Address %s Shared Key %s",
                           bin2hstr(self.addr_from),
                           bin2hstr(self.shared_key),
                           bin2hstr(vote_stats.multi_sig_address))
            return False

        index = vote_stats.get_address_index(self.addr_from)
        if vote_stats.unvotes[index] == self.unvote:
            logger.warning("[MultiSigVote] Invalid as Vote type already executed")
            logger.warning("Vote type %s", self.unvote)
            return False

        tx_balance = addr_from_state.balance

        if tx_balance < self.fee:
            logger.warning('[MultiSigVote] State validation failed for %s because: Insufficient funds',
                           bin2hstr(self.txhash))
            logger.warning('balance: %s, fee: %s', tx_balance, self.fee)
            return False

        return True

    def set_affected_address(self, addresses_set: set):
        super().set_affected_address(addresses_set)

    def apply(self,
              state: State,
              state_container: StateContainer) -> bool:
        address_state = state_container.addresses_state[self.addr_from]
        address_state.update_balance(state_container, self.fee, subtract=True)
        state_container.paginated_tx_hash.insert(address_state, self.txhash)
        vote_stats = state_container.votes_stats[self.shared_key]
        multi_sig_address = vote_stats.multi_sig_address

        weight, found = state_container.addresses_state[multi_sig_address].get_weight_by_signatory(self.addr_from)
        if not found:
            logger.info("[MultiSigVote] Address is not the signatory for the multi sig address")
            return False

        self.set_prev_tx_hash(vote_stats.get_vote_tx_hash_by_signatory_address(self.addr_from))

        if not vote_stats.apply_vote_stats(self, weight, state_container):
            logger.info("[MultiSigVote] Failed to apply vote_stats")
            return False

        return self._apply_state_changes_for_PK(state_container)

    def revert(self,
               state: State,
               state_container: StateContainer) -> bool:
        address_state = state_container.addresses_state[self.addr_from]
        address_state.update_balance(state_container, self.fee)
        state_container.paginated_tx_hash.remove(address_state, self.txhash)
        vote_stats = state_container.votes_stats[self.shared_key]
        multi_sig_address = vote_stats.multi_sig_address

        weight, found = state_container.addresses_state[multi_sig_address].get_weight_by_signatory(self.addr_from)
        if not found:
            logger.info("[MultiSigVote] Address is not the signatory for the multi sig address")
            return False

        if not vote_stats.revert_vote_stats(self, weight, state_container):
            logger.info("[MultiSigVote] Failed to revert vote_stats")
            return False

        return self._revert_state_changes_for_PK(state_container)
<|endoftext|>


<commit_before># #!/usr/bin/python

import fcntl
import os
import sys
import signal
import resource
import logging
import atexit
from logging import handlers

class Daemonize(object):
    """ Daemonize object
    Object constructor expects three arguments:
    - app: contains the application name which will be sent to syslog.
    - pid: path to the pidfile.
    - action: your custom function which will be executed after daemonization.
    - keep_fds: optional list of fds which should not be closed.
    """
    def __init__(self, app, pid, action, keep_fds=None):
        self.app = app
        self.pid = pid
        self.action = action
        self.keep_fds = keep_fds if keep_fds else []
        # Initialize logging.
        self.logger = logging.getLogger(self.app)
        self.logger.setLevel(logging.DEBUG)
        # Display log messages only on defined handlers.
        self.logger.propagate = False

    def sigterm(self, signum, frame):
        """ sigterm method
        These actions will be done after SIGTERM.
        """
        self.logger.warn("Caught signal %s. Stopping daemon." % signum)
        os.remove(self.pid)
        sys.exit(0)

    def exit(self):
        """ exit method
        Cleanup pid file at exit.
        """
        self.logger.warn("Stopping daemon.")
        os.remove(self.pid)
        sys.exit(0)

    def start(self):
        """ start method
        Main daemonization process.
        """
        # Fork, creating a new process for the child.
        process_id = os.fork()
        if process_id < 0:
            # Fork error. Exit badly.
            sys.exit(1)
        elif process_id != 0:
            # This is the parent process. Exit.
            sys.exit(0)
        # This is the child process. Continue.

        # Stop listening for signals that the parent process receives.
        # This is done by getting a new process id.
        # setpgrp() is an alternative to setsid().
        # setsid puts the process in a new parent group and detaches its controlling terminal.
        process_id = os.setsid()
        if process_id == -1:
            # Uh oh, there was a problem.
            sys.exit(1)

        # Close all file descriptors, except the ones mentioned in self.keep_fds.
        devnull = "/dev/null"
        if hasattr(os, "devnull"):
            # Python has set os.devnull on this system, use it instead as it might be different
            # than /dev/null.
            devnull = os.devnull

        for fd in range(resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
            if fd not in self.keep_fds:
                try:
                    os.close(fd)
                except OSError:
                    pass

        os.open(devnull, os.O_RDWR)
        os.dup(0)
        os.dup(0)

        # Initialize syslog.
        # It will work on OS X and Linux. No FreeBSD support, guys, I don't want to import re here
        # to parse your peculiar platform string.
        if sys.platform == "darwin":
            syslog_address = "/var/run/syslog"
        else:
            syslog_address = "/dev/log"
        syslog = handlers.SysLogHandler(syslog_address)
        syslog.setLevel(logging.INFO)
        # Try to mimic to normal syslog messages.
        formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
                                      "%b %e %H:%M:%S")
        syslog.setFormatter(formatter)

        self.logger.addHandler(syslog)

        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
        # octal number which we are typing as 0o27 for Python3 compatibility.
        os.umask(0o27)

        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
        # needs to be deleted results in "directory busy" errors.
        os.chdir("/")

        try:
            # Create a lockfile so that only one instance of this daemon is running at any time.
            lockfile = open(self.pid, "w")
        except IOError:
            self.logger.error("Unable to create a pidfile.")
            sys.exit(1)
        try:
            # Try to get an exclusive lock on the file. This will fail if another process has the file
            # locked.
            fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
            # Record the process id to the lockfile. This is standard practice for daemons.
            lockfile.write("%s" % (os.getpid()))
            lockfile.flush()
        except IOError:
            self.logger.error("Unable to lock on the pidfile.")
            os.remove(self.pid)
            sys.exit(1)

        # Set custom action on SIGTERM.
        signal.signal(signal.SIGTERM, self.sigterm)
        atexit.register(self.exit)

        self.logger.warn("Starting daemon.")

        self.action()
<commit_msg>Minor refactoring of minor refactoring<commit_after># #!/usr/bin/python

import fcntl
import os
import sys
import signal
import resource
import logging
import atexit
from logging import handlers

class Daemonize(object):
    """ Daemonize object
    Object constructor expects three arguments:
    - app: contains the application name which will be sent to syslog.
    - pid: path to the pidfile.
    - action: your custom function which will be executed after daemonization.
    - keep_fds: optional list of fds which should not be closed.
    """
    def __init__(self, app, pid, action, keep_fds=None):
        self.app = app
        self.pid = pid
        self.action = action
        self.keep_fds = keep_fds or []
        # Initialize logging.
        self.logger = logging.getLogger(self.app)
        self.logger.setLevel(logging.DEBUG)
        # Display log messages only on defined handlers.
        self.logger.propagate = False

    def sigterm(self, signum, frame):
        """ sigterm method
        These actions will be done after SIGTERM.
        """
        self.logger.warn("Caught signal %s. Stopping daemon." % signum)
        os.remove(self.pid)
        sys.exit(0)

    def exit(self):
        """ exit method
        Cleanup pid file at exit.
        """
        self.logger.warn("Stopping daemon.")
        os.remove(self.pid)
        sys.exit(0)

    def start(self):
        """ start method
        Main daemonization process.
        """
        # Fork, creating a new process for the child.
        process_id = os.fork()
        if process_id < 0:
            # Fork error. Exit badly.
            sys.exit(1)
        elif process_id != 0:
            # This is the parent process. Exit.
            sys.exit(0)
        # This is the child process. Continue.

        # Stop listening for signals that the parent process receives.
        # This is done by getting a new process id.
        # setpgrp() is an alternative to setsid().
        # setsid puts the process in a new parent group and detaches its controlling terminal.
        process_id = os.setsid()
        if process_id == -1:
            # Uh oh, there was a problem.
            sys.exit(1)

        # Close all file descriptors, except the ones mentioned in self.keep_fds.
        devnull = "/dev/null"
        if hasattr(os, "devnull"):
            # Python has set os.devnull on this system, use it instead as it might be different
            # than /dev/null.
            devnull = os.devnull

        for fd in range(resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
            if fd not in self.keep_fds:
                try:
                    os.close(fd)
                except OSError:
                    pass

        os.open(devnull, os.O_RDWR)
        os.dup(0)
        os.dup(0)

        # Initialize syslog.
        # It will work on OS X and Linux. No FreeBSD support, guys, I don't want to import re here
        # to parse your peculiar platform string.
        if sys.platform == "darwin":
            syslog_address = "/var/run/syslog"
        else:
            syslog_address = "/dev/log"
        syslog = handlers.SysLogHandler(syslog_address)
        syslog.setLevel(logging.INFO)
        # Try to mimic to normal syslog messages.
        formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
                                      "%b %e %H:%M:%S")
        syslog.setFormatter(formatter)

        self.logger.addHandler(syslog)

        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
        # octal number which we are typing as 0o27 for Python3 compatibility.
        os.umask(0o27)

        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
        # needs to be deleted results in "directory busy" errors.
        os.chdir("/")

        try:
            # Create a lockfile so that only one instance of this daemon is running at any time.
            lockfile = open(self.pid, "w")
        except IOError:
            self.logger.error("Unable to create a pidfile.")
            sys.exit(1)
        try:
            # Try to get an exclusive lock on the file. This will fail if another process has the file
            # locked.
            fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
            # Record the process id to the lockfile. This is standard practice for daemons.
            lockfile.write("%s" % (os.getpid()))
            lockfile.flush()
        except IOError:
            self.logger.error("Unable to lock on the pidfile.")
            os.remove(self.pid)
            sys.exit(1)

        # Set custom action on SIGTERM.
        signal.signal(signal.SIGTERM, self.sigterm)
        atexit.register(self.exit)

        self.logger.warn("Starting daemon.")

        self.action()
<|endoftext|>


<commit_before>import os
import PIL.Image as Image

HTML_HEADER = r'''
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

<meta http-equiv="content-type" content="text/html; charset=utf-8">
<title>Vdiff Test Results</title>

vdiff tests

'''

HTML_FOOTER = r'''

'''

def format_stats(stats):
    if stats.number_of_differences > 0:
        clz = " different"
    else:
        clz = ""

    html  = """
%s
\n""" % stats.name
    html += """
""" % clz
    html += """\n""" % (stats.fname1, stats.fname1)
    html += """\n""" % (stats.fname2, stats.fname2)
    if stats.comparison_image_fname is not None:
        html += """\n""" % (stats.comparison_image_fname, stats.comparison_image_fname)
    html += """
\n"""
    html += """
Differences:	%i
\n""" % len(stats.differences)
    html += """
Total delta:	%i
\n""" % stats.total_delta
    html += """
Max delta:	%i
\n""" % stats.max_delta
    html += """
Mean:	%.4f
\n""" % stats.mean
    html += """
Stdev:	%.4f
\n""" % stats.stdev
    html += """
\n"""
    html += """
"""
    return html

def format_stats_list(stats_list):
    html = HTML_HEADER
    for stats in stats_list:
        html += format_stats(stats)
    html += HTML_FOOTER
    return html

def compare_pixel(px1, px2):
    if px1 == px2:
        return 0
    r1, g1, b1, a1 = px1
    r2, g2, b2, a2 = px2
    return abs(r1-r2) + abs(g1-g2) + abs(b1-b2) + abs(a1-a2)

def visual_diff(img1, img2, threshold=0, stop_on_diff=False):
    if isinstance(img1, str) or isinstance(img1, unicode):
        img1 = Image.open(img1)
        img1 = img1.convert("RGBA")
    if isinstance(img2, str) or isinstance(img2, unicode):
        img2 = Image.open(img2)
        img2 = img2.convert("RGBA")
    assert img1.size == img2.size
    w, h = img1.size
    data1 = img1.getdata()
    data2 = img2.getdata()
    size = len(data1)
    differences = []
    for i in xrange(size):
        delta = compare_pixel(data1[i], data2[i])
        if delta > threshold:
            x = i % w
            y = i / w
            differences.append( ( (x, y), data1[i], data2[i], delta ) )
            if stop_on_diff:
                # print data1[i], data2[i]
                break
    return differences

def make_comparison_image(size, differences):
    img = Image.new("L", size, color=255)
    for pos, d1, d2, delta in differences:
        img.putpixel(pos, 255-delta)
    return img

def isEqual(fname1, fname2, threshold=0):
    diff = visual_diff(fname1, fname2, threshold, stop_on_diff=True)
    if len(diff) == 0:
        return True
    return False

class Statistics(object):
    def __init__(self, fname1, fname2, differences=None, name=""):
        self.fname1 = fname1
        self.fname2 = fname2
        if differences is None:
            differences = visual_diff(fname1, fname2)
        self.differences = differences
        self.name = name

        img1 = Image.open(fname1)
        self.width, self.height = img1.size

        self._comparison_image = None
        self.comparison_image_fname = None
        self.calculate()

    def calculate(self):
        diff = self.differences

        total_delta = 0
        max_delta = 0
        for pos, d1, d2, delta in diff:
            total_delta += delta
            max_delta = max(max_delta, delta)
        self.total_delta = total_delta
        self.max_delta = max_delta
        self.mean = mean = total_delta / float(self.width * self.height)

        stdev = 0
        for pos, d1, d2, delta in diff:
            stdev += pow(delta-mean, 2)
        stdev /= float(self.width * self.height)
        self.stdev = stdev

    def _get_size(self):
        return self.width, self.height
    size = property(_get_size)

    def _get_number_of_differences(self):
        return len(self.differences)
    number_of_differences = property(_get_number_of_differences)

    def _get_comparison_image(self):
        if self._comparison_image is None:
            self._comparison_image = make_comparison_image(self.size, self.differences)
        return self._comparison_image
    comparison_image = property(_get_comparison_image)

    def save_comparison_image(self, fname):
        self.comparison_image.save(fname)
        self.comparison_image_fname = fname

    def __str__(self):
        return "<Statistics diff:%s total_delta:%s max_delta:%s mean:%.4f stdev:%.4f>" % (
            len(self.differences), self.total_delta, self.max_delta, self.mean, self.stdev)

def statistics(fname1, fname2, threshold=0):
    diff = visual_diff(fname1, fname2)
    stats = Statistics(fname1, fname2, diff)

    print "Differences:", len(stats.differences)
    print "Total delta:", stats.total_delta
    print "Max delta:", stats.max_delta
    print "Mean:", stats.mean
    print "Stdev:", stats.stdev

    stats.comparison_image.save('cmp.png')

def test_vdiff(self):
    #fname1 = 'vdiff-tests/001-added-square/original.png'
    #fname2 = 'vdiff-tests/001-added-square/bluesquare.png'

    #fname1 = 'vdiff-tests/002-antialiased-text/preview.png'
    #fname2 = 'vdiff-tests/002-antialiased-text/photoshop.png'

    #fname1 = 'vdiff-tests/003-movement/original.png'
    #fname2 = 'vdiff-tests/003-movement/moved.png'

    #fname1 = 'vdiff-tests/004-color/original.png'
    #fname2 = 'vdiff-tests/004-color/darker.png'

    #fname1 = 'vdiff-tests/005-antialiased-text/none.png'
    #fname2 = 'vdiff-tests/005-antialiased-text/smooth.png'

    #fname1 = 'vdiff-tests/006-totally-different/ant.png'
    #fname2 = 'vdiff-tests/006-totally-different/people.png'

    fname1 = 'vdiff-tests/007-black-white/black.png'
    fname2 = 'vdiff-tests/007-black-white/white.png'

    statistics(fname1, fname2)

def usage():
    print """vdiff -- visually compare images
Usage: vdiff <image1> <image2> [threshold]"""

if __name__=='__main__':
    import sys
    if len(sys.argv) < 3:
        usage()
    else:
        fname1 = sys.argv[1]
        fname2 = sys.argv[2]
        try:
            threshold = int(sys.argv[3])
        except:
            threshold = 0
        statistics(fname1, fname2, threshold)
<commit_msg>Killed xrange<commit_after>import os
import PIL.Image as Image

HTML_HEADER = r'''
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

<meta http-equiv="content-type" content="text/html; charset=utf-8">
<title>Vdiff Test Results</title>

vdiff tests

'''

HTML_FOOTER = r'''

'''

def format_stats(stats):
    if stats.number_of_differences > 0:
        clz = " different"
    else:
        clz = ""

    html  = """
%s
\n""" % stats.name
    html += """
""" % clz
    html += """\n""" % (stats.fname1, stats.fname1)
    html += """\n""" % (stats.fname2, stats.fname2)
    if stats.comparison_image_fname is not None:
        html += """\n""" % (stats.comparison_image_fname, stats.comparison_image_fname)
    html += """
\n"""
    html += """
Differences:	%i
\n""" % len(stats.differences)
    html += """
Total delta:	%i
\n""" % stats.total_delta
    html += """
Max delta:	%i
\n""" % stats.max_delta
    html += """
Mean:	%.4f
\n""" % stats.mean
    html += """
Stdev:	%.4f
\n""" % stats.stdev
    html += """
\n"""
    html += """
"""
    return html

def format_stats_list(stats_list):
    html = HTML_HEADER
    for stats in stats_list:
        html += format_stats(stats)
    html += HTML_FOOTER
    return html

def compare_pixel(px1, px2):
    if px1 == px2:
        return 0
    r1, g1, b1, a1 = px1
    r2, g2, b2, a2 = px2
    return abs(r1-r2) + abs(g1-g2) + abs(b1-b2) + abs(a1-a2)

def visual_diff(img1, img2, threshold=0, stop_on_diff=False):
    if isinstance(img1, str) or isinstance(img1, unicode):
        img1 = Image.open(img1)
        img1 = img1.convert("RGBA")
    if isinstance(img2, str) or isinstance(img2, unicode):
        img2 = Image.open(img2)
        img2 = img2.convert("RGBA")
    assert img1.size == img2.size
    w, h = img1.size
    data1 = img1.getdata()
    data2 = img2.getdata()
    size = len(data1)
    differences = []
    for i in range(size):
        delta = compare_pixel(data1[i], data2[i])
        if delta > threshold:
            x = i % w
            y = i / w
            differences.append( ( (x, y), data1[i], data2[i], delta ) )
            if stop_on_diff:
                # print data1[i], data2[i]
                break
    return differences

def make_comparison_image(size, differences):
    img = Image.new("L", size, color=255)
    for pos, d1, d2, delta in differences:
        img.putpixel(pos, 255-delta)
    return img

def isEqual(fname1, fname2, threshold=0):
    diff = visual_diff(fname1, fname2, threshold, stop_on_diff=True)
    if len(diff) == 0:
        return True
    return False

class Statistics(object):
    def __init__(self, fname1, fname2, differences=None, name=""):
        self.fname1 = fname1
        self.fname2 = fname2
        if differences is None:
            differences = visual_diff(fname1, fname2)
        self.differences = differences
        self.name = name

        img1 = Image.open(fname1)
        self.width, self.height = img1.size

        self._comparison_image = None
        self.comparison_image_fname = None
        self.calculate()

    def calculate(self):
        diff = self.differences

        total_delta = 0
        max_delta = 0
        for pos, d1, d2, delta in diff:
            total_delta += delta
            max_delta = max(max_delta, delta)
        self.total_delta = total_delta
        self.max_delta = max_delta
        self.mean = mean = total_delta / float(self.width * self.height)

        stdev = 0
        for pos, d1, d2, delta in diff:
            stdev += pow(delta-mean, 2)
        stdev /= float(self.width * self.height)
        self.stdev = stdev

    def _get_size(self):
        return self.width, self.height
    size = property(_get_size)

    def _get_number_of_differences(self):
        return len(self.differences)
    number_of_differences = property(_get_number_of_differences)

    def _get_comparison_image(self):
        if self._comparison_image is None:
            self._comparison_image = make_comparison_image(self.size, self.differences)
        return self._comparison_image
    comparison_image = property(_get_comparison_image)

    def save_comparison_image(self, fname):
        self.comparison_image.save(fname)
        self.comparison_image_fname = fname

    def __str__(self):
        return "<Statistics diff:%s total_delta:%s max_delta:%s mean:%.4f stdev:%.4f>" % (
            len(self.differences), self.total_delta, self.max_delta, self.mean, self.stdev)

def statistics(fname1, fname2, threshold=0):
    diff = visual_diff(fname1, fname2)
    stats = Statistics(fname1, fname2, diff)

    print "Differences:", len(stats.differences)
    print "Total delta:", stats.total_delta
    print "Max delta:", stats.max_delta
    print "Mean:", stats.mean
    print "Stdev:", stats.stdev

    stats.comparison_image.save('cmp.png')

def test_vdiff(self):
    #fname1 = 'vdiff-tests/001-added-square/original.png'
    #fname2 = 'vdiff-tests/001-added-square/bluesquare.png'

    #fname1 = 'vdiff-tests/002-antialiased-text/preview.png'
    #fname2 = 'vdiff-tests/002-antialiased-text/photoshop.png'

    #fname1 = 'vdiff-tests/003-movement/original.png'
    #fname2 = 'vdiff-tests/003-movement/moved.png'

    #fname1 = 'vdiff-tests/004-color/original.png'
    #fname2 = 'vdiff-tests/004-color/darker.png'

    #fname1 = 'vdiff-tests/005-antialiased-text/none.png'
    #fname2 = 'vdiff-tests/005-antialiased-text/smooth.png'

    #fname1 = 'vdiff-tests/006-totally-different/ant.png'
    #fname2 = 'vdiff-tests/006-totally-different/people.png'

    fname1 = 'vdiff-tests/007-black-white/black.png'
    fname2 = 'vdiff-tests/007-black-white/white.png'

    statistics(fname1, fname2)

def usage():
    print """vdiff -- visually compare images
Usage: vdiff <image1> <image2> [threshold]"""

if __name__=='__main__':
    import sys
    if len(sys.argv) < 3:
        usage()
    else:
        fname1 = sys.argv[1]
        fname2 = sys.argv[2]
        try:
            threshold = int(sys.argv[3])
        except:
            threshold = 0
        statistics(fname1, fname2, threshold)
<|endoftext|>


<commit_before>import normalize.coll
import normalize.record

def record_id(object_, type_=None):
    """Implementation of id() which is overridable and knows about record's
    primary_key property.  Returns if the two objects may be the "same";
    returns None for other types, meaning all bets about identity are off.

    Curiously, this function resembles conversion between a "record" and a
    "tuple": stripping the logical names from the atomic values.
    """
    if type_ is None:
        type_ = type(object_)

    key_vals = list()
    if not type_.primary_key and issubclass(
        type_, normalize.coll.Collection
    ):
        return tuple(
            record_id(v, type_.itemtype) for k, v in object_.iteritems()
        )

    for prop in type_.primary_key or type_._sorted_properties:
        val = getattr(object_, prop.name, None)
        if val is not None and prop.valuetype:
            value_type_list = (
                prop.valuetype if isinstance(prop.valuetype, tuple) else
                (prop.valuetype,)
            )
            val_pk = ()
            set_elements = 0
            for value_type in value_type_list:
                if issubclass(value_type, normalize.record.Record):
                    pk = record_id(val, value_type)
                    pk_elements = len(x for x in pk if x is not None)
                    if not val_pk or pk_elements > set_elements:
                        val_pk = pk
                        set_elements = pk_elements

            val_pk = val_pk or val
            key_vals.append(val_pk)
    return tuple(key_vals)
<commit_msg>record_id: throw a more helpful error if record identity is not hashable<commit_after>import normalize.coll
import normalize.record

def record_id(object_, type_=None):
    """Implementation of id() which is overridable and knows about record's
    primary_key property.  Returns if the two objects may be the "same";
    returns None for other types, meaning all bets about identity are off.

    Curiously, this function resembles conversion between a "record" and a
    "tuple": stripping the logical names from the atomic values.
    """
    if type_ is None:
        type_ = type(object_)

    key_vals = list()
    if not type_.primary_key and issubclass(
        type_, normalize.coll.Collection
    ):
        return tuple(
            record_id(v, type_.itemtype) for k, v in object_.iteritems()
        )

    for prop in type_.primary_key or type_._sorted_properties:
        val = getattr(object_, prop.name, None)
        if val is not None and prop.valuetype:
            value_type_list = (
                prop.valuetype if isinstance(prop.valuetype, tuple) else
                (prop.valuetype,)
            )
            val_pk = ()
            set_elements = 0
            for value_type in value_type_list:
                if issubclass(value_type, normalize.record.Record):
                    pk = record_id(val, value_type)
                    pk_elements = len(x for x in pk if x is not None)
                    if not val_pk or pk_elements > set_elements:
                        val_pk = pk
                        set_elements = pk_elements

            val_pk = val_pk or val
            try:
                val_pk.__hash__()
            except TypeError:
                raise Exception(
                    "PK for %s returned unhashable type; try setting a "
                    "primary key in type %s" % (prop, type_)
                )
            key_vals.append(val_pk)
    return tuple(key_vals)
<|endoftext|>


<commit_before>__version__ = '0.1.7.1'

from normalizr.normalizr import Normalizr<commit_msg>Update version to 0.1.8<commit_after>__version__ = '0.1.8'

from normalizr.normalizr import Normalizr<|endoftext|>


<commit_before>from collections import OrderedDict
from django.shortcuts import render
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect
from django.contrib.auth import logout as auth_logout
from django.contrib.auth.decorators import login_required
from django.contrib.auth.views import login as django_login
from social_auth.models import UserSocialAuth
from social_auth.utils import setting
from ksp_login.context_processors import login_providers
from ksp_login.forms import KspUserCreationForm

def login(request):
    if request.user.is_authenticated():
        return redirect('account_info')
    return django_login(request, template_name='ksp_login/login.html',
                        extra_context=login_providers(request))

@login_required
def info(request):
    context = login_providers(request)
    context['account_associations'] = UserSocialAuth.get_social_auth_for_user(request.user)
    return render(request, 'ksp_login/info.html', context)

def logout(request):
    auth_logout(request)
    return HttpResponseRedirect(reverse('account_info'))

def register(request):
    """
    As the name suggests, registers a new user.

    Can replace the create_user function in the SOCIAL_AUTH pipeline
    (through ksp_login.pipeline.register_user, with the corresponding
    save_status_to_session, of course) to make it possible for the user to
    pick a username.
    """
    try:
        pipeline_state = request.session[setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY',
                                                 'partial_pipeline')]
        backend = pipeline_state['backend']
        pipeline_state = pipeline_state['kwargs']
        standalone = False;
    except KeyError:
        standalone = True
    if request.method == "POST":
        form = KspUserCreationForm(standalone, request.POST)
        if form.is_valid():
            user = form.save()
            if standalone:
                return redirect('account_login')
            pipeline_state['user'] = user
            request.session.modified = True
            return redirect('socialauth_complete', backend=backend)
    else:
        initial_data = None if standalone else {
            'username': pipeline_state['username'],
            'first_name': pipeline_state['details']['first_name'],
            'last_name': pipeline_state['details']['last_name'],
            'email': pipeline_state['details']['email'],
        }
        form = KspUserCreationForm(initial=initial_data,
                                   password_required=standalone)

    return render(request, "ksp_login/registration.html", {
        'form': form,
    })

def __activate_social_auth_monkeypatch():
    from social_auth.backends import (OPENID_ID_FIELD, SocialAuthBackend,
        OpenIDBackend)
    from social_auth.backends.contrib.livejournal import (
        LIVEJOURNAL_USER_FIELD, LiveJournalBackend)
    from social_auth.backends.yahoo import YahooBackend
    from social_auth.backends.google import GoogleBackend
    from social_auth.backends.contrib.yandex import YandexBackend
    from social_auth import backends

    SocialAuthBackend.REQUIRED_FIELD_NAME = None
    SocialAuthBackend.REQUIRED_FIELD_VERBOSE_NAME = None

    OpenIDBackend.REQUIRED_FIELD_NAME = OPENID_ID_FIELD
    OpenIDBackend.REQUIRED_FIELD_VERBOSE_NAME = _('OpenID identity')

    LiveJournalBackend.REQUIRED_FIELD_NAME = LIVEJOURNAL_USER_FIELD
    LiveJournalBackend.REQUIRED_FIELD_VERBOSE_NAME = _('LiveJournal username')

    # Reset to None in those OpenID backends where nothing is required.
    GoogleBackend.REQUIRED_FIELD_NAME = None
    GoogleBackend.REQUIRED_FIELD_VERBOSE_NAME = None
    YahooBackend.REQUIRED_FIELD_NAME = None
    YahooBackend.REQUIRED_FIELD_VERBOSE_NAME = None
    YandexBackend.REQUIRED_FIELD_NAME = None
    YandexBackend.REQUIRED_FIELD_VERBOSE_NAME = None

    # We replace the regular dict in social_auth.backends.BACKENDSCACHE
    # with an OrderedDict in order to remember the order in which they
    # were imported based on settings.AUTHENTICATION_BACKENDS.
    backends.BACKENDSCACHE = OrderedDict()

__activate_social_auth_monkeypatch()
<commit_msg>ksp_login; use redirect instead of HttpResponseRedirect<commit_after>from collections import OrderedDict
from django.shortcuts import render
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import redirect
from django.contrib.auth import logout as auth_logout
from django.contrib.auth.decorators import login_required
from django.contrib.auth.views import login as django_login
from social_auth.models import UserSocialAuth
from social_auth.utils import setting
from ksp_login.context_processors import login_providers
from ksp_login.forms import KspUserCreationForm

def login(request):
    if request.user.is_authenticated():
        return redirect('account_info')
    return django_login(request, template_name='ksp_login/login.html',
                        extra_context=login_providers(request))

@login_required
def info(request):
    context = login_providers(request)
    context['account_associations'] = UserSocialAuth.get_social_auth_for_user(request.user)
    return render(request, 'ksp_login/info.html', context)

def logout(request):
    auth_logout(request)
    return redirect('account_info')

def register(request):
    """
    As the name suggests, registers a new user.

    Can replace the create_user function in the SOCIAL_AUTH pipeline
    (through ksp_login.pipeline.register_user, with the corresponding
    save_status_to_session, of course) to make it possible for the user to
    pick a username.
    """
    try:
        pipeline_state = request.session[setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY',
                                                 'partial_pipeline')]
        backend = pipeline_state['backend']
        pipeline_state = pipeline_state['kwargs']
        standalone = False;
    except KeyError:
        standalone = True
    if request.method == "POST":
        form = KspUserCreationForm(standalone, request.POST)
        if form.is_valid():
            user = form.save()
            if standalone:
                return redirect('account_login')
            pipeline_state['user'] = user
            request.session.modified = True
            return redirect('socialauth_complete', backend=backend)
    else:
        initial_data = None if standalone else {
            'username': pipeline_state['username'],
            'first_name': pipeline_state['details']['first_name'],
            'last_name': pipeline_state['details']['last_name'],
            'email': pipeline_state['details']['email'],
        }
        form = KspUserCreationForm(initial=initial_data,
                                   password_required=standalone)

    return render(request, "ksp_login/registration.html", {
        'form': form,
    })

def __activate_social_auth_monkeypatch():
    from social_auth.backends import (OPENID_ID_FIELD, SocialAuthBackend,
        OpenIDBackend)
    from social_auth.backends.contrib.livejournal import (
        LIVEJOURNAL_USER_FIELD, LiveJournalBackend)
    from social_auth.backends.yahoo import YahooBackend
    from social_auth.backends.google import GoogleBackend
    from social_auth.backends.contrib.yandex import YandexBackend
    from social_auth import backends

    SocialAuthBackend.REQUIRED_FIELD_NAME = None
    SocialAuthBackend.REQUIRED_FIELD_VERBOSE_NAME = None

    OpenIDBackend.REQUIRED_FIELD_NAME = OPENID_ID_FIELD
    OpenIDBackend.REQUIRED_FIELD_VERBOSE_NAME = _('OpenID identity')

    LiveJournalBackend.REQUIRED_FIELD_NAME = LIVEJOURNAL_USER_FIELD
    LiveJournalBackend.REQUIRED_FIELD_VERBOSE_NAME = _('LiveJournal username')

    # Reset to None in those OpenID backends where nothing is required.
    GoogleBackend.REQUIRED_FIELD_NAME = None
    GoogleBackend.REQUIRED_FIELD_VERBOSE_NAME = None
    YahooBackend.REQUIRED_FIELD_NAME = None
    YahooBackend.REQUIRED_FIELD_VERBOSE_NAME = None
    YandexBackend.REQUIRED_FIELD_NAME = None
    YandexBackend.REQUIRED_FIELD_VERBOSE_NAME = None

    # We replace the regular dict in social_auth.backends.BACKENDSCACHE
    # with an OrderedDict in order to remember the order in which they
    # were imported based on settings.AUTHENTICATION_BACKENDS.
    backends.BACKENDSCACHE = OrderedDict()

__activate_social_auth_monkeypatch()
<|endoftext|>


<commit_before>from django import forms
from .models import DeviceToken

class CertFileUploadForm(forms.Form):
    cert_file = forms.FileField()
    target = forms.ChoiceField(choices=((0, 'Develop'), (1, 'Distribute')), required=True, widget=forms.RadioSelect)

class NotificationSendForm(forms.Form):

    target = forms.ChoiceField(choices=((0, 'Develop'), (1, 'Distribute')),
                               required=True,
                               widget=forms.RadioSelect,
                               initial=0)
    device_token = forms.ModelMultipleChoiceField(label='Device Token',
                                                  queryset=DeviceToken.objects.all().values_list('device_token',
                                                                                flat=True)
                                                  )
    title = forms.CharField(required=True)
    subtitle = forms.CharField()
    body = forms.CharField()
    sound = forms.CharField(initial='default', required=True)
    badge = forms.IntegerField(initial=1, required=True)
    content_available = forms.BooleanField(initial=False)
    mutable_content = forms.BooleanField(initial=False)
    extra = forms.CharField(widget=forms.Textarea)
<commit_msg>Modify required parameter of NotificaitonSendForm #19<commit_after>from django import forms
from .models import DeviceToken

class CertFileUploadForm(forms.Form):
    cert_file = forms.FileField()
    target = forms.ChoiceField(choices=((0, 'Develop'), (1, 'Distribute')), required=True, widget=forms.RadioSelect)

class NotificationSendForm(forms.Form):

    target = forms.ChoiceField(choices=((0, 'Develop'), (1, 'Distribute')),
                               required=True,
                               widget=forms.RadioSelect,
                               initial=0)
    device_token = forms.ModelMultipleChoiceField(label='Device Token',
                                                  queryset=DeviceToken.objects.all().values_list('device_token',
                                                                                flat=True)
                                                  )
    title = forms.CharField(required=True)
    subtitle = forms.CharField(required=False)
    body = forms.CharField(required=False)
    sound = forms.CharField(initial='default', required=True)
    badge = forms.IntegerField(initial=1, required=True)
    content_available = forms.BooleanField(initial=False, required=False)
    mutable_content = forms.BooleanField(initial=False, required=False)
    extra = forms.CharField(widget=forms.Textarea, required=False)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

from datetime import timedelta
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.api.fields import UserField
from sentry.api.serializers import serialize
from sentry.db.models.query import create_or_update
from sentry.constants import STATUS_CHOICES
from sentry.models import (
    Activity, Group, GroupAssignee, GroupBookmark, GroupSeen, GroupStatus,
    GroupTagValue
)
from sentry.plugins import plugins
from sentry.utils.safe import safe_execute

class GroupSerializer(serializers.Serializer):
    status = serializers.ChoiceField(choices=zip(
        STATUS_CHOICES.keys(), STATUS_CHOICES.keys()
    ))
    isBookmarked = serializers.BooleanField()
    hasSeen = serializers.BooleanField()
    assignedTo = UserField()

class GroupDetailsEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    def _get_activity(self, request, group, num):
        activity_items = set()
        activity = []
        activity_qs = Activity.objects.filter(
            group=group,
        ).order_by('-datetime').select_related('user')
        # we select excess so we can filter dupes
        for item in activity_qs[:num * 2]:
            sig = (item.event_id, item.type, item.ident, item.user_id)
            # TODO: we could just generate a signature (hash(text)) for notes
            # so there's no special casing
            if item.type == Activity.NOTE:
                activity.append(item)
            elif sig not in activity_items:
                activity_items.add(sig)
                activity.append(item)

        activity.append(Activity(
            project=group.project,
            group=group,
            type=Activity.FIRST_SEEN,
            datetime=group.first_seen,
        ))

        return activity[:num]

    def _get_seen_by(self, request, group):
        seen_by = sorted([
            (gs.user, gs.last_seen)
            for gs in GroupSeen.objects.filter(
                group=group
            ).select_related('user')
        ], key=lambda ls: ls[1], reverse=True)
        return [s[0] for s in seen_by]

    def _get_actions(self, request, group):
        project = group.project

        action_list = []
        for plugin in plugins.for_project(project, version=1):
            results = safe_execute(plugin.actions, request, group, action_list)

            if not results:
                continue

            action_list = results

        for plugin in plugins.for_project(project, version=2):
            for action in (safe_execute(plugin.get_actions, request, group) or ()):
                action_list.append(action)

        return action_list

    def get(self, request, group):
        """
        Retrieve an aggregate

        Return details on an individual aggregate.

            {method} {path}

        """
        data = serialize(group, request.user)

        # TODO: these probably should be another endpoint
        activity = self._get_activity(request, group, num=100)
        seen_by = self._get_seen_by(request, group)

        # find first seen release
        try:
            first_release = GroupTagValue.objects.filter(
                group=group,
                key='sentry:release',
            ).order_by('first_seen')[0]
        except IndexError:
            first_release = None
        else:
            first_release = {
                'version': first_release.value,
                # TODO(dcramer): this should look it up in Release
                'dateCreated': first_release.first_seen,
            }

        action_list = self._get_actions(request, group)

        now = timezone.now()
        hourly_stats = tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=1),
            rollup=3600,
        )[group.id]
        daily_stats = tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=30),
            rollup=3600 * 24,
        )[group.id]

        data.update({
            'firstRelease': first_release,
            'activity': serialize(activity, request.user),
            'seenBy': serialize(seen_by, request.user),
            'pluginActions': action_list,
            'stats': {
                '24h': hourly_stats,
                '30d': daily_stats,
            }
        })

        return Response(data)

    def put(self, request, group):
        """
        Update an aggregate

        Updates an individual aggregate's attributes.

            {method} {path}
            {{
              "status": "resolved"
            }}

        Attributes:

        - status: resolved, unresolved, muted
        - hasSeen: true, false
        - isBookmarked: true, false
        - assignedTo: user

        """
        serializer = GroupSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        # TODO(dcramer): we should allow assignment to anyone who has membership
        # even if that membership is not SSO linked
        if result.get('assignedTo') and not group.project.member_set.filter(user=result['assignedTo']).exists():
            return Response({'detail': 'Cannot assign to non-team member'}, status=400)

        if result.get('status') == 'resolved':
            now = timezone.now()

            group.resolved_at = now
            group.status = GroupStatus.RESOLVED

            happened = Group.objects.filter(
                id=group.id,
            ).exclude(status=GroupStatus.RESOLVED).update(
                status=GroupStatus.RESOLVED,
                resolved_at=now,
            )

            if happened:
                create_or_update(
                    Activity,
                    project=group.project,
                    group=group,
                    type=Activity.SET_RESOLVED,
                    user=request.user,
                )
        elif result.get('status'):
            group.status = STATUS_CHOICES[result['status']]
            group.save()

        if result.get('hasSeen'):
            instance, created = create_or_update(
                GroupSeen,
                group=group,
                user=request.user,
                project=group.project,
                defaults={
                    'last_seen': timezone.now(),
                }
            )
        elif result.get('hasSeen') is False:
            GroupSeen.objects.filter(
                group=group,
                user=request.user,
            ).delete()

        if result.get('isBookmarked'):
            GroupBookmark.objects.get_or_create(
                project=group.project,
                group=group,
                user=request.user,
            )
        elif result.get('isBookmarked') is False:
            GroupBookmark.objects.filter(
                group=group,
                user=request.user,
            ).delete()

        if 'assignedTo' in result:
            now = timezone.now()

            if result['assignedTo']:
                assignee, created = GroupAssignee.objects.get_or_create(
                    group=group,
                    defaults={
                        'project': group.project,
                        'user': result['assignedTo'],
                        'date_added': now,
                    }
                )

                if not created:
                    affected = GroupAssignee.objects.filter(
                        group=group,
                    ).exclude(
                        user=result['assignedTo'],
                    ).update(
                        user=result['assignedTo'],
                        date_added=now
                    )
                else:
                    affected = True

                if affected:
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.ASSIGNED,
                        user=request.user,
                        data={
                            'assignee': result['assignedTo'].id,
                        }
                    )
                    activity.send_notification()

            else:
                affected = GroupAssignee.objects.filter(
                    group=group,
                )[:1].count()
                GroupAssignee.objects.filter(
                    group=group,
                ).delete()

                if affected > 0:
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.UNASSIGNED,
                        user=request.user,
                    )
                    activity.send_notification()

        return Response(serialize(group, request.user))

    def delete(self, request, group):
        """
        Delete an aggregate

        Deletes an individual aggregate.

            {method} {path}
        """
        from sentry.tasks.deletion import delete_group

        delete_group.delay(object_id=group.id)

        return Response(status=202)
<commit_msg>Add lastRelease to group details<commit_after>from __future__ import absolute_import, print_function

from datetime import timedelta
from django.utils import timezone
from rest_framework import serializers
from rest_framework.response import Response

from sentry.app import tsdb
from sentry.api.base import DocSection
from sentry.api.bases.group import GroupEndpoint
from sentry.api.fields import UserField
from sentry.api.serializers import serialize
from sentry.db.models.query import create_or_update
from sentry.constants import STATUS_CHOICES
from sentry.models import (
    Activity, Group, GroupAssignee, GroupBookmark, GroupSeen, GroupStatus,
    GroupTagValue
)
from sentry.plugins import plugins
from sentry.utils.safe import safe_execute

class GroupSerializer(serializers.Serializer):
    status = serializers.ChoiceField(choices=zip(
        STATUS_CHOICES.keys(), STATUS_CHOICES.keys()
    ))
    isBookmarked = serializers.BooleanField()
    hasSeen = serializers.BooleanField()
    assignedTo = UserField()

class GroupDetailsEndpoint(GroupEndpoint):
    doc_section = DocSection.EVENTS

    def _get_activity(self, request, group, num):
        activity_items = set()
        activity = []
        activity_qs = Activity.objects.filter(
            group=group,
        ).order_by('-datetime').select_related('user')
        # we select excess so we can filter dupes
        for item in activity_qs[:num * 2]:
            sig = (item.event_id, item.type, item.ident, item.user_id)
            # TODO: we could just generate a signature (hash(text)) for notes
            # so there's no special casing
            if item.type == Activity.NOTE:
                activity.append(item)
            elif sig not in activity_items:
                activity_items.add(sig)
                activity.append(item)

        activity.append(Activity(
            project=group.project,
            group=group,
            type=Activity.FIRST_SEEN,
            datetime=group.first_seen,
        ))

        return activity[:num]

    def _get_seen_by(self, request, group):
        seen_by = sorted([
            (gs.user, gs.last_seen)
            for gs in GroupSeen.objects.filter(
                group=group
            ).select_related('user')
        ], key=lambda ls: ls[1], reverse=True)
        return [s[0] for s in seen_by]

    def _get_actions(self, request, group):
        project = group.project

        action_list = []
        for plugin in plugins.for_project(project, version=1):
            results = safe_execute(plugin.actions, request, group, action_list)

            if not results:
                continue

            action_list = results

        for plugin in plugins.for_project(project, version=2):
            for action in (safe_execute(plugin.get_actions, request, group) or ()):
                action_list.append(action)

        return action_list

    def get(self, request, group):
        """
        Retrieve an aggregate

        Return details on an individual aggregate.

            {method} {path}

        """
        data = serialize(group, request.user)

        # TODO: these probably should be another endpoint
        activity = self._get_activity(request, group, num=100)
        seen_by = self._get_seen_by(request, group)

        # find first seen release
        try:
            first_release = GroupTagValue.objects.filter(
                group=group,
                key='sentry:release',
            ).order_by('first_seen')[0]
        except IndexError:
            first_release = None
        else:
            first_release = {
                'version': first_release.value,
                # TODO(dcramer): this should look it up in Release
                'dateCreated': first_release.first_seen,
            }

        if first_release is not None:
            # find last seen release
            try:
                last_release = GroupTagValue.objects.filter(
                    group=group,
                    key='sentry:release',
                ).order_by('-last_seen')[0]
            except IndexError:
                last_release = None
            else:
                last_release = {
                    'version': last_release.value,
                    # TODO(dcramer): this should look it up in Release
                    'dateCreated': last_release.first_seen,
                }
        else:
            last_release = None

        action_list = self._get_actions(request, group)

        now = timezone.now()
        hourly_stats = tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=1),
            rollup=3600,
        )[group.id]
        daily_stats = tsdb.get_range(
            model=tsdb.models.group,
            keys=[group.id],
            end=now,
            start=now - timedelta(days=30),
            rollup=3600 * 24,
        )[group.id]

        data.update({
            'firstRelease': first_release,
            'lastRelease': last_release,
            'activity': serialize(activity, request.user),
            'seenBy': serialize(seen_by, request.user),
            'pluginActions': action_list,
            'stats': {
                '24h': hourly_stats,
                '30d': daily_stats,
            }
        })

        return Response(data)

    def put(self, request, group):
        """
        Update an aggregate

        Updates an individual aggregate's attributes.

            {method} {path}
            {{
              "status": "resolved"
            }}

        Attributes:

        - status: resolved, unresolved, muted
        - hasSeen: true, false
        - isBookmarked: true, false
        - assignedTo: user

        """
        serializer = GroupSerializer(data=request.DATA, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors, status=400)

        result = serializer.object

        # TODO(dcramer): we should allow assignment to anyone who has membership
        # even if that membership is not SSO linked
        if result.get('assignedTo') and not group.project.member_set.filter(user=result['assignedTo']).exists():
            return Response({'detail': 'Cannot assign to non-team member'}, status=400)

        if result.get('status') == 'resolved':
            now = timezone.now()

            group.resolved_at = now
            group.status = GroupStatus.RESOLVED

            happened = Group.objects.filter(
                id=group.id,
            ).exclude(status=GroupStatus.RESOLVED).update(
                status=GroupStatus.RESOLVED,
                resolved_at=now,
            )

            if happened:
                create_or_update(
                    Activity,
                    project=group.project,
                    group=group,
                    type=Activity.SET_RESOLVED,
                    user=request.user,
                )
        elif result.get('status'):
            group.status = STATUS_CHOICES[result['status']]
            group.save()

        if result.get('hasSeen'):
            instance, created = create_or_update(
                GroupSeen,
                group=group,
                user=request.user,
                project=group.project,
                defaults={
                    'last_seen': timezone.now(),
                }
            )
        elif result.get('hasSeen') is False:
            GroupSeen.objects.filter(
                group=group,
                user=request.user,
            ).delete()

        if result.get('isBookmarked'):
            GroupBookmark.objects.get_or_create(
                project=group.project,
                group=group,
                user=request.user,
            )
        elif result.get('isBookmarked') is False:
            GroupBookmark.objects.filter(
                group=group,
                user=request.user,
            ).delete()

        if 'assignedTo' in result:
            now = timezone.now()

            if result['assignedTo']:
                assignee, created = GroupAssignee.objects.get_or_create(
                    group=group,
                    defaults={
                        'project': group.project,
                        'user': result['assignedTo'],
                        'date_added': now,
                    }
                )

                if not created:
                    affected = GroupAssignee.objects.filter(
                        group=group,
                    ).exclude(
                        user=result['assignedTo'],
                    ).update(
                        user=result['assignedTo'],
                        date_added=now
                    )
                else:
                    affected = True

                if affected:
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.ASSIGNED,
                        user=request.user,
                        data={
                            'assignee': result['assignedTo'].id,
                        }
                    )
                    activity.send_notification()

            else:
                affected = GroupAssignee.objects.filter(
                    group=group,
                )[:1].count()
                GroupAssignee.objects.filter(
                    group=group,
                ).delete()

                if affected > 0:
                    activity = Activity.objects.create(
                        project=group.project,
                        group=group,
                        type=Activity.UNASSIGNED,
                        user=request.user,
                    )
                    activity.send_notification()

        return Response(serialize(group, request.user))

    def delete(self, request, group):
        """
        Delete an aggregate

        Deletes an individual aggregate.

            {method} {path}
        """
        from sentry.tasks.deletion import delete_group

        delete_group.delay(object_id=group.id)

        return Response(status=202)
<|endoftext|>