<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os

# from pilot.common.exception import PilotException
from pilot.user.atlas.setup import should_pilot_prepare_asetup, is_user_analysis_job, get_platform, get_asetup, \
    get_asetup_options, is_standard_atlas_job

import logging
logger = logging.getLogger(__name__)

def get_payload_command(job):
    """
    Return the full command for execuring the payload, including the sourcing of all setup files and setting of
    environment variables.

    :param job: job object
    :return: command (string)
    """

    # Should the pilot do the asetup or do the jobPars already contain the information?
    prepareasetup = should_pilot_prepare_asetup(job.get('noExecStrCnv', None), job['jobPars'])

    # Is it a user job or not?
    userjob = is_user_analysis_job(job['transformation'])

    # Get the platform value
    platform = get_platform(job['cmtConfig'])

    # Define the setup for asetup, i.e. including full path to asetup and setting of ATLAS_LOCAL_ROOT_BASE
    asetuppath = get_asetup(asetup=prepareasetup)
    asetupoptions = " "

    if is_standard_atlas_job(job['swRelease']):

        # Normal setup (production and user jobs)
        logger.info("preparing normal production/analysis job setup command")

        cmd = asetuppath
        if prepareasetup:
            options = get_asetup_options(job['swRelease'], job['homepackage'])
            asetupoptions = " " + options + " --platform " + platform

            # Always set the --makeflags option (to prevent asetup from overwriting it)
            asetupoptions += ' --makeflags=\"$MAKEFLAGS\"'

            # Verify that the setup works
            # exitcode, output = timedCommand(cmd, timeout=5 * 60)
            # if exitcode != 0:
            #     if "No release candidates found" in output:
            #         pilotErrorDiag = "No release candidates found"
            #         logger.warning(pilotErrorDiag)
            #         return self.__error.ERR_NORELEASEFOUND, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig
            # else:
            #     logger.info("verified setup command")

            cmd += asetupoptions

            if userjob:
                pass
            else:
                # Add Database commands if they are set by the local site
                cmd += os.environ.get('PILOT_DB_LOCAL_SETUP_CMD', '')
                # Add the transform and the job parameters (production jobs)
                if prepareasetup:
                    cmd += ";%s %s" % (job['transformation'], job['jobPars'])
                else:
                    cmd += "; " + job['jobPars']

            cmd = cmd.replace(';;', ';')

    else:  # Generic, non-ATLAS specific jobs, or at least a job with undefined swRelease

        logger.info("generic job (non-ATLAS specific or with undefined swRelease)")

        cmd = ""

    return cmd

def update_job_data(job):
    """
    This function can be used to update/add data to the job object.
    E.g. user specific information can be extracted from other job object fields. In the case of ATLAS, information
    is extracted from the metaData field and added to other job object fields.

    :param job: job object
    :return:
    """

    stageout = "all"

    # handle any error codes
    if 'exeErrorCode' in job['metaData']:
        job['exeErrorCode'] = job['metaData']['exeErrorCode']
        if job['exeErrorCode'] == 0:
            stageout = "all"
        else:
            logger.info('payload failed: exeErrorCode=%d' % job['exeErrorCode'])
            stageout = "log"
    if 'exeErrorDiag' in job['metaData']:
        job['exeErrorDiag'] = job['metaData']['exeErrorDiag']
        if job['exeErrorDiag'] != "":
            logger.warning('payload failed: exeErrorDiag=%s' % job['exeErrorDiag'])

    # determine what should be staged out
    job['stageout'] = stageout  # output and log file or only log file

    # extract the number of events
    job['nEvents'] = get_number_of_events(job['metaData'])

def get_number_of_events(jobreport_dictionary):
    """
    Extract the number of events from the job report.

    :param jobreport_dictionary:
    :return:
    """

    nevents = {} # FORMAT: { format : total_events, .. }

    if jobreport_dictionary != {}:

        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('nevents'):
                        if nevents.has_key(format):
                            print executor_dictionary[format]['nevents']
                            nevents[format] += executor_dictionary[format]['nevents']
                        else:
                            nevents[format] = executor_dictionary[format]['nevents']
                    else:
                        logger.warning("format %s has no such key: nevents" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    # Now find the largest number of events among the different formats
    if nevents != {}:
        try:
            nmax = max(nevents.values())
        except Exception, e:
            logger.warning("exception caught: %s" % (e))
            nmax = 0
    else:
        logger.warning("did not find the number of events in the job report")
        nmax = 0

    return nmax

def get_db_info(jobreport_dictionary):
    """
    Extract and add up the DB info from the job report.

    :param jobreport_dictionary:
    :return: db_time_s, db_data_s [converted strings, from e.g. "dbData=105077960 dbTime=251.42"]
    """

    db_time = 0
    db_data = 0L

    if jobreport_dictionary != {}:

        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('dbData'):
                        try:
                            db_data += executor_dictionary[format]['dbData']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: dbData" % (format))
                    if executor_dictionary[format].has_key('dbTime'):
                        try:
                            db_time += executor_dictionary[format]['dbTime']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: dbTime" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    if db_data != 0L:
        db_data_s = "%s" % (db_data)
    else:
        db_data_s = ""
    if db_time != 0:
        db_time_s = "%.2f" % (db_time)
    else:
        db_time_s = ""

    return db_time_s, db_data_s

def get_cpu_times(jobreport_dictionary):
    """
    Extract and add up the total CPU times from the job report.
    Note: this function is used with Event Service jobs

    :param jobreport_dictionary:
    :return: cpu_conversion_unit (unit), total_cpu_time, conversion_factor (output consistent with set_time_consumed())
    """

    total_cpu_time = 0L

    if jobreport_dictionary != {}:
        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('cpuTime'):
                        try:
                            total_cpu_time += executor_dictionary[format]['cpuTime']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: cpuTime" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    conversion_factor = 1.0
    cpu_conversion_unit = "s"

    return cpu_conversion_unit, total_cpu_time, conversion_factor
<commit_msg>Updated comments<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os

# from pilot.common.exception import PilotException
from pilot.user.atlas.setup import should_pilot_prepare_asetup, is_user_analysis_job, get_platform, get_asetup, \
    get_asetup_options, is_standard_atlas_job

import logging
logger = logging.getLogger(__name__)

def get_payload_command(job):
    """
    Return the full command for execuring the payload, including the sourcing of all setup files and setting of
    environment variables.

    :param job: job object
    :return: command (string)
    """

    # Should the pilot do the asetup or do the jobPars already contain the information?
    prepareasetup = should_pilot_prepare_asetup(job.get('noExecStrCnv', None), job['jobPars'])

    # Is it a user job or not?
    userjob = is_user_analysis_job(job['transformation'])

    # Get the platform value
    platform = get_platform(job['cmtConfig'])

    # Define the setup for asetup, i.e. including full path to asetup and setting of ATLAS_LOCAL_ROOT_BASE
    asetuppath = get_asetup(asetup=prepareasetup)
    asetupoptions = " "

    if is_standard_atlas_job(job['swRelease']):

        # Normal setup (production and user jobs)
        logger.info("preparing normal production/analysis job setup command")

        cmd = asetuppath
        if prepareasetup:
            options = get_asetup_options(job['swRelease'], job['homepackage'])
            asetupoptions = " " + options + " --platform " + platform

            # Always set the --makeflags option (to prevent asetup from overwriting it)
            asetupoptions += ' --makeflags=\"$MAKEFLAGS\"'

            # Verify that the setup works
            # exitcode, output = timedCommand(cmd, timeout=5 * 60)
            # if exitcode != 0:
            #     if "No release candidates found" in output:
            #         pilotErrorDiag = "No release candidates found"
            #         logger.warning(pilotErrorDiag)
            #         return self.__error.ERR_NORELEASEFOUND, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig
            # else:
            #     logger.info("verified setup command")

            cmd += asetupoptions

            if userjob:
                pass
            else:
                # Add Database commands if they are set by the local site
                cmd += os.environ.get('PILOT_DB_LOCAL_SETUP_CMD', '')
                # Add the transform and the job parameters (production jobs)
                if prepareasetup:
                    cmd += ";%s %s" % (job['transformation'], job['jobPars'])
                else:
                    cmd += "; " + job['jobPars']

            cmd = cmd.replace(';;', ';')

    else:  # Generic, non-ATLAS specific jobs, or at least a job with undefined swRelease

        logger.info("generic job (non-ATLAS specific or with undefined swRelease)")

        cmd = ""

    return cmd

def update_job_data(job):
    """
    This function can be used to update/add data to the job object.
    E.g. user specific information can be extracted from other job object fields. In the case of ATLAS, information
    is extracted from the metaData field and added to other job object fields.

    :param job: job object
    :return:
    """

    stageout = "all"

    # handle any error codes
    if 'exeErrorCode' in job['metaData']:
        job['exeErrorCode'] = job['metaData']['exeErrorCode']
        if job['exeErrorCode'] == 0:
            stageout = "all"
        else:
            logger.info('payload failed: exeErrorCode=%d' % job['exeErrorCode'])
            stageout = "log"
    if 'exeErrorDiag' in job['metaData']:
        job['exeErrorDiag'] = job['metaData']['exeErrorDiag']
        if job['exeErrorDiag'] != "":
            logger.warning('payload failed: exeErrorDiag=%s' % job['exeErrorDiag'])

    # determine what should be staged out
    job['stageout'] = stageout  # output and log file or only log file

    # extract the number of events
    job['nEvents'] = get_number_of_events(job['metaData'])

def get_number_of_events(jobreport_dictionary):
    """
    Extract the number of events from the job report.

    :param jobreport_dictionary:
    :return:
    """

    nevents = {} # FORMAT: { format : total_events, .. }

    if jobreport_dictionary != {}:

        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('nevents'):
                        if nevents.has_key(format):
                            print executor_dictionary[format]['nevents']
                            nevents[format] += executor_dictionary[format]['nevents']
                        else:
                            nevents[format] = executor_dictionary[format]['nevents']
                    else:
                        logger.warning("format %s has no such key: nevents" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    # Now find the largest number of events among the different formats
    if nevents != {}:
        try:
            nmax = max(nevents.values())
        except Exception, e:
            logger.warning("exception caught: %s" % (e))
            nmax = 0
    else:
        logger.warning("did not find the number of events in the job report")
        nmax = 0

    return nmax

def get_db_info(jobreport_dictionary):
    """
    Extract and add up the DB info from the job report.
    This information is reported with the jobMetrics.

    :param jobreport_dictionary:
    :return: db_time_s, db_data_s [converted strings, from e.g. "dbData=105077960 dbTime=251.42"]
    """

    db_time = 0
    db_data = 0L

    if jobreport_dictionary != {}:

        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('dbData'):
                        try:
                            db_data += executor_dictionary[format]['dbData']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: dbData" % (format))
                    if executor_dictionary[format].has_key('dbTime'):
                        try:
                            db_time += executor_dictionary[format]['dbTime']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: dbTime" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    if db_data != 0L:
        db_data_s = "%s" % (db_data)
    else:
        db_data_s = ""
    if db_time != 0:
        db_time_s = "%.2f" % (db_time)
    else:
        db_time_s = ""

    return db_time_s, db_data_s

def get_cpu_times(jobreport_dictionary):
    """
    Extract and add up the total CPU times from the job report.
    Note: this function is used with Event Service jobs

    :param jobreport_dictionary:
    :return: cpu_conversion_unit (unit), total_cpu_time, conversion_factor (output consistent with set_time_consumed())
    """

    total_cpu_time = 0L

    if jobreport_dictionary != {}:
        if jobreport_dictionary.has_key('resource'):
            resource_dictionary = jobreport_dictionary['resource']
            if resource_dictionary.has_key('executor'):
                executor_dictionary = resource_dictionary['executor']
                for format in executor_dictionary.keys(): # "RAWtoESD", ..
                    if executor_dictionary[format].has_key('cpuTime'):
                        try:
                            total_cpu_time += executor_dictionary[format]['cpuTime']
                        except:
                            pass
                    else:
                        logger.warning("format %s has no such key: cpuTime" % (format))
            else:
                logger.warning("no such key: executor")
        else:
            logger.warning("no such key: resource")

    conversion_factor = 1.0
    cpu_conversion_unit = "s"

    return cpu_conversion_unit, total_cpu_time, conversion_factor
<|endoftext|>


<commit_before>"""Constants for August devices."""

from datetime import timedelta

from homeassistant.const import Platform

DEFAULT_TIMEOUT = 15

CONF_ACCESS_TOKEN_CACHE_FILE = "access_token_cache_file"
CONF_LOGIN_METHOD = "login_method"
CONF_INSTALL_ID = "install_id"

VERIFICATION_CODE_KEY = "verification_code"

NOTIFICATION_ID = "august_notification"
NOTIFICATION_TITLE = "August"

MANUFACTURER = "August Home Inc."

DEFAULT_AUGUST_CONFIG_FILE = ".august.conf"

DEFAULT_NAME = "August"
DOMAIN = "august"

OPERATION_METHOD_AUTORELOCK = "autorelock"
OPERATION_METHOD_REMOTE = "remote"
OPERATION_METHOD_KEYPAD = "keypad"
OPERATION_METHOD_MOBILE_DEVICE = "mobile"

ATTR_OPERATION_AUTORELOCK = "autorelock"
ATTR_OPERATION_METHOD = "method"
ATTR_OPERATION_REMOTE = "remote"
ATTR_OPERATION_KEYPAD = "keypad"

# Limit battery, online, and hardware updates to hourly
# in order to reduce the number of api requests and
# avoid hitting rate limits
MIN_TIME_BETWEEN_DETAIL_UPDATES = timedelta(hours=1)

# Activity needs to be checked more frequently as the
# doorbell motion and rings are included here
ACTIVITY_UPDATE_INTERVAL = timedelta(seconds=10)

LOGIN_METHODS = ["phone", "email"]

PLATFORMS = [
    Platform.BUTTON,
    Platform.CAMERA,
    Platform.BINARY_SENSOR,
    Platform.LOCK,
    Platform.SENSOR,
]
<commit_msg>Increase default august timeout (#77762)<commit_after>"""Constants for August devices."""

from datetime import timedelta

from homeassistant.const import Platform

DEFAULT_TIMEOUT = 25

CONF_ACCESS_TOKEN_CACHE_FILE = "access_token_cache_file"
CONF_LOGIN_METHOD = "login_method"
CONF_INSTALL_ID = "install_id"

VERIFICATION_CODE_KEY = "verification_code"

NOTIFICATION_ID = "august_notification"
NOTIFICATION_TITLE = "August"

MANUFACTURER = "August Home Inc."

DEFAULT_AUGUST_CONFIG_FILE = ".august.conf"

DEFAULT_NAME = "August"
DOMAIN = "august"

OPERATION_METHOD_AUTORELOCK = "autorelock"
OPERATION_METHOD_REMOTE = "remote"
OPERATION_METHOD_KEYPAD = "keypad"
OPERATION_METHOD_MOBILE_DEVICE = "mobile"

ATTR_OPERATION_AUTORELOCK = "autorelock"
ATTR_OPERATION_METHOD = "method"
ATTR_OPERATION_REMOTE = "remote"
ATTR_OPERATION_KEYPAD = "keypad"

# Limit battery, online, and hardware updates to hourly
# in order to reduce the number of api requests and
# avoid hitting rate limits
MIN_TIME_BETWEEN_DETAIL_UPDATES = timedelta(hours=1)

# Activity needs to be checked more frequently as the
# doorbell motion and rings are included here
ACTIVITY_UPDATE_INTERVAL = timedelta(seconds=10)

LOGIN_METHODS = ["phone", "email"]

PLATFORMS = [
    Platform.BUTTON,
    Platform.CAMERA,
    Platform.BINARY_SENSOR,
    Platform.LOCK,
    Platform.SENSOR,
]
<|endoftext|>


<commit_before>"""Support for Honeywell Lyric sensor platform."""
from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import cast

from aiolyric.objects.device import LyricDevice
from aiolyric.objects.location import LyricLocation

from homeassistant.components.sensor import (
    STATE_CLASS_MEASUREMENT,
    SensorEntity,
    SensorEntityDescription,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (
    DEVICE_CLASS_HUMIDITY,
    DEVICE_CLASS_TEMPERATURE,
    DEVICE_CLASS_TIMESTAMP,
)
from homeassistant.core import HomeAssistant
from homeassistant.helpers.typing import StateType
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator
from homeassistant.util import dt as dt_util

from . import LyricDeviceEntity
from .const import (
    DOMAIN,
    PRESET_HOLD_UNTIL,
    PRESET_NO_HOLD,
    PRESET_PERMANENT_HOLD,
    PRESET_TEMPORARY_HOLD,
    PRESET_VACATION_HOLD,
)

LYRIC_SETPOINT_STATUS_NAMES = {
    PRESET_NO_HOLD: "Following Schedule",
    PRESET_PERMANENT_HOLD: "Held Permanently",
    PRESET_TEMPORARY_HOLD: "Held Temporarily",
    PRESET_VACATION_HOLD: "Holiday",
}

@dataclass
class LyricSensorEntityDescription(SensorEntityDescription):
    """Class describing Honeywell Lyric sensor entities."""

    value: Callable[[LyricDevice], StateType] = round

def get_datetime_from_future_time(time: str) -> datetime:
    """Get datetime from future time provided."""
    time = dt_util.parse_time(time)
    now = dt_util.utcnow()
    if time <= now.time():
        now = now + timedelta(days=1)
    return dt_util.as_utc(datetime.combine(now.date(), time))

async def async_setup_entry(
    hass: HomeAssistant, entry: ConfigEntry, async_add_entities
) -> None:
    """Set up the Honeywell Lyric sensor platform based on a config entry."""
    coordinator: DataUpdateCoordinator = hass.data[DOMAIN][entry.entry_id]

    entities = []

    def get_setpoint_status(status: str, time: str) -> str:
        if status == PRESET_HOLD_UNTIL:
            return f"Held until {time}"
        return LYRIC_SETPOINT_STATUS_NAMES.get(status, None)

    for location in coordinator.data.locations:
        for device in location.devices:
            if device.indoorTemperature:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_indoor_temperature",
                            name="Indoor Temperature",
                            device_class=DEVICE_CLASS_TEMPERATURE,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement=hass.config.units.temperature_unit,
                            value=lambda device: device.indoorTemperature,
                        ),
                        location,
                        device,
                    )
                )
            if device.outdoorTemperature:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_outdoor_temperature",
                            name="Outdoor Temperature",
                            device_class=DEVICE_CLASS_TEMPERATURE,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement=hass.config.units.temperature_unit,
                            value=lambda device: device.outdoorTemperature,
                        ),
                        location,
                        device,
                    )
                )
            if device.displayedOutdoorHumidity:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_outdoor_humidity",
                            name="Outdoor Humidity",
                            device_class=DEVICE_CLASS_HUMIDITY,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement="%",
                            value=lambda device: device.displayedOutdoorHumidity,
                        ),
                        location,
                        device,
                    )
                )
            if device.changeableValues:
                if device.changeableValues.nextPeriodTime:
                    entities.append(
                        LyricSensor(
                            coordinator,
                            LyricSensorEntityDescription(
                                key=f"{device.macID}_next_period_time",
                                name="Next Period Time",
                                device_class=DEVICE_CLASS_TIMESTAMP,
                                value=lambda device: get_datetime_from_future_time(
                                    device.changeableValues.nextPeriodTime
                                ).isoformat(),
                            ),
                            location,
                            device,
                        )
                    )
                if device.changeableValues.thermostatSetpointStatus:
                    entities.append(
                        LyricSensor(
                            coordinator,
                            LyricSensorEntityDescription(
                                key=f"{device.macID}_setpoint_status",
                                name="Setpoint Status",
                                icon="mdi:thermostat",
                                value=lambda device: get_setpoint_status(
                                    device.changeableValues.thermostatSetpointStatus,
                                    device.changeableValues.nextPeriodTime,
                                ),
                            ),
                            location,
                            device,
                        )
                    )

    async_add_entities(entities, True)

class LyricSensor(LyricDeviceEntity, SensorEntity):
    """Define a Honeywell Lyric sensor."""

    coordinator: DataUpdateCoordinator
    entity_description: LyricSensorEntityDescription

    def __init__(
        self,
        coordinator: DataUpdateCoordinator,
        description: LyricSensorEntityDescription,
        location: LyricLocation,
        device: LyricDevice,
    ) -> None:
        """Initialize."""
        super().__init__(
            coordinator,
            location,
            device,
            description.key,
        )
        self.entity_description = description

    @property
    def native_value(self) -> StateType:
        """Return the state."""
        device: LyricDevice = self.device
        try:
            return cast(StateType, self.entity_description.value(device))
        except TypeError:
            return None
<commit_msg>Fix lyric timestamp sensor (#61209)<commit_after>"""Support for Honeywell Lyric sensor platform."""
from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import cast

from aiolyric.objects.device import LyricDevice
from aiolyric.objects.location import LyricLocation

from homeassistant.components.sensor import (
    STATE_CLASS_MEASUREMENT,
    SensorEntity,
    SensorEntityDescription,
)
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import (
    DEVICE_CLASS_HUMIDITY,
    DEVICE_CLASS_TEMPERATURE,
    DEVICE_CLASS_TIMESTAMP,
)
from homeassistant.core import HomeAssistant
from homeassistant.helpers.typing import StateType
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator
from homeassistant.util import dt as dt_util

from . import LyricDeviceEntity
from .const import (
    DOMAIN,
    PRESET_HOLD_UNTIL,
    PRESET_NO_HOLD,
    PRESET_PERMANENT_HOLD,
    PRESET_TEMPORARY_HOLD,
    PRESET_VACATION_HOLD,
)

LYRIC_SETPOINT_STATUS_NAMES = {
    PRESET_NO_HOLD: "Following Schedule",
    PRESET_PERMANENT_HOLD: "Held Permanently",
    PRESET_TEMPORARY_HOLD: "Held Temporarily",
    PRESET_VACATION_HOLD: "Holiday",
}

@dataclass
class LyricSensorEntityDescription(SensorEntityDescription):
    """Class describing Honeywell Lyric sensor entities."""

    value: Callable[[LyricDevice], StateType | datetime] = round

def get_datetime_from_future_time(time: str) -> datetime:
    """Get datetime from future time provided."""
    time = dt_util.parse_time(time)
    now = dt_util.utcnow()
    if time <= now.time():
        now = now + timedelta(days=1)
    return dt_util.as_utc(datetime.combine(now.date(), time))

async def async_setup_entry(
    hass: HomeAssistant, entry: ConfigEntry, async_add_entities
) -> None:
    """Set up the Honeywell Lyric sensor platform based on a config entry."""
    coordinator: DataUpdateCoordinator = hass.data[DOMAIN][entry.entry_id]

    entities = []

    def get_setpoint_status(status: str, time: str) -> str:
        if status == PRESET_HOLD_UNTIL:
            return f"Held until {time}"
        return LYRIC_SETPOINT_STATUS_NAMES.get(status, None)

    for location in coordinator.data.locations:
        for device in location.devices:
            if device.indoorTemperature:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_indoor_temperature",
                            name="Indoor Temperature",
                            device_class=DEVICE_CLASS_TEMPERATURE,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement=hass.config.units.temperature_unit,
                            value=lambda device: device.indoorTemperature,
                        ),
                        location,
                        device,
                    )
                )
            if device.outdoorTemperature:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_outdoor_temperature",
                            name="Outdoor Temperature",
                            device_class=DEVICE_CLASS_TEMPERATURE,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement=hass.config.units.temperature_unit,
                            value=lambda device: device.outdoorTemperature,
                        ),
                        location,
                        device,
                    )
                )
            if device.displayedOutdoorHumidity:
                entities.append(
                    LyricSensor(
                        coordinator,
                        LyricSensorEntityDescription(
                            key=f"{device.macID}_outdoor_humidity",
                            name="Outdoor Humidity",
                            device_class=DEVICE_CLASS_HUMIDITY,
                            state_class=STATE_CLASS_MEASUREMENT,
                            native_unit_of_measurement="%",
                            value=lambda device: device.displayedOutdoorHumidity,
                        ),
                        location,
                        device,
                    )
                )
            if device.changeableValues:
                if device.changeableValues.nextPeriodTime:
                    entities.append(
                        LyricSensor(
                            coordinator,
                            LyricSensorEntityDescription(
                                key=f"{device.macID}_next_period_time",
                                name="Next Period Time",
                                device_class=DEVICE_CLASS_TIMESTAMP,
                                value=lambda device: get_datetime_from_future_time(
                                    device.changeableValues.nextPeriodTime
                                ),
                            ),
                            location,
                            device,
                        )
                    )
                if device.changeableValues.thermostatSetpointStatus:
                    entities.append(
                        LyricSensor(
                            coordinator,
                            LyricSensorEntityDescription(
                                key=f"{device.macID}_setpoint_status",
                                name="Setpoint Status",
                                icon="mdi:thermostat",
                                value=lambda device: get_setpoint_status(
                                    device.changeableValues.thermostatSetpointStatus,
                                    device.changeableValues.nextPeriodTime,
                                ),
                            ),
                            location,
                            device,
                        )
                    )

    async_add_entities(entities, True)

class LyricSensor(LyricDeviceEntity, SensorEntity):
    """Define a Honeywell Lyric sensor."""

    coordinator: DataUpdateCoordinator
    entity_description: LyricSensorEntityDescription

    def __init__(
        self,
        coordinator: DataUpdateCoordinator,
        description: LyricSensorEntityDescription,
        location: LyricLocation,
        device: LyricDevice,
    ) -> None:
        """Initialize."""
        super().__init__(
            coordinator,
            location,
            device,
            description.key,
        )
        self.entity_description = description

    @property
    def native_value(self) -> StateType:
        """Return the state."""
        device: LyricDevice = self.device
        try:
            return cast(StateType, self.entity_description.value(device))
        except TypeError:
            return None
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from pipeline.conf import settings
from pipeline.compilers import SubProcessCompiler

class DustCompiler(SubProcessCompiler):
    output_extension = 'js'

    def match_file(self, filename):
        print filename
        return filename.endswith('.dust')

    def compile_file(self, infile, outfile, outdated=False, force=False):
        if not outdated and not force:
            return  # File doesn't need to be recompiled

        command = ' '.join(filter(None, [
            settings.PIPELINE_DUST_BINARY,
            '--name="%s"' % infile.split('/')[-1].split('.')[0],
            infile,
            outfile
        ]))

        return self.execute_command(command)
<commit_msg>removed print<commit_after>from __future__ import unicode_literals

from pipeline.conf import settings
from pipeline.compilers import SubProcessCompiler

class DustCompiler(SubProcessCompiler):
    output_extension = 'js'

    def match_file(self, filename):
        return filename.endswith('.dust')

    def compile_file(self, infile, outfile, outdated=False, force=False):
        if not outdated and not force:
            return  # File doesn't need to be recompiled

        command = ' '.join(filter(None, [
            settings.PIPELINE_DUST_BINARY,
            '--name="%s"' % infile.split('/')[-1].split('.')[0],
            infile,
            outfile
        ]))

        return self.execute_command(command)
<|endoftext|>


<commit_before>from collections import defaultdict
from itertools import chain

def flatten(list_of_lists):
    """Flatten an iterable of iterables."""
    return chain.from_iterable(list_of_lists)

class Spec(object):
    @classmethod
    def from_line(cls, line):
        """Parses a spec line from a requirements file and returns a Spec."""
        from pkg_resources import Requirement
        req = Requirement.parse(line)
        return cls(req.project_name, req.specs)

    def __init__(self, name, specs, source=None):
        """The Spec class represents a package version specification,
        typically given by a single line in a requirements.txt file.

        Each Spec belongs to a single package name, and can have multiple
        'specs' (lowercase), which are the famous (qualifier, version) tuples.

        The source is an instance of SpecSource, and defines where this spec
        comes from.
        """
        self.name = name
        self.specs = specs if specs else []
        self.source = source

    def __unicode__(self):
        specs_string = ','.join(map(''.join, self.specs))
        return '%s%s' % (self.name, specs_string)

    def __str__(self):
        return str(unicode(self))

    def __repr__(self):
        return str(self)

class SpecSource(object):
    def __str__(self):
        return str(unicode(self))

class FileSource(SpecSource):
    def __init__(self, filename, lineno):
        """Records a given filename and line number as a spec source."""
        self.filename = filename
        self.lineno = lineno

    def __unicode__(self):
        return '%s:%s' % (self.filename, self.lineno)

class InferredSource(SpecSource):
    def __init__(self):
        """Records that this spec source was inferred."""
        pass

    def __unicode__(self):
        return 'inferred'

class RequiredBySource(SpecSource):
    def __init__(self, package):
        """Records a dependency of another package."""
        self.required_by = package

    def __unicode__(self):
        return unicode(self.required_by)

class SpecSet(object):
    def __init__(self):
        """A collection of Spec instances that can be normalized and used for
        conflict detection.
        """
        self._byname = defaultdict(list)
        self._items = []

    def add_specs(self, iterable):
        for spec in iterable:
            self.add_spec(spec)

    def add_spec(self, spec):
        self._items.append(spec)
        self._byname[spec.name].append(spec)

    def normalize_specs_for_name(self, name):
        # TODO: This method should not lose source information, as it does
        # right now.  When normalizing, we might drop a few specs, but for the
        # ones we are keeping, the source should remain clear.

        """Normalizes specs for the given package name.

        Example before normalizing:
            [
             Spec('Django', [('>=', '1.3.1'), ('<=', '1.5.0')]),
             Spec('Django', [('>', '1.3.0'), ('<=', '1.3.2')])
            ]

        After normalizing:
            [Spec('Django', [('>=', '1.3.1'), ('<=', '1.3.2')])]

        Example before normalizing:
            [
             Spec('Django', [('>=', '1.3.2'), ('<=', '1.5.0')]),
             Spec('Django', [('>', '1.3.0'), ('<=', '1.3.2')])
            ]

        After normalizing:
            [Spec('Django', [('==', '1.3.2')])]
        """
        specs = list(flatten(map(lambda s: s.specs, self._byname[name])))

        # First, group the flattened spec list by qualifier
        by_qualifiers = defaultdict(list)
        for spec in specs:
            qualifier, version = spec
            by_qualifiers[qualifier].append(version)

        # For each qualifier type, apply selection logic.  For the unequality
        # qualifiers, select the value that yields the narrowest range
        # possible.

        # Pick the smallest less-than spec
        if '<' in by_qualifiers:
            by_qualifiers['<'] = sorted(by_qualifiers['<'])[0]
        if '<=' in by_qualifiers:
            by_qualifiers['<='] = sorted(by_qualifiers['<='])[0]

        if '<' in by_qualifiers and '<=' in by_qualifiers:
            if by_qualifiers['<'] <= by_qualifiers['<=']:
                # < xyz wins over <= xyz
                del by_qualifiers['<=']
            else:
                del by_qualifiers['<']

        # Pick the highest greater-than spec
        if '>' in by_qualifiers:
            by_qualifiers['>'] = sorted(by_qualifiers['>'])[-1]
        if '>=' in by_qualifiers:
            by_qualifiers['>='] = sorted(by_qualifiers['>='])[-1]

        if '>' in by_qualifiers and '>=' in by_qualifiers:
            if by_qualifiers['>'] >= by_qualifiers['>=']:
                # > xyz wins over >= xyz
                del by_qualifiers['>=']
            else:
                del by_qualifiers['>']

        # Normalize less-than/greater-than in the specific case where they
        # overlap on a specific version
        if '>=' in by_qualifiers and '<=' in by_qualifiers:
            if by_qualifiers['>='] == by_qualifiers['<=']:
                by_qualifiers['=='].append(by_qualifiers['<='])
                del by_qualifiers['>=']
                del by_qualifiers['<=']

        # Detect any conflicts
        if '==' in by_qualifiers:
            # Multiple '==' keys are conflicts
            assert len(set(by_qualifiers['=='])) <= 1, 'Conflict! %s' % (' with '.join(map(lambda v: '%s==%s' % (name, v), by_qualifiers['=='],)))  # noqa

            # Pick the only == qualifier
            by_qualifiers['=='] = by_qualifiers['=='][0]

            # Any non-'==' key is a conflict if the pinned version does not
            # fall in that range.  Otherwise, the unequality variant can be
            # removed from the spec set.
            pinned_version = by_qualifiers['==']
            for qual, value in by_qualifiers.items():
                if qual == '==':
                    continue

                # Perform conflict checks
                if qual == '>':
                    assert pinned_version > value, 'Conflict: %s==%s with %s>%s' % (name, pinned_version, name, value)
                if qual == '>=':
                    assert pinned_version >= value, 'Conflict: %s==%s with %s>=%s' % (name, pinned_version, name, value)
                if qual == '<':
                    assert pinned_version < value, 'Conflict: %s==%s with %s<%s' % (name, pinned_version, name, value)
                if qual == '<=':
                    assert pinned_version <= value, 'Conflict: %s==%s with %s<=%s' % (name, pinned_version, name, value)

                # If no conflicts are found, prefer the pinned version and
                # discard the inequality spec
                del by_qualifiers[qual]
        else:
            # Checks for conflicts due to non-overlapping ranges
            less_than = None
            if '<' in by_qualifiers:
                less_than = by_qualifiers['<']
                less_than_op = '<'
            elif '<=' in by_qualifiers:
                less_than = by_qualifiers['<=']
                less_than_op = '<='

            greater_than = None
            if '>' in by_qualifiers:
                greater_than = by_qualifiers['>']
                greater_than_op = '>'
            elif '>=' in by_qualifiers:
                greater_than = by_qualifiers['>=']
                greater_than_op = '>='

            if less_than and greater_than:
                assert less_than > greater_than, 'Conflict: %s%s and %s%s' % (less_than_op, less_than, greater_than_op, greater_than)  # noqa

        inferred_spec = Spec(name, by_qualifiers.items(), source=InferredSource())
        return inferred_spec

    def normalize(self):
        # TODO: Would it be nicer if this function returns a new SpecSet
        # instance?  I think so.

        self._normalized_by_name = {}
        for name in self._byname:
            self._normalized_by_name[name] = self.normalize_specs_for_name(name)

    def __str__(self):
        self.normalize()
        lines = []
        for spec in self._normalized_by_name.values():
            lines.append(unicode(spec))
        return '\n'.join(lines)
<commit_msg>Minor fix.<commit_after>from collections import defaultdict
from itertools import chain

def flatten(list_of_lists):
    """Flatten an iterable of iterables."""
    return chain.from_iterable(list_of_lists)

class Spec(object):
    @classmethod
    def from_line(cls, line):
        """Parses a spec line from a requirements file and returns a Spec."""
        from pkg_resources import Requirement
        req = Requirement.parse(line)
        return cls(req.project_name, req.specs)

    def __init__(self, name, specs, source=None):
        """The Spec class represents a package version specification,
        typically given by a single line in a requirements.txt file.

        Each Spec belongs to a single package name, and can have multiple
        'specs' (lowercase), which are the famous (qualifier, version) tuples.

        The source is an instance of SpecSource, and defines where this spec
        comes from.
        """
        self.name = name
        self.specs = specs if specs else []
        self.source = source

    def __str__(self):
        specs_string = ','.join(map(''.join, self.specs))
        return '%s%s' % (self.name, specs_string)

    def __unicode__(self):
        return unicode(str(self))

    def __repr__(self):
        return str(self)

class SpecSource(object):
    def __str__(self):
        return str(unicode(self))

class FileSource(SpecSource):
    def __init__(self, filename, lineno):
        """Records a given filename and line number as a spec source."""
        self.filename = filename
        self.lineno = lineno

    def __unicode__(self):
        return '%s:%s' % (self.filename, self.lineno)

class InferredSource(SpecSource):
    def __init__(self):
        """Records that this spec source was inferred."""
        pass

    def __unicode__(self):
        return 'inferred'

class RequiredBySource(SpecSource):
    def __init__(self, package):
        """Records a dependency of another package."""
        self.required_by = package

    def __unicode__(self):
        return unicode(self.required_by)

class SpecSet(object):
    def __init__(self):
        """A collection of Spec instances that can be normalized and used for
        conflict detection.
        """
        self._byname = defaultdict(list)
        self._items = []

    def add_specs(self, iterable):
        for spec in iterable:
            self.add_spec(spec)

    def add_spec(self, spec):
        self._items.append(spec)
        self._byname[spec.name].append(spec)

    def normalize_specs_for_name(self, name):
        # TODO: This method should not lose source information, as it does
        # right now.  When normalizing, we might drop a few specs, but for the
        # ones we are keeping, the source should remain clear.

        """Normalizes specs for the given package name.

        Example before normalizing:
            [
             Spec('Django', [('>=', '1.3.1'), ('<=', '1.5.0')]),
             Spec('Django', [('>', '1.3.0'), ('<=', '1.3.2')])
            ]

        After normalizing:
            [Spec('Django', [('>=', '1.3.1'), ('<=', '1.3.2')])]

        Example before normalizing:
            [
             Spec('Django', [('>=', '1.3.2'), ('<=', '1.5.0')]),
             Spec('Django', [('>', '1.3.0'), ('<=', '1.3.2')])
            ]

        After normalizing:
            [Spec('Django', [('==', '1.3.2')])]
        """
        specs = list(flatten(map(lambda s: s.specs, self._byname[name])))

        # First, group the flattened spec list by qualifier
        by_qualifiers = defaultdict(list)
        for spec in specs:
            qualifier, version = spec
            by_qualifiers[qualifier].append(version)

        # For each qualifier type, apply selection logic.  For the unequality
        # qualifiers, select the value that yields the narrowest range
        # possible.

        # Pick the smallest less-than spec
        if '<' in by_qualifiers:
            by_qualifiers['<'] = sorted(by_qualifiers['<'])[0]
        if '<=' in by_qualifiers:
            by_qualifiers['<='] = sorted(by_qualifiers['<='])[0]

        if '<' in by_qualifiers and '<=' in by_qualifiers:
            if by_qualifiers['<'] <= by_qualifiers['<=']:
                # < xyz wins over <= xyz
                del by_qualifiers['<=']
            else:
                del by_qualifiers['<']

        # Pick the highest greater-than spec
        if '>' in by_qualifiers:
            by_qualifiers['>'] = sorted(by_qualifiers['>'])[-1]
        if '>=' in by_qualifiers:
            by_qualifiers['>='] = sorted(by_qualifiers['>='])[-1]

        if '>' in by_qualifiers and '>=' in by_qualifiers:
            if by_qualifiers['>'] >= by_qualifiers['>=']:
                # > xyz wins over >= xyz
                del by_qualifiers['>=']
            else:
                del by_qualifiers['>']

        # Normalize less-than/greater-than in the specific case where they
        # overlap on a specific version
        if '>=' in by_qualifiers and '<=' in by_qualifiers:
            if by_qualifiers['>='] == by_qualifiers['<=']:
                by_qualifiers['=='].append(by_qualifiers['<='])
                del by_qualifiers['>=']
                del by_qualifiers['<=']

        # Detect any conflicts
        if '==' in by_qualifiers:
            # Multiple '==' keys are conflicts
            assert len(set(by_qualifiers['=='])) <= 1, 'Conflict! %s' % (' with '.join(map(lambda v: '%s==%s' % (name, v), by_qualifiers['=='],)))  # noqa

            # Pick the only == qualifier
            by_qualifiers['=='] = by_qualifiers['=='][0]

            # Any non-'==' key is a conflict if the pinned version does not
            # fall in that range.  Otherwise, the unequality variant can be
            # removed from the spec set.
            pinned_version = by_qualifiers['==']
            for qual, value in by_qualifiers.items():
                if qual == '==':
                    continue

                # Perform conflict checks
                if qual == '>':
                    assert pinned_version > value, 'Conflict: %s==%s with %s>%s' % (name, pinned_version, name, value)
                if qual == '>=':
                    assert pinned_version >= value, 'Conflict: %s==%s with %s>=%s' % (name, pinned_version, name, value)
                if qual == '<':
                    assert pinned_version < value, 'Conflict: %s==%s with %s<%s' % (name, pinned_version, name, value)
                if qual == '<=':
                    assert pinned_version <= value, 'Conflict: %s==%s with %s<=%s' % (name, pinned_version, name, value)

                # If no conflicts are found, prefer the pinned version and
                # discard the inequality spec
                del by_qualifiers[qual]
        else:
            # Checks for conflicts due to non-overlapping ranges
            less_than = None
            if '<' in by_qualifiers:
                less_than = by_qualifiers['<']
                less_than_op = '<'
            elif '<=' in by_qualifiers:
                less_than = by_qualifiers['<=']
                less_than_op = '<='

            greater_than = None
            if '>' in by_qualifiers:
                greater_than = by_qualifiers['>']
                greater_than_op = '>'
            elif '>=' in by_qualifiers:
                greater_than = by_qualifiers['>=']
                greater_than_op = '>='

            if less_than and greater_than:
                assert less_than > greater_than, 'Conflict: %s%s and %s%s' % (less_than_op, less_than, greater_than_op, greater_than)  # noqa

        inferred_spec = Spec(name, by_qualifiers.items(), source=InferredSource())
        return inferred_spec

    def normalize(self):
        # TODO: Would it be nicer if this function returns a new SpecSet
        # instance?  I think so.

        self._normalized_by_name = {}
        for name in self._byname:
            self._normalized_by_name[name] = self.normalize_specs_for_name(name)

    def __str__(self):
        self.normalize()
        lines = []
        for spec in self._normalized_by_name.values():
            lines.append(unicode(spec))
        return '\n'.join(lines)
<|endoftext|>


<commit_before># Copyright 2014 Dave Kludt
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pitchfork import app
from flask import g, render_template, request, redirect
from flask import url_for, flash, jsonify
from flask.ext.classy import FlaskView, route
from models import Product, Call
from pitchfork.adminbp.decorators import check_perms
from bson.objectid import ObjectId

import global_helper
import global_forms
import pymongo

class ProductsView(FlaskView):
    decorators = [check_perms(request)]
    route_base = '/'

    @route('/<product>/')
    @route('/<product>/<testing>')
    def index(self, product, testing=None):
        testing_calls = False
        if testing is not None:
            testing_calls = True

        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        api_calls = global_helper.gather_api_calls(
            found_product,
            testing_calls
        )
        restrict_dcs, data_centers = global_helper.check_for_product_dcs(
            found_product
        )
        return render_template(
            'product_front.html',
            api_calls=api_calls,
            api_settings=found_product,
            data_centers=data_centers,
            title=found_product.title,
            api_process='/%s/api/call/process' % product,
            testing=testing_calls,
            require_dc=found_product.require_dc,
            restrict_dcs=restrict_dcs
        )

    """ Product API Call Fire """

    @route('/<product>/api/call/process', methods=['POST'])
    def execute_api_call(self, product):
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        if request.json.get('testing') or request.json.get('mock'):
            api_call = getattr(g.db, found_product.db_name).find_one(
                {'_id': ObjectId(request.json.get('api_id'))}
            )
        else:
            api_call = getattr(g.db, found_product.db_name).find_and_modify(
                query={'_id': ObjectId(request.json.get('api_id'))},
                update={'$inc': {'accessed': 1}}
            )

        """ Change this to a jsonify call and have jquery handle it """
        if not api_call:
            flash('API Call was not found')
            return redirect(url_for('index'))

        """ Retrieve all of the elements for the call """
        api_url, header, data_package = global_helper.generate_vars_for_call(
            found_product,
            api_call,
            request
        )

        """ Send off the request and retrieve the data elements """
        if request.json.get('mock'):
            response_headers, response_body, response_code = None, None, None
            request_headers = header
        else:
            request_headers, response_headers, response_body, response_code = (
                global_helper.process_api_request(
                    api_url,
                    request.json.get('api_verb'),
                    data_package,
                    header
                )
            )

        if not (request.json.get('testing') or request.json.get('mock')):
            global_helper.log_api_call_request(
                request_headers,
                response_headers,
                response_body,
                response_code,
                api_call,
                request.json,
                data_package,
                api_url,
                found_product.title
            )

        """ Send the data structure back to the browser """
        return jsonify(
            request_headers=request_headers,
            response_headers=response_headers,
            response_body=response_body,
            response_code=response_code,
            api_url=api_url,
            data_package=data_package
        )

    """ Product Management """

    @route('/<product>/manage', methods=['GET', 'POST'])
    def manage_settings(self, product):
        product_data = self.retrieve_product(product)
        if type(product_data) is str:
            title = "%s Manage Settings" % product.title()
            post_url = "/%s/manage" % product
            product_data = None
            form = global_forms.ManageProduct()
        else:
            title = "%s Manage Settings" % product_data.title
            post_url = "%s/manage" % product_data.app_url
            form = global_forms.ManageProduct(obj=product_data)

        if request.method == 'POST' and form.validate_on_submit():
            to_save = Product(request.form.to_dict())
            to_save.set_db_name()
            g.db.api_settings.update(
                {}, {
                    '$set': {
                        product: to_save.__dict__
                    }
                }
            )
            if to_save.active:
                g.db.api_settings.update(
                    {}, {
                        '$addToSet': {'active_products': product}
                    }
                )
            else:
                g.db.api_settings.update(
                    {}, {
                        '$pull': {'active_products': product}
                    }
                )

            flash('Product was successfully updated', 'success')
            return redirect('/%s/manage' % product)
        else:
            if request.method == 'POST':
                flash('Form was not saved successfully', 'error')

            return render_template(
                'manage/manage_product.html',
                title=title,
                form=form,
                product=product_data,
                post_url=post_url
            )

    @route('/<product>/manage/api')
    def manage_api_calls(self, product):
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash(
                'Could not find product, please check the URL and try again',
                'error'
            )
            return redirect(url_for('index'))

        product_url = "/%s/manage/api" % product
        try:
            api_commands = getattr(g.db, found_product.db_name).find().sort(
                [
                    ['tested', pymongo.DESCENDING],
                    ['title', pymongo.ASCENDING]
                ]
            )
        except:
            api_commands = []

        return render_template(
            'manage/manage_api_calls.html',
            title="%s - API Calls" % found_product.title,
            api_commands=api_commands,
            product_url=product_url
        )

    @route('/<product>/manage/api/add', methods=['GET', 'POST'])
    @route('/<product>/manage/api/edit/<api_id>', methods=['GET', 'POST'])
    def manage_add_edit_call(self, product, api_id=None):
        edit, count, title = False, 1, 'Add API Call'
        api_settings = g.db.api_settings.find_one()
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        if api_id:
            found_call = self.retrieve_api_call(found_product, api_id)
            if not found_call:
                flash('API Call was not found', 'error')
                return redirect('/%s/manage/api' % product)

            title = 'Edit API Call'
            edit = True
            post_url = "/%s/manage/api/edit/%s" % (product, api_id)
            form, count = global_helper.generate_edit_call_form(
                found_product,
                found_call,
                api_id
            )
        else:
            post_url = "/%s/manage/api/add" % product
            form = global_helper.add_fields_to_form(count)
            for i in range(count):
                temp = getattr(form, 'variable_%i' % i)
                temp.form.id_value.data = i

        form.verb.choices = [
            (verb.get('name'), verb.get('name'))
            for verb in api_settings.get('verbs')
        ]
        form.product.data = product
        if request.method == 'POST' and form.validate_on_submit():
            api_call = Call(request.form.to_dict())
            api_call.variables = global_helper.get_vars_for_call(
                list(request.form.iterlists())
            )
            if api_id:
                getattr(g.db, found_product.db_name).update(
                    {
                        '_id': ObjectId(api_id)
                    }, {
                        '$set': api_call.__dict__
                    }
                )
                flash('API Call was successfully updated', 'success')
            else:
                try:
                    getattr(g.db, found_product.db_name).insert(
                        api_call.__dict__
                    )
                    flash('API Call was added successfully', 'success')
                except:
                    flash(
                        'There was an issue storing the API Call. Check '
                        'the product and ensure the db_name is specified',
                        'error'
                    )

            return redirect('/%s/manage/api' % product)
        else:
            if request.method == 'POST':
                flash(
                    'Form validation error, please check the '
                    'form and try again',
                    'error'
                )

            return render_template(
                'manage/manage_call_add_edit.html',
                title=title,
                form=form,
                count=count,
                edit=edit,
                post_url=post_url,
                cancel_return='/%s/manage/api' % product,
            )

    @route('/<product>/manage/api/<action>/<api_id>')
    def manage_api_actions(self, product, action, api_id):
        action_map = {'delete': True, 'confirm': True, 'unconfirm': False}
        found_product = self.retrieve_product(product)
        if type(found_product) is not str:
            if action in action_map:
                if getattr(g.db, found_product.db_name).find_one(
                    {'_id': ObjectId(api_id)}
                ):
                    if action == 'delete':
                        getattr(g.db, found_product.db_name).remove(
                            {'_id': ObjectId(api_id)}
                        )
                        message = 'API call was successfully removed'
                    else:
                        getattr(g.db, found_product.db_name).update(
                            {
                                '_id': ObjectId(api_id)
                            }, {
                                '$set': {
                                    'tested': action_map.get(action)
                                }
                            }
                        )
                        message = 'API call was successfully updated'
                else:
                    flash(
                        'API call was not found and nothing removed',
                        'error'
                    )
                    return redirect('/%s/manage/api' % product)

                flash(message, 'success')
                return redirect('/%s/manage/api' % product)
            else:
                message = 'Invalid action provided, so no action taken'
        else:
            message = 'Product was not found, so no action taken'

        flash(message, 'error')
        return redirect('/%s/manage/api' % product)

    def retrieve_product(self, product):
        temp_product = g.db.api_settings.find_one()
        if temp_product and temp_product.get(product):
            return Product(temp_product.get(product))

        return str(product)

    def retrieve_api_call(self, product, api_id):
        temp_call = getattr(g.db, product.db_name).find_one(
            {'_id': ObjectId(api_id)}
        )
        if temp_call:
            return Call(temp_call)

        return temp_call

ProductsView.register(app)
<commit_msg>Change logic around whether to log call or not<commit_after># Copyright 2014 Dave Kludt
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pitchfork import app
from flask import g, render_template, request, redirect
from flask import url_for, flash, jsonify
from flask.ext.classy import FlaskView, route
from models import Product, Call
from pitchfork.adminbp.decorators import check_perms
from bson.objectid import ObjectId

import global_helper
import global_forms
import pymongo

class ProductsView(FlaskView):
    decorators = [check_perms(request)]
    route_base = '/'

    @route('/<product>/')
    @route('/<product>/<testing>')
    def index(self, product, testing=None):
        testing_calls = False
        if testing is not None:
            testing_calls = True

        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        api_calls = global_helper.gather_api_calls(
            found_product,
            testing_calls
        )
        restrict_dcs, data_centers = global_helper.check_for_product_dcs(
            found_product
        )
        return render_template(
            'product_front.html',
            api_calls=api_calls,
            api_settings=found_product,
            data_centers=data_centers,
            title=found_product.title,
            api_process='/%s/api/call/process' % product,
            testing=testing_calls,
            require_dc=found_product.require_dc,
            restrict_dcs=restrict_dcs
        )

    """ Product API Call Fire """

    @route('/<product>/api/call/process', methods=['POST'])
    def execute_api_call(self, product):
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        if request.json.get('testing') or request.json.get('mock'):
            api_call = getattr(g.db, found_product.db_name).find_one(
                {'_id': ObjectId(request.json.get('api_id'))}
            )
        else:
            api_call = getattr(g.db, found_product.db_name).find_and_modify(
                query={'_id': ObjectId(request.json.get('api_id'))},
                update={'$inc': {'accessed': 1}}
            )

        """ Change this to a jsonify call and have jquery handle it """
        if not api_call:
            flash('API Call was not found')
            return redirect(url_for('index'))

        """ Retrieve all of the elements for the call """
        api_url, header, data_package = global_helper.generate_vars_for_call(
            found_product,
            api_call,
            request
        )

        """ Send off the request and retrieve the data elements """
        if request.json.get('mock'):
            response_headers, response_body, response_code = None, None, None
            request_headers = header
        else:
            request_headers, response_headers, response_body, response_code = (
                global_helper.process_api_request(
                    api_url,
                    request.json.get('api_verb'),
                    data_package,
                    header
                )
            )

        if (
            request.json.get('mock') is None and
            request.json.get('testing') == 'false'
        ):
            global_helper.log_api_call_request(
                request_headers,
                response_headers,
                response_body,
                response_code,
                api_call,
                request.json,
                data_package,
                api_url,
                found_product.title
            )

        """ Send the data structure back to the browser """
        return jsonify(
            request_headers=request_headers,
            response_headers=response_headers,
            response_body=response_body,
            response_code=response_code,
            api_url=api_url,
            data_package=data_package
        )

    """ Product Management """

    @route('/<product>/manage', methods=['GET', 'POST'])
    def manage_settings(self, product):
        product_data = self.retrieve_product(product)
        if type(product_data) is str:
            title = "%s Manage Settings" % product.title()
            post_url = "/%s/manage" % product
            product_data = None
            form = global_forms.ManageProduct()
        else:
            title = "%s Manage Settings" % product_data.title
            post_url = "%s/manage" % product_data.app_url
            form = global_forms.ManageProduct(obj=product_data)

        if request.method == 'POST' and form.validate_on_submit():
            to_save = Product(request.form.to_dict())
            to_save.set_db_name()
            g.db.api_settings.update(
                {}, {
                    '$set': {
                        product: to_save.__dict__
                    }
                }
            )
            if to_save.active:
                g.db.api_settings.update(
                    {}, {
                        '$addToSet': {'active_products': product}
                    }
                )
            else:
                g.db.api_settings.update(
                    {}, {
                        '$pull': {'active_products': product}
                    }
                )

            flash('Product was successfully updated', 'success')
            return redirect('/%s/manage' % product)
        else:
            if request.method == 'POST':
                flash('Form was not saved successfully', 'error')

            return render_template(
                'manage/manage_product.html',
                title=title,
                form=form,
                product=product_data,
                post_url=post_url
            )

    @route('/<product>/manage/api')
    def manage_api_calls(self, product):
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash(
                'Could not find product, please check the URL and try again',
                'error'
            )
            return redirect(url_for('index'))

        product_url = "/%s/manage/api" % product
        try:
            api_commands = getattr(g.db, found_product.db_name).find().sort(
                [
                    ['tested', pymongo.DESCENDING],
                    ['title', pymongo.ASCENDING]
                ]
            )
        except:
            api_commands = []

        return render_template(
            'manage/manage_api_calls.html',
            title="%s - API Calls" % found_product.title,
            api_commands=api_commands,
            product_url=product_url
        )

    @route('/<product>/manage/api/add', methods=['GET', 'POST'])
    @route('/<product>/manage/api/edit/<api_id>', methods=['GET', 'POST'])
    def manage_add_edit_call(self, product, api_id=None):
        edit, count, title = False, 1, 'Add API Call'
        api_settings = g.db.api_settings.find_one()
        found_product = self.retrieve_product(product)
        if type(found_product) is str:
            flash('Product not found, please check the URL and try again')
            return redirect(url_for('index'))

        if api_id:
            found_call = self.retrieve_api_call(found_product, api_id)
            if not found_call:
                flash('API Call was not found', 'error')
                return redirect('/%s/manage/api' % product)

            title = 'Edit API Call'
            edit = True
            post_url = "/%s/manage/api/edit/%s" % (product, api_id)
            form, count = global_helper.generate_edit_call_form(
                found_product,
                found_call,
                api_id
            )
        else:
            post_url = "/%s/manage/api/add" % product
            form = global_helper.add_fields_to_form(count)
            for i in range(count):
                temp = getattr(form, 'variable_%i' % i)
                temp.form.id_value.data = i

        form.verb.choices = [
            (verb.get('name'), verb.get('name'))
            for verb in api_settings.get('verbs')
        ]
        form.product.data = product
        if request.method == 'POST' and form.validate_on_submit():
            api_call = Call(request.form.to_dict())
            api_call.variables = global_helper.get_vars_for_call(
                list(request.form.iterlists())
            )
            if api_id:
                getattr(g.db, found_product.db_name).update(
                    {
                        '_id': ObjectId(api_id)
                    }, {
                        '$set': api_call.__dict__
                    }
                )
                flash('API Call was successfully updated', 'success')
            else:
                try:
                    getattr(g.db, found_product.db_name).insert(
                        api_call.__dict__
                    )
                    flash('API Call was added successfully', 'success')
                except:
                    flash(
                        'There was an issue storing the API Call. Check '
                        'the product and ensure the db_name is specified',
                        'error'
                    )

            return redirect('/%s/manage/api' % product)
        else:
            if request.method == 'POST':
                flash(
                    'Form validation error, please check the '
                    'form and try again',
                    'error'
                )

            return render_template(
                'manage/manage_call_add_edit.html',
                title=title,
                form=form,
                count=count,
                edit=edit,
                post_url=post_url,
                cancel_return='/%s/manage/api' % product,
            )

    @route('/<product>/manage/api/<action>/<api_id>')
    def manage_api_actions(self, product, action, api_id):
        action_map = {'delete': True, 'confirm': True, 'unconfirm': False}
        found_product = self.retrieve_product(product)
        if type(found_product) is not str:
            if action in action_map:
                if getattr(g.db, found_product.db_name).find_one(
                    {'_id': ObjectId(api_id)}
                ):
                    if action == 'delete':
                        getattr(g.db, found_product.db_name).remove(
                            {'_id': ObjectId(api_id)}
                        )
                        message = 'API call was successfully removed'
                    else:
                        getattr(g.db, found_product.db_name).update(
                            {
                                '_id': ObjectId(api_id)
                            }, {
                                '$set': {
                                    'tested': action_map.get(action)
                                }
                            }
                        )
                        message = 'API call was successfully updated'
                else:
                    flash(
                        'API call was not found and nothing removed',
                        'error'
                    )
                    return redirect('/%s/manage/api' % product)

                flash(message, 'success')
                return redirect('/%s/manage/api' % product)
            else:
                message = 'Invalid action provided, so no action taken'
        else:
            message = 'Product was not found, so no action taken'

        flash(message, 'error')
        return redirect('/%s/manage/api' % product)

    def retrieve_product(self, product):
        temp_product = g.db.api_settings.find_one()
        if temp_product and temp_product.get(product):
            return Product(temp_product.get(product))

        return str(product)

    def retrieve_api_call(self, product, api_id):
        temp_call = getattr(g.db, product.db_name).find_one(
            {'_id': ObjectId(api_id)}
        )
        if temp_call:
            return Call(temp_call)

        return temp_call

ProductsView.register(app)
<|endoftext|>


<commit_before>import os
import pwd
import grp
import sys
import signal
import daemon.pidfile
import argparse
import datetime
import threading

from pandaharvester.harvesterconfig import harvester_config

# the master class which runs the main process
class Master:
    # constructor
    def __init__(self, single_mode=False, stop_event=None):
        # initialize database and config
        self.singleMode = single_mode
        self.stopEvent = stop_event
        from pandaharvester.harvestercore.communicator_pool import CommunicatorPool
        self.communicatorPool = CommunicatorPool()
        from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
        self.queueConfigMapper = QueueConfigMapper()
        from pandaharvester.harvestercore.db_proxy import DBProxy
        dbProxy = DBProxy()
        dbProxy.make_tables(self.queueConfigMapper)

    # main loop
    def start(self):
        # send first heartbeat
        self.communicatorPool.is_alive({'startTime': datetime.datetime.utcnow()})
        # thread list
        thrList = []
        # Command manager
        from pandaharvester.harvesterbody.command_manager import CommandManager
        thr = CommandManager(self.communicatorPool, single_mode=True)
        thr.set_stop_event(self.stopEvent)
        thr.run()
        thr.set_single_mode(self.singleMode)
        thr.start()
        thrList.append(thr)
        # Cacher
        from pandaharvester.harvesterbody.cacher import Cacher
        thr = Cacher(single_mode=True)
        thr.set_stop_event(self.stopEvent)
        thr.run()
        thr.set_single_mode(self.singleMode)
        thr.start()
        thrList.append(thr)
        # Credential Manager
        from pandaharvester.harvesterbody.cred_manager import CredManager
        thr = CredManager(single_mode=self.singleMode)
        thr.set_stop_event(self.stopEvent)
        thr.execute()
        thr.start()
        thrList.append(thr)
        # Job Fetcher
        from pandaharvester.harvesterbody.job_fetcher import JobFetcher
        nThr = harvester_config.jobfetcher.nThreads
        for iThr in range(nThr):
            thr = JobFetcher(self.communicatorPool,
                             self.queueConfigMapper,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Propagator
        from pandaharvester.harvesterbody.propagator import Propagator
        nThr = harvester_config.propagator.nThreads
        for iThr in range(nThr):
            thr = Propagator(self.communicatorPool,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Monitor
        from pandaharvester.harvesterbody.monitor import Monitor
        nThr = harvester_config.monitor.nThreads
        for iThr in range(nThr):
            thr = Monitor(self.queueConfigMapper,
                          single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Preparator
        from pandaharvester.harvesterbody.preparator import Preparator
        nThr = harvester_config.preparator.nThreads
        for iThr in range(nThr):
            thr = Preparator(self.communicatorPool,
                             self.queueConfigMapper,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Submitter
        from pandaharvester.harvesterbody.submitter import Submitter
        nThr = harvester_config.submitter.nThreads
        for iThr in range(nThr):
            thr = Submitter(self.queueConfigMapper,
                            single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Stager
        from pandaharvester.harvesterbody.stager import Stager
        nThr = harvester_config.stager.nThreads
        for iThr in range(nThr):
            thr = Stager(self.queueConfigMapper,
                         single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # EventFeeder
        from pandaharvester.harvesterbody.event_feeder import EventFeeder
        nThr = harvester_config.eventfeeder.nThreads
        for iThr in range(nThr):
            thr = EventFeeder(self.communicatorPool,
                              self.queueConfigMapper,
                              single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        ##################
        # loop on stop event to be interruptable since thr.join blocks signal capture in python 2.7
        while True:
            if self.singleMode:
                break
            self.stopEvent.wait(1)
            if self.stopEvent.is_set():
                break
        ##################
        # join
        for thr in thrList:
            thr.join()

# main
if __name__ == "__main__":
    # parse option
    parser = argparse.ArgumentParser()
    parser.add_argument('--pid', action='store', dest='pid', default=None,
                        help='pid filename')
    parser.add_argument('--single', action='store_true', dest='singleMode', default=False,
                        help='use single mode')

    options = parser.parse_args()
    uid = pwd.getpwuid(harvester_config.master.uname).pw_uid
    gid = grp.getgrgid(harvester_config.master.gname).gr_gid
    timeNow = datetime.datetime.utcnow()
    print "{0} Master: INFO    start".format(str(timeNow))

    # make daemon context
    dc = daemon.DaemonContext(stdout=sys.stdout,
                              stderr=sys.stderr,
                              uid=uid,
                              gid=gid,
                              pidfile=daemon.pidfile.PIDLockFile(options.pid))
    with dc:
        # stop event
        stopEvent = threading.Event()

        # signal handlers
        def catch_sigkill(sig, frame):
            os.killpg(os.getpgrp(), signal.SIGKILL)

        def catch_sigterm(sig, frame):
            stopEvent.set()

        # set handler
        signal.signal(signal.SIGINT, catch_sigkill)
        signal.signal(signal.SIGHUP, catch_sigkill)
        signal.signal(signal.SIGTERM, catch_sigterm)
        signal.signal(signal.SIGUSR2, catch_sigterm)

        # start master
        master = Master(single_mode=options.singleMode, stop_event=stopEvent)
        master.start()

    timeNow = datetime.datetime.utcnow()
    print "{0} Master: INFO    terminated".format(str(timeNow))
<commit_msg>backout change<commit_after>import os
import pwd
import grp
import sys
import signal
import daemon.pidfile
import argparse
import datetime
import threading

from pandaharvester.harvesterconfig import harvester_config

# the master class which runs the main process
class Master:
    # constructor
    def __init__(self, single_mode=False, stop_event=None):
        # initialize database and config
        self.singleMode = single_mode
        self.stopEvent = stop_event
        from pandaharvester.harvestercore.communicator_pool import CommunicatorPool
        self.communicatorPool = CommunicatorPool()
        from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
        self.queueConfigMapper = QueueConfigMapper()
        from pandaharvester.harvestercore.db_proxy import DBProxy
        dbProxy = DBProxy()
        dbProxy.make_tables(self.queueConfigMapper)

    # main loop
    def start(self):
        # send first heartbeat
        self.communicatorPool.is_alive({'startTime': datetime.datetime.utcnow()})
        # thread list
        thrList = []
        # Command manager
        from pandaharvester.harvesterbody.command_manager import CommandManager
        thr = CommandManager(self.communicatorPool, single_mode=True)
        thr.set_stop_event(self.stopEvent)
        thr.run()
        thr.set_single_mode(self.singleMode)
        thr.start()
        thrList.append(thr)
        # Cacher
        from pandaharvester.harvesterbody.cacher import Cacher
        thr = Cacher(single_mode=True)
        thr.set_stop_event(self.stopEvent)
        thr.run()
        thr.set_single_mode(self.singleMode)
        thr.start()
        thrList.append(thr)
        # Credential Manager
        from pandaharvester.harvesterbody.cred_manager import CredManager
        thr = CredManager(single_mode=self.singleMode)
        thr.set_stop_event(self.stopEvent)
        thr.execute()
        thr.start()
        thrList.append(thr)
        # Job Fetcher
        from pandaharvester.harvesterbody.job_fetcher import JobFetcher
        nThr = harvester_config.jobfetcher.nThreads
        for iThr in range(nThr):
            thr = JobFetcher(self.communicatorPool,
                             self.queueConfigMapper,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Propagator
        from pandaharvester.harvesterbody.propagator import Propagator
        nThr = harvester_config.propagator.nThreads
        for iThr in range(nThr):
            thr = Propagator(self.communicatorPool,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Monitor
        from pandaharvester.harvesterbody.monitor import Monitor
        nThr = harvester_config.monitor.nThreads
        for iThr in range(nThr):
            thr = Monitor(self.queueConfigMapper,
                          single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Preparator
        from pandaharvester.harvesterbody.preparator import Preparator
        nThr = harvester_config.preparator.nThreads
        for iThr in range(nThr):
            thr = Preparator(self.communicatorPool,
                             self.queueConfigMapper,
                             single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Submitter
        from pandaharvester.harvesterbody.submitter import Submitter
        nThr = harvester_config.submitter.nThreads
        for iThr in range(nThr):
            thr = Submitter(self.queueConfigMapper,
                            single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # Stager
        from pandaharvester.harvesterbody.stager import Stager
        nThr = harvester_config.stager.nThreads
        for iThr in range(nThr):
            thr = Stager(self.queueConfigMapper,
                         single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        # EventFeeder
        from pandaharvester.harvesterbody.event_feeder import EventFeeder
        nThr = harvester_config.eventfeeder.nThreads
        for iThr in range(nThr):
            thr = EventFeeder(self.communicatorPool,
                              self.queueConfigMapper,
                              single_mode=self.singleMode)
            thr.set_stop_event(self.stopEvent)
            thr.start()
            thrList.append(thr)
        ##################
        # loop on stop event to be interruptable since thr.join blocks signal capture in python 2.7
        while True:
            if self.singleMode:
                break
            self.stopEvent.wait(1)
            if self.stopEvent.is_set():
                break
        ##################
        # join
        for thr in thrList:
            thr.join()

# main
if __name__ == "__main__":
    # parse option
    parser = argparse.ArgumentParser()
    parser.add_argument('--pid', action='store', dest='pid', default=None,
                        help='pid filename')
    parser.add_argument('--single', action='store_true', dest='singleMode', default=False,
                        help='use single mode')

    options = parser.parse_args()
    uid = pwd.getpwnam(harvester_config.master.uname).pw_uid
    gid = grp.getgrnam(harvester_config.master.gname).gr_gid
    timeNow = datetime.datetime.utcnow()
    print "{0} Master: INFO    start".format(str(timeNow))

    # make daemon context
    dc = daemon.DaemonContext(stdout=sys.stdout,
                              stderr=sys.stderr,
                              uid=uid,
                              gid=gid,
                              pidfile=daemon.pidfile.PIDLockFile(options.pid))
    with dc:
        # stop event
        stopEvent = threading.Event()

        # signal handlers
        def catch_sigkill(sig, frame):
            os.killpg(os.getpgrp(), signal.SIGKILL)

        def catch_sigterm(sig, frame):
            stopEvent.set()

        # set handler
        signal.signal(signal.SIGINT, catch_sigkill)
        signal.signal(signal.SIGHUP, catch_sigkill)
        signal.signal(signal.SIGTERM, catch_sigterm)
        signal.signal(signal.SIGUSR2, catch_sigterm)

        # start master
        master = Master(single_mode=options.singleMode, stop_event=stopEvent)
        master.start()

    timeNow = datetime.datetime.utcnow()
    print "{0} Master: INFO    terminated".format(str(timeNow))
<|endoftext|>