<commit_before># (C) Datadog, Inc. 2015-2017
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

"""Mesos Slave check

Collects metrics from mesos slave node.
"""
# stdlib
from urlparse import urlparse

# 3rd party
import requests

# project
from checks import AgentCheck, CheckException
from config import _is_affirmative

DEFAULT_MASTER_PORT = 5050

class MesosSlave(AgentCheck):
    GAUGE = AgentCheck.gauge
    MONOTONIC_COUNT = AgentCheck.monotonic_count
    SERVICE_CHECK_NAME = "mesos_slave.can_connect"
    service_check_needed = True

    TASK_STATUS = {
        'TASK_STARTING'     : AgentCheck.OK,
        'TASK_RUNNING'      : AgentCheck.OK,
        'TASK_FINISHED'     : AgentCheck.OK,
        'TASK_FAILED'       : AgentCheck.CRITICAL,
        'TASK_KILLED'       : AgentCheck.WARNING,
        'TASK_LOST'         : AgentCheck.CRITICAL,
        'TASK_STAGING'      : AgentCheck.OK,
        'TASK_ERROR'        : AgentCheck.CRITICAL,
    }

    TASK_METRICS = {
        'cpus'                              : ('mesos.state.task.cpu', GAUGE),
        'mem'                               : ('mesos.state.task.mem', GAUGE),
        'disk'                              : ('mesos.state.task.disk', GAUGE),
    }

    SLAVE_TASKS_METRICS = {
        'slave/tasks_failed'                : ('mesos.slave.tasks_failed', MONOTONIC_COUNT),
        'slave/tasks_finished'              : ('mesos.slave.tasks_finished', MONOTONIC_COUNT),
        'slave/tasks_killed'                : ('mesos.slave.tasks_killed', MONOTONIC_COUNT),
        'slave/tasks_lost'                  : ('mesos.slave.tasks_lost', MONOTONIC_COUNT),
        'slave/tasks_running'               : ('mesos.slave.tasks_running', GAUGE),
        'slave/tasks_staging'               : ('mesos.slave.tasks_staging', GAUGE),
        'slave/tasks_starting'              : ('mesos.slave.tasks_starting', GAUGE),
    }

    SYSTEM_METRICS = {
        'system/cpus_total'                 : ('mesos.stats.system.cpus_total', GAUGE),
        'system/load_15min'                 : ('mesos.stats.system.load_15min', GAUGE),
        'system/load_1min'                  : ('mesos.stats.system.load_1min', GAUGE),
        'system/load_5min'                  : ('mesos.stats.system.load_5min', GAUGE),
        'system/mem_free_bytes'             : ('mesos.stats.system.mem_free_bytes', GAUGE),
        'system/mem_total_bytes'            : ('mesos.stats.system.mem_total_bytes', GAUGE),
        'slave/registered'                  : ('mesos.stats.registered', GAUGE),
        'slave/uptime_secs'                 : ('mesos.stats.uptime_secs', GAUGE),
    }

    SLAVE_RESOURCE_METRICS = {
        'slave/cpus_percent'                : ('mesos.slave.cpus_percent', GAUGE),
        'slave/cpus_total'                  : ('mesos.slave.cpus_total', GAUGE),
        'slave/cpus_used'                   : ('mesos.slave.cpus_used', GAUGE),
        'slave/gpus_percent'                : ('mesos.slave.gpus_percent', GAUGE),
        'slave/gpus_total'                  : ('mesos.slave.gpus_total', GAUGE),
        'slave/gpus_used'                   : ('mesos.slave.gpus_used', GAUGE),
        'slave/disk_percent'                : ('mesos.slave.disk_percent', GAUGE),
        'slave/disk_total'                  : ('mesos.slave.disk_total', GAUGE),
        'slave/disk_used'                   : ('mesos.slave.disk_used', GAUGE),
        'slave/mem_percent'                 : ('mesos.slave.mem_percent', GAUGE),
        'slave/mem_total'                   : ('mesos.slave.mem_total', GAUGE),
        'slave/mem_used'                    : ('mesos.slave.mem_used', GAUGE),
    }

    SLAVE_EXECUTORS_METRICS = {
        'slave/executors_registering'       : ('mesos.slave.executors_registering', GAUGE),
        'slave/executors_running'           : ('mesos.slave.executors_running', GAUGE),
        'slave/executors_terminated'        : ('mesos.slave.executors_terminated', GAUGE),
        'slave/executors_terminating'       : ('mesos.slave.executors_terminating', GAUGE),
    }

    STATS_METRICS = {
        'slave/frameworks_active'           : ('mesos.slave.frameworks_active', GAUGE),
        'slave/invalid_framework_messages'  : ('mesos.slave.invalid_framework_messages', GAUGE),
        'slave/invalid_status_updates'      : ('mesos.slave.invalid_status_updates', GAUGE),
        'slave/recovery_errors'             : ('mesos.slave.recovery_errors', GAUGE),
        'slave/valid_framework_messages'    : ('mesos.slave.valid_framework_messages', GAUGE),
        'slave/valid_status_updates'        : ('mesos.slave.valid_status_updates', GAUGE),
    }

    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self.cluster_name = None
        for instance in instances or []:
            url = instance.get('url', '')
            parsed_url = urlparse(url)
            ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))
            if not ssl_verify and parsed_url.scheme == 'https':
                self.log.warning('Skipping SSL cert validation for %s based on configuration.' % url)

    def _get_json(self, url, timeout, verify):
        tags = ["url:%s" % url]
        msg = None
        status = None
        try:
            r = requests.get(url, timeout=timeout, verify=verify)
            if r.status_code != 200:
                status = AgentCheck.CRITICAL
                msg = "Got %s when hitting %s" % (r.status_code, url)
            else:
                status = AgentCheck.OK
                msg = "Mesos master instance detected at %s " % url
        except requests.exceptions.Timeout as e:
            # If there's a timeout
            msg = "%s seconds timeout when hitting %s" % (timeout, url)
            status = AgentCheck.CRITICAL
        except Exception as e:
            msg = str(e)
            status = AgentCheck.CRITICAL
        finally:
            if self.service_check_needed:
                self.service_check(self.SERVICE_CHECK_NAME, status, tags=tags, message=msg)
                self.service_check_needed = False
            if status is AgentCheck.CRITICAL:
                raise CheckException("Cannot connect to mesos, please check your configuration.")

        if r.encoding is None:
            r.encoding = 'UTF8'

        return r.json()

    def _get_state(self, url, timeout, verify):
        return self._get_json(url + '/state.json', timeout, verify)

    def _get_stats(self, url, timeout, verify):
        if self.version >= [0, 22, 0]:
            endpoint = '/metrics/snapshot'
        else:
            endpoint = '/stats.json'
        return self._get_json(url + endpoint, timeout, verify)

    def _get_constant_attributes(self, url, timeout, master_port, verify):
        state_metrics = None
        parsed_url = urlparse(url)
        if self.cluster_name is None:
            state_metrics = self._get_state(url, timeout, verify)
            if state_metrics is not None:
                self.version = map(int, state_metrics['version'].split('.'))

                if 'master_hostname' in state_metrics:
                    master_state = self._get_state(
                        '{0}://{1}:{2}'.format(parsed_url.scheme,
                                               state_metrics['master_hostname'],
                                               master_port),
                        timeout,
                        verify
                    )
                    if master_state is not None:
                        self.cluster_name = master_state.get('cluster')

        return state_metrics

    def check(self, instance):
        if 'url' not in instance:
            raise Exception('Mesos instance missing "url" value.')

        url = instance['url']
        instance_tags = instance.get('tags', [])
        tasks = instance.get('tasks', [])
        default_timeout = self.init_config.get('default_timeout', 5)
        timeout = float(instance.get('timeout', default_timeout))
        master_port = instance.get("master_port", DEFAULT_MASTER_PORT)
        ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))

        state_metrics = self._get_constant_attributes(url, timeout, master_port, ssl_verify)
        tags = None

        if state_metrics is None:
            state_metrics = self._get_state(url, timeout, ssl_verify)
        if state_metrics:
            tags = [
                'mesos_pid:{0}'.format(state_metrics['pid']),
                'mesos_node:slave',
            ]
            if self.cluster_name:
                tags.append('mesos_cluster:{0}'.format(self.cluster_name))

            tags += instance_tags
            for task in tasks:
                for framework in state_metrics['frameworks']:
                    for executor in framework['executors']:
                        for t in executor['tasks']:
                            if task.lower() in t['name'].lower() and t['slave_id'] == state_metrics['id']:
                                task_tags = ['task_name:' + t['name']] + tags
                                self.service_check(t['name'] + '.ok', self.TASK_STATUS[t['state']], tags=task_tags)
                                for key_name, (metric_name, metric_func) in self.TASK_METRICS.iteritems():
                                    metric_func(self, metric_name, t['resources'][key_name], tags=task_tags)

        stats_metrics = self._get_stats(url, timeout, ssl_verify)
        if stats_metrics:
            tags = tags if tags else instance_tags
            metrics = [self.SLAVE_TASKS_METRICS, self.SYSTEM_METRICS, self.SLAVE_RESOURCE_METRICS,
                      self.SLAVE_EXECUTORS_METRICS, self.STATS_METRICS]
            for m in metrics:
                for key_name, (metric_name, metric_func) in m.iteritems():
                    metric_func(self, metric_name, stats_metrics[key_name], tags=tags)

        self.service_check_needed = True
<commit_msg>Checks if metric exists before submitting<commit_after># (C) Datadog, Inc. 2015-2017
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

"""Mesos Slave check

Collects metrics from mesos slave node.
"""
# stdlib
from urlparse import urlparse

# 3rd party
import requests

# project
from checks import AgentCheck, CheckException
from config import _is_affirmative

DEFAULT_MASTER_PORT = 5050

class MesosSlave(AgentCheck):
    GAUGE = AgentCheck.gauge
    MONOTONIC_COUNT = AgentCheck.monotonic_count
    SERVICE_CHECK_NAME = "mesos_slave.can_connect"
    service_check_needed = True

    TASK_STATUS = {
        'TASK_STARTING'     : AgentCheck.OK,
        'TASK_RUNNING'      : AgentCheck.OK,
        'TASK_FINISHED'     : AgentCheck.OK,
        'TASK_FAILED'       : AgentCheck.CRITICAL,
        'TASK_KILLED'       : AgentCheck.WARNING,
        'TASK_LOST'         : AgentCheck.CRITICAL,
        'TASK_STAGING'      : AgentCheck.OK,
        'TASK_ERROR'        : AgentCheck.CRITICAL,
    }

    TASK_METRICS = {
        'cpus'                              : ('mesos.state.task.cpu', GAUGE),
        'mem'                               : ('mesos.state.task.mem', GAUGE),
        'disk'                              : ('mesos.state.task.disk', GAUGE),
    }

    SLAVE_TASKS_METRICS = {
        'slave/tasks_failed'                : ('mesos.slave.tasks_failed', MONOTONIC_COUNT),
        'slave/tasks_finished'              : ('mesos.slave.tasks_finished', MONOTONIC_COUNT),
        'slave/tasks_killed'                : ('mesos.slave.tasks_killed', MONOTONIC_COUNT),
        'slave/tasks_lost'                  : ('mesos.slave.tasks_lost', MONOTONIC_COUNT),
        'slave/tasks_running'               : ('mesos.slave.tasks_running', GAUGE),
        'slave/tasks_staging'               : ('mesos.slave.tasks_staging', GAUGE),
        'slave/tasks_starting'              : ('mesos.slave.tasks_starting', GAUGE),
    }

    SYSTEM_METRICS = {
        'system/cpus_total'                 : ('mesos.stats.system.cpus_total', GAUGE),
        'system/load_15min'                 : ('mesos.stats.system.load_15min', GAUGE),
        'system/load_1min'                  : ('mesos.stats.system.load_1min', GAUGE),
        'system/load_5min'                  : ('mesos.stats.system.load_5min', GAUGE),
        'system/mem_free_bytes'             : ('mesos.stats.system.mem_free_bytes', GAUGE),
        'system/mem_total_bytes'            : ('mesos.stats.system.mem_total_bytes', GAUGE),
        'slave/registered'                  : ('mesos.stats.registered', GAUGE),
        'slave/uptime_secs'                 : ('mesos.stats.uptime_secs', GAUGE),
    }

    SLAVE_RESOURCE_METRICS = {
        'slave/cpus_percent'                : ('mesos.slave.cpus_percent', GAUGE),
        'slave/cpus_total'                  : ('mesos.slave.cpus_total', GAUGE),
        'slave/cpus_used'                   : ('mesos.slave.cpus_used', GAUGE),
        'slave/gpus_percent'                : ('mesos.slave.gpus_percent', GAUGE),
        'slave/gpus_total'                  : ('mesos.slave.gpus_total', GAUGE),
        'slave/gpus_used'                   : ('mesos.slave.gpus_used', GAUGE),
        'slave/disk_percent'                : ('mesos.slave.disk_percent', GAUGE),
        'slave/disk_total'                  : ('mesos.slave.disk_total', GAUGE),
        'slave/disk_used'                   : ('mesos.slave.disk_used', GAUGE),
        'slave/mem_percent'                 : ('mesos.slave.mem_percent', GAUGE),
        'slave/mem_total'                   : ('mesos.slave.mem_total', GAUGE),
        'slave/mem_used'                    : ('mesos.slave.mem_used', GAUGE),
    }

    SLAVE_EXECUTORS_METRICS = {
        'slave/executors_registering'       : ('mesos.slave.executors_registering', GAUGE),
        'slave/executors_running'           : ('mesos.slave.executors_running', GAUGE),
        'slave/executors_terminated'        : ('mesos.slave.executors_terminated', GAUGE),
        'slave/executors_terminating'       : ('mesos.slave.executors_terminating', GAUGE),
    }

    STATS_METRICS = {
        'slave/frameworks_active'           : ('mesos.slave.frameworks_active', GAUGE),
        'slave/invalid_framework_messages'  : ('mesos.slave.invalid_framework_messages', GAUGE),
        'slave/invalid_status_updates'      : ('mesos.slave.invalid_status_updates', GAUGE),
        'slave/recovery_errors'             : ('mesos.slave.recovery_errors', GAUGE),
        'slave/valid_framework_messages'    : ('mesos.slave.valid_framework_messages', GAUGE),
        'slave/valid_status_updates'        : ('mesos.slave.valid_status_updates', GAUGE),
    }

    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self.cluster_name = None
        for instance in instances or []:
            url = instance.get('url', '')
            parsed_url = urlparse(url)
            ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))
            if not ssl_verify and parsed_url.scheme == 'https':
                self.log.warning('Skipping SSL cert validation for %s based on configuration.' % url)

    def _get_json(self, url, timeout, verify):
        tags = ["url:%s" % url]
        msg = None
        status = None
        try:
            r = requests.get(url, timeout=timeout, verify=verify)
            if r.status_code != 200:
                status = AgentCheck.CRITICAL
                msg = "Got %s when hitting %s" % (r.status_code, url)
            else:
                status = AgentCheck.OK
                msg = "Mesos master instance detected at %s " % url
        except requests.exceptions.Timeout as e:
            # If there's a timeout
            msg = "%s seconds timeout when hitting %s" % (timeout, url)
            status = AgentCheck.CRITICAL
        except Exception as e:
            msg = str(e)
            status = AgentCheck.CRITICAL
        finally:
            if self.service_check_needed:
                self.service_check(self.SERVICE_CHECK_NAME, status, tags=tags, message=msg)
                self.service_check_needed = False
            if status is AgentCheck.CRITICAL:
                raise CheckException("Cannot connect to mesos, please check your configuration.")

        if r.encoding is None:
            r.encoding = 'UTF8'

        return r.json()

    def _get_state(self, url, timeout, verify):
        return self._get_json(url + '/state.json', timeout, verify)

    def _get_stats(self, url, timeout, verify):
        if self.version >= [0, 22, 0]:
            endpoint = '/metrics/snapshot'
        else:
            endpoint = '/stats.json'
        return self._get_json(url + endpoint, timeout, verify)

    def _get_constant_attributes(self, url, timeout, master_port, verify):
        state_metrics = None
        parsed_url = urlparse(url)
        if self.cluster_name is None:
            state_metrics = self._get_state(url, timeout, verify)
            if state_metrics is not None:
                self.version = map(int, state_metrics['version'].split('.'))

                if 'master_hostname' in state_metrics:
                    master_state = self._get_state(
                        '{0}://{1}:{2}'.format(parsed_url.scheme,
                                               state_metrics['master_hostname'],
                                               master_port),
                        timeout,
                        verify
                    )
                    if master_state is not None:
                        self.cluster_name = master_state.get('cluster')

        return state_metrics

    def check(self, instance):
        if 'url' not in instance:
            raise Exception('Mesos instance missing "url" value.')

        url = instance['url']
        instance_tags = instance.get('tags', [])
        tasks = instance.get('tasks', [])
        default_timeout = self.init_config.get('default_timeout', 5)
        timeout = float(instance.get('timeout', default_timeout))
        master_port = instance.get("master_port", DEFAULT_MASTER_PORT)
        ssl_verify = not _is_affirmative(instance.get('disable_ssl_validation', False))

        state_metrics = self._get_constant_attributes(url, timeout, master_port, ssl_verify)
        tags = None

        if state_metrics is None:
            state_metrics = self._get_state(url, timeout, ssl_verify)
        if state_metrics:
            tags = [
                'mesos_pid:{0}'.format(state_metrics['pid']),
                'mesos_node:slave',
            ]
            if self.cluster_name:
                tags.append('mesos_cluster:{0}'.format(self.cluster_name))

            tags += instance_tags
            for task in tasks:
                for framework in state_metrics['frameworks']:
                    for executor in framework['executors']:
                        for t in executor['tasks']:
                            if task.lower() in t['name'].lower() and t['slave_id'] == state_metrics['id']:
                                task_tags = ['task_name:' + t['name']] + tags
                                self.service_check(t['name'] + '.ok', self.TASK_STATUS[t['state']], tags=task_tags)
                                for key_name, (metric_name, metric_func) in self.TASK_METRICS.iteritems():
                                    metric_func(self, metric_name, t['resources'][key_name], tags=task_tags)

        stats_metrics = self._get_stats(url, timeout, ssl_verify)
        if stats_metrics:
            tags = tags if tags else instance_tags
            metrics = [self.SLAVE_TASKS_METRICS, self.SYSTEM_METRICS, self.SLAVE_RESOURCE_METRICS,
                      self.SLAVE_EXECUTORS_METRICS, self.STATS_METRICS]
            for m in metrics:
                for key_name, (metric_name, metric_func) in m.iteritems():
                    if key_name in stats_metrics:
                        metric_func(self, metric_name, stats_metrics[key_name], tags=tags)

        self.service_check_needed = True
<|endoftext|>


<commit_before># *****************************************************************************
# Copyright (c) 2015 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
#   Amit M Mangalvedkar - Initial Contribution
# *****************************************************************************

import json
import sys
import time
from pip._vendor.distlib.compat import raw_input

try:
	import ibmiotf
	import ibmiotf.application
except ImportError:
	# This part is only required to run the sample from within the samples
	# directory when the module itself is not installed.
	#
	# If you have the module installed, just use "import ibmiotf"
	import os
	import inspect
	cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"../../src")))
	if cmd_subfolder not in sys.path:
		sys.path.insert(0, cmd_subfolder)
	import ibmiotf.application

# Initialize the application client.
try:
	apiOptions = {"org": "uguhsp", "id": "myapp", "auth-method": "apikey", "auth-key": "SOME KEY", "auth-token": "SOME TOKEN"}
	apiCli = ibmiotf.api.ApiClient(apiOptions)
	deviceInfo1 = {"serialNumber": "100087", "manufacturer": "ACME Co.", "model": "7865", "deviceClass": "A", "description": "My shiny device", "fwVersion": "1.0.0", "hwVersion": "1.0", "descriptiveLocation": "Office 5, D Block"}
	metadata1 = {"customField1": "customValue1", "customField2": "customValue2"}
	deviceTypeId = "iotsample-arduino"

	deviceId = "200020002000"
	authToken = "password"
	metadata2 = {"customField1": "customValue3", "customField2": "customValue4"}
	deviceInfo = {"serialNumber": "001", "manufacturer": "Blueberry", "model": "e2", "deviceClass": "A", "descriptiveLocation" : "Bangalore", "fwVersion" : "1.0.1", "hwVersion" : "12.01"}
	location = {"longitude" : "12.78", "latitude" : "45.90", "elevation" : "2000", "accuracy" : "0", "measuredDateTime" : "2015-10-28T08:45:11.662Z"}

	print("Registering a new device")
	print("Registered Device = ", apiCli.registerDevice(deviceTypeId, deviceId, authToken, deviceInfo, location, metadata2))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id whose status needs to be found out for a device of a device type= ")
	print("Retrieving the single device status")
	print("Status retrieved = ", apiCli.getDeviceManagementRequestStatusByDevice(requestId, deviceTypeId, deviceId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id whose status needs to be found out = ")
	print("Retrieving the status")
	print("Status retrieved = ", apiCli.getDeviceManagementRequestStatus(requestId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id to be deleted = ")
	print("Retrieving the request id")
	print("Request id retrieved = ", apiCli.getDeviceManagementRequest(requestId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id whose status needs to be found out = ")
	print("Retrieving the status")
	print("Status retrieved = ", apiCli.getAllDeviceManagementRequestStatus(requestId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id to be deleted = ")
	print("Removing the request id")
	print("Request id removed = ", apiCli.deleteDeviceManagementRequest(requestId))
	time.sleep(1)

	print("\nInitiating device management requests")
	mgmtRequest = {"action": "device/reboot", "parameters": [{"name": "string","value": "string" }], "devices": [{ "typeId": deviceTypeId, "deviceId": deviceId }]}
	print("Device Management Requests = ", apiCli.initiateDeviceManagementRequest(mgmtRequest))
	time.sleep(1)

	print("\nRetrieving device management requests")
	print("Device Management Requests = ", apiCli.getAllDeviceManagementRequests())
	time.sleep(1)

	print("\nDeleting an existing device")
	deleted = apiCli.deleteDevice(deviceTypeId, deviceId)
	print("Device deleted = ", deleted)

except ibmiotf.IoTFCReSTException as e:
	print(e.httpCode)
	print(str(e))
	sys.exit()
<commit_msg>Removed an incorrect method call from deviceManagement examples<commit_after># *****************************************************************************
# Copyright (c) 2015 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
#   Amit M Mangalvedkar - Initial Contribution
# *****************************************************************************

import json
import sys
import time
from pip._vendor.distlib.compat import raw_input

try:
	import ibmiotf
	import ibmiotf.application
except ImportError:
	# This part is only required to run the sample from within the samples
	# directory when the module itself is not installed.
	#
	# If you have the module installed, just use "import ibmiotf"
	import os
	import inspect
	cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"../../src")))
	if cmd_subfolder not in sys.path:
		sys.path.insert(0, cmd_subfolder)
	import ibmiotf.application

# Initialize the application client.
try:
	apiOptions = {"org": "uguhsp", "id": "myapp", "auth-method": "apikey", "auth-key": "SOME KEY", "auth-token": "SOME TOKEN"}
	apiCli = ibmiotf.api.ApiClient(apiOptions)
	deviceInfo1 = {"serialNumber": "100087", "manufacturer": "ACME Co.", "model": "7865", "deviceClass": "A", "description": "My shiny device", "fwVersion": "1.0.0", "hwVersion": "1.0", "descriptiveLocation": "Office 5, D Block"}
	metadata1 = {"customField1": "customValue1", "customField2": "customValue2"}
	deviceTypeId = "iotsample-arduino"

	deviceId = "200020002000"
	authToken = "password"
	metadata2 = {"customField1": "customValue3", "customField2": "customValue4"}
	deviceInfo = {"serialNumber": "001", "manufacturer": "Blueberry", "model": "e2", "deviceClass": "A", "descriptiveLocation" : "Bangalore", "fwVersion" : "1.0.1", "hwVersion" : "12.01"}
	location = {"longitude" : "12.78", "latitude" : "45.90", "elevation" : "2000", "accuracy" : "0", "measuredDateTime" : "2015-10-28T08:45:11.662Z"}

	print("Registering a new device")
	print("Registered Device = ", apiCli.registerDevice(deviceTypeId, deviceId, authToken, deviceInfo, location, metadata2))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id whose status needs to be found out for a device of a device type= ")
	print("Retrieving the single device status")
	print("Status retrieved = ", apiCli.getDeviceManagementRequestStatusByDevice(requestId, deviceTypeId, deviceId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id whose status needs to be found out = ")
	print("Retrieving the status")
	print("Status retrieved = ", apiCli.getDeviceManagementRequestStatus(requestId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id to be deleted = ")
	print("Retrieving the request id")
	print("Request id retrieved = ", apiCli.getDeviceManagementRequest(requestId))
	time.sleep(1)

	requestId = raw_input("\nEnter the request Id to be deleted = ")
	print("Removing the request id")
	print("Request id removed = ", apiCli.deleteDeviceManagementRequest(requestId))
	time.sleep(1)

	print("\nInitiating device management requests")
	mgmtRequest = {"action": "device/reboot", "parameters": [{"name": "string","value": "string" }], "devices": [{ "typeId": deviceTypeId, "deviceId": deviceId }]}
	print("Device Management Requests = ", apiCli.initiateDeviceManagementRequest(mgmtRequest))
	time.sleep(1)

	print("\nRetrieving device management requests")
	print("Device Management Requests = ", apiCli.getAllDeviceManagementRequests())
	time.sleep(1)

	print("\nDeleting an existing device")
	deleted = apiCli.deleteDevice(deviceTypeId, deviceId)
	print("Device deleted = ", deleted)

except ibmiotf.IoTFCReSTException as e:
	print(e.httpCode)
	print(str(e))
	sys.exit()
<|endoftext|>


<commit_before>import argparse
import os

from obographs import Graph, Node
from wikidataintegrator import wdi_login
from scheduled_bots import PROPS

class DONode(Node):
    def set_aliases(self, wd_item):
        # filter out aliases containing these strings
        bad_things = {'(disorder)', '[obs]', '(finding)', 'subac.', 'morphologic abnormality', '(qualifier value)', '[dup]'}
        if self.synonyms is not None:
            aliases = [x for x in self.synonyms if all(y not in x for y in bad_things)]
            wd_item.set_aliases(aliases=aliases, append=True)

    def _pre_create(self):
        # remove SNOMEDCT_US_2016_03_01 xrefs
        prefixes = {'SNOMED', 'EFO:', 'KEGG:', 'MEDDRA:', 'CSP:', 'URL:'}
        self.xrefs = set(x for x in self.xrefs if all(not x.lower().startswith(prefix.lower()) for prefix in prefixes))

class DOGraph(Graph):
    NAME = "Disease Ontology"
    GRAPH_URI = 'http://purl.obolibrary.org/obo/doid.owl'
    QID = "Q5282129"
    DEFAULT_DESCRIPTION = "human disease"
    APPEND_PROPS = {PROPS['subclass of'], PROPS['instance of'],
                    PROPS['has cause'], PROPS['anatomical location'],
                    PROPS['has phenotype'], PROPS['pathogen transmission process'],
                    PROPS['OMIM ID'], PROPS['Orphanet ID'],
                    PROPS['MeSH ID'], PROPS['ICD-10-CM'],
                    PROPS['ICD-10'], PROPS['ICD-9-CM'],
                    PROPS['ICD-9'], PROPS['NCI Thesaurus ID'],
                    PROPS['UMLS CUI'], PROPS['Disease Ontology ID'],
                    PROPS['GARD rare disease ID'], PROPS['Human Phenotype Ontology ID']}
    FAST_RUN = True

    PRED_PID_MAP = {'http://purl.obolibrary.org/obo/RO_0001025': PROPS['anatomical location'],#anatomical location doesn't have a RO#. This RO# is 'located_in'
                    'http://purl.obolibrary.org/obo/RO_0002200': PROPS['has phenotype'],  # has phenotype
                    # 'http://purl.obolibrary.org/obo/IDO_0000664': PROPS[],  # has_material_basis_in
                    # 'http://purl.obolibrary.org/obo/RO_0003304': PROPS[],  # contributes to condition
                    'http://purl.obolibrary.org/obo/RO_0002451': PROPS['pathogen transmission process'],  # transmitted by
                    # 'http://purl.obolibrary.org/obo/RO_0001020': PROPS[],  # is allergic trigger for
                    'is_a': PROPS['subclass of']}

    NODE_CLASS = DONode

    def filter_nodes(self):
        super(DOGraph, self).filter_nodes()
        self.nodes = [x for x in self.nodes if "DOID:" in x.id_curie]

    def _post_parse_graph(self):
        # we need to get edges out of the logicalDefinitionAxioms, and im not sure how this will generalize ...
        # also, this is not going to work on, for example: http://purl.obolibrary.org/obo/DOID_5520
        # which has multiple values. will have to do with sparql at some point # todo

        # location
        edges_location = []
        for axiom in self.json_graph['logicalDefinitionAxioms']:
            for restriction in axiom['restrictions']:
                if restriction and (restriction['propertyId'] == "http://purl.obolibrary.org/obo/RO_0001025"):
                    edges_location.append({'sub': axiom['definedClassId'],
                                           'pred': restriction['propertyId'],
                                           'obj': restriction['fillerId']})
        self.edges.extend(edges_location)
        # logical definition axioms are not all encompassing. A lot of information is stored in 'edges'.

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='run wikidata disease ontology bot')
    parser.add_argument("json_path", help="Path to json file")
    parser.add_argument("--local", help="preconfigured local wikibase port 7171 and 7272", action='store_true')
    args = parser.parse_args()

    if args.local:
        mediawiki_api_url = "http://localhost:7171/w/api.php"
        sparql_endpoint_url = "http://localhost:7272/proxy/wdqs/bigdata/namespace/wdq/sparql"
        login = wdi_login.WDLogin("testbot", "password", mediawiki_api_url=mediawiki_api_url)
    else:
        try:
            from scheduled_bots.local import WDUSER, WDPASS
        except ImportError:
            if "WDUSER" in os.environ and "WDPASS" in os.environ:
                WDUSER = os.environ['WDUSER']
                WDPASS = os.environ['WDPASS']
            else:
                raise ValueError("WDUSER and WDPASS must be specified in local.py or as environment variables")

        mediawiki_api_url = 'https://www.wikidata.org/w/api.php'
        sparql_endpoint_url = 'https://query.wikidata.org/sparql'
        login = wdi_login.WDLogin(WDUSER, WDPASS)

print("Done running obographs_DO")

g = DOGraph(args.json_path, mediawiki_api_url=mediawiki_api_url, sparql_endpoint_url=sparql_endpoint_url)
g.run(login)<commit_msg>Added back scheduled_bots.ontology. to line 4<commit_after>import argparse
import os

from scheduled_bots.ontology.obographs import Graph, Node
from wikidataintegrator import wdi_login
from scheduled_bots import PROPS

class DONode(Node):
    def set_aliases(self, wd_item):
        # filter out aliases containing these strings
        bad_things = {'(disorder)', '[obs]', '(finding)', 'subac.', 'morphologic abnormality', '(qualifier value)', '[dup]'}
        if self.synonyms is not None:
            aliases = [x for x in self.synonyms if all(y not in x for y in bad_things)]
            wd_item.set_aliases(aliases=aliases, append=True)

    def _pre_create(self):
        # remove SNOMEDCT_US_2016_03_01 xrefs
        prefixes = {'SNOMED', 'EFO:', 'KEGG:', 'MEDDRA:', 'CSP:', 'URL:'}
        self.xrefs = set(x for x in self.xrefs if all(not x.lower().startswith(prefix.lower()) for prefix in prefixes))

class DOGraph(Graph):
    NAME = "Disease Ontology"
    GRAPH_URI = 'http://purl.obolibrary.org/obo/doid.owl'
    QID = "Q5282129"
    DEFAULT_DESCRIPTION = "human disease"
    APPEND_PROPS = {PROPS['subclass of'], PROPS['instance of'],
                    PROPS['has cause'], PROPS['anatomical location'],
                    PROPS['has phenotype'], PROPS['pathogen transmission process'],
                    PROPS['OMIM ID'], PROPS['Orphanet ID'],
                    PROPS['MeSH ID'], PROPS['ICD-10-CM'],
                    PROPS['ICD-10'], PROPS['ICD-9-CM'],
                    PROPS['ICD-9'], PROPS['NCI Thesaurus ID'],
                    PROPS['UMLS CUI'], PROPS['Disease Ontology ID'],
                    PROPS['GARD rare disease ID'], PROPS['Human Phenotype Ontology ID']}
    FAST_RUN = True

    PRED_PID_MAP = {'http://purl.obolibrary.org/obo/RO_0001025': PROPS['anatomical location'],#anatomical location doesn't have a RO#. This RO# is 'located_in'
                    'http://purl.obolibrary.org/obo/RO_0002200': PROPS['has phenotype'],  # has phenotype
                    # 'http://purl.obolibrary.org/obo/IDO_0000664': PROPS[],  # has_material_basis_in
                    # 'http://purl.obolibrary.org/obo/RO_0003304': PROPS[],  # contributes to condition
                    'http://purl.obolibrary.org/obo/RO_0002451': PROPS['pathogen transmission process'],  # transmitted by
                    # 'http://purl.obolibrary.org/obo/RO_0001020': PROPS[],  # is allergic trigger for
                    'is_a': PROPS['subclass of']}

    NODE_CLASS = DONode

    def filter_nodes(self):
        super(DOGraph, self).filter_nodes()
        self.nodes = [x for x in self.nodes if "DOID:" in x.id_curie]

    def _post_parse_graph(self):
        # we need to get edges out of the logicalDefinitionAxioms, and im not sure how this will generalize ...
        # also, this is not going to work on, for example: http://purl.obolibrary.org/obo/DOID_5520
        # which has multiple values. will have to do with sparql at some point # todo

        # location
        edges_location = []
        for axiom in self.json_graph['logicalDefinitionAxioms']:
            for restriction in axiom['restrictions']:
                if restriction and (restriction['propertyId'] == "http://purl.obolibrary.org/obo/RO_0001025"):
                    edges_location.append({'sub': axiom['definedClassId'],
                                           'pred': restriction['propertyId'],
                                           'obj': restriction['fillerId']})
        self.edges.extend(edges_location)
        # logical definition axioms are not all encompassing. A lot of information is stored in 'edges'.

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='run wikidata disease ontology bot')
    parser.add_argument("json_path", help="Path to json file")
    parser.add_argument("--local", help="preconfigured local wikibase port 7171 and 7272", action='store_true')
    args = parser.parse_args()

    if args.local:
        mediawiki_api_url = "http://localhost:7171/w/api.php"
        sparql_endpoint_url = "http://localhost:7272/proxy/wdqs/bigdata/namespace/wdq/sparql"
        login = wdi_login.WDLogin("testbot", "password", mediawiki_api_url=mediawiki_api_url)
    else:
        try:
            from scheduled_bots.local import WDUSER, WDPASS
        except ImportError:
            if "WDUSER" in os.environ and "WDPASS" in os.environ:
                WDUSER = os.environ['WDUSER']
                WDPASS = os.environ['WDPASS']
            else:
                raise ValueError("WDUSER and WDPASS must be specified in local.py or as environment variables")

        mediawiki_api_url = 'https://www.wikidata.org/w/api.php'
        sparql_endpoint_url = 'https://query.wikidata.org/sparql'
        login = wdi_login.WDLogin(WDUSER, WDPASS)

print("Done running obographs_DO")

g = DOGraph(args.json_path, mediawiki_api_url=mediawiki_api_url, sparql_endpoint_url=sparql_endpoint_url)
g.run(login)
<|endoftext|>


<commit_before>"""
=======
Entropy
=======

In information theory, information entropy is the log-base-2 of the number of
possible outcomes for a message.

For an image, local entropy is related to the complexity contained in a given
neighborhood, typically defined by a structuring element. A large number of
various gray levels has a higher entropy than an homogeneous neighborhood.

Entropy filter can detect subtle variations of local gray level distribution,
in the example, the image is composed of two surfaces with two slightly
different distributions.

Image center has a random distribution in the range [-14,+14] centered on 128,
while the borders has a random distribution in the range [-15,+15] centered
on 128.

We apply the local entropy measure using a circular structuring element of
radius 10. As a result, one can detect the central square. Radius should be big
enough to efficiently sample the local gray level distribution.

In the second example, the local entropy is used to detect image texture.

"""
import matplotlib.pyplot as plt
import numpy as np

from skimage import data
from skimage.util import img_as_ubyte
from skimage.filters.rank import entropy
from skimage.morphology import disk

noise_mask = 28 * np.ones((128, 128), dtype=np.uint8)
noise_mask[32:-32, 32:-32] = 30

noise = (noise_mask * np.random.random(noise_mask.shape) - .5 *
         noise_mask).astype(np.uint8)
img = noise + 128

radius = 10
e = entropy(img, disk(radius))

plt.figure(figsize=[15, 5])
plt.subplot(1, 3, 1)
plt.imshow(noise_mask, cmap=plt.cm.gray)
plt.xlabel('noise mask')
plt.colorbar()
plt.subplot(1, 3, 2)
plt.imshow(img, cmap=plt.cm.gray)
plt.xlabel('noised image')
plt.colorbar()
plt.subplot(1, 3, 3)
plt.imshow(e)
plt.xlabel('image local entropy ($r=%d$)' % radius)
plt.colorbar()

# second example: texture detection

image = img_as_ubyte(data.camera())

fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(10, 4))

img0 = ax0.imshow(image, cmap=plt.cm.gray)
ax0.set_title('Image')
ax0.axis('off')
fig.colorbar(img0, ax=ax0)

img1 = ax1.imshow(entropy(image, disk(5)), cmap=plt.cm.jet)
ax1.set_title('Entropy')
ax1.axis('off')
fig.colorbar(img1, ax=ax1)

plt.show()
<commit_msg>fix entropy example doc, fix figure syntax<commit_after>"""
=======
Entropy
=======

In information theory, information entropy is the log-base-2 of the number of
possible outcomes for a message.

For an image, local entropy is related to the complexity contained in a given
neighborhood, typically defined by a structuring element. A large number of
various gray levels has a higher entropy than an homogeneous neighborhood.

The entropy filter can detect subtle variations of local gray level distribution.
In the example, the image is composed of two surfaces with two slightly
different distributions.

Image has a uniform random distribution in the range [-14, +14] in the middle of the
image and a uniform random distribution in the range [-15, 15] at
the image borders, both centered at a gray value of 128.

We apply the local entropy measure using a circular structuring element of
radius 10. As a result, one can detect the central square. The radius is
big enough to efficiently sample the local gray level distribution.

In the second example, the local entropy is used to detect image texture.

"""
import matplotlib.pyplot as plt
import numpy as np

from skimage import data
from skimage.util import img_as_ubyte
from skimage.filters.rank import entropy
from skimage.morphology import disk

noise_mask = 28 * np.ones((128, 128), dtype=np.uint8)
noise_mask[32:-32, 32:-32] = 30

noise = (noise_mask * np.random.random(noise_mask.shape) - .5 *
         noise_mask).astype(np.uint8)
img = noise + 128

radius = 10
e = entropy(img, disk(radius))

fig, ax = plt.subplots(1, 3, figsize=(8, 5))
ax1, ax2, ax3 = ax.ravel()

ax1.imshow(noise_mask, cmap=plt.cm.gray)
ax1.set_xlabel('Noise mask')
ax2.imshow(img, cmap=plt.cm.gray)
ax2.set_xlabel('Noised image')
ax3.imshow(e)
ax3.set_xlabel('Local entropy ($r=%d$)' % radius)

# second example: texture detection

image = img_as_ubyte(data.camera())

fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(10, 4))

img0 = ax0.imshow(image, cmap=plt.cm.gray)
ax0.set_title('Image')
ax0.axis('off')
fig.colorbar(img0, ax=ax0)

img1 = ax1.imshow(entropy(image, disk(5)), cmap=plt.cm.jet)
ax1.set_title('Entropy')
ax1.axis('off')
fig.colorbar(img1, ax=ax1)

plt.show()
<|endoftext|>


<commit_before>"""
Django settings for scotland_yard project.

Generated by 'django-admin startproject' using Django 1.9.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import dj_database_url

DEBUG = True

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = os.environ.get('SECRET_KEY')

CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True
CORS_EXPOSE_HEADERS = ("authToken")

ALLOWED_HOSTS = []

INSTALLED_APPS = [
    'syard_main.apps.SyardMainConfig',
    'syard_api.apps.SyardApiConfig',
    'corsheaders',
    'rest_framework',
    'rest_framework.authtoken',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = [
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'scotland_yard.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'scotland_yard.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get('DJANGO_DATABASE_URL'))
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, "scotland_yard", "static")
<commit_msg>made tuple<commit_after>"""
Django settings for scotland_yard project.

Generated by 'django-admin startproject' using Django 1.9.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import dj_database_url

DEBUG = True

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = os.environ.get('SECRET_KEY')

CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True
CORS_EXPOSE_HEADERS = ("authToken",)

ALLOWED_HOSTS = []

INSTALLED_APPS = [
    'syard_main.apps.SyardMainConfig',
    'syard_api.apps.SyardApiConfig',
    'corsheaders',
    'rest_framework',
    'rest_framework.authtoken',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = [
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'scotland_yard.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'scotland_yard.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get('DJANGO_DATABASE_URL'))
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, "scotland_yard", "static")
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import numpy as np
import sep
import sys
from operator import add

from pyspark import SparkContext

if __name__ == "__main__":
	sc = SparkContext(appName="SourceExtractor")
	rdd = sc.fitsData("/Users/zhaozhang/projects/SDSS/data")
	#rdd = sc.fitsData("/Users/zhaozhang/projects/Montage/m101/rawdir")
	#rdd = sc.fitsData("/Users/zhaozhang/projects/Kira/scratch/spark-ec2/data/")
	catalog = rdd.map(lambda (key, fits): (key, extract(np.copy(fits))))
	catalog.saveAsTextFile("temp-output")

	sc.stop()

def extract(data):
	bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3)
	bkg.subfrom(data)
	objs = sep.extract(data, 1.5*bkg.globalrms)
	flux, fluxerr, flag = sep.sum_circle(data, objs['x'], objs['y'], 5.,
                                         err=bkg.globalrms)
	kr, flag = sep.kron_radius(data, objs['x'], objs['y'], objs['a'],
                               			objs['b'], objs['theta'], 6.0)
	eflux, efluxerr, eflag = sep.sum_ellipse(data, objs['x'], objs['y'],
                                          objs['a'], objs['b'],
                                          objs['theta'], r=2.5 * kr,
                                          err=bkg.globalrms, subpix=1)
	retstr = ""
	for i in range(len(objs['x'])):
		retstr = retstr+(str(objs['x'][i])+"\t"+str(objs['y'][i])+"\t"+str(flux[i])+"\t"+str(fluxerr[i])+"\t"+str(kr[i])+"\t"+str(eflux[i])+"\t"+str(efluxerr[i])+"\t"+str(flag[i])+"\n")
	return retstr
<commit_msg>updates with source extractor<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import numpy as np
import sep
import sys
from operator import add

from pyspark import SparkContext

def extract(data):
	bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3)
	bkg.subfrom(data)
	objs = sep.extract(data, 1.5*bkg.globalrms)
	flux, fluxerr, flag = sep.sum_circle(data, objs['x'], objs['y'], 5.,
                                         err=bkg.globalrms)
	kr, flag = sep.kron_radius(data, objs['x'], objs['y'], objs['a'],
                               			objs['b'], objs['theta'], 6.0)
	eflux, efluxerr, eflag = sep.sum_ellipse(data, objs['x'], objs['y'],
                                          objs['a'], objs['b'],
                                          objs['theta'], r=2.5 * kr,
                                          err=bkg.globalrms, subpix=1)
	retstr = ""
	for i in range(len(objs['x'])):
		retstr = retstr+(str(objs['x'][i])+"\t"+str(objs['y'][i])+"\t"+str(flux[i])+"\t"+str(fluxerr[i])+"\t"+str(kr[i])+"\t"+str(eflux[i])+"\t"+str(efluxerr[i])+"\t"+str(flag[i])+"\n")
	return retstr

if __name__ == "__main__":
	sc = SparkContext(appName="SourceExtractor")
	rdd = sc.fitsData("/Users/zhaozhang/projects/SDSS/data")
	#rdd = sc.fitsData("/Users/zhaozhang/projects/Montage/m101/rawdir")
	#rdd = sc.fitsData("/Users/zhaozhang/projects/Kira/scratch/spark-ec2/data/")
	catalog = rdd.map(lambda (key, fits): (key, extract(np.copy(fits))))
	catalog.saveAsTextFile("temp-output")

	sc.stop()
<|endoftext|>


<commit_before>
from collections import namedtuple
from nac.workflows.initialization import split_trajectory
from os.path import join
from qmworks import Settings
from qmworks.utils import settings2Dict

import getpass
import os
import shutil
import string
import subprocess

SLURM = namedtuple("SLURM", ("nodes", "tasks", "time", "name"))

def main():
    """
    THE USER MUST CHANGES THESE VARIABLES ACCORDING TO HER/HIS NEEDS:
      * project_name
      * path to the basis and Cp2k Potential
      * CP2K:
          - Range of Molecular oribtals printed by CP2K
          - Cell parameter
      * Settings to Run Cp2k simulations
      * Path to the trajectory in XYZ

    The slurm configuration is optional but the user can edit it:
        property  default
       * nodes         2
       * tasks        24
       * time   48:00:00
       * name       namd

    """
    # USER DEFINED CONFIGURATION
    project_name = 'distribute_Cd33Se33'  # name use to create folders

    # Path to the basis set used by Cp2k
    home = os.path.expanduser('~')
    basisCP2K = join(home, "Cp2k/cp2k_basis/BASIS_MOLOPT")
    potCP2K = join(home, "Cp2k/cp2k_basis/GTH_POTENTIALS")
    lower_orbital, upper_orbital = 278, 317
    cp2k_main, cp2k_guess = cp2k_input(lower_orbital, upper_orbital,
                                       cell_parameters=28)

    # Trajectory splitting
    path_to_trajectory = "traj1000.xyz"
    blocks = 5  # Number of chunks to split the trajectory

    # SLURM Configuration
    slurm = SLURM(
        nodes=2,
        tasks=24,
        time="48:00:00",
        name="namd"
    )

    distribute_computations(project_name, basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, path_to_trajectory, blocks, slurm)

def cp2k_input(lower_orbital, upper_orbital, cell_parameters=None):
    """
    # create ``Settings`` for the Cp2K Jobs.
    """
    # Main Cp2k Jobs
    cp2k_args = Settings()
    cp2k_args.basis = "DZVP-MOLOPT-SR-GTH"
    cp2k_args.potential = "GTH-PBE"
    cp2k_args.cell_parameters = [cell_parameters] * 3
    main_dft = cp2k_args.specific.cp2k.force_eval.dft
    main_dft.scf.added_mos = 20
    main_dft.scf.max_scf = 200
    main_dft.scf.eps_scf = 1e-5
    main_dft['print']['mo']['mo_index_range'] = "{} {}".format(lower_orbital,
                                                               upper_orbital)
    cp2k_args.specific.cp2k.force_eval.subsys.cell.periodic = 'None'

    # Setting to calculate the wave function used as guess
    cp2k_OT = Settings()
    cp2k_OT.basis = "DZVP-MOLOPT-SR-GTH"
    cp2k_OT.potential = "GTH-PBE"
    cp2k_OT.cell_parameters = [cell_parameters] * 3
    ot_dft = cp2k_OT.specific.cp2k.force_eval.dft
    ot_dft.scf.scf_guess = 'atomic'
    ot_dft.scf.ot.minimizer = 'DIIS'
    ot_dft.scf.ot.n_diis = 7
    ot_dft.scf.ot.preconditioner = 'FULL_SINGLE_INVERSE'
    ot_dft.scf.added_mos = 0
    ot_dft.scf.eps_scf = 1e-05
    ot_dft.scf.scf_guess = 'restart'
    cp2k_OT.specific.cp2k.force_eval.subsys.cell.periodic = 'None'

    return cp2k_args, cp2k_OT

# ============================> Distribution <=================================

def distribute_computations(project_name, basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, path_to_trajectory, blocks, slurm):

    script_name = "script_remote_function.py"
    # Split the trajectory in Chunks and move each chunk to its corresponding
    # directory.
    chunks_trajectory = split_trajectory(path_to_trajectory, blocks, '.')
    chunks_trajectory.sort()
    enumerate_from = 0
    for  file_xyz, l in zip(chunks_trajectory, string.ascii_lowercase):
        folder = 'chunk_{}'.format(l)
        os.mkdir(folder)
        shutil.move(file_xyz, folder)
        # function to be execute remotely
        write_python_script(folder, file_xyz, project_name,
                            basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, enumerate_from, script_name, project_name)
        write_slurm_script(folder, slurm, script_name)
        enumerate_from += number_of_geometries(join(folder, file_xyz))

def write_python_script(folder, file_xyz, project_name, basisCP2K, potCP2K, cp2k_main,
                        cp2k_guess, enumerate_from, script_name):
    """ Write the python script to compute the PYXAID hamiltonians"""
    scratch = '/scratch-shared'
    user = getpass.getuser()
    path_hdf5 = join(scratch, user, project_name, '{}.hdf5'.format(folder))
    path = join(scratch, user, project_name)
    if not os.path.exists(path):
        os.makedirs(path)
    nCouplings = 40
    xs = """
from nac.workflows.workflow_coupling import generate_pyxaid_hamiltonians
from nac.workflows.initialization import initialize
from qmworks.utils import dict2Setting
import plams

plams.init()

project_name = '{}'
path_basis = '{}'
path_potential = '{}'
path_hdf5 = '{}'
path_traj_xyz = '{}'
basisname = '{}'

initial_config = initialize(project_name, path_traj_xyz,
                            basisname=basisname,
                            path_basis=path_basis,
                            path_potential=path_potential,
                            enumerate_from={},
                            calculate_guesses='first',
                            path_hdf5=path_hdf5)

cp2k_main = dict2Setting({})
cp2k_guess = dict2Setting({})

generate_pyxaid_hamiltonians('cp2k', project_name, cp2k_main,
                             guess_args=cp2k_guess, path='{}',
                             nCouplings={},
                             **initial_config)
plams.finish()
 """.format(project_name, basisCP2K, potCP2K, path_hdf5, file_xyz, cp2k_main.basis,
            enumerate_from, settings2Dict(cp2k_main), settings2Dict(cp2k_guess),
            path, nCouplings)

    with open(join(folder, script_name), 'w') as f:
        f.write(xs)

    return script_name

def write_slurm_script(folder, slurm, python_script):
    """
    write an Slurm launch script
    """
    sbatch = lambda x, y: "#SBATCH -{} {}\n".format(x, y)

    header = "#! /bin/bash\n"
    modules = "\nmodule load cp2k/3.0\nsource activate qmworks\n\n"
    time = sbatch('t', slurm.time)
    nodes = sbatch('N', slurm.nodes)
    tasks = sbatch('n', slurm.tasks)
    name = sbatch('J', slurm.name)
    python = "python {}".format(python_script)

    # Script content
    content = header + time + nodes + tasks + name + modules + python

    file_name = join(folder, "launch.sh")

    with open(file_name, 'w') as f:
        f.write(content)
    return file_name

def number_of_geometries(file_name):
    """
    Count the number of geometries in XYZ formant in a given file.
    """

    with open(file_name, 'r') as f:
        numat = int(f.readline())

    cmd = "wc -l {}".format(file_name)
    wc = subprocess.getoutput(cmd).split()[0]

    lines_per_geometry = numat + 2

    return int(wc) // lines_per_geometry

if __name__ == "__main__":
    main()
<commit_msg>fixed bug in distribute jobs<commit_after>
from collections import namedtuple
from nac.workflows.initialization import split_trajectory
from os.path import join
from qmworks import Settings
from qmworks.utils import settings2Dict

import getpass
import os
import shutil
import string
import subprocess

SLURM = namedtuple("SLURM", ("nodes", "tasks", "time", "name"))

def main():
    """
    THE USER MUST CHANGES THESE VARIABLES ACCORDING TO HER/HIS NEEDS:
      * project_name
      * path to the basis and Cp2k Potential
      * CP2K:
          - Range of Molecular oribtals printed by CP2K
          - Cell parameter
      * Settings to Run Cp2k simulations
      * Path to the trajectory in XYZ

    The slurm configuration is optional but the user can edit it:
        property  default
       * nodes         2
       * tasks        24
       * time   48:00:00
       * name       namd

    """
    # USER DEFINED CONFIGURATION
    project_name = 'distribute_Cd33Se33'  # name use to create folders

    # Path to the basis set used by Cp2k
    home = os.path.expanduser('~')
    basisCP2K = join(home, "Cp2k/cp2k_basis/BASIS_MOLOPT")
    potCP2K = join(home, "Cp2k/cp2k_basis/GTH_POTENTIALS")
    lower_orbital, upper_orbital = 278, 317
    cp2k_main, cp2k_guess = cp2k_input(lower_orbital, upper_orbital,
                                       cell_parameters=28)

    # Trajectory splitting
    path_to_trajectory = "traj1000.xyz"
    blocks = 5  # Number of chunks to split the trajectory

    # SLURM Configuration
    slurm = SLURM(
        nodes=2,
        tasks=24,
        time="48:00:00",
        name="namd"
    )

    distribute_computations(project_name, basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, path_to_trajectory, blocks, slurm)

def cp2k_input(lower_orbital, upper_orbital, cell_parameters=None):
    """
    # create ``Settings`` for the Cp2K Jobs.
    """
    # Main Cp2k Jobs
    cp2k_args = Settings()
    cp2k_args.basis = "DZVP-MOLOPT-SR-GTH"
    cp2k_args.potential = "GTH-PBE"
    cp2k_args.cell_parameters = [cell_parameters] * 3
    main_dft = cp2k_args.specific.cp2k.force_eval.dft
    main_dft.scf.added_mos = 20
    main_dft.scf.max_scf = 200
    main_dft.scf.eps_scf = 1e-5
    main_dft['print']['mo']['mo_index_range'] = "{} {}".format(lower_orbital,
                                                               upper_orbital)
    cp2k_args.specific.cp2k.force_eval.subsys.cell.periodic = 'None'

    # Setting to calculate the wave function used as guess
    cp2k_OT = Settings()
    cp2k_OT.basis = "DZVP-MOLOPT-SR-GTH"
    cp2k_OT.potential = "GTH-PBE"
    cp2k_OT.cell_parameters = [cell_parameters] * 3
    ot_dft = cp2k_OT.specific.cp2k.force_eval.dft
    ot_dft.scf.scf_guess = 'atomic'
    ot_dft.scf.ot.minimizer = 'DIIS'
    ot_dft.scf.ot.n_diis = 7
    ot_dft.scf.ot.preconditioner = 'FULL_SINGLE_INVERSE'
    ot_dft.scf.added_mos = 0
    ot_dft.scf.eps_scf = 1e-05
    ot_dft.scf.scf_guess = 'restart'
    cp2k_OT.specific.cp2k.force_eval.subsys.cell.periodic = 'None'

    return cp2k_args, cp2k_OT

# ============================> Distribution <=================================

def distribute_computations(project_name, basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, path_to_trajectory, blocks, slurm):

    script_name = "script_remote_function.py"
    # Split the trajectory in Chunks and move each chunk to its corresponding
    # directory.
    chunks_trajectory = split_trajectory(path_to_trajectory, blocks, '.')
    chunks_trajectory.sort()
    enumerate_from = 0
    for  file_xyz, l in zip(chunks_trajectory, string.ascii_lowercase):
        folder = 'chunk_{}'.format(l)
        os.mkdir(folder)
        shutil.move(file_xyz, folder)
        # function to be execute remotely
        write_python_script(folder, file_xyz, project_name,
                            basisCP2K, potCP2K, cp2k_main,
                            cp2k_guess, enumerate_from, script_name)
        write_slurm_script(folder, slurm, script_name)
        enumerate_from += number_of_geometries(join(folder, file_xyz))

def write_python_script(folder, file_xyz, project_name, basisCP2K, potCP2K, cp2k_main,
                        cp2k_guess, enumerate_from, script_name):
    """ Write the python script to compute the PYXAID hamiltonians"""
    scratch = '/scratch-shared'
    user = getpass.getuser()
    path_hdf5 = join(scratch, user, project_name, '{}.hdf5'.format(folder))
    path = join(scratch, user, project_name)
    if not os.path.exists(path):
        os.makedirs(path)
    nCouplings = 40
    xs = """
from nac.workflows.workflow_coupling import generate_pyxaid_hamiltonians
from nac.workflows.initialization import initialize
from qmworks.utils import dict2Setting
import plams

plams.init()

project_name = '{}'
path_basis = '{}'
path_potential = '{}'
path_hdf5 = '{}'
path_traj_xyz = '{}'
basisname = '{}'

initial_config = initialize(project_name, path_traj_xyz,
                            basisname=basisname,
                            path_basis=path_basis,
                            path_potential=path_potential,
                            enumerate_from={},
                            calculate_guesses='first',
                            path_hdf5=path_hdf5)

cp2k_main = dict2Setting({})
cp2k_guess = dict2Setting({})

generate_pyxaid_hamiltonians('cp2k', project_name, cp2k_main,
                             guess_args=cp2k_guess, path='{}',
                             nCouplings={},
                             **initial_config)
plams.finish()
 """.format(project_name, basisCP2K, potCP2K, path_hdf5, file_xyz, cp2k_main.basis,
            enumerate_from, settings2Dict(cp2k_main), settings2Dict(cp2k_guess),
            path, nCouplings)

    with open(join(folder, script_name), 'w') as f:
        f.write(xs)

    return script_name

def write_slurm_script(folder, slurm, python_script):
    """
    write an Slurm launch script
    """
    sbatch = lambda x, y: "#SBATCH -{} {}\n".format(x, y)

    header = "#! /bin/bash\n"
    modules = "\nmodule load cp2k/3.0\nsource activate qmworks\n\n"
    time = sbatch('t', slurm.time)
    nodes = sbatch('N', slurm.nodes)
    tasks = sbatch('n', slurm.tasks)
    name = sbatch('J', slurm.name)
    python = "python {}".format(python_script)

    # Script content
    content = header + time + nodes + tasks + name + modules + python

    file_name = join(folder, "launch.sh")

    with open(file_name, 'w') as f:
        f.write(content)
    return file_name

def number_of_geometries(file_name):
    """
    Count the number of geometries in XYZ formant in a given file.
    """

    with open(file_name, 'r') as f:
        numat = int(f.readline())

    cmd = "wc -l {}".format(file_name)
    wc = subprocess.getoutput(cmd).split()[0]

    lines_per_geometry = numat + 2

    return int(wc) // lines_per_geometry

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from pathlib import Path
import wave

import h5py

DIR_PATH = Path('/Users/harold/Desktop/Clips')
INPUT_FILE_PATH = DIR_PATH / 'Clips.h5'
CLIP_COUNT = 5

def main():

    with h5py.File(INPUT_FILE_PATH, 'r') as file_:

        clip_group = file_['clips']

        for i, clip_id in enumerate(clip_group):

            if i == CLIP_COUNT:
                break

            samples, sample_rate = read_clip(clip_group, clip_id)

            print(clip_id, len(samples), samples.dtype, sample_rate)

            write_wave_file(clip_id, samples, sample_rate)

def read_clip(clip_group, clip_id):
    clip = clip_group[clip_id]
    samples = clip[:]
    sample_rate = clip.attrs['sample_rate']
    return samples, sample_rate

def write_wave_file(i, samples, sample_rate):
    file_name = f'{i}.wav'
    file_path = DIR_PATH / file_name
    with wave.open(str(file_path), 'wb') as file_:
        file_.setparams((1, 2, sample_rate, len(samples), 'NONE', ''))
        file_.writeframes(samples.tobytes())

if __name__ == '__main__':
    main()
<commit_msg>Add attribute display to clip extraction script.<commit_after>from pathlib import Path
import wave

import h5py

DIR_PATH = Path('/Users/harold/Desktop/Clips')
INPUT_FILE_PATH = DIR_PATH / 'Clips.h5'
CLIP_COUNT = 5

def main():

    with h5py.File(INPUT_FILE_PATH, 'r') as file_:

        clip_group = file_['clips']

        for i, clip_id in enumerate(clip_group):

            if i == CLIP_COUNT:
                break

            samples, attributes = read_clip(clip_group, clip_id)

            show_clip(clip_id, samples, attributes)

            write_wave_file(clip_id, samples, attributes['sample_rate'])

def read_clip(clip_group, clip_id):
    clip = clip_group[clip_id]
    samples = clip[:]
    attributes = dict((name, value) for name, value in clip.attrs.items())
    return samples, attributes

def show_clip(clip_id, samples, attributes):
    print(f'clip {clip_id}:')
    print(f'    length: {len(samples)}')
    print('    attributes:')
    for key in sorted(attributes.keys()):
        value = attributes[key]
        print(f'        {key}: {value}')
    print()

def write_wave_file(i, samples, sample_rate):
    file_name = f'{i}.wav'
    file_path = DIR_PATH / file_name
    with wave.open(str(file_path), 'wb') as file_:
        file_.setparams((1, 2, sample_rate, len(samples), 'NONE', ''))
        file_.writeframes(samples.tobytes())

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Leach Wunderground's archive

import urllib2, cookielib
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
asos = i['asos']
mesosite = i['mesosite']
import mx.DateTime, csv, time
import sys
from metar import Metar
"""
    EGI  2008-12-18 21:55
"""

cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
r = opener.open("http://www.wunderground.com/cgi-bin/findweather/getForecast?setpref=SHOWMETAR&value=1").read()

def doit(stid, now):
  metarPass = 0
  metarFail = 0
  processed = 0
  sqlFail = 0
  url = "http://www.wunderground.com/history/airport/K%s/%s/%-i/%-i/DailyHistory.html?req_city=NA&req_state=NA&req_statename=NA&format=1" % (stid, now.year, now.month, now.day)
  try:
    data = opener.open(url).read()
  except:
    print "Fail STID: %s NOW: %s" % (stid, now)
    return 0
  lines = data.split("\n")
  headers = {}
  for line in lines: # Skip header
    line = line.replace("
", "")
    if line.strip() == "":
      continue
    tokens = line.split(",")
    if len(tokens) < 3:
      continue
    if len(headers.keys()) == 0:
      for i in range(len(tokens)):
        headers[ tokens[i] ] = i
      continue
    if not headers.has_key("FullMetar"):
      continue
    mstr = (tokens[headers["FullMetar"]]).strip().replace("'","")
    if mstr[:5] != "METAR":
      continue
    mtr = None
    try:
      mtr = Metar.Metar(mstr, now.month, now.year)
    except:
      pass
    sky = {'skyc1': "", 'skyc2': "", 'skyc3': "",
           'skyl1': "Null", 'skyl2': "Null", 'skyl3': "Null"}
    if mtr is not None and mtr.time is not None:
      metarPass += 1
      gts = mx.DateTime.DateTime( mtr.time.year, mtr.time.month,
                mtr.time.day, mtr.time.hour, mtr.time.minute)
      # When processing data on the last day of the month, we get GMT times
      # for the next month, need to account for that
      if gts.day == 1 and now.day > 10:
          gts += mx.DateTime.RelativeDateTime(months=1)
      #print now, gts
      tmpf = "Null"
      if (mtr.temp):
        tmpf = mtr.temp.value("F")
      dwpf = "Null"
      if (mtr.dewpt):
        dwpf = mtr.dewpt.value("F")

      sknt = "Null"
      if mtr.wind_speed:
        sknt = mtr.wind_speed.value("KT")
      gust = "Null"
      if mtr.wind_gust:
        gust = mtr.wind_gust.value("KT")

      drct = "Null"
      if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
        drct = mtr.wind_dir.value()

      vsby = "Null"
      if mtr.vis:
        vsby = mtr.vis.value("SM")

      alti = "Null"
      if mtr.press:
        alti = mtr.press.value("IN")

      p01m = 0
      if mtr.precip_1hr:
        p01m = mtr.precip_1hr.value("CM") * 10.0

      # Do something with sky coverage
      for i in range(len(mtr.sky)):
        (c,h,b) = mtr.sky[i]
        sky['skyc%s' % (i+1)] = c
        if h is not None:
          sky['skyl%s' % (i+1)] = h.value("FT")

    else:
      metarFail += 1
      tmpf = tokens[ headers['TemperatureF'] ]
      addon = 0
      if headers.has_key('TimeCST'):
        tstr = tokens[ headers['TimeCST'] ]
      elif headers.has_key('TimeCDT'):
        tstr = tokens[ headers['TimeCDT'] ]
      elif headers.has_key('TimeEDT'):
        tstr = tokens[ headers['TimeEDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=-1)
      elif headers.has_key('TimeEST'):
        tstr = tokens[ headers['TimeEST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=-1)
      elif headers.has_key('TimeMST'):
        tstr = tokens[ headers['TimeMST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=1)
      elif headers.has_key('TimeMDT'):
        tstr = tokens[ headers['TimeMDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=1)
      elif headers.has_key('TimePST'):
        tstr = tokens[ headers['TimePST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=2)
      elif headers.has_key('TimePDT'):
        tstr = tokens[ headers['TimePDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=2)
      else:
        print 'Unknown TimeZone!', headers.keys()
        continue
      ts = mx.DateTime.strptime( '%s %s' % (now.strftime('%Y-%m-%d'),
           tstr), '%Y-%m-%d %I:%M %p' ) + addon
      gts = ts.gmtime()
      dwpf = tokens[ headers['Dew PointF'] ]
      alti = tokens[ headers['Sea Level PressureIn'] ]
      vsby = tokens[ headers['VisibilityMPH'] ]
      d = tokens[ headers['Wind Direction'] ]
      if d in ["Calm","N/A","Variable"]:
        drct = "Null"
      else:
        drct = mesonet.txt2drct[ tokens[ headers['Wind Direction'] ] ]
      s = tokens[ headers['Wind SpeedMPH'] ]
      if s == "Calm" or s == "N/A":
        sknt = 0
      else:
        sknt = float( s ) / 1.15
      g = tokens[ headers['Gust SpeedMPH'] ]
      if g == "-" or g.strip() == "":
        gust = "Null"
      else:
        gust = float(g) * 1.15
      p = tokens[ headers['PrecipitationIn'] ]
      if p == "N/A" or p.strip() == "":
        p01m = "Null"
      else:
        p01m = float(p) * 25.4

    sql = """INSERT into t%s (station, valid, tmpf, dwpf, vsby,
        drct, sknt, gust, p01m, alti, skyc1, skyc2, skyc3,
        skyl1, skyl2, skyl3, metar) values ('%s','%s+00', %s, %s, %s,
        %s, %s, %s, %s, %s, '%s', '%s', '%s', %s, %s, %s,'%s')""" % (now.year,
        stid, gts.strftime("%Y-%m-%d %H:%M"), tmpf, dwpf, vsby, drct,
        sknt, gust, p01m, alti,
        sky['skyc1'], sky['skyc2'], sky['skyc3'],
        sky['skyl1'], sky['skyl2'], sky['skyl3'], mstr)
    try:
      asos.query(sql)
      processed += 1
    except:
      sqlFail += 1

  #print "%s %s MPASS: %s MFAIL: %s SQLFAIL: %s" % (stid, now, metarPass,
  #      metarFail, sqlFail)
  return processed

def recover():
    """
    Account for a stupid processing error I did with not accounting for obs
    at the end of the month
    """
    network = sys.argv[1]
    rs = mesosite.query("SELECT * from stations WHERE network = '%s' ORDER by id DESC" % (
                                                                                network,)).dictresult()
    for i in range(len(rs)):
        sid = rs[i]['id']
        # Delete all obs from the last day of the month, sigh
        asos.query("""DELETE from alldata where station = '%s' and
        (
        (extract(month from valid) = 1 and extract(day from valid) = 31) or
        (extract(month from valid) = 2 and extract(day from valid) = 28) or
        (extract(month from valid) = 2 and extract(day from valid) = 29) or
        (extract(month from valid) = 3 and extract(day from valid) = 31) or
        (extract(month from valid) = 4 and extract(day from valid) = 30) or
        (extract(month from valid) = 5 and extract(day from valid) = 31) or
        (extract(month from valid) = 6 and extract(day from valid) = 30) or
        (extract(month from valid) = 7 and extract(day from valid) = 31) or
        (extract(month from valid) = 8 and extract(day from valid) = 31) or
        (extract(month from valid) = 9 and extract(day from valid) = 30) or
        (extract(month from valid) = 10 and extract(day from valid) = 31) or
        (extract(month from valid) = 11 and extract(day from valid) = 30) or
        (extract(month from valid) = 12 and extract(day from valid) = 31)
        ) and valid < '2010-02-15'
        """ % (sid,))
        now = mx.DateTime.DateTime(1948,1,1)
        ets = mx.DateTime.DateTime(2010,3,1)
        interval = mx.DateTime.RelativeDateTime(months=1)
        while now < ets:
            ts = now - mx.DateTime.RelativeDateTime(days=1)
            obs = doit(sid, ts)
            #print sid, ts, obs
            now += interval

def normal():
    network = sys.argv[1]
    # 1. Query for a list of stations to iterate over
    rs = mesosite.query("SELECT * from stations WHERE network = '%s' ORDER by id DESC" % (
                                                                                network,)).dictresult()
    for i in range(len(rs)):
        sid = rs[i]['id']
  # 2. Look in the database for earliest ob with METAR
        rs2 = asos.query("""SELECT min(valid) from alldata WHERE station = '%s'
     and metar is not null""" % (sid,)).dictresult()
        if len(rs2) == 0 or rs2[0]['min'] is None:
            tend = mx.DateTime.now()
        else:
            tend = mx.DateTime.strptime(rs2[0]['min'][:16], '%Y-%m-%d %H:%M')

        # 3. Move old data to old table
        asos.query("""INSERT into alldata_save SELECT * from alldata WHERE
   station = '%s' and valid < '%s'""" % (sid, tend.strftime("%Y-%m-%d %H:%M")))
        asos.query("""DELETE from alldata WHERE
   station = '%s' and valid < '%s'""" % (sid, tend.strftime("%Y-%m-%d %H:%M")))

#     4. Figure out when wunder archive starts and then process till end
        sts = mx.DateTime.DateTime(1948,1,1)
        ets = tend
        now = sts
        tbegin = None
        processed = 0
        while now < ets:
            interval = mx.DateTime.RelativeDateTime(days=1)
            time.sleep(0.5)
            obs = doit(sid, now)
            if obs == 0 and processed == 0:
                interval = mx.DateTime.RelativeDateTime(months=1)
            elif obs > 0 and processed == 0:
                tbegin = now
            processed += obs
            now += interval
            if tbegin is not None:
                print "Station %s Begin: %s Obs: %s" % (sid, tbegin.strftime("%Y-%m-%d"),
      processed)

recover()<commit_msg>print out bug and run normal as default<commit_after># Leach Wunderground's archive

import urllib2, cookielib
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
asos = i['asos']
mesosite = i['mesosite']
import mx.DateTime, csv, time
import sys
from metar import Metar
"""
    EGI  2008-12-18 21:55
"""

cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
r = opener.open("http://www.wunderground.com/cgi-bin/findweather/getForecast?setpref=SHOWMETAR&value=1").read()

def doit(stid, now):
  metarPass = 0
  metarFail = 0
  processed = 0
  sqlFail = 0
  url = "http://www.wunderground.com/history/airport/K%s/%s/%-i/%-i/DailyHistory.html?req_city=NA&req_state=NA&req_statename=NA&format=1" % (stid, now.year, now.month, now.day)
  try:
    data = opener.open(url).read()
  except:
    print "Fail STID: %s NOW: %s" % (stid, now)
    return 0
  lines = data.split("\n")
  headers = {}
  for line in lines: # Skip header
    line = line.replace("
", "")
    if line.strip() == "":
      continue
    tokens = line.split(",")
    if len(tokens) < 3:
      continue
    if len(headers.keys()) == 0:
      for i in range(len(tokens)):
        headers[ tokens[i] ] = i
      continue
    if not headers.has_key("FullMetar"):
      continue
    mstr = (tokens[headers["FullMetar"]]).strip().replace("'","")
    if mstr[:5] != "METAR":
      continue
    mtr = None
    try:
      mtr = Metar.Metar(mstr, now.month, now.year)
    except:
      pass
    sky = {'skyc1': "", 'skyc2': "", 'skyc3': "",
           'skyl1': "Null", 'skyl2': "Null", 'skyl3': "Null"}
    if mtr is not None and mtr.time is not None:
      metarPass += 1
      gts = mx.DateTime.DateTime( mtr.time.year, mtr.time.month,
                mtr.time.day, mtr.time.hour, mtr.time.minute)
      # When processing data on the last day of the month, we get GMT times
      # for the next month, need to account for that
      if gts.day == 1 and now.day > 10:
          gts += mx.DateTime.RelativeDateTime(months=1)
      #print now, gts
      tmpf = "Null"
      if (mtr.temp):
        tmpf = mtr.temp.value("F")
      dwpf = "Null"
      if (mtr.dewpt):
        dwpf = mtr.dewpt.value("F")

      sknt = "Null"
      if mtr.wind_speed:
        sknt = mtr.wind_speed.value("KT")
      gust = "Null"
      if mtr.wind_gust:
        gust = mtr.wind_gust.value("KT")

      drct = "Null"
      if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
        drct = mtr.wind_dir.value()

      vsby = "Null"
      if mtr.vis:
        vsby = mtr.vis.value("SM")

      alti = "Null"
      if mtr.press:
        alti = mtr.press.value("IN")

      p01m = 0
      if mtr.precip_1hr:
        p01m = mtr.precip_1hr.value("CM") * 10.0

      # Do something with sky coverage
      for i in range(len(mtr.sky)):
        (c,h,b) = mtr.sky[i]
        sky['skyc%s' % (i+1)] = c
        if h is not None:
          sky['skyl%s' % (i+1)] = h.value("FT")

    else:
      metarFail += 1
      tmpf = tokens[ headers['TemperatureF'] ]
      addon = 0
      if headers.has_key('TimeCST'):
        tstr = tokens[ headers['TimeCST'] ]
      elif headers.has_key('TimeCDT'):
        tstr = tokens[ headers['TimeCDT'] ]
      elif headers.has_key('TimeEDT'):
        tstr = tokens[ headers['TimeEDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=-1)
      elif headers.has_key('TimeEST'):
        tstr = tokens[ headers['TimeEST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=-1)
      elif headers.has_key('TimeMST'):
        tstr = tokens[ headers['TimeMST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=1)
      elif headers.has_key('TimeMDT'):
        tstr = tokens[ headers['TimeMDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=1)
      elif headers.has_key('TimePST'):
        tstr = tokens[ headers['TimePST'] ]
        addon = mx.DateTime.RelativeDateTime(hours=2)
      elif headers.has_key('TimePDT'):
        tstr = tokens[ headers['TimePDT'] ]
        addon = mx.DateTime.RelativeDateTime(hours=2)
      else:
        print 'Unknown TimeZone!', headers.keys()
        continue
      ts = mx.DateTime.strptime( '%s %s' % (now.strftime('%Y-%m-%d'),
           tstr), '%Y-%m-%d %I:%M %p' ) + addon
      gts = ts.gmtime()
      dwpf = tokens[ headers['Dew PointF'] ]
      alti = tokens[ headers['Sea Level PressureIn'] ]
      vsby = tokens[ headers['VisibilityMPH'] ]
      d = tokens[ headers['Wind Direction'] ]
      if d in ["Calm","N/A","Variable"]:
        drct = "Null"
      else:
        drct = mesonet.txt2drct[ tokens[ headers['Wind Direction'] ] ]
      s = tokens[ headers['Wind SpeedMPH'] ]
      if s == "Calm" or s == "N/A":
        sknt = 0
      else:
        sknt = float( s ) / 1.15
      g = tokens[ headers['Gust SpeedMPH'] ]
      if g == "-" or g.strip() == "":
        gust = "Null"
      else:
        gust = float(g) * 1.15
      p = tokens[ headers['PrecipitationIn'] ]
      if p == "N/A" or p.strip() == "":
        p01m = "Null"
      else:
        p01m = float(p) * 25.4

    sql = """INSERT into t%s (station, valid, tmpf, dwpf, vsby,
        drct, sknt, gust, p01m, alti, skyc1, skyc2, skyc3,
        skyl1, skyl2, skyl3, metar) values ('%s','%s+00', %s, %s, %s,
        %s, %s, %s, %s, %s, '%s', '%s', '%s', %s, %s, %s,'%s')""" % (now.year,
        stid, gts.strftime("%Y-%m-%d %H:%M"), tmpf, dwpf, vsby, drct,
        sknt, gust, p01m, alti,
        sky['skyc1'], sky['skyc2'], sky['skyc3'],
        sky['skyl1'], sky['skyl2'], sky['skyl3'], mstr)
    try:
      asos.query(sql)
      processed += 1
    except:
      sqlFail += 1

  #print "%s %s MPASS: %s MFAIL: %s SQLFAIL: %s" % (stid, now, metarPass,
  #      metarFail, sqlFail)
  return processed

def recover():
    """
    Account for a stupid processing error I did with not accounting for obs
    at the end of the month
    """
    network = sys.argv[1]
    rs = mesosite.query("SELECT * from stations WHERE network = '%s' ORDER by id DESC" % (
                                                                                network,)).dictresult()
    for i in range(len(rs)):
        sid = rs[i]['id']
        # Delete all obs from the last day of the month, sigh
        asos.query("""DELETE from alldata where station = '%s' and
        (
        (extract(month from valid) = 1 and extract(day from valid) = 31) or
        (extract(month from valid) = 2 and extract(day from valid) = 28) or
        (extract(month from valid) = 2 and extract(day from valid) = 29) or
        (extract(month from valid) = 3 and extract(day from valid) = 31) or
        (extract(month from valid) = 4 and extract(day from valid) = 30) or
        (extract(month from valid) = 5 and extract(day from valid) = 31) or
        (extract(month from valid) = 6 and extract(day from valid) = 30) or
        (extract(month from valid) = 7 and extract(day from valid) = 31) or
        (extract(month from valid) = 8 and extract(day from valid) = 31) or
        (extract(month from valid) = 9 and extract(day from valid) = 30) or
        (extract(month from valid) = 10 and extract(day from valid) = 31) or
        (extract(month from valid) = 11 and extract(day from valid) = 30) or
        (extract(month from valid) = 12 and extract(day from valid) = 31)
        ) and valid < '2010-02-15'
        """ % (sid,))
        now = mx.DateTime.DateTime(1948,1,1)
        ets = mx.DateTime.DateTime(2010,3,1)
        interval = mx.DateTime.RelativeDateTime(months=1)
        while now < ets:
            ts = now - mx.DateTime.RelativeDateTime(days=1)
            obs = doit(sid, ts)
            #print sid, ts, obs
            now += interval

def normal():
    network = sys.argv[1]
    # 1. Query for a list of stations to iterate over
    rs = mesosite.query("SELECT * from stations WHERE network = '%s' ORDER by id DESC" % (
                                                                                network,)).dictresult()
    for i in range(len(rs)):
        sid = rs[i]['id']
  # 2. Look in the database for earliest ob with METAR
        rs2 = asos.query("""SELECT min(valid) from alldata WHERE station = '%s'
     and metar is not null""" % (sid,)).dictresult()
        if len(rs2) == 0 or rs2[0]['min'] is None:
            tend = mx.DateTime.now()
        else:
            tend = mx.DateTime.strptime(rs2[0]['min'][:16], '%Y-%m-%d %H:%M')

        # 3. Move old data to old table
        asos.query("""INSERT into alldata_save SELECT * from alldata WHERE
   station = '%s' and valid < '%s'""" % (sid, tend.strftime("%Y-%m-%d %H:%M")))
        asos.query("""DELETE from alldata WHERE
   station = '%s' and valid < '%s'""" % (sid, tend.strftime("%Y-%m-%d %H:%M")))

#     4. Figure out when wunder archive starts and then process till end
        sts = mx.DateTime.DateTime(1948,1,1)
        ets = tend
        now = sts
        tbegin = None
        processed = 0
        while now < ets:
            interval = mx.DateTime.RelativeDateTime(days=1)
            time.sleep(0.5)
            obs = doit(sid, now)
            if obs == 0 and processed == 0:
                interval = mx.DateTime.RelativeDateTime(months=1)
            elif obs > 0 and processed == 0:
                tbegin = now
            processed += obs
            now += interval
        if tbegin is not None:
            print "Station %s Begin: %s Obs: %s" % (sid,
                                tbegin.strftime("%Y-%m-%d"), processed)

#recover()
normal()<|endoftext|>


<commit_before>#	extractMADIS.py
#  Get the latest MADIS numbers from the data file!

import string, mx.DateTime, os, sys
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
iemaccess = i['iem']
import netCDF3

now = mx.DateTime.gmt() + mx.DateTime.RelativeDateTime(hours=-1)

# IOC Test
if len(sys.argv) == 2:
  fp = "/mnt/mesonet/data/madis/mesonet/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
  table = "current_qc_ioc"
else:
  fp = "/mesonet/data/madis/mesonet/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
  table = "current_qc"

if (not os.path.isfile(fp)):
  sys.exit()

nc = netCDF3.Dataset(fp)

providers = nc.variables["dataProvider"]

nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkQCD = nc.variables["temperatureQCD"]
dwpkQCD = nc.variables["dewpointQCD"]
altiQCD = nc.variables["altimeterQCD"]

def figure(val, qcval):
  if (qcval > 1000):
    return 'Null'
  return mesonet.k2f(val + qcval) -  mesonet.k2f(val)

def figureAlti(val, qcval):
  if (qcval > 100000.):
    return 'Null'
  return qcval / 100.0

def check(val):
  if (val >  1000000.):
    return 'Null'
  return val

i = 0
found =0
for p in range(len(providers)):
  provider = providers[p]
  if (provider.tostring()[:3] == 'IEM' or provider.tostring()[:5] == 'IADOT' ):
    found = 1
    ts = mx.DateTime.gmtime( nc.variables["observationTime"][i] )

    tmpf = check( mesonet.k2f( nc_tmpk[i] ) )
    dwpf = check( mesonet.k2f( nc_dwpk[i] ) )
    alti = check( (nc_alti[i][0] / 100.0 ) * 0.0295298)
    tmpf_qc_av = figure(nc_tmpk[i], tmpkQCD[i,0])
    tmpf_qc_sc = figure(nc_tmpk[i], tmpkQCD[i,6])
    dwpf_qc_av = figure(nc_dwpk[i], dwpkQCD[i,0])
    dwpf_qc_sc = figure(nc_dwpk[i], dwpkQCD[i,6])
    alti_qc_av = figureAlti(alti, altiQCD[i,0] * 0.0295298 )
    alti_qc_sc = figureAlti(alti, altiQCD[i,6] * 0.0295298 )
    #print id, tmpf, tmpkQCD[i]
    sql = """UPDATE %s SET tmpf = %s, tmpf_qc_av = %s,
     tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s,
     dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s,
     alti_qc_sc = %s, valid = '%s' WHERE station = '%s' """ % (table, tmpf,
     tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av,
     dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc,
     ts.strftime("%Y-%m-%d %H:%M"), (nc.variables["stationId"][i]).tostring()[:5] )
    iemaccess.query(sql)

  i += 1

nc.close()
<commit_msg>add 3 hour delay, sigh<commit_after>#	extractMADIS.py
#  Get the latest MADIS numbers from the data file!

import string, mx.DateTime, os, sys
from pyIEM import iemdb, mesonet
i = iemdb.iemdb()
iemaccess = i['iem']
import netCDF3

now = mx.DateTime.gmt() + mx.DateTime.RelativeDateTime(hours=-3)

# IOC Test
if len(sys.argv) == 2:
  fp = "/mnt/mesonet/data/madis/mesonet/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
  table = "current_qc_ioc"
else:
  fp = "/mesonet/data/madis/mesonet/%s.nc" % (now.strftime("%Y%m%d_%H00"), )
  table = "current_qc"

if (not os.path.isfile(fp)):
  sys.exit()

nc = netCDF3.Dataset(fp)

providers = nc.variables["dataProvider"]

nc_tmpk = nc.variables["temperature"]
nc_dwpk = nc.variables["dewpoint"]
nc_alti = nc.variables["altimeter"]
tmpkQCD = nc.variables["temperatureQCD"]
dwpkQCD = nc.variables["dewpointQCD"]
altiQCD = nc.variables["altimeterQCD"]

def figure(val, qcval):
  if (qcval > 1000):
    return 'Null'
  return mesonet.k2f(val + qcval) -  mesonet.k2f(val)

def figureAlti(val, qcval):
  if (qcval > 100000.):
    return 'Null'
  return qcval / 100.0

def check(val):
  if (val >  1000000.):
    return 'Null'
  return val

i = 0
found =0
for p in range(len(providers)):
  provider = providers[p]
  if (provider.tostring()[:3] == 'IEM' or provider.tostring()[:5] == 'IADOT' ):
    found = 1
    ts = mx.DateTime.gmtime( nc.variables["observationTime"][i] )

    tmpf = check( mesonet.k2f( nc_tmpk[i] ) )
    dwpf = check( mesonet.k2f( nc_dwpk[i] ) )
    alti = check( (nc_alti[i][0] / 100.0 ) * 0.0295298)
    tmpf_qc_av = figure(nc_tmpk[i], tmpkQCD[i,0])
    tmpf_qc_sc = figure(nc_tmpk[i], tmpkQCD[i,6])
    dwpf_qc_av = figure(nc_dwpk[i], dwpkQCD[i,0])
    dwpf_qc_sc = figure(nc_dwpk[i], dwpkQCD[i,6])
    alti_qc_av = figureAlti(alti, altiQCD[i,0] * 0.0295298 )
    alti_qc_sc = figureAlti(alti, altiQCD[i,6] * 0.0295298 )
    #print id, tmpf, tmpkQCD[i]
    sql = """UPDATE %s SET tmpf = %s, tmpf_qc_av = %s,
     tmpf_qc_sc = %s, dwpf = %s, dwpf_qc_av = %s,
     dwpf_qc_sc = %s, alti = %s, alti_qc_av = %s,
     alti_qc_sc = %s, valid = '%s' WHERE station = '%s' """ % (table, tmpf,
     tmpf_qc_av, tmpf_qc_sc, dwpf, dwpf_qc_av,
     dwpf_qc_sc, alti, alti_qc_av, alti_qc_sc,
     ts.strftime("%Y-%m-%d %H:%M"), (nc.variables["stationId"][i]).tostring()[:5] )
    iemaccess.query(sql)

  i += 1

nc.close()
<|endoftext|>