<commit_before># coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .once import OnceIE
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    ExtractorError,
    int_or_none,
)

class VoxMediaVolumeIE(OnceIE):
    _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P<id>[0-9a-f]{9})'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        setup = self._parse_json(self._search_regex(
            r'setup\s*=\s*({.+});', webpage, 'setup'), video_id)
        video_data = setup.get('video') or {}
        info = {
            'id': video_id,
            'title': video_data.get('title_short'),
            'description': video_data.get('description_long') or video_data.get('description_short'),
            'thumbnail': video_data.get('brightcove_thumbnail')
        }
        asset = setup.get('asset') or setup.get('params') or {}

        formats = []
        hls_url = asset.get('hls_url')
        if hls_url:
            formats.extend(self._extract_m3u8_formats(
                hls_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
        mp4_url = asset.get('mp4_url')
        if mp4_url:
            tbr = self._search_regex(r'-(\d+)k\.', mp4_url, 'bitrate', default=None)
            format_id = 'http'
            if tbr:
                format_id += '-' + tbr
            formats.append({
                'format_id': format_id,
                'url': mp4_url,
                'tbr': int_or_none(tbr),
            })
        if formats:
            self._sort_formats(formats)
            info['formats'] = formats
            return info

        for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
            provider_video_id = video_data.get('%s_id' % provider_video_type)
            if not provider_video_id:
                continue
            if provider_video_type == 'brightcove':
                info['formats'] = self._extract_once_formats(provider_video_id)
                self._sort_formats(info['formats'])
            else:
                info.update({
                    '_type': 'url_transparent',
                    'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
                    'ie_key': provider_video_type.capitalize(),
                })
            return info
        raise ExtractorError('Unable to find provider video id')

class VoxMediaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:(?:theverge|vox|sbnation|eater|polygon|curbed|racked|funnyordie)\.com|recode\.net)/(?:[^/]+/)*(?P<id>[^/?]+)'
    _TESTS = [{
        # Volume embed, Youtube
        'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
        'info_dict': {
            'id': 'j4mLW6x17VM',
            'ext': 'mp4',
            'title': 'Material world: how Google discovered what software is made of',
            'description': 'md5:dfc17e7715e3b542d66e33a109861382',
            'upload_date': '20190710',
            'uploader_id': 'TheVerge',
            'uploader': 'The Verge',
        },
        'add_ie': ['Youtube'],
    }, {
        # Volume embed, Youtube
        'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
        'md5': '4c8f4a0937752b437c3ebc0ed24802b5',
        'info_dict': {
            'id': 'Gy8Md3Eky38',
            'ext': 'mp4',
            'title': 'The Nexus 6: hands-on with Google\'s phablet',
            'description': 'md5:d9f0216e5fb932dd2033d6db37ac3f1d',
            'uploader_id': 'TheVerge',
            'upload_date': '20141021',
            'uploader': 'The Verge',
        },
        'add_ie': ['Youtube'],
        'skip': 'similar to the previous test',
    }, {
        # Volume embed, Youtube
        'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
        'info_dict': {
            'id': 'YCjDnX-Xzhg',
            'ext': 'mp4',
            'title': "Mississippi's laws are so bad that its anti-LGBTQ law isn't needed to allow discrimination",
            'description': 'md5:fc1317922057de31cd74bce91eb1c66c',
            'uploader_id': 'voxdotcom',
            'upload_date': '20150915',
            'uploader': 'Vox',
        },
        'add_ie': ['Youtube'],
        'skip': 'similar to the previous test',
    }, {
        # youtube embed
        'url': 'http://www.vox.com/2016/3/24/11291692/robot-dance',
        'md5': '83b3080489fb103941e549352d3e0977',
        'info_dict': {
            'id': 'FcNHTJU1ufM',
            'ext': 'mp4',
            'title': 'How "the robot" became the greatest novelty dance of all time',
            'description': 'md5:b081c0d588b8b2085870cda55e6da176',
            'upload_date': '20160324',
            'uploader_id': 'voxdotcom',
            'uploader': 'Vox',
        },
        'add_ie': ['Youtube'],
        'skip': 'Page no longer contain videos',
    }, {
        # SBN.VideoLinkset.entryGroup multiple ooyala embeds
        'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
        'info_dict': {
            'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
            'title': '25 lies you will tell yourself on National Signing Day',
            'description': 'It\'s the most self-delusional time of the year, and everyone\'s gonna tell the same lies together!',
        },
        'playlist': [{
            'md5': '721fededf2ab74ae4176c8c8cbfe092e',
            'info_dict': {
                'id': 'p3cThlMjE61VDi_SD9JlIteSNPWVDBB9',
                'ext': 'mp4',
                'title': 'Buddy Hield vs Steph Curry (and the world)',
                'description': 'Let’s dissect only the most important Final Four storylines.',
            },
        }, {
            'md5': 'bf0c5cc115636af028be1bab79217ea9',
            'info_dict': {
                'id': 'BmbmVjMjE6esPHxdALGubTrouQ0jYLHj',
                'ext': 'mp4',
                'title': 'Chasing Cinderella 2016: Syracuse basketball',
                'description': 'md5:e02d56b026d51aa32c010676765a690d',
            },
        }],
        'skip': 'Page no longer contain videos',
    }, {
        # volume embed, Brightcove Once
        'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya',
        'md5': '2dbc77b8b0bff1894c2fce16eded637d',
        'info_dict': {
            'id': '1231c973d',
            'ext': 'mp4',
            'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
            'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
        },
        'add_ie': ['VoxMediaVolume'],
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))

        def create_entry(provider_video_id, provider_video_type, title=None, description=None):
            video_url = {
                'youtube': '%s',
                'ooyala': 'ooyala:%s',
                'volume': 'http://volume.vox-cdn.com/embed/%s',
            }[provider_video_type] % provider_video_id
            return {
                '_type': 'url_transparent',
                'url': video_url,
                'title': title or self._og_search_title(webpage),
                'description': description or self._og_search_description(webpage),
            }

        entries = []
        entries_data = self._search_regex([
            r'Chorus\.VideoContext\.addVideo\((\[{.+}\])\);',
            r'var\s+entry\s*=\s*({.+});',
            r'SBN\.VideoLinkset\.entryGroup\(\s*(\[.+\])',
        ], webpage, 'video data', default=None)
        if entries_data:
            entries_data = self._parse_json(entries_data, display_id)
            if isinstance(entries_data, dict):
                entries_data = [entries_data]
            for video_data in entries_data:
                provider_video_id = video_data.get('provider_video_id')
                provider_video_type = video_data.get('provider_video_type')
                if provider_video_id and provider_video_type:
                    entries.append(create_entry(
                        provider_video_id, provider_video_type,
                        video_data.get('title'), video_data.get('description')))

        provider_video_id = self._search_regex(
            r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
        if provider_video_id:
            entries.append(create_entry(provider_video_id, 'ooyala'))

        volume_uuid = self._search_regex(
            r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
        if volume_uuid:
            entries.append(create_entry(volume_uuid, 'volume'))

        if len(entries) == 1:
            return entries[0]
        else:
            return self.playlist_result(entries, display_id, self._og_search_title(webpage), self._og_search_description(webpage))
<commit_msg>[voxmedia] fix volume embed extraction(closes #28338)<commit_after># coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from .once import OnceIE
from ..compat import compat_urllib_parse_unquote
from ..utils import (
    ExtractorError,
    int_or_none,
    try_get,
    unified_timestamp,
)

class VoxMediaVolumeIE(OnceIE):
    _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P<id>[0-9a-f]{9})'

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        setup = self._parse_json(self._search_regex(
            r'setup\s*=\s*({.+});', webpage, 'setup'), video_id)
        player_setup = setup.get('player_setup') or setup
        video_data = player_setup.get('video') or {}
        formatted_metadata = video_data.get('formatted_metadata') or {}
        info = {
            'id': video_id,
            'title': player_setup.get('title') or video_data.get('title_short'),
            'description': video_data.get('description_long') or video_data.get('description_short'),
            'thumbnail': formatted_metadata.get('thumbnail') or video_data.get('brightcove_thumbnail'),
            'timestamp': unified_timestamp(formatted_metadata.get('video_publish_date')),
        }
        asset = try_get(setup, lambda x: x['embed_assets']['chorus'], dict) or {}

        formats = []
        hls_url = asset.get('hls_url')
        if hls_url:
            formats.extend(self._extract_m3u8_formats(
                hls_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
        mp4_url = asset.get('mp4_url')
        if mp4_url:
            tbr = self._search_regex(r'-(\d+)k\.', mp4_url, 'bitrate', default=None)
            format_id = 'http'
            if tbr:
                format_id += '-' + tbr
            formats.append({
                'format_id': format_id,
                'url': mp4_url,
                'tbr': int_or_none(tbr),
            })
        if formats:
            self._sort_formats(formats)
            info['formats'] = formats
            info['duration'] = int_or_none(asset.get('duration'))
            return info

        for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
            provider_video_id = video_data.get('%s_id' % provider_video_type)
            if not provider_video_id:
                continue
            if provider_video_type == 'brightcove':
                info['formats'] = self._extract_once_formats(provider_video_id)
                self._sort_formats(info['formats'])
            else:
                info.update({
                    '_type': 'url_transparent',
                    'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
                    'ie_key': provider_video_type.capitalize(),
                })
            return info
        raise ExtractorError('Unable to find provider video id')

class VoxMediaIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?(?:(?:theverge|vox|sbnation|eater|polygon|curbed|racked|funnyordie)\.com|recode\.net)/(?:[^/]+/)*(?P<id>[^/?]+)'
    _TESTS = [{
        # Volume embed, Youtube
        'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
        'info_dict': {
            'id': 'j4mLW6x17VM',
            'ext': 'mp4',
            'title': 'Material world: how Google discovered what software is made of',
            'description': 'md5:dfc17e7715e3b542d66e33a109861382',
            'upload_date': '20190710',
            'uploader_id': 'TheVerge',
            'uploader': 'The Verge',
        },
        'add_ie': ['Youtube'],
    }, {
        # Volume embed, Youtube
        'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
        'md5': 'fd19aa0cf3a0eea515d4fd5c8c0e9d68',
        'info_dict': {
            'id': 'Gy8Md3Eky38',
            'ext': 'mp4',
            'title': 'The Nexus 6: hands-on with Google\'s phablet',
            'description': 'md5:d9f0216e5fb932dd2033d6db37ac3f1d',
            'uploader_id': 'TheVerge',
            'upload_date': '20141021',
            'uploader': 'The Verge',
            'timestamp': 1413907200,
        },
        'add_ie': ['Youtube'],
        'skip': 'similar to the previous test',
    }, {
        # Volume embed, Youtube
        'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
        'info_dict': {
            'id': '22986359b',
            'ext': 'mp4',
            'title': "Mississippi's laws are so bad that its anti-LGBTQ law isn't needed to allow discrimination",
            'description': 'md5:fc1317922057de31cd74bce91eb1c66c',
            'upload_date': '20150915',
            'timestamp': 1442332800,
            'duration': 285,
        },
        'add_ie': ['Youtube'],
        'skip': 'similar to the previous test',
    }, {
        # youtube embed
        'url': 'http://www.vox.com/2016/3/24/11291692/robot-dance',
        'md5': '83b3080489fb103941e549352d3e0977',
        'info_dict': {
            'id': 'FcNHTJU1ufM',
            'ext': 'mp4',
            'title': 'How "the robot" became the greatest novelty dance of all time',
            'description': 'md5:b081c0d588b8b2085870cda55e6da176',
            'upload_date': '20160324',
            'uploader_id': 'voxdotcom',
            'uploader': 'Vox',
        },
        'add_ie': ['Youtube'],
        'skip': 'Page no longer contain videos',
    }, {
        # SBN.VideoLinkset.entryGroup multiple ooyala embeds
        'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
        'info_dict': {
            'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
            'title': '25 lies you will tell yourself on National Signing Day',
            'description': 'It\'s the most self-delusional time of the year, and everyone\'s gonna tell the same lies together!',
        },
        'playlist': [{
            'md5': '721fededf2ab74ae4176c8c8cbfe092e',
            'info_dict': {
                'id': 'p3cThlMjE61VDi_SD9JlIteSNPWVDBB9',
                'ext': 'mp4',
                'title': 'Buddy Hield vs Steph Curry (and the world)',
                'description': 'Let’s dissect only the most important Final Four storylines.',
            },
        }, {
            'md5': 'bf0c5cc115636af028be1bab79217ea9',
            'info_dict': {
                'id': 'BmbmVjMjE6esPHxdALGubTrouQ0jYLHj',
                'ext': 'mp4',
                'title': 'Chasing Cinderella 2016: Syracuse basketball',
                'description': 'md5:e02d56b026d51aa32c010676765a690d',
            },
        }],
        'skip': 'Page no longer contain videos',
    }, {
        # volume embed, Brightcove Once
        'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya',
        'md5': '2dbc77b8b0bff1894c2fce16eded637d',
        'info_dict': {
            'id': '1231c973d',
            'ext': 'mp4',
            'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
            'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
            'timestamp': 1402938000,
            'upload_date': '20140616',
            'duration': 4114,
        },
        'add_ie': ['VoxMediaVolume'],
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))

        def create_entry(provider_video_id, provider_video_type, title=None, description=None):
            video_url = {
                'youtube': '%s',
                'ooyala': 'ooyala:%s',
                'volume': 'http://volume.vox-cdn.com/embed/%s',
            }[provider_video_type] % provider_video_id
            return {
                '_type': 'url_transparent',
                'url': video_url,
                'title': title or self._og_search_title(webpage),
                'description': description or self._og_search_description(webpage),
            }

        entries = []
        entries_data = self._search_regex([
            r'Chorus\.VideoContext\.addVideo\((\[{.+}\])\);',
            r'var\s+entry\s*=\s*({.+});',
            r'SBN\.VideoLinkset\.entryGroup\(\s*(\[.+\])',
        ], webpage, 'video data', default=None)
        if entries_data:
            entries_data = self._parse_json(entries_data, display_id)
            if isinstance(entries_data, dict):
                entries_data = [entries_data]
            for video_data in entries_data:
                provider_video_id = video_data.get('provider_video_id')
                provider_video_type = video_data.get('provider_video_type')
                if provider_video_id and provider_video_type:
                    entries.append(create_entry(
                        provider_video_id, provider_video_type,
                        video_data.get('title'), video_data.get('description')))

        provider_video_id = self._search_regex(
            r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
        if provider_video_id:
            entries.append(create_entry(provider_video_id, 'ooyala'))

        volume_uuid = self._search_regex(
            r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
        if volume_uuid:
            entries.append(create_entry(volume_uuid, 'volume'))

        if len(entries) == 1:
            return entries[0]
        else:
            return self.playlist_result(entries, display_id, self._og_search_title(webpage), self._og_search_description(webpage))
<|endoftext|>


<commit_before>"""
distribution.py
Author: Roger Danilek
Credit: Adam Pikielny, Nils , Stack Overflow

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
import string
a = string.ascii_lowercase

b = input("Please enter a string of text (the bigger the better): ")

print('The distribution of characters in "'+ b +'" is: ')

counts = []
for i in list(a):
    counts.append(b.count(i))
zipper = list(zip(counts,a))
zipper.sort(key = lambda f: (-f[0],f[1]))
for h in zipper:
    if h[0]!=0:
        print(h[0]*h[1])

<commit_msg>Updated from Brython Server: October 2, 2015 2:52:18 PM GMT-4<commit_after>"""
distribution.py
Author: Roger Danilek
Credit: Adam Pikielny, Nils , Stack Overflow

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
import string
a = string.ascii_lowercase

b = input("Please enter a string of text (the bigger the better): ")

print('The distribution of characters in "'+ b +'" is: ')

counts = []
a.lower()
for i in list(a):
    counts.append(b.count(i))
zipper = list(zip(counts,a))
zipper.sort(key = lambda f: (-f[0],f[1]))
for h in zipper:
    if h[0]!=0:
        print(h[0]*h[1])

<|endoftext|>


<commit_before># coding=utf-8
"""A (fairly) simple host/service monitor."""

import os
import sys
import time
import logging

from envconfig import EnvironmentAwareConfigParser

from optparse import OptionParser, SUPPRESS_HELP

from socket import gethostname

from util import get_config_dict

import Monitors.monitor
import Monitors.network
import Monitors.service
import Monitors.host
import Monitors.file
import Monitors.compound
import Monitors.hass

from simplemonitor import SimpleMonitor

import Loggers.file
import Loggers.db
import Loggers.network
import Loggers.mqtt

import Alerters.mail
import Alerters.ses
import Alerters.bulksms
import Alerters.fortysixelks
import Alerters.syslogger
import Alerters.execute
import Alerters.slack
import Alerters.pushover
import Alerters.pushbullet
import Alerters.telegram
import Alerters.nc

try:
    import colorlog
except ImportError:
    pass

VERSION = "1.8"

main_logger = logging.getLogger("simplemonitor")

def load_monitors(m, filename):
    """Load all the monitors from the config file and return a populated SimpleMonitor."""
    config = EnvironmentAwareConfigParser()
    config.read(filename)
    monitors = config.sections()
    if "defaults" in monitors:
        default_config = get_config_dict(config, "defaults")
        monitors.remove("defaults")
    else:
        default_config = {}

    myhostname = gethostname().lower()

    main_logger.info("=== Loading monitors")
    for monitor in monitors:
        if config.has_option(monitor, "runon"):
            if myhostname != config.get(monitor, "runon").lower():
                main_logger.warning(
                    "Ignoring monitor %s because it's only for host %s",
                    monitor,
                    config.get(monitor, "runon"),
                )
                continue
        monitor_type = config.get(monitor, "type")
        new_monitor = None
        config_options = default_config.copy()
        config_options.update(get_config_dict(config, monitor))

        try:
            cls = Monitors.monitor.get_class(monitor_type)
        except KeyError:
            main_logger.error(
                "Unknown monitor type %s; valid types are: %s",
                monitor_type,
                ", ".join(Monitors.monitor.all_types()),
            )
            continue
        new_monitor = cls(monitor, config_options)
        new_monitor.set_mon_refs(m)

        main_logger.info("Adding %s monitor %s: %s", monitor_type, monitor, new_monitor)
        m.add_monitor(monitor, new_monitor)

    for i in list(m.monitors.keys()):
        m.monitors[i].post_config_setup()
    main_logger.info("--- Loaded %d monitors", m.count_monitors())
    return m

def load_loggers(m, config):
    """Load the loggers listed in the config object."""

    if config.has_option("reporting", "loggers"):
        loggers = config.get("reporting", "loggers").split(",")
    else:
        loggers = []

    main_logger.info("=== Loading loggers")
    for config_logger in loggers:
        logger_type = config.get(config_logger, "type")
        config_options = get_config_dict(config, config_logger)
        config_options["_name"] = config_logger
        try:
            logger_cls = Loggers.logger.get_class(logger_type)
        except KeyError:
            main_logger.error(
                "Unknown logger type %s; valid types are: %s",
                logger_type,
                ", ".join(Loggers.logger.all_types()),
            )
            continue
        new_logger = logger_cls(config_options)
        main_logger.info(
            "Adding %s logger %s: %s", logger_type, config_logger, new_logger
        )
        m.add_logger(config_logger, new_logger)
        del new_logger
    main_logger.info("--- Loaded %d loggers", len(m.loggers))
    return m

def load_alerters(m, config):
    """Load the alerters listed in the config object."""
    if config.has_option("reporting", "alerters"):
        alerters = config.get("reporting", "alerters").split(",")
    else:
        alerters = []

    main_logger.info("=== Loading alerters")
    for alerter in alerters:
        alerter_type = config.get(alerter, "type")
        config_options = get_config_dict(config, alerter)
        try:
            alerter_cls = Alerters.alerter.get_class(alerter_type)
        except KeyError:
            main_logger.error(
                "Unknown alerter type %s; valid types are: %s",
                alerter_type,
                ", ".join(Alerters.alerter.all_types()),
            )
            continue
        new_alerter = alerter_cls(config_options)
        main_logger.info("Adding %s alerter %s", alerter_type, alerter)
        new_alerter.name = alerter
        m.add_alerter(alerter, new_alerter)
        del new_alerter
    main_logger.info("--- Loaded %d alerters", len(m.alerters))
    return m

def main():
    r"""This is where it happens \o/"""

    parser = OptionParser()
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-q",
        "--quiet",
        action="store_true",
        dest="quiet",
        default=False,
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-t",
        "--test",
        action="store_true",
        dest="test",
        default=False,
        help="Test config and exit",
    )
    parser.add_option(
        "-p", "--pidfile", dest="pidfile", default=None, help="Write PID into this file"
    )
    parser.add_option(
        "-N",
        "--no-network",
        dest="no_network",
        default=False,
        action="store_true",
        help="Disable network listening socket",
    )
    parser.add_option(
        "-d",
        "--debug",
        dest="debug",
        default=False,
        action="store_true",
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-f",
        "--config",
        dest="config",
        default="monitor.ini",
        help="configuration file",
    )
    parser.add_option(
        "-H",
        "--no-heartbeat",
        action="store_true",
        dest="no_heartbeat",
        default=False,
        help="Omit printing the '.' character when running checks",
    )
    parser.add_option(
        "-1",
        "--one-shot",
        action="store_true",
        dest="one_shot",
        default=False,
        help='Run the monitors once only, without alerting. Require monitors without "fail" in the name to succeed. Exit zero or non-zero accordingly.',
    )
    parser.add_option("--loops", dest="loops", default=-1, help=SUPPRESS_HELP, type=int)
    parser.add_option(
        "-l",
        "--log-level",
        dest="loglevel",
        default="warn",
        help="Log level: critical, error, warn, info, debug",
    )
    parser.add_option(
        "-C",
        "--no-colour",
        "--no-color",
        action="store_true",
        dest="no_colour",
        default=False,
        help="Do not colourise log output",
    )
    parser.add_option(
        "--no-timestamps",
        action="store_true",
        dest="no_timestamps",
        default=False,
        help="Do not prefix log output with timestamps",
    )
    parser.add_option(
        "--dump-known-resources",
        action="store_true",
        dest="dump_resources",
        default=False,
        help=SUPPRESS_HELP,
    )

    (options, _) = parser.parse_args()

    if options.dump_resources:
        import pprint

        print("Monitors:")
        pprint.pprint(sorted(Monitors.monitor.all_types()), compact=True)
        print("Loggers:")
        pprint.pprint(sorted(Loggers.logger.all_types()), compact=True)
        print("Alerters:")
        pprint.pprint(sorted(Alerters.alerter.all_types()), compact=True)
        sys.exit(0)

    if options.quiet:
        print("Warning: --quiet is deprecated; use --log-level=critical")
        options.loglevel = "critical"

    if options.verbose:
        print("Warning: --verbose is deprecated; use --log-level=info")
        options.loglevel = "info"

    if options.debug:
        print("Warning: --debug is deprecated; use --log-level=debug")
        options.loglevel = "debug"

    if options.no_timestamps:
        logging_timestamp = ""
    else:
        logging_timestamp = "%(asctime)s "

    try:
        log_level = getattr(logging, options.loglevel.upper())
    except AttributeError:
        print("Log level {0} is unknown".format(options.loglevel))
        sys.exit(1)

    log_datefmt = "%Y-%m-%d %H:%M:%S"
    log_plain_format = logging_timestamp + "%(levelname)8s (%(name)s) %(message)s"
    if not options.no_colour:
        try:
            handler = colorlog.StreamHandler()
            handler.setFormatter(
                colorlog.ColoredFormatter(
                    logging_timestamp
                    + "%(log_color)s%(levelname)8s%(reset)s (%(name)s) %(message)s",
                    datefmt="%Y-%m-%d %H:%M:%S",
                )
            )
            main_logger.addHandler(handler)
        except NameError:
            logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)
            main_logger.error("Could not enable colorlog")
    else:
        logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)

    main_logger.setLevel(log_level)

    if not options.quiet:
        main_logger.info("=== SimpleMonitor v%s", VERSION)
        main_logger.info("Loading main config from %s", options.config)

    config = EnvironmentAwareConfigParser()
    if not os.path.exists(options.config):
        main_logger.critical('Configuration file "%s" does not exist!', options.config)
        sys.exit(1)
    try:
        config.read(options.config)
    except Exception as e:
        main_logger.critical("Unable to read configuration file")
        main_logger.critical(e)
        sys.exit(1)

    try:
        interval = config.getint("monitor", "interval")
    except Exception:
        main_logger.critical(
            'Missing [monitor] section from config file, or missing the "interval" setting in it'
        )
        sys.exit(1)

    pidfile = config.get("monitor", "pidfile", fallback=None)

    if options.pidfile:
        pidfile = options.pidfile

    if pidfile:
        my_pid = os.getpid()
        try:
            with open(pidfile, "w") as file_handle:
                file_handle.write("%d\n" % my_pid)
        except Exception:
            main_logger.error("Couldn't write to pidfile!")
            pidfile = None

    if config.has_option("monitor", "monitors"):
        monitors_file = config.get("monitor", "monitors")
    else:
        monitors_file = "monitors.ini"

    main_logger.info("Loading monitor config from %s", monitors_file)

    try:
        allow_pickle = config.getboolean("monitor", "allow_pickle", fallback="true")
    except ValueError:
        main_logger.critical('allow_pickle should be "true" or "false".')
        sys.exit(1)

    m = SimpleMonitor(allow_pickle=allow_pickle)

    m = load_monitors(m, monitors_file)

    count = m.count_monitors()
    if count == 0:
        main_logger.critical("No monitors loaded :(")
        sys.exit(2)

    m = load_loggers(m, config)
    m = load_alerters(m, config)

    try:
        if config.get("monitor", "remote") == "1":
            if not options.no_network:
                enable_remote = True
                remote_port = int(config.get("monitor", "remote_port"))
            else:
                enable_remote = False
        else:
            enable_remote = False
    except Exception:
        enable_remote = False

    if not m.verify_dependencies():
        sys.exit(1)

    if options.test:
        main_logger.warning("Config test complete. Exiting.")
        sys.exit(0)

    if options.one_shot:
        main_logger.warning(
            "One-shot mode: expecting monitors without 'fail' in the name to succeed, and with to fail. Will exit zero or non-zero accordingly."
        )

    try:
        key = config.get("monitor", "key")
    except Exception:
        key = None

    if enable_remote:
        if not options.quiet:
            if not allow_pickle:
                allowing_pickle = "not "
            else:
                allowing_pickle = ""
            main_logger.info(
                "Starting remote listener thread ({0}allowing pickle data)".format(
                    allowing_pickle
                )
            )
        remote_listening_thread = Loggers.network.Listener(
            m, remote_port, key, allow_pickle=allow_pickle
        )
        remote_listening_thread.daemon = True
        remote_listening_thread.start()

    if not options.quiet:
        main_logger.info(
            "=== Starting... (loop runs every %ds) Hit ^C to stop", interval
        )
    loop = True
    heartbeat = 0

    loops = int(options.loops)

    while loop:
        try:
            if loops > 0:
                loops -= 1
                if loops == 0:
                    main_logger.warning(
                        "Ran out of loop counter, will stop after this one"
                    )
                    loop = False
            m.run_loop()

            if (
                options.loglevel in ["error", "critical", "warn"]
                and not options.no_heartbeat
            ):
                heartbeat += 1
                if heartbeat == 2:
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    heartbeat = 0
        except KeyboardInterrupt:
            main_logger.info("Received ^C")
            loop = False
        except Exception:
            sys.exc_info()
            main_logger.exception("Caught unhandled exception during main loop")
        if loop and enable_remote:
            if not remote_listening_thread.isAlive():
                main_logger.error("Listener thread died :(")
                remote_listening_thread = Loggers.network.Listener(
                    m, remote_port, key, allow_pickle=allow_pickle
                )
                remote_listening_thread.start()

        if options.one_shot:
            break

        try:
            if loop:
                time.sleep(interval)
        except Exception:
            main_logger.info("Quitting")
            loop = False

    if enable_remote:
        remote_listening_thread.running = False
        main_logger.info("Waiting for listener thread to exit")
        remote_listening_thread.join(0)

    if pidfile:
        try:
            os.unlink(pidfile)
        except Exception:
            main_logger.error("Couldn't remove pidfile!")

    if not options.quiet:
        main_logger.info("Finished.")

    if options.one_shot:  # pragma: no cover
        ok = True
        print("\n--> One-shot results:")
        for monitor in sorted(m.monitors.keys()):
            if "fail" in monitor:
                if m.monitors[monitor].error_count == 0:
                    print("    Monitor {0} should have failed".format(monitor))
                    ok = False
                else:
                    print("    Monitor {0} was ok (failed)".format(monitor))
            elif "skip" in monitor:
                if m.monitors[monitor].skipped():
                    print("    Monitor {0} was ok (skipped)".format(monitor))
                else:
                    print("    Monitor {0} should have been skipped".format(monitor))
                    ok = False
            else:
                if m.monitors[monitor].error_count > 0:
                    print("    Monitor {0} failed and shouldn't have".format(monitor))
                    ok = False
                else:
                    print("    Monitor {0} was ok".format(monitor))
        if not ok:
            print("Not all non-'fail' succeeded, or not all 'fail' monitors failed.")
            sys.exit(1)

    logging.shutdown()

if __name__ == "__main__":
    main()
<commit_msg>Use fallback for monitors filename config<commit_after># coding=utf-8
"""A (fairly) simple host/service monitor."""

import os
import sys
import time
import logging

from envconfig import EnvironmentAwareConfigParser

from optparse import OptionParser, SUPPRESS_HELP

from socket import gethostname

from util import get_config_dict

import Monitors.monitor
import Monitors.network
import Monitors.service
import Monitors.host
import Monitors.file
import Monitors.compound
import Monitors.hass

from simplemonitor import SimpleMonitor

import Loggers.file
import Loggers.db
import Loggers.network
import Loggers.mqtt

import Alerters.mail
import Alerters.ses
import Alerters.bulksms
import Alerters.fortysixelks
import Alerters.syslogger
import Alerters.execute
import Alerters.slack
import Alerters.pushover
import Alerters.pushbullet
import Alerters.telegram
import Alerters.nc

try:
    import colorlog
except ImportError:
    pass

VERSION = "1.8"

main_logger = logging.getLogger("simplemonitor")

def load_monitors(m, filename):
    """Load all the monitors from the config file and return a populated SimpleMonitor."""
    config = EnvironmentAwareConfigParser()
    config.read(filename)
    monitors = config.sections()
    if "defaults" in monitors:
        default_config = get_config_dict(config, "defaults")
        monitors.remove("defaults")
    else:
        default_config = {}

    myhostname = gethostname().lower()

    main_logger.info("=== Loading monitors")
    for monitor in monitors:
        if config.has_option(monitor, "runon"):
            if myhostname != config.get(monitor, "runon").lower():
                main_logger.warning(
                    "Ignoring monitor %s because it's only for host %s",
                    monitor,
                    config.get(monitor, "runon"),
                )
                continue
        monitor_type = config.get(monitor, "type")
        new_monitor = None
        config_options = default_config.copy()
        config_options.update(get_config_dict(config, monitor))

        try:
            cls = Monitors.monitor.get_class(monitor_type)
        except KeyError:
            main_logger.error(
                "Unknown monitor type %s; valid types are: %s",
                monitor_type,
                ", ".join(Monitors.monitor.all_types()),
            )
            continue
        new_monitor = cls(monitor, config_options)
        new_monitor.set_mon_refs(m)

        main_logger.info("Adding %s monitor %s: %s", monitor_type, monitor, new_monitor)
        m.add_monitor(monitor, new_monitor)

    for i in list(m.monitors.keys()):
        m.monitors[i].post_config_setup()
    main_logger.info("--- Loaded %d monitors", m.count_monitors())
    return m

def load_loggers(m, config):
    """Load the loggers listed in the config object."""

    if config.has_option("reporting", "loggers"):
        loggers = config.get("reporting", "loggers").split(",")
    else:
        loggers = []

    main_logger.info("=== Loading loggers")
    for config_logger in loggers:
        logger_type = config.get(config_logger, "type")
        config_options = get_config_dict(config, config_logger)
        config_options["_name"] = config_logger
        try:
            logger_cls = Loggers.logger.get_class(logger_type)
        except KeyError:
            main_logger.error(
                "Unknown logger type %s; valid types are: %s",
                logger_type,
                ", ".join(Loggers.logger.all_types()),
            )
            continue
        new_logger = logger_cls(config_options)
        main_logger.info(
            "Adding %s logger %s: %s", logger_type, config_logger, new_logger
        )
        m.add_logger(config_logger, new_logger)
        del new_logger
    main_logger.info("--- Loaded %d loggers", len(m.loggers))
    return m

def load_alerters(m, config):
    """Load the alerters listed in the config object."""
    if config.has_option("reporting", "alerters"):
        alerters = config.get("reporting", "alerters").split(",")
    else:
        alerters = []

    main_logger.info("=== Loading alerters")
    for alerter in alerters:
        alerter_type = config.get(alerter, "type")
        config_options = get_config_dict(config, alerter)
        try:
            alerter_cls = Alerters.alerter.get_class(alerter_type)
        except KeyError:
            main_logger.error(
                "Unknown alerter type %s; valid types are: %s",
                alerter_type,
                ", ".join(Alerters.alerter.all_types()),
            )
            continue
        new_alerter = alerter_cls(config_options)
        main_logger.info("Adding %s alerter %s", alerter_type, alerter)
        new_alerter.name = alerter
        m.add_alerter(alerter, new_alerter)
        del new_alerter
    main_logger.info("--- Loaded %d alerters", len(m.alerters))
    return m

def main():
    r"""This is where it happens \o/"""

    parser = OptionParser()
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        dest="verbose",
        default=False,
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-q",
        "--quiet",
        action="store_true",
        dest="quiet",
        default=False,
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-t",
        "--test",
        action="store_true",
        dest="test",
        default=False,
        help="Test config and exit",
    )
    parser.add_option(
        "-p", "--pidfile", dest="pidfile", default=None, help="Write PID into this file"
    )
    parser.add_option(
        "-N",
        "--no-network",
        dest="no_network",
        default=False,
        action="store_true",
        help="Disable network listening socket",
    )
    parser.add_option(
        "-d",
        "--debug",
        dest="debug",
        default=False,
        action="store_true",
        help=SUPPRESS_HELP,
    )
    parser.add_option(
        "-f",
        "--config",
        dest="config",
        default="monitor.ini",
        help="configuration file",
    )
    parser.add_option(
        "-H",
        "--no-heartbeat",
        action="store_true",
        dest="no_heartbeat",
        default=False,
        help="Omit printing the '.' character when running checks",
    )
    parser.add_option(
        "-1",
        "--one-shot",
        action="store_true",
        dest="one_shot",
        default=False,
        help='Run the monitors once only, without alerting. Require monitors without "fail" in the name to succeed. Exit zero or non-zero accordingly.',
    )
    parser.add_option("--loops", dest="loops", default=-1, help=SUPPRESS_HELP, type=int)
    parser.add_option(
        "-l",
        "--log-level",
        dest="loglevel",
        default="warn",
        help="Log level: critical, error, warn, info, debug",
    )
    parser.add_option(
        "-C",
        "--no-colour",
        "--no-color",
        action="store_true",
        dest="no_colour",
        default=False,
        help="Do not colourise log output",
    )
    parser.add_option(
        "--no-timestamps",
        action="store_true",
        dest="no_timestamps",
        default=False,
        help="Do not prefix log output with timestamps",
    )
    parser.add_option(
        "--dump-known-resources",
        action="store_true",
        dest="dump_resources",
        default=False,
        help=SUPPRESS_HELP,
    )

    (options, _) = parser.parse_args()

    if options.dump_resources:
        import pprint

        print("Monitors:")
        pprint.pprint(sorted(Monitors.monitor.all_types()), compact=True)
        print("Loggers:")
        pprint.pprint(sorted(Loggers.logger.all_types()), compact=True)
        print("Alerters:")
        pprint.pprint(sorted(Alerters.alerter.all_types()), compact=True)
        sys.exit(0)

    if options.quiet:
        print("Warning: --quiet is deprecated; use --log-level=critical")
        options.loglevel = "critical"

    if options.verbose:
        print("Warning: --verbose is deprecated; use --log-level=info")
        options.loglevel = "info"

    if options.debug:
        print("Warning: --debug is deprecated; use --log-level=debug")
        options.loglevel = "debug"

    if options.no_timestamps:
        logging_timestamp = ""
    else:
        logging_timestamp = "%(asctime)s "

    try:
        log_level = getattr(logging, options.loglevel.upper())
    except AttributeError:
        print("Log level {0} is unknown".format(options.loglevel))
        sys.exit(1)

    log_datefmt = "%Y-%m-%d %H:%M:%S"
    log_plain_format = logging_timestamp + "%(levelname)8s (%(name)s) %(message)s"
    if not options.no_colour:
        try:
            handler = colorlog.StreamHandler()
            handler.setFormatter(
                colorlog.ColoredFormatter(
                    logging_timestamp
                    + "%(log_color)s%(levelname)8s%(reset)s (%(name)s) %(message)s",
                    datefmt="%Y-%m-%d %H:%M:%S",
                )
            )
            main_logger.addHandler(handler)
        except NameError:
            logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)
            main_logger.error("Could not enable colorlog")
    else:
        logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)

    main_logger.setLevel(log_level)

    if not options.quiet:
        main_logger.info("=== SimpleMonitor v%s", VERSION)
        main_logger.info("Loading main config from %s", options.config)

    config = EnvironmentAwareConfigParser()
    if not os.path.exists(options.config):
        main_logger.critical('Configuration file "%s" does not exist!', options.config)
        sys.exit(1)
    try:
        config.read(options.config)
    except Exception as e:
        main_logger.critical("Unable to read configuration file")
        main_logger.critical(e)
        sys.exit(1)

    try:
        interval = config.getint("monitor", "interval")
    except Exception:
        main_logger.critical(
            'Missing [monitor] section from config file, or missing the "interval" setting in it'
        )
        sys.exit(1)

    pidfile = config.get("monitor", "pidfile", fallback=None)

    if options.pidfile:
        pidfile = options.pidfile

    if pidfile:
        my_pid = os.getpid()
        try:
            with open(pidfile, "w") as file_handle:
                file_handle.write("%d\n" % my_pid)
        except Exception:
            main_logger.error("Couldn't write to pidfile!")
            pidfile = None

    monitors_file = config.get("monitor", "monitors", "monitors.ini")
    main_logger.info("Loading monitor config from %s", monitors_file)

    try:
        allow_pickle = config.getboolean("monitor", "allow_pickle", fallback="true")
    except ValueError:
        main_logger.critical('allow_pickle should be "true" or "false".')
        sys.exit(1)

    m = SimpleMonitor(allow_pickle=allow_pickle)

    m = load_monitors(m, monitors_file)

    count = m.count_monitors()
    if count == 0:
        main_logger.critical("No monitors loaded :(")
        sys.exit(2)

    m = load_loggers(m, config)
    m = load_alerters(m, config)

    try:
        if config.get("monitor", "remote") == "1":
            if not options.no_network:
                enable_remote = True
                remote_port = int(config.get("monitor", "remote_port"))
            else:
                enable_remote = False
        else:
            enable_remote = False
    except Exception:
        enable_remote = False

    if not m.verify_dependencies():
        sys.exit(1)

    if options.test:
        main_logger.warning("Config test complete. Exiting.")
        sys.exit(0)

    if options.one_shot:
        main_logger.warning(
            "One-shot mode: expecting monitors without 'fail' in the name to succeed, and with to fail. Will exit zero or non-zero accordingly."
        )

    try:
        key = config.get("monitor", "key")
    except Exception:
        key = None

    if enable_remote:
        if not options.quiet:
            if not allow_pickle:
                allowing_pickle = "not "
            else:
                allowing_pickle = ""
            main_logger.info(
                "Starting remote listener thread ({0}allowing pickle data)".format(
                    allowing_pickle
                )
            )
        remote_listening_thread = Loggers.network.Listener(
            m, remote_port, key, allow_pickle=allow_pickle
        )
        remote_listening_thread.daemon = True
        remote_listening_thread.start()

    if not options.quiet:
        main_logger.info(
            "=== Starting... (loop runs every %ds) Hit ^C to stop", interval
        )
    loop = True
    heartbeat = 0

    loops = int(options.loops)

    while loop:
        try:
            if loops > 0:
                loops -= 1
                if loops == 0:
                    main_logger.warning(
                        "Ran out of loop counter, will stop after this one"
                    )
                    loop = False
            m.run_loop()

            if (
                options.loglevel in ["error", "critical", "warn"]
                and not options.no_heartbeat
            ):
                heartbeat += 1
                if heartbeat == 2:
                    sys.stdout.write(".")
                    sys.stdout.flush()
                    heartbeat = 0
        except KeyboardInterrupt:
            main_logger.info("Received ^C")
            loop = False
        except Exception:
            sys.exc_info()
            main_logger.exception("Caught unhandled exception during main loop")
        if loop and enable_remote:
            if not remote_listening_thread.isAlive():
                main_logger.error("Listener thread died :(")
                remote_listening_thread = Loggers.network.Listener(
                    m, remote_port, key, allow_pickle=allow_pickle
                )
                remote_listening_thread.start()

        if options.one_shot:
            break

        try:
            if loop:
                time.sleep(interval)
        except Exception:
            main_logger.info("Quitting")
            loop = False

    if enable_remote:
        remote_listening_thread.running = False
        main_logger.info("Waiting for listener thread to exit")
        remote_listening_thread.join(0)

    if pidfile:
        try:
            os.unlink(pidfile)
        except Exception:
            main_logger.error("Couldn't remove pidfile!")

    if not options.quiet:
        main_logger.info("Finished.")

    if options.one_shot:  # pragma: no cover
        ok = True
        print("\n--> One-shot results:")
        for monitor in sorted(m.monitors.keys()):
            if "fail" in monitor:
                if m.monitors[monitor].error_count == 0:
                    print("    Monitor {0} should have failed".format(monitor))
                    ok = False
                else:
                    print("    Monitor {0} was ok (failed)".format(monitor))
            elif "skip" in monitor:
                if m.monitors[monitor].skipped():
                    print("    Monitor {0} was ok (skipped)".format(monitor))
                else:
                    print("    Monitor {0} should have been skipped".format(monitor))
                    ok = False
            else:
                if m.monitors[monitor].error_count > 0:
                    print("    Monitor {0} failed and shouldn't have".format(monitor))
                    ok = False
                else:
                    print("    Monitor {0} was ok".format(monitor))
        if not ok:
            print("Not all non-'fail' succeeded, or not all 'fail' monitors failed.")
            sys.exit(1)

    logging.shutdown()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import pwd

from dockerspawner import DockerSpawner
from textwrap import dedent
from traitlets import (
    Integer,
    Unicode,
)

class SystemUserSpawner(DockerSpawner):

    container_image = Unicode("jupyterhub/systemuser", config=True)

    host_homedir_format_string = Unicode(
        "/home/{username}",
        config=True,
        help=dedent(
            """
            Format string for the path to the user's home directory on the host.
            The format string should include a `username` variable, which will
            be formatted with the user's username.
            """
        )
    )

    image_homedir_format_string = Unicode(
        "/home/{username}",
        config=True,
        help=dedent(
            """
            Format string for the path to the user's home directory
            inside the image.  The format string should include a
            `username` variable, which will be formatted with the
            user's username.
            """
        )
    )

    user_id = Integer(-1,
        help=dedent(
            """
            If system users are being used, then we need to know their user id
            in order to mount the home directory.

            User IDs are looked up in two ways:

            1. stored in the state dict (authenticator can write here)
            2. lookup via pwd
            """
        )
    )

    @property
    def host_homedir(self):
        """
        Path to the volume containing the user's home directory on the host.
        """
        return self.host_homedir_format_string.format(username=self.user.name)

    @property
    def homedir(self):
        """
        Path to the user's home directory in the docker image.
        """
        return self.image_homedir_format_string.format(username=self.user.name)

    @property
    def volume_mount_points(self):
        """
        Volumes are declared in docker-py in two stages.  First, you declare
        all the locations where you're going to mount volumes when you call
        create_container.

        Returns a list of all the values in self.volumes or
        self.read_only_volumes.
        """
        mount_points = super(SystemUserSpawner, self).volume_mount_points
        mount_points.append(self.homedir)
        return mount_points

    @property
    def volume_binds(self):
        """
        The second half of declaring a volume with docker-py happens when you
        actually call start().  The required format is a dict of dicts that
        looks like:

        {
            host_location: {'bind': container_location, 'ro': True}
        }
        """
        volumes = super(SystemUserSpawner, self).volume_binds
        volumes[self.host_homedir] = {
            'bind': self.homedir,
            'ro': False
        }
        return volumes

    def get_env(self):
        env = super(SystemUserSpawner, self).get_env()
        env.update(dict(
            USER=self.user.name,
            USER_ID=self.user_id,
            HOME=self.homedir
        ))
        return env

    def _user_id_default(self):
        """
        Get user_id from pwd lookup by name

        If the authenticator stores user_id in the user state dict,
        this will never be called, which is necessary if
        the system users are not on the Hub system (i.e. Hub itself is in a container).
        """
        return pwd.getpwnam(self.user.name).pw_uid

    def load_state(self, state):
        super().load_state(state)
        if 'user_id' in state:
            self.user_id = state['user_id']

    def get_state(self):
        state = super().get_state()
        if self.user_id >= 0:
            state['user_id'] = self.user_id
        return state

    def start(self, image=None, extra_create_kwargs=None,
        extra_start_kwargs=None, extra_host_config=None):
        """start the single-user server in a docker container"""
        if extra_create_kwargs is None:
            extra_create_kwargs = {}

        if 'working_dir' not in extra_create_kwargs:
            extra_create_kwargs['working_dir'] = self.homedir

        return super(SystemUserSpawner, self).start(
            image=image,
            extra_create_kwargs=extra_create_kwargs,
            extra_start_kwargs=extra_start_kwargs,
            extra_host_config=extra_host_config
        )
<commit_msg>Windows fix<commit_after>from dockerspawner import DockerSpawner
from textwrap import dedent
from traitlets import (
    Integer,
    Unicode,
)

class SystemUserSpawner(DockerSpawner):

    container_image = Unicode("jupyterhub/systemuser", config=True)

    host_homedir_format_string = Unicode(
        "/home/{username}",
        config=True,
        help=dedent(
            """
            Format string for the path to the user's home directory on the host.
            The format string should include a `username` variable, which will
            be formatted with the user's username.
            """
        )
    )

    image_homedir_format_string = Unicode(
        "/home/{username}",
        config=True,
        help=dedent(
            """
            Format string for the path to the user's home directory
            inside the image.  The format string should include a
            `username` variable, which will be formatted with the
            user's username.
            """
        )
    )

    user_id = Integer(-1,
        help=dedent(
            """
            If system users are being used, then we need to know their user id
            in order to mount the home directory.

            User IDs are looked up in two ways:

            1. stored in the state dict (authenticator can write here)
            2. lookup via pwd
            """
        )
    )

    @property
    def host_homedir(self):
        """
        Path to the volume containing the user's home directory on the host.
        """
        return self.host_homedir_format_string.format(username=self.user.name)

    @property
    def homedir(self):
        """
        Path to the user's home directory in the docker image.
        """
        return self.image_homedir_format_string.format(username=self.user.name)

    @property
    def volume_mount_points(self):
        """
        Volumes are declared in docker-py in two stages.  First, you declare
        all the locations where you're going to mount volumes when you call
        create_container.

        Returns a list of all the values in self.volumes or
        self.read_only_volumes.
        """
        mount_points = super(SystemUserSpawner, self).volume_mount_points
        mount_points.append(self.homedir)
        return mount_points

    @property
    def volume_binds(self):
        """
        The second half of declaring a volume with docker-py happens when you
        actually call start().  The required format is a dict of dicts that
        looks like:

        {
            host_location: {'bind': container_location, 'ro': True}
        }
        """
        volumes = super(SystemUserSpawner, self).volume_binds
        volumes[self.host_homedir] = {
            'bind': self.homedir,
            'ro': False
        }
        return volumes

    def get_env(self):
        env = super(SystemUserSpawner, self).get_env()
        env.update(dict(
            USER=self.user.name,
            USER_ID=self.user_id,
            HOME=self.homedir
        ))
        return env

    def _user_id_default(self):
        """
        Get user_id from pwd lookup by name

        If the authenticator stores user_id in the user state dict,
        this will never be called, which is necessary if
        the system users are not on the Hub system (i.e. Hub itself is in a container).
        """
        import pwd
        return pwd.getpwnam(self.user.name).pw_uid

    def load_state(self, state):
        super().load_state(state)
        if 'user_id' in state:
            self.user_id = state['user_id']

    def get_state(self):
        state = super().get_state()
        if self.user_id >= 0:
            state['user_id'] = self.user_id
        return state

    def start(self, image=None, extra_create_kwargs=None,
        extra_start_kwargs=None, extra_host_config=None):
        """start the single-user server in a docker container"""
        if extra_create_kwargs is None:
            extra_create_kwargs = {}

        if 'working_dir' not in extra_create_kwargs:
            extra_create_kwargs['working_dir'] = self.homedir

        return super(SystemUserSpawner, self).start(
            image=image,
            extra_create_kwargs=extra_create_kwargs,
            extra_start_kwargs=extra_start_kwargs,
            extra_host_config=extra_host_config
        )
<|endoftext|>


<commit_before>"""
distribution.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
letters = input("Please enter a string of text (the bigger the better): ")

def compare(a, b):
    """
    compare - generic comparison function for testing two elements.
    """
    return b >= a

def bsort(seq, cmp):
    """
    bsort - simple sorting algorithm that uses any comparison function
    seq - a list to be sorted
    cmp - a function for comparing two elements of seq
    """
    sorted = False  # assume the seq is not sorted to start with
    while not sorted:
        sorted = True   # assume it's already sorted correctly
        for index, value in enumerate(seq): # for every element in seq
            if index > 0:                   # past the first..
                if not cmp(seq[index-1], value):  # if this element is out of order
                    sorted = False          # then the list is not sorted yet
                    seq[index-1], seq[index] = seq[index], seq[index-1] # and swap it

tosort = [4, 10, 3, -1000, 30]
bsort(tosort, compare)
print(tosort)

acount=str(letters).count('a')
bcount=str(letters).count('b')
ccount=str(letters).count('c')
dcount=str(letters).count('d')
ecount=str(letters).count('e')
fcount=str(letters).count('f')
gcount=str(letters).count('g')
hcount=str(letters).count('h')
icount=str(letters).count('i')
jcount=str(letters).count('j')
kcount=str(letters).count('k')
lcount=str(letters).count('l')
mcount=str(letters).count('m')
ncount=str(letters).count('n')
ocount=str(letters).count('o')
pcount=str(letters).count('p')
qcount=str(letters).count('q')
rcount=str(letters).count('r')
scount=str(letters).count('s')
tcount=str(letters).count('t')
ucount=str(letters).count('u')
vcount=str(letters).count('v')
wcount=str(letters).count('w')
xcount=str(letters).count('x')
ycount=str(letters).count('y')
zcount=str(letters).count('z')

tosort = [acount, bcount, ccount,dcount,ecount,fcount, gcount,hcount,icount]
bsort(tosort, compare)
print(tosort)<commit_msg>Updated from Brython Server: 9/27/2016 1:51:44 PM<commit_after>"""
distribution.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit a Python program (distribution.py) that computes and displays
the distribution of characters in a given sample of text.

Output of your program should look like this:

Please enter a string of text (the bigger the better): The rain in Spain stays mainly in the plain.
The distribution of characters in "The rain in Spain stays mainly in the plain." is:
iiiiii
nnnnnn
aaaaa
sss
ttt
ee
hh
ll
pp
yy
m
r

Notice about this example:

* The text: 'The rain ... plain' is provided by the user as input to your program.
* Uppercase characters are converted to lowercase
* Spaces and punctuation marks are ignored completely.
* Characters that are more common appear first in the list.
* Where the same number of characters occur, the lines are ordered alphabetically.
  For example, in the printout above, the letters e, h, l, p and y both occur twice
  in the text and they are listed in the output in alphabetical order.
* Letters that do not occur in the text are not listed in the output at all.
"""
letters = input("Please enter a string of text (the bigger the better): ")

def compare(a, b):
    """
    compare - generic comparison function for testing two elements.
    """
    return b >= a

def bsort(seq, cmp):
    """
    bsort - simple sorting algorithm that uses any comparison function
    seq - a list to be sorted
    cmp - a function for comparing two elements of seq
    """
    sorted = False  # assume the seq is not sorted to start with
    while not sorted:
        sorted = True   # assume it's already sorted correctly
        for index, value in enumerate(seq): # for every element in seq
            if index > 0:                   # past the first..
                if not cmp(seq[index-1], value):  # if this element is out of order
                    sorted = False          # then the list is not sorted yet
                    seq[index-1], seq[index] = seq[index], seq[index-1] # and swap it

tosort = [4, 10, 3, -1000, 30]
bsort(tosort, compare)
print(tosort)

acount=str(letters).count('a')
bcount=str(letters).count('b')
ccount=str(letters).count('c')
dcount=str(letters).count('d')
ecount=str(letters).count('e')
fcount=str(letters).count('f')
gcount=str(letters).count('g')
hcount=str(letters).count('h')
icount=str(letters).count('i')
jcount=str(letters).count('j')
kcount=str(letters).count('k')
lcount=str(letters).count('l')
mcount=str(letters).count('m')
ncount=str(letters).count('n')
ocount=str(letters).count('o')
pcount=str(letters).count('p')
qcount=str(letters).count('q')
rcount=str(letters).count('r')
scount=str(letters).count('s')
tcount=str(letters).count('t')
ucount=str(letters).count('u')
vcount=str(letters).count('v')
wcount=str(letters).count('w')
xcount=str(letters).count('x')
ycount=str(letters).count('y')
zcount=str(letters).count('z')

tosort = [acount, bcount, ccount,dcount,ecount,fcount, gcount,hcount,icount,jcount,kcount,lcount,mcount,ncount,ocount,pcount, qcount,rcount,scount,tcount,ucount,vcount,wcount,xcount,ycount,zcount]
bsort(tosort, compare)
print(tosort)<|endoftext|>


<commit_before>"""
Perform regular monitoring of the COS FUV and NUV dark rates

"""
import os
import sqlite3
import logging
import numpy as np
import scipy
import matplotlib.pyplot as plt
import shutil
import glob

from astropy.io import fits
from astropy.time import Time
import datetime

#from .support import corrtag_image
from support import corrtag_image
from solar import get_solar_data
import plotting

base_dir = '/grp/hst/cos/Monitors/Darks/'
web_directory = '/grp/webpages/COS/'

DB_NAME = "/grp/hst/cos/Monitors/DB/cos_darkrates.db"
PHD_TABLE = 'phd'

#-------------------------------------------------------------------------------

def pull_darks(base, detector):
    '''
    Recursively find all darks in the given base directory taken with the
    given detector

    '''

    for root, dirs, files in os.walk(base):

        if 'gzip' in root: continue
        if 'experimental' in root: continue

        print root

        for filename in files:
            if not '.fits' in filename:
                continue
            elif not '_corrtag' in filename:
                continue

            full_filename = os.path.join( root, filename )

            if not fits.getval(full_filename, 'exptype', ext=0) == 'DARK':
                continue
            if not fits.getval(full_filename, 'detector', ext=0) == detector:
                continue

            yield full_filename

#-------------------------------------------------------------------------------

def get_temp(filename):

    detector = fits.getval( filename, 'DETECTOR' )
    segment = fits.getval( filename, 'SEGMENT' )

    if detector == 'FUV' and segment == 'FUVA':
        temp_keyword = 'LDCAMPAT'
    elif detector == 'FUV' and segment == 'FUVB':
        temp_keyword = 'LDCAMPBT'
    elif detector == 'NUV':
        temp_keyword = 'LMMCETMP'
    else:
        raise ValueError('What??? {} {}'.format(detector, segment))

    path, name = os.path.split(filename)
    rootname = name[:9]
    spt_file = os.path.join(path, rootname + '_spt.fits')

    try:
        temperature = fits.getval(spt_file, temp_keyword, ext=2)
    except IOError:
        temperature = fits.getval(spt_file + '.gz', temp_keyword, ext=2)

    return temperature

#-------------------------------------------------------------------------------

def mjd_to_decyear( time_array ):
    """ pull this out when you get it into astropy.time

    """

    times = Time( time_array, scale='tt', format='mjd' )

    out_times = []
    for value in times:
        year = value.datetime.year
        n_days = (value.datetime - datetime.datetime(value.datetime.year, 1, 1)).total_seconds()
        total_days = (datetime.datetime(value.datetime.year+1, 1, 1) - datetime.datetime(value.datetime.year, 1, 1)).total_seconds()

        fraction = float(n_days) / total_days

        out_times.append( year + fraction )

    return np.array(out_times)

#-------------------------------------------------------------------------------

def pull_orbital_info( dataset, step=1 ):
    """ Pull second by second orbital information from the dataset

    """

    SECOND_PER_MJD = 1.15741e-5

    hdu = fits.open( dataset )
    timeline = hdu['timeline'].data
    segment = hdu[0].header['segment']

    if segment == 'N/A':
        xlim = (0, 1024)
        ylim = (0, 1204)
        pha = (-1, 1)
    elif segment == 'FUVA':
        xlim = (1200, 15099)
        ylim = (380, 680)
        pha = (2, 23)
    elif segment == 'FUVB':
        xlim = (950, 15049)
        ylim = (440, 720)
        pha = (2, 23)
    else:
        raise ValueError('What segment is this? {}'.format(segment))

    times = timeline['time'][::step].copy()
    lat = timeline['latitude'][:-1][::step].copy().astype(np.float64)
    lon = timeline['longitude'][:-1][::step].copy().astype(np.float64)
    try:
        sun_lat = timeline['sun_lat'][:-1][::step].copy().astype(np.float64)
        sun_lon = timeline['sun_lon'][:-1][::step].copy().astype(np.float64)
    except KeyError:
        sun_lat = lat.copy() * 0
        sun_lon = lat.copy() * 0

    mjd = hdu[1].header['EXPSTART']  + \
        times.copy()[:-1].astype( np.float64 ) * \
        SECOND_PER_MJD

    decyear = mjd_to_decyear( mjd )

    if not len( times ):
        blank = np.array( [0] )
        return blank, blank, blank, blank, blank, blank

    events = hdu['events'].data
    filtered_index = np.where( (events['PHA'] > pha[0]) &
                               (events['PHA'] < pha[1]) &
                               (events['XCORR'] > xlim[0]) &
                               (events['XCORR'] < xlim[1]) &
                               (events['YCORR'] > ylim[0]) &
                               (events['YCORR'] < ylim[1])
                               )

    ta_index = np.where( (events['XCORR'] > xlim[0]) &
                         (events['XCORR'] < xlim[1]) &
                         (events['YCORR'] > ylim[0]) &
                         (events['YCORR'] < ylim[1])
                         )

    counts = np.histogram( events[filtered_index]['time'], bins=times )[0]
    ta_counts = np.histogram( events[ta_index]['time'], bins=times )[0]

    npix = float((xlim[1] - xlim[0]) * (ylim[1] - ylim[0]))
    counts = counts / npix / step
    ta_counts = ta_counts / npix / step

    if not len( lat ) == len(counts):
        lat = lat[:-1]
        lon = lon[:-1]
        sun_lat = sun_lat[:-1]
        sun_lon = sun_lon[:-1]

    assert len(lat) == len(counts), \
        'Arrays are not equal in length {}:{}'.format( len(lat), len(counts) )

    return counts, ta_counts, decyear, lat, lon, sun_lat, sun_lon

#-------------------------------------------------------------------------------

def compile_phd():
   db = sqlite3.connect(DB_NAME)
   c = db.cursor()

   #-- Find available datasets from master table
   table = 'FUV_stats'
   c.execute( """SELECT obsname FROM %s """ %(table))
   available = set( [str(item[0]) for item in c] )

   #-- populate PHD table
   table = PHD_TABLE
   try:
       columns = ', '.join(['bin{} real'.format(pha) for pha in range(0,31)])
       c.execute("""CREATE TABLE {} ( obsname text, {})""".format(table, columns ))
   except sqlite3.OperationalError:
       pass

   c.execute( """SELECT obsname FROM %s """ %(table))
   already_done = set( [str(item[0]) for item in c] )

   for filename in available:
       obsname = os.path.split(filename)[-1]
       if obsname in already_done:
           print filename, 'done'
       else:
           print filename, 'running'

       counts = pha_hist(filename)
       table_values = (obsname, ) + tuple(list(counts) )

       c.execute( """INSERT INTO %s VALUES (?{})""" % (table, ',?'*31 ),
                  table_values)

       db.commit()

#-------------------------------------------------------------------------------

def pha_hist(filename):
    hdu = fits.open( filename )
    pha_list_all = hdu[1].data['PHA']
    counts, bins = np.histogram(pha_list_all, bins=31, range=(0, 31))

    return counts

#-------------------------------------------------------------------------------

def compile_darkrates(detector='FUV'):
    db = sqlite3.connect(DB_NAME)

    c = db.cursor()
    table = '{}_stats'.format(detector)
    try:
        c.execute("""CREATE TABLE {} ( obsname text, date real, dark real, ta_dark real, latitude real, longitude real, sun_lat real, sun_lon real, temp real)""".format(table))
    except sqlite3.OperationalError:
        pass

    location = '/smov/cos/Data/'
    c.execute( """SELECT DISTINCT obsname FROM %s """ %(table))
    already_done = set( [str(item[0]) for item in c] )

    for filename in pull_darks(location, detector):
        obsname = os.path.split(filename)[-1]

        if obsname in already_done:
            print filename, 'done'
        else:
            print filename, 'running'

        counts, ta_counts, date, lat, lon, sun_lat, sun_lon = pull_orbital_info( filename, 25 )

        temp = get_temp(filename)

        for i in range(len(counts)):
            c.execute( """INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?)""" % (table),
                       (obsname,
                        date[i],
                        counts[i],
                        ta_counts[i],
                        lat[i],
                        lon[i],
                        sun_lat[i],
                        sun_lon[i],
                        temp))

        db.commit()

#-------------------------------------------------------------------------------

def compile_darkimage(dataset_list, pha=(2,23) ):
    """ Make a superdark from everything"""
    output_image = 0

    for filename in dataset_list:
        hdu = pyfits.open(filename)
        current_image = corrtag_image(hdu['events'].data, pha=pha)
        output_image += current_image

    return output_image

#-------------------------------------------------------------------------------

def make_plots( detector, TA=False ):
    print '#-------------------#'
    print 'Making plots for {}'.format(detector)
    print '#-------------------#'
    plt.ioff()
    db = sqlite3.connect(DB_NAME)

    cursor = db.cursor()
    table = '{}_stats'.format(detector)

    if detector == 'FUV':
        search_strings = ['_corrtag_a.fits', '_corrtag_b.fits']
        segments = ['FUVA', 'FUVB']
    elif detector == 'NUV':
        search_strings = ['_corrtag.fits']
        segments = ['NUV']
    else:
        raise ValueError('Only FUV or NUV allowed.  NOT:{}'.format(detector) )

    try:
        solar_data = np.genfromtxt(base_dir + 'solar_flux.txt', dtype=None)
        solar_date = np.array( mjd_to_decyear([line[0] for line in solar_data]) )
        solar_flux = np.array([line[1] for line in solar_data])
    except TypeError:
        solar_date = np.ones(1000)
        solar_flux = np.ones(1000)

    dark_key = 'dark'
    if TA:
        dark_key = 'ta_dark'

    for key, segment in zip(search_strings, segments):
        #-- Plot vs time
        print 'Plotting Time'
        cursor.execute( """SELECT date,{},temp,latitude,longitude FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))

        data = [ item for item in cursor ]
        mjd = np.array( [item[0] for item in data] )
        dark = np.array( [item[1] for item in data] )
        temp = np.array( [item[2] for item in data] )
        latitude = np.array( [item[3] for item in data] )
        longitude = np.array( [item[4] for item in data] )

        index = np.argsort(mjd)
        mjd = mjd[index]
        dark = dark[index]
        temp = temp[index]
        latitude = latitude[index]
        longitude = longitude[index]

        index_keep = np.where( (longitude < 250) | (latitude > 10) )[0]
        mjd = mjd[index_keep]
        dark = dark[index_keep]
        temp = temp[index_keep]

        outname = os.path.join(base_dir, detector, '{}_vs_time_{}.png'.format(dark_key, segment) )
        plotting.plot_time( detector, dark, mjd, temp, solar_flux, solar_date, outname )

        #-- Plot vs orbit
        print 'Plotting Orbit'
        cursor.execute( """SELECT {},latitude,longitude,sun_lat,sun_lon,date FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))
        data = [ item for item in cursor ]
        dark = np.array( [item[0] for item in data] )
        latitude = np.array( [item[1] for item in data] )
        longitude = np.array( [item[2] for item in data] )
        sun_lat = np.array( [item[3] for item in data] )
        sun_lon = np.array( [item[4] for item in data] )
        date = np.array( [item[5] for item in data] )

        index = np.argsort(date)
        dark = dark[index]
        latitude = latitude[index]
        longitude = longitude[index]
        sun_lat = sun_lat[index]
        sun_lon = sun_lon[index]

        outname = os.path.join(base_dir, detector, '{}_vs_orbit_{}.png'.format(dark_key, segment) )
        plotting.plot_orbital_rate(longitude, latitude, dark, sun_lon, sun_lat, outname )

        #-- Plot histogram of darkrates
        print 'Plotting Hist'
        cursor.execute( """SELECT {},date FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))
        data = [ item for item in cursor ]
        dark = np.array( [item[0] for item in data] )
        date = np.array( [item[1] for item in data] )

        index = np.argsort(date)
        date = date[index]
        dark = dark[index]

        for year in set(map(int, date)):
            index = np.where( (date >= year) &
                              (date < year + 1))

            outname = os.path.join(base_dir, detector, '{}_hist_{}_{}.pdf'.format(dark_key, year, segment) )
            plotting.plot_histogram(dark[index], outname )

        index = np.where(date >= date.max() - .5)
        outname = os.path.join(base_dir, detector, '{}_hist_-6mo_{}.pdf'.format(dark_key, segment) )
        plotting.plot_histogram(dark[index], outname )

        outname = os.path.join(base_dir, detector, '{}_hist_{}.pdf'.format(dark_key, segment) )
        plotting.plot_histogram(dark, outname )

#-------------------------------------------------------------------------------

def move_products():
    '''
    Move created pdf files to webpage directory
    '''
    print '#-------------------#'
    print 'Moving products into'
    print 'webpage directory'
    print '#-------------------#'

    for detector in ['FUV', 'NUV']:

        write_dir = web_directory + detector.lower() + '_darks/'
        move_list = glob.glob(base_dir + detector + '/*.p??')

        for item in move_list:
            path, file_to_move = os.path.split(item)
            shutil.copy(item, write_dir + file_to_move)
            print 'Moving: %s' % (file_to_move)

        os.system('chmod 777 ' + write_dir + '*.pdf')

#-------------------------------------------------------------------------------

def monitor():
    """ main monitoring pipeline"""

    get_solar_data('/grp/hst/cos/Monitors/Darks/')

    for detector in ['FUV', 'NUV']:
        compile_darkrates(detector)
        #if detector == 'FUV':
        #    compile_phd()
        make_plots(detector)

        if detector == 'FUV':
            make_plots(detector, TA=True)

    move_products()

#-------------------------------------------------------------------------------

if __name__ == '__main__':
    monitor()
<commit_msg>fixing missing timeline error<commit_after>"""
Perform regular monitoring of the COS FUV and NUV dark rates

"""
import os
import sqlite3
import logging
import numpy as np
import scipy
import matplotlib.pyplot as plt
import shutil
import glob

from astropy.io import fits
from astropy.time import Time
import datetime

#from .support import corrtag_image
from support import corrtag_image
from solar import get_solar_data
import plotting

base_dir = '/grp/hst/cos/Monitors/Darks/'
web_directory = '/grp/webpages/COS/'

DB_NAME = "/grp/hst/cos/Monitors/DB/cos_darkrates.db"
PHD_TABLE = 'phd'

#-------------------------------------------------------------------------------

def pull_darks(base, detector):
    '''
    Recursively find all darks in the given base directory taken with the
    given detector

    '''

    for root, dirs, files in os.walk(base):

        if 'gzip' in root: continue
        if 'experimental' in root: continue

        print root

        for filename in files:
            if not '.fits' in filename:
                continue
            elif not '_corrtag' in filename:
                continue

            full_filename = os.path.join( root, filename )

            if not fits.getval(full_filename, 'exptype', ext=0) == 'DARK':
                continue
            if not fits.getval(full_filename, 'detector', ext=0) == detector:
                continue

            yield full_filename

#-------------------------------------------------------------------------------

def get_temp(filename):

    detector = fits.getval( filename, 'DETECTOR' )
    segment = fits.getval( filename, 'SEGMENT' )

    if detector == 'FUV' and segment == 'FUVA':
        temp_keyword = 'LDCAMPAT'
    elif detector == 'FUV' and segment == 'FUVB':
        temp_keyword = 'LDCAMPBT'
    elif detector == 'NUV':
        temp_keyword = 'LMMCETMP'
    else:
        raise ValueError('What??? {} {}'.format(detector, segment))

    path, name = os.path.split(filename)
    rootname = name[:9]
    spt_file = os.path.join(path, rootname + '_spt.fits')

    try:
        temperature = fits.getval(spt_file, temp_keyword, ext=2)
    except IOError:
        temperature = fits.getval(spt_file + '.gz', temp_keyword, ext=2)

    return temperature

#-------------------------------------------------------------------------------

def mjd_to_decyear( time_array ):
    """ pull this out when you get it into astropy.time

    """

    times = Time( time_array, scale='tt', format='mjd' )

    out_times = []
    for value in times:
        year = value.datetime.year
        n_days = (value.datetime - datetime.datetime(value.datetime.year, 1, 1)).total_seconds()
        total_days = (datetime.datetime(value.datetime.year+1, 1, 1) - datetime.datetime(value.datetime.year, 1, 1)).total_seconds()

        fraction = float(n_days) / total_days

        out_times.append( year + fraction )

    return np.array(out_times)

#-------------------------------------------------------------------------------

def pull_orbital_info( dataset, step=1 ):
    """ Pull second by second orbital information from the dataset

    """

    SECOND_PER_MJD = 1.15741e-5

    hdu = fits.open( dataset )
    try:
        timeline = hdu['timeline'].data
        segment = hdu[0].header['segment']
    except KeyError:
        return None, None, None, None, None, None, None

    if segment == 'N/A':
        xlim = (0, 1024)
        ylim = (0, 1204)
        pha = (-1, 1)
    elif segment == 'FUVA':
        xlim = (1200, 15099)
        ylim = (380, 680)
        pha = (2, 23)
    elif segment == 'FUVB':
        xlim = (950, 15049)
        ylim = (440, 720)
        pha = (2, 23)
    else:
        raise ValueError('What segment is this? {}'.format(segment))

    times = timeline['time'][::step].copy()
    lat = timeline['latitude'][:-1][::step].copy().astype(np.float64)
    lon = timeline['longitude'][:-1][::step].copy().astype(np.float64)
    try:
        sun_lat = timeline['sun_lat'][:-1][::step].copy().astype(np.float64)
        sun_lon = timeline['sun_lon'][:-1][::step].copy().astype(np.float64)
    except KeyError:
        sun_lat = lat.copy() * 0
        sun_lon = lat.copy() * 0

    mjd = hdu[1].header['EXPSTART']  + \
        times.copy()[:-1].astype( np.float64 ) * \
        SECOND_PER_MJD

    decyear = mjd_to_decyear( mjd )

    if not len( times ):
        blank = np.array( [0] )
        return blank, blank, blank, blank, blank, blank

    events = hdu['events'].data
    filtered_index = np.where( (events['PHA'] > pha[0]) &
                               (events['PHA'] < pha[1]) &
                               (events['XCORR'] > xlim[0]) &
                               (events['XCORR'] < xlim[1]) &
                               (events['YCORR'] > ylim[0]) &
                               (events['YCORR'] < ylim[1])
                               )

    ta_index = np.where( (events['XCORR'] > xlim[0]) &
                         (events['XCORR'] < xlim[1]) &
                         (events['YCORR'] > ylim[0]) &
                         (events['YCORR'] < ylim[1])
                         )

    counts = np.histogram( events[filtered_index]['time'], bins=times )[0]
    ta_counts = np.histogram( events[ta_index]['time'], bins=times )[0]

    npix = float((xlim[1] - xlim[0]) * (ylim[1] - ylim[0]))
    counts = counts / npix / step
    ta_counts = ta_counts / npix / step

    if not len( lat ) == len(counts):
        lat = lat[:-1]
        lon = lon[:-1]
        sun_lat = sun_lat[:-1]
        sun_lon = sun_lon[:-1]

    assert len(lat) == len(counts), \
        'Arrays are not equal in length {}:{}'.format( len(lat), len(counts) )

    return counts, ta_counts, decyear, lat, lon, sun_lat, sun_lon

#-------------------------------------------------------------------------------

def compile_phd():
   db = sqlite3.connect(DB_NAME)
   c = db.cursor()

   #-- Find available datasets from master table
   table = 'FUV_stats'
   c.execute( """SELECT obsname FROM %s """ %(table))
   available = set( [str(item[0]) for item in c] )

   #-- populate PHD table
   table = PHD_TABLE
   try:
       columns = ', '.join(['bin{} real'.format(pha) for pha in range(0,31)])
       c.execute("""CREATE TABLE {} ( obsname text, {})""".format(table, columns ))
   except sqlite3.OperationalError:
       pass

   c.execute( """SELECT obsname FROM %s """ %(table))
   already_done = set( [str(item[0]) for item in c] )

   for filename in available:
       obsname = os.path.split(filename)[-1]
       if obsname in already_done:
           print filename, 'done'
       else:
           print filename, 'running'

       counts = pha_hist(filename)
       table_values = (obsname, ) + tuple(list(counts) )

       c.execute( """INSERT INTO %s VALUES (?{})""" % (table, ',?'*31 ),
                  table_values)

       db.commit()

#-------------------------------------------------------------------------------

def pha_hist(filename):
    hdu = fits.open( filename )
    pha_list_all = hdu[1].data['PHA']
    counts, bins = np.histogram(pha_list_all, bins=31, range=(0, 31))

    return counts

#-------------------------------------------------------------------------------

def compile_darkrates(detector='FUV'):
    db = sqlite3.connect(DB_NAME)

    c = db.cursor()
    table = '{}_stats'.format(detector)
    try:
        c.execute("""CREATE TABLE {} ( obsname text, date real, dark real, ta_dark real, latitude real, longitude real, sun_lat real, sun_lon real, temp real)""".format(table))
    except sqlite3.OperationalError:
        pass

    location = '/smov/cos/Data/'
    c.execute( """SELECT DISTINCT obsname FROM %s """ %(table))
    already_done = set( [str(item[0]) for item in c] )

    for filename in pull_darks(location, detector):
        obsname = os.path.split(filename)[-1]

        if obsname in already_done:
            print filename, 'done'
        else:
            print filename, 'running'

        counts, ta_counts, date, lat, lon, sun_lat, sun_lon = pull_orbital_info( filename, 25 )
        if counts == None:
            print "bad File {}".format(filename)
            continue

        temp = get_temp(filename)

        for i in range(len(counts)):
            c.execute( """INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?)""" % (table),
                       (obsname,
                        date[i],
                        counts[i],
                        ta_counts[i],
                        lat[i],
                        lon[i],
                        sun_lat[i],
                        sun_lon[i],
                        temp))

        db.commit()

#-------------------------------------------------------------------------------

def compile_darkimage(dataset_list, pha=(2,23) ):
    """ Make a superdark from everything"""
    output_image = 0

    for filename in dataset_list:
        hdu = pyfits.open(filename)
        current_image = corrtag_image(hdu['events'].data, pha=pha)
        output_image += current_image

    return output_image

#-------------------------------------------------------------------------------

def make_plots( detector, TA=False ):
    print '#-------------------#'
    print 'Making plots for {}'.format(detector)
    print '#-------------------#'
    plt.ioff()
    db = sqlite3.connect(DB_NAME)

    cursor = db.cursor()
    table = '{}_stats'.format(detector)

    if detector == 'FUV':
        search_strings = ['_corrtag_a.fits', '_corrtag_b.fits']
        segments = ['FUVA', 'FUVB']
    elif detector == 'NUV':
        search_strings = ['_corrtag.fits']
        segments = ['NUV']
    else:
        raise ValueError('Only FUV or NUV allowed.  NOT:{}'.format(detector) )

    try:
        solar_data = np.genfromtxt(base_dir + 'solar_flux.txt', dtype=None)
        solar_date = np.array( mjd_to_decyear([line[0] for line in solar_data]) )
        solar_flux = np.array([line[1] for line in solar_data])
    except TypeError:
        solar_date = np.ones(1000)
        solar_flux = np.ones(1000)

    dark_key = 'dark'
    if TA:
        dark_key = 'ta_dark'

    for key, segment in zip(search_strings, segments):
        #-- Plot vs time
        print 'Plotting Time'
        cursor.execute( """SELECT date,{},temp,latitude,longitude FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))

        data = [ item for item in cursor ]
        mjd = np.array( [item[0] for item in data] )
        dark = np.array( [item[1] for item in data] )
        temp = np.array( [item[2] for item in data] )
        latitude = np.array( [item[3] for item in data] )
        longitude = np.array( [item[4] for item in data] )

        index = np.argsort(mjd)
        mjd = mjd[index]
        dark = dark[index]
        temp = temp[index]
        latitude = latitude[index]
        longitude = longitude[index]

        index_keep = np.where( (longitude < 250) | (latitude > 10) )[0]
        mjd = mjd[index_keep]
        dark = dark[index_keep]
        temp = temp[index_keep]

        outname = os.path.join(base_dir, detector, '{}_vs_time_{}.png'.format(dark_key, segment) )
        plotting.plot_time( detector, dark, mjd, temp, solar_flux, solar_date, outname )

        #-- Plot vs orbit
        print 'Plotting Orbit'
        cursor.execute( """SELECT {},latitude,longitude,sun_lat,sun_lon,date FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))
        data = [ item for item in cursor ]
        dark = np.array( [item[0] for item in data] )
        latitude = np.array( [item[1] for item in data] )
        longitude = np.array( [item[2] for item in data] )
        sun_lat = np.array( [item[3] for item in data] )
        sun_lon = np.array( [item[4] for item in data] )
        date = np.array( [item[5] for item in data] )

        index = np.argsort(date)
        dark = dark[index]
        latitude = latitude[index]
        longitude = longitude[index]
        sun_lat = sun_lat[index]
        sun_lon = sun_lon[index]

        outname = os.path.join(base_dir, detector, '{}_vs_orbit_{}.png'.format(dark_key, segment) )
        plotting.plot_orbital_rate(longitude, latitude, dark, sun_lon, sun_lat, outname )

        #-- Plot histogram of darkrates
        print 'Plotting Hist'
        cursor.execute( """SELECT {},date FROM {} WHERE obsname LIKE '%{}%'""".format(dark_key, table, key))
        data = [ item for item in cursor ]
        dark = np.array( [item[0] for item in data] )
        date = np.array( [item[1] for item in data] )

        index = np.argsort(date)
        date = date[index]
        dark = dark[index]

        for year in set(map(int, date)):
            index = np.where( (date >= year) &
                              (date < year + 1))

            outname = os.path.join(base_dir, detector, '{}_hist_{}_{}.pdf'.format(dark_key, year, segment) )
            plotting.plot_histogram(dark[index], outname )

        index = np.where(date >= date.max() - .5)
        outname = os.path.join(base_dir, detector, '{}_hist_-6mo_{}.pdf'.format(dark_key, segment) )
        plotting.plot_histogram(dark[index], outname )

        outname = os.path.join(base_dir, detector, '{}_hist_{}.pdf'.format(dark_key, segment) )
        plotting.plot_histogram(dark, outname )

#-------------------------------------------------------------------------------

def move_products():
    '''
    Move created pdf files to webpage directory
    '''
    print '#-------------------#'
    print 'Moving products into'
    print 'webpage directory'
    print '#-------------------#'

    for detector in ['FUV', 'NUV']:

        write_dir = web_directory + detector.lower() + '_darks/'
        move_list = glob.glob(base_dir + detector + '/*.p??')

        for item in move_list:
            path, file_to_move = os.path.split(item)
            shutil.copy(item, write_dir + file_to_move)
            print 'Moving: %s' % (file_to_move)

        os.system('chmod 777 ' + write_dir + '*.pdf')

#-------------------------------------------------------------------------------

def monitor():
    """ main monitoring pipeline"""

    get_solar_data('/grp/hst/cos/Monitors/Darks/')

    for detector in ['FUV', 'NUV']:
        compile_darkrates(detector)
        #if detector == 'FUV':
        #    compile_phd()
        make_plots(detector)

        if detector == 'FUV':
            make_plots(detector, TA=True)

    move_products()

#-------------------------------------------------------------------------------

if __name__ == '__main__':
    monitor()
<|endoftext|>