<commit_before># Copyright 2015 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for ping_benchmark."""

import time
import unittest
import mock

from perfkitbenchmarker.linux_benchmarks import object_storage_service_benchmark
from tests import mock_flags

class TestBuildCommands(unittest.TestCase):
  def setUp(self):
    mocked_flags = mock_flags.PatchTestCaseFlags(self)

    mocked_flags.object_storage_multistream_objects_per_stream = 100
    mocked_flags.object_storage_object_sizes = {'1KB': '100%'}
    mocked_flags.object_storage_multistream_num_streams = 1
    mocked_flags.num_vms = 1

  def testBuildCommands(self):
    vm = mock.MagicMock()
    vm.RobustRemoteCommand = mock.MagicMock(return_value=('', ''))

    command_builder = mock.MagicMock()

    with mock.patch(time.__name__ + '.time', return_value=1.0):
      with mock.patch(object_storage_service_benchmark.__name__ +
                      '._ProcessMultiStreamResults'):
        object_storage_service_benchmark.MultiStreamRWBenchmark(
            [], {}, vm, command_builder, None, 'bucket', 'regional-bucket')

    self.assertEqual(
        command_builder.BuildCommand.call_args_list[0],
        mock.call(['--bucket=bucket',
                   '--objects_per_stream=100',
                   '--object_sizes="{1000: 100.0}"',
                   '--num_streams=1',
                   '--start_time=176.0',
                   '--objects_written_file=/tmp/pkb/pkb-objects-written',
                   '--scenario=MultiStreamWrite',
                   '--stream_num_start=0']))

    self.assertEqual(
        command_builder.BuildCommand.call_args_list[1],
        mock.call(['--bucket=bucket',
                   '--objects_per_stream=100',
                   '--num_streams=1',
                   '--start_time=176.0',
                   '--objects_written_file=/tmp/pkb/pkb-objects-written',
                   '--scenario=MultiStreamRead',
                   '--stream_num_start=0']))

class TestDistributionToBackendFormat(unittest.TestCase):
  def testPointDistribution(self):
    dist = {'100KB': '100%'}

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {100000: 100.0})

  def testMultiplePointsDistribution(self):
    dist = {'1B': '10%',
            '2B': '20%',
            '4B': '70%'}

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {1: 10.0,
         2: 20.0,
         4: 70.0})

  def testAbbreviatedPointDistribution(self):
    dist = '100KB'

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {100000: 100.0})

  def testBadPercentages(self):
    dist = {'1B': '50%'}

    with self.assertRaises(ValueError):
      object_storage_service_benchmark._DistributionToBackendFormat(dist)

if __name__ == '__main__':
  unittest.main()
<commit_msg>Fix object storage benchmark tests<commit_after># Copyright 2015 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for ping_benchmark."""

import time
import unittest
import mock

from perfkitbenchmarker.linux_benchmarks import object_storage_service_benchmark
from tests import mock_flags

class TestBuildCommands(unittest.TestCase):
  def setUp(self):
    mocked_flags = mock_flags.PatchTestCaseFlags(self)

    mocked_flags.object_storage_multistream_objects_per_stream = 100
    mocked_flags.object_storage_object_sizes = {'1KB': '100%'}
    mocked_flags.object_storage_multistream_num_streams = 1
    mocked_flags.num_vms = 1

  def testBuildCommands(self):
    vm = mock.MagicMock()
    vm.RobustRemoteCommand = mock.MagicMock(return_value=('', ''))

    command_builder = mock.MagicMock()

    with mock.patch(time.__name__ + '.time', return_value=1.0):
      with mock.patch(object_storage_service_benchmark.__name__ +
                      '._ProcessMultiStreamResults'):
        object_storage_service_benchmark.MultiStreamRWBenchmark(
            [], {}, vm, command_builder, None, 'bucket', 'regional-bucket')

    self.assertEqual(
        command_builder.BuildCommand.call_args_list[0],
        mock.call(['--bucket=bucket',
                   '--objects_per_stream=100',
                   '--object_sizes="{1000: 100.0}"',
                   '--num_streams=1',
                   '--start_time=16.1',
                   '--objects_written_file=/tmp/pkb/pkb-objects-written',
                   '--scenario=MultiStreamWrite',
                   '--stream_num_start=0']))

    self.assertEqual(
        command_builder.BuildCommand.call_args_list[1],
        mock.call(['--bucket=bucket',
                   '--objects_per_stream=100',
                   '--num_streams=1',
                   '--start_time=16.1',
                   '--objects_written_file=/tmp/pkb/pkb-objects-written',
                   '--scenario=MultiStreamRead',
                   '--stream_num_start=0']))

class TestDistributionToBackendFormat(unittest.TestCase):
  def testPointDistribution(self):
    dist = {'100KB': '100%'}

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {100000: 100.0})

  def testMultiplePointsDistribution(self):
    dist = {'1B': '10%',
            '2B': '20%',
            '4B': '70%'}

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {1: 10.0,
         2: 20.0,
         4: 70.0})

  def testAbbreviatedPointDistribution(self):
    dist = '100KB'

    self.assertEqual(
        object_storage_service_benchmark._DistributionToBackendFormat(dist),
        {100000: 100.0})

  def testBadPercentages(self):
    dist = {'1B': '50%'}

    with self.assertRaises(ValueError):
      object_storage_service_benchmark._DistributionToBackendFormat(dist)

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env pythen

"""ShutIt utility functions.
"""

# The MIT License (MIT)
#
# Copyright (C) 2014 OpenBet Limited
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import print_function
import binascii
import getpass
import logging
import os
import random
import re
import readline
import stat
import string
import sys
import threading
import time
import shutit_assets
import shutit_class
import shutit_global

PY3 = (sys.version_info[0] >= 3)

if PY3:
	from builtins import input

def is_file_secure(file_name):
	"""Returns false if file is considered insecure, true if secure.
	If file doesn't exist, it's considered secure!
	"""
	if not os.path.isfile(file_name):
		return True
	file_mode = os.stat(file_name).st_mode
	if file_mode & (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH):
		return False
	return True

def colourise(code, msg):
	"""Colourize the given string for a terminal.
	"""
	return '\033[%sm%s\033[0m' % (code, msg) if code else msg

def random_id(size=8, chars=string.ascii_letters + string.digits):
	"""Generates a random string of given size from the given chars.

	@param size:  The size of the random string.
	@param chars: Constituent pool of characters to draw random characters from.
	@type size:   number
	@type chars:  string
	@rtype:       string
	@return:      The string of random characters.
	"""
	return ''.join(random.choice(chars) for _ in range(size))

def random_word(size=6):
	"""Returns a random word in lower case.
	"""
	words = shutit_assets.get_words().splitlines()
	word = ''
	while len(word) != size or "'" in word:
		word = words[int(random.random() * (len(words) - 1))]
	return word.lower()

def get_hash(string_to_hash):
	"""Helper function to get preceding integer
	eg com.openbet == 1003189494
	>>> import binascii
	>>> abs(binascii.crc32(b'shutit.tk'))
	782914092

	Recommended means of determining run order integer part.
	"""
	return abs(binascii.crc32(string_to_hash.encode()))

# get the ordinal for a given char, in a friendly way
def get_wide_hex(char):
	if len(char) != 2:
		return r'\x' + hex(ord(char))[2:]
	return r'\u' + hex(0x10000 + (ord(char[0]) - 0xD800) * 0x400 + (ord(char[1]) - 0xDC00))[2:]

# CTRL-\ HANDLING CODE STARTS
def ctrl_quit_signal_handler(_,frame):
	print(r'CRTL-\ caught, hard-exiting ShutIt')
	shutit_frame = get_shutit_frame(frame)
	if shutit_frame:
		shutit_class.do_finalize()
	shutit_global.shutit_global_object.handle_exit(exit_code=1)
# CTRL-\ HANDLING CODE ENDS

# CTRL-C HANDLING CODE STARTS
in_ctrlc = False
def ctrlc_background():
	global ctrl_c_calls
	global in_ctrlc
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	in_ctrlc = True
	time.sleep(1)
	in_ctrlc = False

def ctrl_c_signal_handler(_, frame):
	"""CTRL-c signal handler - enters a pause point if it can.
	"""
	global ctrl_c_calls
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	shutit_frame = get_shutit_frame(frame)
	if in_ctrlc:
		msg = 'CTRL-C hit twice, quitting'
		if shutit_frame:
			print('\n')
			shutit = shutit_frame.f_locals['shutit']
			shutit.log(msg,level=logging.CRITICAL)
		else:
			print(msg)
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	if shutit_frame:
		shutit = shutit_frame.f_locals['shutit']
		if shutit.build['ctrlc_passthrough']:
			shutit.self.get_current_shutit_pexpect_session().pexpect_child.sendline(r'')
			return
		print(colourise(31,"\r" + r"You may need to wait for a command to complete before a pause point is available. Alternatively, CTRL-\ to quit."))
		shutit.build['ctrlc_stop'] = True
		t = threading.Thread(target=ctrlc_background)
		t.daemon = True
		t.start()
		# Reset the ctrl-c calls
		ctrl_c_calls = 0
		return
	print(colourise(31,'\n' + '*' * 80))
	print(colourise(31,"CTRL-c caught, CTRL-c twice to quit."))
	print(colourise(31,'*' * 80))
	t = threading.Thread(target=ctrlc_background)
	t.daemon = True
	t.start()
	# Reset the ctrl-c calls
	ctrl_c_calls = 0

def get_shutit_frame(frame):
	global ctrl_c_calls
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	if not frame.f_back:
		return None
	else:
		if 'shutit' in frame.f_locals:
			return frame
		return get_shutit_frame(frame.f_back)
ctrl_c_calls = 0
# CTRL-C HANDLING CODE ENDS

def print_frame_recurse(frame):
	if frame.f_back:
		print('=' * 77)
		print(frame.f_locals)
		print_frame_recurse(frame.f_back)

def check_regexp(regex):
	if regex is None:
		# Is this ok?
		return True
	try:
		re.compile(regex)
		return True
	except re.error:
		return False

def sendline(child, line):
	"""Handles sending of line to pexpect object.
	"""
	child.sendline(line)

def sanitize_terminal():
	os.system('stty sane')

def util_raw_input(prompt='', default=None, ispass=False, use_readline=True):
	"""Handles raw_input calls, and switches off interactivity if there is apparently
	no controlling terminal (or there are any other problems)
	"""
	if use_readline:
		try:
			readline.read_init_file('/etc/inputrc')
		except IOError:
			pass
		readline.parse_and_bind('tab: complete')
	prompt = '\r\n' + prompt
	if ispass:
		prompt += '\r\nInput Secret: '
	sanitize_terminal()
	if shutit_global.shutit_global_object.interactive == 0:
		return default
	if not shutit_global.shutit_global_object.determine_interactive():
		return default
	# See: https//github.com/ianmiell/shutit/issues/299 - python3 made input == python 2's raw_input
	try:
		input = raw_input
	except NameError:
		pass
	while True:
		try:
			if ispass:
				return getpass.getpass(prompt=prompt)
			else:
				return input(prompt).strip() or default
		except KeyboardInterrupt:
			continue
		except IOError:
			msg = 'Problems getting raw input, assuming no controlling terminal.'
	if ispass:
		return getpass.getpass(prompt=prompt)
	else:
		return input(prompt).strip() or default
	shutit_global.shutit_global_object.set_noninteractive(msg=msg)
	return default

def get_input(msg, default='', valid=None, boolean=False, ispass=False, colour='32'):
	"""Gets input from the user, and returns the answer.

	@param msg:       message to send to user
	@param default:   default value if nothing entered
	@param valid:     valid input values (default == empty list == anything allowed)
	@param boolean:   whether return value should be boolean
	@param ispass:    True if this is a password (ie whether to not echo input)
	"""
	if boolean and valid is None:
		valid = ('yes','y','Y','1','true','no','n','N','0','false')
	answer = util_raw_input(prompt=colourise(colour,msg),ispass=ispass)
	if boolean and answer in ('', None) and default != '':
		return default
	if valid is not None:
		while answer not in valid:
			shutit_global.shutit_global_object.log('Answer must be one of: ' + str(valid),transient=True)
			answer = util_raw_input(prompt=colourise(colour,msg),ispass=ispass)
	if boolean:
		if answer.lower() in ('yes','y','1','true','t'):
		        return True
		elif answer.lower() in ('no','n','0','false','f'):
		        return False
	return answer or default
<commit_msg>fix Python 3 compatibility for "shutit skeleton"<commit_after>#!/usr/bin/env pythen

"""ShutIt utility functions.
"""

# The MIT License (MIT)
#
# Copyright (C) 2014 OpenBet Limited
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import print_function
import binascii
import getpass
import logging
import os
import random
import re
import readline
import stat
import string
import sys
import threading
import time
import shutit_assets
import shutit_class
import shutit_global

PY3 = (sys.version_info[0] >= 3)

if PY3:
	from builtins import input

def is_file_secure(file_name):
	"""Returns false if file is considered insecure, true if secure.
	If file doesn't exist, it's considered secure!
	"""
	if not os.path.isfile(file_name):
		return True
	file_mode = os.stat(file_name).st_mode
	if file_mode & (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH):
		return False
	return True

def colourise(code, msg):
	"""Colourize the given string for a terminal.
	"""
	return '\033[%sm%s\033[0m' % (code, msg) if code else msg

def random_id(size=8, chars=string.ascii_letters + string.digits):
	"""Generates a random string of given size from the given chars.

	@param size:  The size of the random string.
	@param chars: Constituent pool of characters to draw random characters from.
	@type size:   number
	@type chars:  string
	@rtype:       string
	@return:      The string of random characters.
	"""
	return ''.join(random.choice(chars) for _ in range(size))

def random_word(size=6):
	"""Returns a random word in lower case.
	"""
	words = shutit_assets.get_words().splitlines()
	word = ''
	while len(word) != size or "'" in word:
		word = words[int(random.random() * (len(words) - 1))]
	return word.lower()

def get_hash(string_to_hash):
	"""Helper function to get preceding integer
	eg com.openbet == 1003189494
	>>> import binascii
	>>> abs(binascii.crc32(b'shutit.tk'))
	782914092

	Recommended means of determining run order integer part.
	"""
	return abs(binascii.crc32(string_to_hash.encode()))

# get the ordinal for a given char, in a friendly way
def get_wide_hex(char):
	if len(char) != 2:
		return r'\x' + hex(ord(char))[2:]
	return r'\u' + hex(0x10000 + (ord(char[0]) - 0xD800) * 0x400 + (ord(char[1]) - 0xDC00))[2:]

# CTRL-\ HANDLING CODE STARTS
def ctrl_quit_signal_handler(_,frame):
	print(r'CRTL-\ caught, hard-exiting ShutIt')
	shutit_frame = get_shutit_frame(frame)
	if shutit_frame:
		shutit_class.do_finalize()
	shutit_global.shutit_global_object.handle_exit(exit_code=1)
# CTRL-\ HANDLING CODE ENDS

# CTRL-C HANDLING CODE STARTS
in_ctrlc = False
def ctrlc_background():
	global ctrl_c_calls
	global in_ctrlc
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	in_ctrlc = True
	time.sleep(1)
	in_ctrlc = False

def ctrl_c_signal_handler(_, frame):
	"""CTRL-c signal handler - enters a pause point if it can.
	"""
	global ctrl_c_calls
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	shutit_frame = get_shutit_frame(frame)
	if in_ctrlc:
		msg = 'CTRL-C hit twice, quitting'
		if shutit_frame:
			print('\n')
			shutit = shutit_frame.f_locals['shutit']
			shutit.log(msg,level=logging.CRITICAL)
		else:
			print(msg)
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	if shutit_frame:
		shutit = shutit_frame.f_locals['shutit']
		if shutit.build['ctrlc_passthrough']:
			shutit.self.get_current_shutit_pexpect_session().pexpect_child.sendline(r'')
			return
		print(colourise(31,"\r" + r"You may need to wait for a command to complete before a pause point is available. Alternatively, CTRL-\ to quit."))
		shutit.build['ctrlc_stop'] = True
		t = threading.Thread(target=ctrlc_background)
		t.daemon = True
		t.start()
		# Reset the ctrl-c calls
		ctrl_c_calls = 0
		return
	print(colourise(31,'\n' + '*' * 80))
	print(colourise(31,"CTRL-c caught, CTRL-c twice to quit."))
	print(colourise(31,'*' * 80))
	t = threading.Thread(target=ctrlc_background)
	t.daemon = True
	t.start()
	# Reset the ctrl-c calls
	ctrl_c_calls = 0

def get_shutit_frame(frame):
	global ctrl_c_calls
	ctrl_c_calls += 1
	if ctrl_c_calls > 10:
		shutit_global.shutit_global_object.handle_exit(exit_code=1)
	if not frame.f_back:
		return None
	else:
		if 'shutit' in frame.f_locals:
			return frame
		return get_shutit_frame(frame.f_back)
ctrl_c_calls = 0
# CTRL-C HANDLING CODE ENDS

def print_frame_recurse(frame):
	if frame.f_back:
		print('=' * 77)
		print(frame.f_locals)
		print_frame_recurse(frame.f_back)

def check_regexp(regex):
	if regex is None:
		# Is this ok?
		return True
	try:
		re.compile(regex)
		return True
	except re.error:
		return False

def sendline(child, line):
	"""Handles sending of line to pexpect object.
	"""
	child.sendline(line)

def sanitize_terminal():
	os.system('stty sane')

def util_raw_input(prompt='', default=None, ispass=False, use_readline=True):
	"""Handles raw_input calls, and switches off interactivity if there is apparently
	no controlling terminal (or there are any other problems)
	"""
	if use_readline:
		try:
			readline.read_init_file('/etc/inputrc')
		except IOError:
			pass
		readline.parse_and_bind('tab: complete')
	prompt = '\r\n' + prompt
	if ispass:
		prompt += '\r\nInput Secret: '
	sanitize_terminal()
	if shutit_global.shutit_global_object.interactive == 0:
		return default
	if not shutit_global.shutit_global_object.determine_interactive():
		return default
	while True:
		try:
			if ispass:
				return getpass.getpass(prompt=prompt)
			else:
				return input(prompt).strip() or default
		except KeyboardInterrupt:
			continue
		except IOError:
			msg = 'Problems getting raw input, assuming no controlling terminal.'
	if ispass:
		return getpass.getpass(prompt=prompt)
	else:
		return input(prompt).strip() or default
	shutit_global.shutit_global_object.set_noninteractive(msg=msg)
	return default

def get_input(msg, default='', valid=None, boolean=False, ispass=False, colour='32'):
	"""Gets input from the user, and returns the answer.

	@param msg:       message to send to user
	@param default:   default value if nothing entered
	@param valid:     valid input values (default == empty list == anything allowed)
	@param boolean:   whether return value should be boolean
	@param ispass:    True if this is a password (ie whether to not echo input)
	"""
	if boolean and valid is None:
		valid = ('yes','y','Y','1','true','no','n','N','0','false')
	answer = util_raw_input(prompt=colourise(colour,msg),ispass=ispass)
	if boolean and answer in ('', None) and default != '':
		return default
	if valid is not None:
		while answer not in valid:
			shutit_global.shutit_global_object.log('Answer must be one of: ' + str(valid),transient=True)
			answer = util_raw_input(prompt=colourise(colour,msg),ispass=ispass)
	if boolean:
		if answer.lower() in ('yes','y','1','true','t'):
		        return True
		elif answer.lower() in ('no','n','0','false','f'):
		        return False
	return answer or default
<|endoftext|>


<commit_before>""":mod:`sider.types` --- Conversion between Python and Redis types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

In Redis all data are byte strings --- *bulks*.  Lists are lists of byte
strings, sets are sets of byte strings, and hashes consist of byte string
keys and byte string values.

To store richier objects into Redis we have to encode Python values and
decode Redis data.  :class:`Bulk` and its subclasses are for that, it
defines two basic methods: :meth:`~Bulk.encode()` and
:meth:`~Bulk.decode()`.  For example, :class:`Integer` encodes Python
:class:`int` ``3`` into Redis bulk ``"3"`` and decodes Redis bulk ``"3"``
into Python :class:`int` ``3``.

"""
from __future__ import absolute_import
import numbers

class Bulk(object):
    """The abstract base class to be subclassed.  You have to implement
    :meth:`encode()` and :meth:`decode()` methods in subclasses.

    """

    def encode(self, value):
        """Encodes a Python ``value`` into Redis bulk.  Every subclass of
        :class:`Bulk` must implement this method.  By default it raises
        :exc:`NotImplementedError`.

        :param value: a Python value to encode into Redis bulk
        :returns: an encoded Redis bulk
        :rtype: :class:`str`
        :raises: :exc:`~exceptions.TypeError` if the type of a given
                 value is not acceptable by this type

        """
        cls = type(self)
        raise NotImplementedError(
            '{0}.{1}.to_redis() method must be overridden by '
            'subclass'.format(cls.__module__, cls.__name__)
        )

    def decode(self, bulk):
        """Decodes a Redis ``bulk`` to Python object.  Every subclass of
        :class:`Bulk` must implement this method.  By default it raises
        :exc:`NotImplementedError`.

        :param bulk: a Redis bullk to decode into Python object
        :type bulk: :class:`str`
        :returns: a decoded Python object

        """
        cls = type(self)
        raise NotImplementedError(
            '{0}.{1}.to_redis() method must be overridden by '
            'subclass'.format(cls.__module__, cls.__name__)
        )

class Integer(Bulk):
    """Stores integers as decimal strings.  For example:

    .. sourcecode:: pycon

       >>> integer = Integer()
       >>> integer.encode(42)
       '42'
       >>> integer.decode('42')
       42

    Why it doesn't store integers as binaries but decimals is that
    Redis provides :redis:`INCR`, :redis:`INCRBY`, :redis:`DECR` and
    :redis:`DECRBY` for decimal strings.  You can simply add and
    subtract integers.

    """

    def encode(self, value):
        if not isinstance(value, numbers.Integral):
            raise TypeError('expected an integer, not ' + repr(value))
        return str(value)

    def decode(self, bulk):
        return int(bulk)

class ByteString(Bulk):
    """Stores byte strings.  It stores the given byte strings as these are.
    It works simply transparently for :class:`str` values.

    .. sourcecode:: pycon

       >>> bytestr = ByteString()
       >>> bytestr.encode('annyeong')
       'annyeong'
       >>> bytestr.decode('sayonara')
       'sayonara'

    """

    def encode(self, value):
        if not isinstance(value, str):
            raise ValueError('expected a byte str, not ' + repr(value))
        return value

    def decode(self, bulk):
        return bulk

class UnicodeString(Bulk):
    r"""Stores Unicode strings (:class:`unicode`), not byte strings
    (:class:`str`).  Internally all Unicode strings are encoded into
    and decoded from UTF-8 byte strings.

    .. sourcecode:: pycon

       >>> unistr = UnicodeString()
       >>> unistr.encode(u'\uc720\ub2c8\ucf54\ub4dc')
       '\xec\x9c\xa0\xeb\x8b\x88\xec\xbd\x94\xeb\x93\x9c'
       >>> unistr.decode(_)
       u'\uc720\ub2c8\ucf54\ub4dc'

    """

    def encode(self, value):
        if not isinstance(value, unicode):
            raise ValueError('expected a unicode string, not ' + repr(value))
        return value.encode('utf-8')

    def decode(self, bulk):
        return bulk.decode('utf-8')

<commit_msg>Added sider.types.Bulk.ensure_type() class method<commit_after>""":mod:`sider.types` --- Conversion between Python and Redis types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

In Redis all data are byte strings --- *bulks*.  Lists are lists of byte
strings, sets are sets of byte strings, and hashes consist of byte string
keys and byte string values.

To store richier objects into Redis we have to encode Python values and
decode Redis data.  :class:`Bulk` and its subclasses are for that, it
defines two basic methods: :meth:`~Bulk.encode()` and
:meth:`~Bulk.decode()`.  For example, :class:`Integer` encodes Python
:class:`int` ``3`` into Redis bulk ``"3"`` and decodes Redis bulk ``"3"``
into Python :class:`int` ``3``.

"""
from __future__ import absolute_import
import numbers

class Bulk(object):
    """The abstract base class to be subclassed.  You have to implement
    :meth:`encode()` and :meth:`decode()` methods in subclasses.

    """

    @classmethod
    def ensure_subtype(cls, subtype, parameter=None):
        """Raises a :exc:`TypeError` if the given ``subtype`` is not
        a subclass of the class.

        .. sourcecode:: pycon

           >>> Integer.ensure_subtype(Bulk)  # doctest: +NORMALIZE_WHITESPACE
           Traceback (most recent call last):
             ...
           TypeError: expected a subtype of sider.types.Integer,
                      but sider.types.Bulk was passed
           >>> Bulk.ensure_subtype(1)  # doctest: +NORMALIZE_WHITESPACE
           Traceback (most recent call last):
             ...
           TypeError: expected a type, not 1

        Otherwise it simply does nothing.

        .. sourcecode:: pycon

           >>> Bulk.ensure_subtype(Bulk)
           >>> Bulk.ensure_subtype(ByteString)
           >>> ByteString.ensure_subtype(ByteString)

        If an optional ``parameter`` name has present, the error message
        becomes better.

        .. sourcecode:: pycon

           >>> Integer.ensure_subtype(Bulk,
           ...   parameter='argname')  # doctest: +NORMALIZE_WHITESPACE
           Traceback (most recent call last):
             ...
           TypeError: argname must be a subtype of sider.types.Integer,
                      but sider.types.Bulk was passed
           >>> Bulk.ensure_subtype(1,
           ...   parameter='argname')  # doctest: +NORMALIZE_WHITESPACE
           Traceback (most recent call last):
             ...
           TypeError: argname must be a type, not 1

        :param subtype: a type expected to be a subtype of the class
        :type subtype: :class:`type`
        :param parameter: an optional parameter name.
                          if present the error message becomes better
        :type parameter: :class:`str`
        :raises: :exc:`TypeError` if the given ``subtype`` is not
                 a subclass of the class

        """
        if not isinstance(subtype, type):
            if parameter:
                msg = '{0} must be a type, not {1!r}'
            else:
                msg = 'expected a type, not {1!r}'
            raise TypeError(msg.format(parameter, subtype))
        elif not issubclass(subtype, cls):
            if parameter:
                msg = '{0} must be a subtype of {1}, but {2} was passed'
            else:
                msg = 'expected a subtype of {1}, but {2} was passed'
            typename = '.'.join((cls.__module__, cls.__name__))
            subname = '.'.join((subtype.__module__, subtype.__name__))
            raise TypeError(msg.format(parameter, typename, subname))

    def encode(self, value):
        """Encodes a Python ``value`` into Redis bulk.  Every subclass of
        :class:`Bulk` must implement this method.  By default it raises
        :exc:`NotImplementedError`.

        :param value: a Python value to encode into Redis bulk
        :returns: an encoded Redis bulk
        :rtype: :class:`str`
        :raises: :exc:`~exceptions.TypeError` if the type of a given
                 value is not acceptable by this type

        """
        cls = type(self)
        raise NotImplementedError(
            '{0}.{1}.to_redis() method must be overridden by '
            'subclass'.format(cls.__module__, cls.__name__)
        )

    def decode(self, bulk):
        """Decodes a Redis ``bulk`` to Python object.  Every subclass of
        :class:`Bulk` must implement this method.  By default it raises
        :exc:`NotImplementedError`.

        :param bulk: a Redis bullk to decode into Python object
        :type bulk: :class:`str`
        :returns: a decoded Python object

        """
        cls = type(self)
        raise NotImplementedError(
            '{0}.{1}.to_redis() method must be overridden by '
            'subclass'.format(cls.__module__, cls.__name__)
        )

class Integer(Bulk):
    """Stores integers as decimal strings.  For example:

    .. sourcecode:: pycon

       >>> integer = Integer()
       >>> integer.encode(42)
       '42'
       >>> integer.decode('42')
       42

    Why it doesn't store integers as binaries but decimals is that
    Redis provides :redis:`INCR`, :redis:`INCRBY`, :redis:`DECR` and
    :redis:`DECRBY` for decimal strings.  You can simply add and
    subtract integers.

    """

    def encode(self, value):
        if not isinstance(value, numbers.Integral):
            raise TypeError('expected an integer, not ' + repr(value))
        return str(value)

    def decode(self, bulk):
        return int(bulk)

class ByteString(Bulk):
    """Stores byte strings.  It stores the given byte strings as these are.
    It works simply transparently for :class:`str` values.

    .. sourcecode:: pycon

       >>> bytestr = ByteString()
       >>> bytestr.encode('annyeong')
       'annyeong'
       >>> bytestr.decode('sayonara')
       'sayonara'

    """

    def encode(self, value):
        if not isinstance(value, str):
            raise ValueError('expected a byte str, not ' + repr(value))
        return value

    def decode(self, bulk):
        return bulk

class UnicodeString(Bulk):
    r"""Stores Unicode strings (:class:`unicode`), not byte strings
    (:class:`str`).  Internally all Unicode strings are encoded into
    and decoded from UTF-8 byte strings.

    .. sourcecode:: pycon

       >>> unistr = UnicodeString()
       >>> unistr.encode(u'\uc720\ub2c8\ucf54\ub4dc')
       '\xec\x9c\xa0\xeb\x8b\x88\xec\xbd\x94\xeb\x93\x9c'
       >>> unistr.decode(_)
       u'\uc720\ub2c8\ucf54\ub4dc'

    """

    def encode(self, value):
        if not isinstance(value, unicode):
            raise ValueError('expected a unicode string, not ' + repr(value))
        return value.encode('utf-8')

    def decode(self, bulk):
        return bulk.decode('utf-8')

<|endoftext|>


<commit_before>def sieve(n):
    if n < 2:
        return []
    not_prime = set()
    prime = [2]
    for i in range(3, n+1, 2):
        if i not in not_prime:
            prime.append(i)
            not_prime.update(range(i*i, n, i))
    return prime
<commit_msg>Fix bug where n is the square of a prime<commit_after>def sieve(n):
    if n < 2:
        return []
    not_prime = set()
    prime = [2]
    for i in range(3, n+1, 2):
        if i not in not_prime:
            prime.append(i)
            not_prime.update(range(i*i, n+1, i))
    return prime
<|endoftext|>


<commit_before>import logging

from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q

from goals.models import CustomAction, UserAction

logger = logging.getLogger("loggly_logs")

class Command(BaseCommand):
    help = 'Updates the next_trigger_date field for stale UserActions and CustomActions.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--user',
            action='store',
            dest='user',
            default=None,
            help=("Restrict this command to the given User. "
                  "Accepts a username, email, or id")
        )
        parser.add_argument(
            '--hours',
            action='store',
            dest='hours',
            default=2,
            type=int,
            help="Limit to objects that are older than the given value (in hours)"
        )
        parser.add_argument(
            '--limit',
            action='store',
            dest='limit',
            default=100,
            type=int,
            help="Limit the number of objects that are modified"
        )

    def _user_kwargs(self, user):
        User = get_user_model()

        kwargs = {}  # Keyword arguments that get passed to our action querysets.
        if user:
            try:
                if user.isnumeric():
                    criteria = Q(id=user)
                else:
                    criteria = (Q(username=user) | Q(email=user))
                kwargs['user'] = User.objects.get(criteria)
            except User.DoesNotExist:
                msg = "Could not find user: {0}".format(user)
                raise CommandError(msg)
        return kwargs

    def handle(self, *args, **options):
        count = 0
        kwargs = self._user_kwargs(options['user'])
        limit = options.pop('limit')

        # Limit the refreshed objects to those that are at least 2 hours old.
        # (or to an age based on limit option)
        hours = options.pop('hours')
        kwargs['hours'] = hours

        # Fetch the set of UserActions to refresh.
        # We want to limit these to reduce the amount of time it takes to
        # complete execution of this script.
        useractions = UserAction.objects.stale(**kwargs).published()
        if limit:
            useractions = useractions[:limit]

        # If we're in staging or dev, only do this for our accounts.
        if settings.STAGING or settings.DEBUG:
            useractions = useractions.filter(user__email__icontains="@tndata.org")

        for ua in useractions:
            count += 1
            ua.save(update_triggers=True)  # fields get refreshed on save.

        msg = "Refreshed Trigger Date for {0} UserActions".format(count)
        logger.error(msg)
        self.stderr.write(msg)

        count = 0
        for ca in CustomAction.objects.stale(**kwargs):
            count += 1
            ca.save()  # fields get refreshed on save.

        msg = "Refreshed Trigger Date for {0} CustomActions".format(count)
        logger.error(msg)
        self.stderr.write(msg)
<commit_msg>reviste refresh_useractions so that it separates objects that are part of sequenced, dynamic notifications from those that have customized triggers<commit_after>import logging

from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q

from goals.sequence import get_next_useractions_in_sequence
from goals.models import CustomAction, UserAction

logger = logging.getLogger("loggly_logs")

class Command(BaseCommand):
    help = 'Updates the next_trigger_date field for stale UserActions and CustomActions.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--user',
            action='store',
            dest='user',
            default=None,
            help=("Restrict this command to the given User. "
                  "Accepts a username, email, or id")
        )
        parser.add_argument(
            '--hours',
            action='store',
            dest='hours',
            default=2,
            type=int,
            help="Limit to objects that are older than the given value (in hours)"
        )
        parser.add_argument(
            '--limit',
            action='store',
            dest='limit',
            default=100,
            type=int,
            help="Limit the number of objects that are modified"
        )

    def _users(self, user):
        """Pull the `user` option return a queryset of of Users."""
        User = get_user_model()
        users = User.objects.filter(is_active=True)
        if user and user.isnumeric():
            users = users.filter(id=user)
        elif user:
            users = users.filter(Q(username=user) | Q(email=user))

        # Restrict to internal users for staging and/or debug
        if settings.STAGING or settings.DEBUG:
            users = users.filter(email__icontains="@tndata.org")
        return users

    def _useraction_kwargs(self, user):
        """Pull the `user` option and create a dict of keyword arguments
        that can be passed to the UserAction query.

        """
        User = get_user_model()

        kwargs = {}
        if user:
            try:
                if user.isnumeric():
                    criteria = Q(id=user)
                else:
                    criteria = (Q(username=user) | Q(email=user))
                kwargs['user'] = User.objects.get(criteria)
            except User.DoesNotExist:
                msg = "Could not find user: {0}".format(user)
                raise CommandError(msg)

        # Restrict to internal users for staging and/or debug
        if settings.STAGING or settings.DEBUG:
            kwargs['user__email__icontains'] = "@tndata.org"

        return kwargs

    def handle(self, *args, **options):
        # Limit the refreshed objects to those that are at least 2 hours old.
        # (or to an age based on limit option)
        limit = options.pop('limit')
        hours = options.pop('hours')

        # 1. Fetch the set of SEQUENCED UserActions to refresh.
        count = 0
        for user in self._users(options['user']):
            useractions = get_next_useractions_in_sequence(user)
            useractions = useractions.stale(hours=hours).published()
            for ua in useractions:
                count += 1
                ua.save(update_triggers=True)  # fields get refreshed on save.

        msg = "Refreshed Trigger Date for {0} UserActions".format(count)
        logger.info(msg)
        self.stderr.write(msg)

        # 2. Refresh stale UserActions that have a custom trigger
        count = 0
        kwargs = self._useraction_kwargs(options['user'])
        kwargs['hours'] = hours
        useractions = UserAction.objects.stale(
            hours=hours,
            custom_trigger__isnull=False
        )
        for ua in useractions.published()[:limit]:
            count += 1
            ua.save(update_triggers=True)

        msg = "Refreshed Custom Triggers for {0} UserActions".format(count)
        logger.info(msg)
        self.stderr.write(msg)

        # 3. Update all custom actions
        count = 0
        for ca in CustomAction.objects.stale(**kwargs):
            count += 1
            ca.save()  # fields get refreshed on save.

        msg = "Refreshed Trigger Date for {0} CustomActions".format(count)
        logger.info(msg)
        self.stderr.write(msg)
<|endoftext|>


<commit_before># Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

from conary.build import macros
from conary.build.recipe import Recipe, RECIPE_TYPE_DERIVEDPKG

class DerivedPackageRecipe(Recipe):

    _recipeType = RECIPE_TYPE_DERIVEDPKG
    internalAbstractBaseClass = 1
    parentVersion = None

    def updateTroves(self, troves):
        for trv in troves:
            rmvList = []
            for (pathId, path, fileId, version) in trv.iterFileList():
                if path in self.removedPaths:
                    rmvList.append(pathId)

            for pathId in rmvList:
                trv.removeFile(pathId)

    def remove(self, pattern):
        self.removedPaths.add(pattern)

    def __init__(self, repos, cfg, branch, flavor, extraMacros={}):
        self.repos = repos
        self.cfg = cfg
        self.branch = branch
        self.flavor = flavor
        self.macros = macros.Macros()
        self.macros.update(extraMacros)
        self.removedPaths = set()
<commit_msg>switch derived packages to inherting from abstractpackagerecipe class instead of being a new top-level class<commit_after># Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

from conary.build import macros
from conary.build.recipe import RECIPE_TYPE_DERIVEDPKG
from conary.build.packagerecipe import _AbstractPackageRecipe

class DerivedPackageRecipe(_AbstractPackageRecipe):

    _recipeType = RECIPE_TYPE_DERIVEDPKG
    internalAbstractBaseClass = 1
    parentVersion = None

    def updateTroves(self, troves):
        for trv in troves:
            rmvList = []
            for (pathId, path, fileId, version) in trv.iterFileList():
                if path in self.removedPaths:
                    rmvList.append(pathId)

            for pathId in rmvList:
                trv.removeFile(pathId)

    def remove(self, pattern):
        self.removedPaths.add(pattern)

    def __init__(self, cfg, laReposCache, srcdirs, extraMacros={},
                 crossCompile=None, lightInstance=False):
        _AbstractPackageRecipe.__init__(self, cfg, laReposCache, srcDirs,
                                        extraMacros = extraMacros,
                                        crossCompile = crossCompile,
                                        lightInstance = lightInstance)
        self.repos = laReposCache.repos
        self.removedPaths = set()
<|endoftext|>


<commit_before>#from __future__ import print_function # good idea for python < 3
#from __future__ import division # good idea for python < 3
import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
'''
reads in a PHOENIX model spectrum (FITS format)
makes a continuum flattened (normalized) spectrum over a specified wavelength range
saves the result to a text file

'specfile' is the FITS HiRes PHOENIX spectrum file which lives in directory 'dir'
'outtxt' is the new text file that will be written
'idxstart' and 'idxend' set the index of the wavelengths that are written out
(you can set the indices to 0 and -1, but that's a much bigger wavelength range than APOGEE)

you can get PHOENIX spectra from here: http://phoenix.astro.physik.uni-goettingen.de
(click "version 2.0 of the spectral library" and login as a guest when prompted)
'''

def moving_average(series, window=100, sigma=50):
    '''
    Calculates the moving average of a series by using a Gaussian kernel to smooth it.
    This function is used by continuumFlattenSpec.
    Borrowed from http://www.nehalemlabs.net/prototype/blog/2014/04/12/how-to-fix-scipys-interpolating-spline-default-behavior/

    Input: series or array, window size, and Gaussian width (sigma).
    Returns: the smoothed data and the smoothed variance.
    '''
    from scipy.signal import gaussian
    from scipy.ndimage import filters
    b = gaussian(window, sigma)
    average = filters.convolve1d(series, b/b.sum())
    var = filters.convolve1d(np.power(series-average,2), b/b.sum())
    return average, var

def continuumFlattenSpec(waves, fluxes, win=50, varwin=60, varsig=100, weight=1.0, fitplot=True):
    '''
    Fits a spline to a spectrum and divides to continuum flatten it.
    Weighting inspired by http://www.nehalemlabs.net/prototype/blog/2014/04/12/how-to-fix-scipys-interpolating-spline-default-behavior/

    !!WARNING!! this function is not one-size-fits-all and the input parameters will
    probably need to be significantly tweaked for your specific case! It is VERY EASY
    to accidentally overfit or underfit your spectrum and get a poor continuum fit!

    More specifically, this function traces the shape of the spectrum in two ways:
    (1) the maximum values of a rolling window function with size 'win'
        --> this traces the TOP of the spectrum and may be an overestimate if it's spikey
    (2) a Gaussian-smoothed version of the spectrum with window size 'varwin' and Gaussian width 'varsig'
        --> this traces the spectrum's shape but is systematically LOWER than the continuum
        --> the variance of the spectrum is also saved during this step

    Then, scipy's UnivariateSpline function is used to fit a 5th order spline to either
    (1) OR a combination of (1) and (2). In both cases, the fit is weighted by
    by 'weight' divided by the square root of /sqrt(variance).
    This kind of weighting helps balance overfitting vs. underfitting, as described
    at the link above, but it is not perfect.

    Finally, a plot of the result is shown (assuming 'fitplot' is True) so the user can
    decide if the continuum fit is acceptable or not.

    Input: wavelength array, flux array, and the parameters described above
    Returns: a pair of numpy arrays corresponding to wavelengths and flattened fluxes.
    Each array has length = original_length - 2*window.
    '''
    from scipy.interpolate import UnivariateSpline
    import pandas as pd
    from PyAstronomy import pyasl
    # Fit the continuum
    window = win
    specsmooth_top = pd.Series(fluxes).rolling(window=window, center=True).max()
    specsmooth_bot, variance = np.array(moving_average(fluxes, window=varwin, sigma=varsig))
    variance = variance[window/2:-window/2]
    w = weight/np.sqrt(variance)
    specsmooth = []
    for top in np.array(specsmooth_top)[window/2:-window/2]:
        specsmooth.append(top) # BETTER CHOICE IF SPECTRUM IS NOT SPIKEY
        #specsmooth.append((top + bot) / 2.) # BETTER CHOICE IF SPECTRUM IS SPIKEY
    waves = waves[window/2:-window/2]
    fluxes = fluxes[window/2:-window/2]
    continuum = UnivariateSpline(waves, specsmooth, k=5, w=w)(waves)
    # Plot the result
    if fitplot == True:
        fig = plt.figure()
        fig.add_subplot(211)
        plt.plot(waves, fluxes, 'b-', label='spectrum')
        plt.plot(waves, np.array(specsmooth_top)[window/2:-window/2], 'k:')
        plt.plot(waves, np.array(specsmooth_bot)[window/2:-window/2], 'k:')
        plt.plot(waves, specsmooth, 'k-', label='smoothed')
        plt.plot(waves, continuum, 'r-', label='continuum')
        plt.legend()
        fig.add_subplot(212)
        plt.plot(waves, fluxes/continuum, 'b-', label='flattened')
        plt.axhline(y=1, ls=':', color='k')
        plt.legend()
        plt.show()
    return waves, fluxes/continuum # numpy arrays

### MAIN PROGRAM BEGINS HERE ###

#######################
## edit values below ##
#######################
# directory where wavelength file lives:
#wavedir = '../../../PHOENIX/PHOENIX-ACES-AGSS-COND-2011/'
# directory where spectrum file lives:
#specdir = wavedir + 'Z-0.0/'
# spectrum file you want to run the program on:
#specfile = 'lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
#specfile = 'lte04800-2.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
specfile = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#specfile = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#specfile = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358

# wavelength region that will be fit with a spline:
#wavestart = 4318; waveend = 7200
#wavestart = 670; waveend = 2470
wavestart = 14000; waveend = 18000
# wavelength region that will be saved (must be no longer than the above):
#truncstart = 700; truncend = 2500
truncstart = 15000; truncend = 17000
# spline fitting parameters (final result is VERY SENSITIVE to these!!):
splinewindow = 100
variancewindow = 70 #60
variancesigma = 200 #150
splineweight = 0.9
#######################
## edit values above ##
#######################

wavefits = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#wavefits = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#wavefits = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358
#infits = 'lte05000-3.00+0.5.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
#infits = wavedir + specfile
#infits = specdir + specfile
infits = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#infits = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#infits = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358
outtxt = specfile[:-5] + '_' + str(truncstart) + '-' + str(truncend) + '-norm.txt'

# read in PHOENIX FITS spectrum and corresponding wavelength FITS file
hdu = fits.open(infits)
spec = hdu[0].data
hduwave = fits.open(wavefits)
wave = hduwave[0].data

# put the wavelength array in goddamn air units
# reference: Huswer et al. 2013
wave = wave / (1.0 + 0.05792105/(238.0185-(1e4/wave)**2) + 0.00167917/(57.362-(1e4/wave)**2))

# truncate spectrum to the range you want to fit
idxstart = np.where(wave > wavestart)[0][0]
idxend = np.where(wave > waveend)[0][0]
wavechunk = wave[idxstart:idxend]
specchunk = spec[idxstart:idxend]

# measure the continuum level and divide by it to flatten the spectrum
print('Flattening spectrum...')
wavenorm, specnorm = continuumFlattenSpec(wavechunk, specchunk, win=splinewindow,
                                            varwin=variancewindow, varsig=variancesigma,
                                            weight=splineweight, fitplot=True)

# truncate the flattened spectrum to final desired wavelength range
idxtruncstart = np.where(wavenorm > truncstart-100)[0][0]
idxtruncend = np.where(wavenorm > truncend+100)[0][0]
wavenorm = wavenorm[idxtruncstart:idxtruncend]
specnorm = specnorm[idxtruncstart:idxtruncend]

# plot the final spectrum
plt.plot(wavenorm, specnorm, ls='None', marker='.', label='final flattened spectrum')
plt.legend()
plt.show()

# write the result to a text file
f = open(outtxt, 'w')
for wentry, sentry in zip(wavenorm, specnorm):
    print(wentry, sentry, file=f)
f.close()
print('New spectrum written to {0}'.format(outtxt))<commit_msg>Poked spline parameters, then returned them to their original settings<commit_after>#from __future__ import print_function # good idea for python < 3
#from __future__ import division # good idea for python < 3
import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
'''
reads in a PHOENIX model spectrum (FITS format)
makes a continuum flattened (normalized) spectrum over a specified wavelength range
saves the result to a text file

'specfile' is the FITS HiRes PHOENIX spectrum file which lives in directory 'dir'
'outtxt' is the new text file that will be written
'idxstart' and 'idxend' set the index of the wavelengths that are written out
(you can set the indices to 0 and -1, but that's a much bigger wavelength range than APOGEE)

you can get PHOENIX spectra from here: http://phoenix.astro.physik.uni-goettingen.de
(click "version 2.0 of the spectral library" and login as a guest when prompted)
'''

def moving_average(series, window=100, sigma=50):
    '''
    Calculates the moving average of a series by using a Gaussian kernel to smooth it.
    This function is used by continuumFlattenSpec.
    Borrowed from http://www.nehalemlabs.net/prototype/blog/2014/04/12/how-to-fix-scipys-interpolating-spline-default-behavior/

    Input: series or array, window size, and Gaussian width (sigma).
    Returns: the smoothed data and the smoothed variance.
    '''
    from scipy.signal import gaussian
    from scipy.ndimage import filters
    b = gaussian(window, sigma)
    average = filters.convolve1d(series, b/b.sum())
    var = filters.convolve1d(np.power(series-average,2), b/b.sum())
    return average, var

def continuumFlattenSpec(waves, fluxes, win=50, varwin=60, varsig=100, weight=1.0, fitplot=True):
    '''
    Fits a spline to a spectrum and divides to continuum flatten it.
    Weighting inspired by http://www.nehalemlabs.net/prototype/blog/2014/04/12/how-to-fix-scipys-interpolating-spline-default-behavior/

    !!WARNING!! this function is not one-size-fits-all and the input parameters will
    probably need to be significantly tweaked for your specific case! It is VERY EASY
    to accidentally overfit or underfit your spectrum and get a poor continuum fit!

    More specifically, this function traces the shape of the spectrum in two ways:
    (1) the maximum values of a rolling window function with size 'win'
        --> this traces the TOP of the spectrum and may be an overestimate if it's spikey
    (2) a Gaussian-smoothed version of the spectrum with window size 'varwin' and Gaussian width 'varsig'
        --> this traces the spectrum's shape but is systematically LOWER than the continuum
        --> the variance of the spectrum is also saved during this step

    Then, scipy's UnivariateSpline function is used to fit a 5th order spline to either
    (1) OR a combination of (1) and (2). In both cases, the fit is weighted by
    by 'weight' divided by the square root of /sqrt(variance).
    This kind of weighting helps balance overfitting vs. underfitting, as described
    at the link above, but it is not perfect.

    Finally, a plot of the result is shown (assuming 'fitplot' is True) so the user can
    decide if the continuum fit is acceptable or not.

    Input: wavelength array, flux array, and the parameters described above
    Returns: a pair of numpy arrays corresponding to wavelengths and flattened fluxes.
    Each array has length = original_length - 2*window.
    '''
    from scipy.interpolate import UnivariateSpline
    import pandas as pd
    from PyAstronomy import pyasl
    # Fit the continuum
    window = win
    specsmooth_top = pd.Series(fluxes).rolling(window=window, center=True).max()
    specsmooth_bot, variance = np.array(moving_average(fluxes, window=varwin, sigma=varsig))
    variance = variance[window/2:-window/2]
    w = weight/np.sqrt(variance)
    specsmooth = []
    for top in np.array(specsmooth_top)[window/2:-window/2]:
        specsmooth.append(top) # BETTER CHOICE IF SPECTRUM IS NOT SPIKEY
        #specsmooth.append((top + bot) / 2.) # BETTER CHOICE IF SPECTRUM IS SPIKEY
    waves = waves[window/2:-window/2]
    fluxes = fluxes[window/2:-window/2]
    continuum = UnivariateSpline(waves, specsmooth, k=5, w=w)(waves)
    # Plot the result
    if fitplot == True:
        fig = plt.figure()
        fig.add_subplot(211)
        plt.plot(waves, fluxes, 'b-', label='spectrum')
        plt.plot(waves, np.array(specsmooth_top)[window/2:-window/2], 'k:')
        plt.plot(waves, np.array(specsmooth_bot)[window/2:-window/2], 'k:')
        plt.plot(waves, specsmooth, 'k-', label='smoothed')
        plt.plot(waves, continuum, 'r-', label='continuum')
        plt.legend()
        fig.add_subplot(212)
        plt.plot(waves, fluxes/continuum, 'b-', label='flattened')
        plt.axhline(y=1, ls=':', color='k')
        plt.legend()
        plt.show()
    return waves, fluxes/continuum # numpy arrays

### MAIN PROGRAM BEGINS HERE ###

#######################
## edit values below ##
#######################
# directory where wavelength file lives:
#wavedir = '../../../PHOENIX/PHOENIX-ACES-AGSS-COND-2011/'
# directory where spectrum file lives:
#specdir = wavedir + 'Z-0.0/'
# spectrum file you want to run the program on:
#specfile = 'lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
#specfile = 'lte04800-2.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
specfile = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#specfile = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#specfile = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358

# wavelength region that will be fit with a spline:
#wavestart = 4318; waveend = 7200
#wavestart = 670; waveend = 2470
wavestart = 14000; waveend = 18000
# wavelength region that will be saved (must be no longer than the above):
#truncstart = 700; truncend = 2500
truncstart = 15000; truncend = 17000
# spline fitting parameters (final result is VERY SENSITIVE to these!!):
splinewindow = 100
variancewindow = 70 #60
variancesigma = 200 #150
splineweight = 0.9 #0.9
#######################
## edit values above ##
#######################

wavefits = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#wavefits = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#wavefits = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358
#infits = 'lte05000-3.00+0.5.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'
#infits = wavedir + specfile
#infits = specdir + specfile
infits = '/Users/revhalzoo/SDSS/A4851217/lte07000-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #A4851217
#infits = '/Users/revhalzoo/SDSS/B5285607/lte06400-5.00-0.0.Alpha=-0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #B5285607
#infits = '/Users/revhalzoo/SDSS/C6449358/lte06500-4.50-0.0.Alpha=+0.20.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' #C6449358
outtxt = specfile[:-5] + '_' + str(truncstart) + '-' + str(truncend) + '-norm.txt'

# read in PHOENIX FITS spectrum and corresponding wavelength FITS file
hdu = fits.open(infits)
spec = hdu[0].data
hduwave = fits.open(wavefits)
wave = hduwave[0].data

# put the wavelength array in goddamn air units
# reference: Huswer et al. 2013
wave = wave / (1.0 + 0.05792105/(238.0185-(1e4/wave)**2) + 0.00167917/(57.362-(1e4/wave)**2))

# truncate spectrum to the range you want to fit
idxstart = np.where(wave > wavestart)[0][0]
idxend = np.where(wave > waveend)[0][0]
wavechunk = wave[idxstart:idxend]
specchunk = spec[idxstart:idxend]

# measure the continuum level and divide by it to flatten the spectrum
print('Flattening spectrum...')
wavenorm, specnorm = continuumFlattenSpec(wavechunk, specchunk, win=splinewindow,
                                            varwin=variancewindow, varsig=variancesigma,
                                            weight=splineweight, fitplot=True)

# truncate the flattened spectrum to final desired wavelength range
idxtruncstart = np.where(wavenorm > truncstart-100)[0][0]
idxtruncend = np.where(wavenorm > truncend+100)[0][0]
wavenorm = wavenorm[idxtruncstart:idxtruncend]
specnorm = specnorm[idxtruncstart:idxtruncend]

# plot the final spectrum
plt.plot(wavenorm, specnorm, ls='None', marker='.', label='final flattened spectrum')
plt.legend()
plt.show()

# write the result to a text file
f = open(outtxt, 'w')
for wentry, sentry in zip(wavenorm, specnorm):
    print(wentry, sentry, file=f)
f.close()
print('New spectrum written to {0}'.format(outtxt))<|endoftext|>


<commit_before>import re
import json
import os
import shutil
import tarfile
import tempfile
import zipfile
from os.path import abspath, basename, dirname, isdir, join

def info_from_fn(fn):
    pat = re.compile(r'([\w\.-]+)-([\w\.]+)\.(win32|win-amd64)-py(\d)\.(\d)\.exe')
    m = pat.match(fn)
    if m is None:
         return
    arch_map = {'win32': 'x86', 'win-amd64': 'x86_64'}
    py_ver = '%s.%s' % (m.group(4), m.group(5))
    return {
        "name": m.group(1).lower(),
        "version": m.group(2),
        "build": "py" + py_ver.replace('.', ''),
        "build_number": 0,
        "depends": ['python %s*' % py_ver],
        "platform": "win",
        "arch": arch_map[m.group(3)],
    }

def extract(src_path, dir_path):
    file_map = [
        ('PLATLIB/', 'Lib/site-packages/'),
        ('PURELIB/', 'Lib/site-packages/'),
        ('SCRIPTS/', 'Scripts/'),
        ('DATA/Lib/site-packages/', 'Lib/site-packages/'),
    ]
    z = zipfile.ZipFile(src_path)
    for src in z.namelist():
        if src.endswith(('/', '\\')):
            continue
        for a, b in file_map:
            if src.startswith(a):
                dst = abspath(join(dir_path, b + src[len(a):]))
                break
        else:
            raise RuntimeError("Don't know how to handle file %s" % src)

        dst_dir = dirname(dst)
        #print 'file %r to %r' % (src, dst_dir)
        if not isdir(dst_dir):
            os.makedirs(dst_dir)
        data = z.read(src)
        with open(dst, 'wb') as fi:
            fi.write(data)
    z.close()

def get_files(dir_path):
    res = set()
    for root, dirs, files in os.walk(dir_path):
        for fn in files:
            res.add(join(root, fn)[len(dir_path) + 1:])
    return sorted(res)

def convert(file, output_repo='.'):
    fn1 = basename(file)
    info = info_from_fn(fn1)
    if info is None:
         print("WARNING: Invalid .exe filename '%s', skipping" % fn1)
         return

    tmp_dir = tempfile.mkdtemp()
    extract(file, tmp_dir)
    info_dir = join(tmp_dir, 'info')
    os.mkdir(info_dir)
    files = get_files(tmp_dir)
    with open(join(info_dir, 'files'), 'w') as fo:
        for f in files:
            fo.write('%s\n' % f)
    with open(join(info_dir, 'index.json'), 'w') as fo:
        json.dump(info, fo, indent=2, sort_keys=True)
    for fn in os.listdir(info_dir):
        files.append('info/' + fn)

    subdir_map = {'x86': 'win-32', 'x86_64': 'win-64'}
    output_dir = join(output_repo, subdir_map[info['arch']])
    if not isdir(output_dir):
        os.makedirs(output_dir)
    fn2 = '%(name)s-%(version)s-%(build)s.tar.bz2' % info
    output_path = join(output_dir, fn2)

    t = tarfile.open(output_path, 'w:bz2')
    for f in files:
        t.add(join(tmp_dir, f), f)
    t.close()

    print("Wrote: %s" % output_path)
    shutil.rmtree(tmp_dir)
<commit_msg>add globals constants<commit_after>import re
import json
import os
import shutil
import tarfile
import tempfile
import zipfile
from os.path import abspath, basename, dirname, isdir, join

arch_map = {'win32': 'x86', 'win-amd64': 'x86_64'}

subdir_map = {'x86': 'win-32', 'x86_64': 'win-64'}

file_map = [
    ('PLATLIB/', 'Lib/site-packages/'),
    ('PURELIB/', 'Lib/site-packages/'),
    ('SCRIPTS/', 'Scripts/'),
    ('DATA/Lib/site-packages/', 'Lib/site-packages/'),
]

def info_from_fn(fn):
    pat = re.compile(r'([\w\.-]+)-([\w\.]+)\.(win32|win-amd64)-py(\d)\.(\d)\.exe')
    m = pat.match(fn)
    if m is None:
         return
    py_ver = '%s.%s' % (m.group(4), m.group(5))
    return {
        "name": m.group(1).lower(),
        "version": m.group(2),
        "build": "py" + py_ver.replace('.', ''),
        "build_number": 0,
        "depends": ['python %s*' % py_ver],
        "platform": "win",
        "arch": arch_map[m.group(3)],
    }

def extract(src_path, dir_path):
    z = zipfile.ZipFile(src_path)
    for src in z.namelist():
        if src.endswith(('/', '\\')):
            continue
        for a, b in file_map:
            if src.startswith(a):
                dst = abspath(join(dir_path, b + src[len(a):]))
                break
        else:
            raise RuntimeError("Don't know how to handle file %s" % src)

        dst_dir = dirname(dst)
        #print 'file %r to %r' % (src, dst_dir)
        if not isdir(dst_dir):
            os.makedirs(dst_dir)
        data = z.read(src)
        with open(dst, 'wb') as fi:
            fi.write(data)
    z.close()

def get_files(dir_path):
    res = set()
    for root, dirs, files in os.walk(dir_path):
        for fn in files:
            res.add(join(root, fn)[len(dir_path) + 1:])
    return sorted(res)

def convert(file, output_repo='.'):
    fn1 = basename(file)
    info = info_from_fn(fn1)
    if info is None:
         print("WARNING: Invalid .exe filename '%s', skipping" % fn1)
         return

    tmp_dir = tempfile.mkdtemp()
    extract(file, tmp_dir)
    info_dir = join(tmp_dir, 'info')
    os.mkdir(info_dir)
    files = get_files(tmp_dir)
    with open(join(info_dir, 'files'), 'w') as fo:
        for f in files:
            fo.write('%s\n' % f)
    with open(join(info_dir, 'index.json'), 'w') as fo:
        json.dump(info, fo, indent=2, sort_keys=True)
    for fn in os.listdir(info_dir):
        files.append('info/' + fn)

    output_dir = join(output_repo, subdir_map[info['arch']])
    if not isdir(output_dir):
        os.makedirs(output_dir)
    fn2 = '%(name)s-%(version)s-%(build)s.tar.bz2' % info
    output_path = join(output_dir, fn2)

    t = tarfile.open(output_path, 'w:bz2')
    for f in files:
        t.add(join(tmp_dir, f), f)
    t.close()

    print("Wrote: %s" % output_path)
    shutil.rmtree(tmp_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import time, datetime, os
import threading
import traceback

import component
import log

class Router (component.Receiver):
    incoming_phases = ('filter', 'parse', 'handle', 'catch', 'cleanup')
    outgoing_phases = ('outgoing',)

    def __init__(self):
        component.Receiver.__init__(self)
        self.backends = []
        self.apps = []
        self.running = False
        self.logger = None

    def log(self, level, msg, *args):
        self.logger.write(self, level, msg, *args)

    def set_logger(self, level, file):
        self.logger = log.Logger(level, file)

    def build_component (self, class_template, conf):
        """Imports and instantiates an module, given a dict with
           the config key/value pairs to pass along."""
        # break the class name off the end of the module template
        # i.e. "apps.%s.app.App" -> ("apps.%s.app", "App")
        module_template, class_name = class_template.rsplit(".",1)

        # make a copy of the conf dict so we can delete from it
        conf = conf.copy()

        # resolve the component name into a real class
        module_name = module_template % (conf.pop("type"))
        module = __import__(module_name, {}, {}, [''])
        component_class = getattr(module, class_name)

        # create the component with an instance of this router
        # and keep hold of it here, so we can communicate both ways
        component = component_class(self)
        try:
            component._configure(**conf)
        except TypeError, e:
            # "__init__() got an unexpected keyword argument '...'"
            if "unexpected keyword" in e.message:
                missing_keyword = e.message.split("'")[1]
                raise Exception("Component '%s' does not support a '%s' option."
                        % (title, missing_keyword))
            else:
                raise
        return component

    def add_backend (self, conf):
        try:
            backend = self.build_component("rapidsms.backends.%s.Backend", conf)
            self.info("Added backend: %r" % conf)
            self.backends.append(backend)

        except:
            self.log_last_exception("Failed to add backend: %r" % conf)

    def get_backend (self, slug):
        '''gets a backend by slug, if it exists'''
        for backend in self.backends:
            if backend.slug == slug:
                return backend
        return None

    def add_app (self, conf):
        try:

            # find the App class via the config, since apps can be
            # loaded via arbitrary module strings now (apps.whatever vs
            # rapidsms.apps.whatever vs rapidsms.contrib.apps.whatever)
            app_module = __import__("%s.app" % conf["module"], {}, {}, ["App"])
            app_class = getattr(app_module, "App")
            app = app_class(self)

            # pass the configuration on to the app. note that
            # we're not watching for keyword argument any more,
            # since Component._configure wraps that nicely now
            app._configure(**dict(conf))
            self.apps.append(app)

            # dump the app config. should this be debug level?
            self.info("Added app: %r" % (conf))

        except:
            self.log_last_exception("Failed to add app: %r" % conf)

    def start_backend (self, backend):
        while self.running:
            try:
                backend.start()

                # if backend execution completed
                # normally (and did not raise),
                # allow the thread to terminate
                break

            except Exception, e:

                # an exception was raised in backend.start()
                # sleep for 5 seconds, then loop and restart it
                self.log_last_exception("Error in the %s backend" % backend.slug)

                # don't bother restarting the backend
                # if the router isn't running any more
                if not self.running:
                    break

                # TODO: where did the 5.0 constant come from?
                # we should probably be doing something more intelligent
                # here, rather than just hoping five seconds is enough
                time.sleep(5.0)
                self.info("Restarting the %s backend" % backend.slug)

    def start_all_apps (self):
        """Calls the _start_ method of each app registed via
           Router.add_app, logging any exceptions raised, but
           not allowing them to propagate. Returns True if all
           of the apps started without raising."""

        raised = False
        for app in self.apps:
            try:
                app.start()

            except Exception:
                self.log_last_exception("The %s app failed to start" % app.slug)
                raised = True

        # if any of the apps raised, we'll return
        # False, to warn that _something_ is wrong
        return not raised

    def start_all_backends (self):
        """Starts all backends registed via Router.add_backend,
           by calling self.start_backend in a new thread for each."""

        for backend in self.backends:
            worker = threading.Thread(
                target=self.start_backend,
                args=(backend,))

            worker.start()

            # attach the worker thread to the backend,
            # so we can check that it's still running
            backend.thread = worker

    def stop_all_backends (self):
        """Notifies all backends registered via Router.add_backend
           that they should stop. This method cannot guarantee that
           backends *will* stop in a timely manner."""

        for backend in self.backends:
            try:
                backend.stop()
                timeout = 5
                step = 0.1

                # wait up to five seconds for the backend's
                # worker thread to terminate, or log failure
                while(backend.thread.is_alive()):
                    if timeout <= 0:
                        raise RuntimeError, "The %s backend's worker thread did not terminate" % backend.slug

                    else:
                        time.sleep(step)
                        timeout -= step

            except Exception:
                self.log_last_exception("The %s backend failed to stop" % backend.slug)

    def start (self):
        self.running = True

        # dump some debug info for now
        self.info("BACKENDS: %r" % (self.backends))
        self.info("APPS: %r" % (self.apps))
        self.info("SERVING FOREVER...")

        self.start_all_backends()
        self.start_all_apps()

        # check for any pending messages
        try:
            fn = "/tmp/rapidsms-pending"
            f = file(fn)

            # trash the file, to prevent
            # these messages being re-sent
            msgs = f.readlines()
            os.unlink(fn)
            f.close()

            # iterate the pending messages,
            for pending_msg in msgs:
                be_name, identity, txt =\
                    pending_msg.strip().split(":")

                # find the backend named by the message,
                # reconstruct the object, and send it
                for backend in self.backends.values():
                    if backend.slug == be_slug:
                        msg = backend.message(identity, txt)
                        self.info("Sending pending message: %r" % msg)
                        msg.send()

        # something went bang. not sure what, and don't
        # particularly care. they'll be re-tried the
        # next time rapidsms starts up
        except:
            pass

        # wait until we're asked to stop
        while self.running:
            try:
                self.run()

            except KeyboardInterrupt:
                self.warning("Caught KeyboardInterrupt")
                break

            except SystemExit:
                self.warning("Caught SystemExit")
                break

        self.info("Stopping all backends...")
        self.stop_all_backends()
        self.running = False

    def stop (self):
        self.running = False

    def run(self):
        msg = self.next_message(timeout=1.0)
        if msg is not None:
            self.incoming(msg)

    def __sorted_apps(self):
        return sorted(self.apps, key=lambda a: a.priority())

    def incoming(self, message):
        #self.info("Incoming message via %s: %s ->'%s'" %\
        #    (message.connection.backend.slug, message.connection.identity, message.text))

        # loop through all of the apps and notify them of
        # the incoming message so that they all get a
        # chance to do what they will with it
        try:
            for phase in self.incoming_phases:
                for app in self.__sorted_apps():
                    self.debug('IN' + ' ' + phase + ' ' + app.slug)
                    responses = len(message.responses)
                    handled = False
                    try:
                        handled = getattr(app, phase)(message)
                    except Exception, e:
                        self.error("%s failed on %s: %r\n%s", app, phase, e, traceback.print_exc())

                    # during the "filter" phase, apps can return True
                    # to abort ALL further processing of this message
                    if phase == 'filter':
                        if handled is True:
                            self.warning('Message filtered by "%s" app', app.slug)
                            raise(StopIteration)

                    elif phase == 'handle' or phase == 'catch':
                        if handled is True:
                            self.debug("%s short-circuited %s phase", app.slug, phase)
                            break

                    elif responses < len(message.responses):
                        self.warning("App '%s' shouldn't send responses in %s()!",
                            app.slug, phase)

        # maybe raised within the loop, when
        # it's aborted during the filter phase
        except StopIteration:
            pass

        # now send the message's responses
        message.flush_responses()

        # we are no longer interested in
        # this message... but some crazy
        # synchronous backends might be!
        message.processed = True

    def outgoing(self, message):
        #self.info("Outgoing message via %s: %s <- '%s'" %\
        #    (message.connection.backend.slug, message.connection.identity, message.text))

        # first notify all of the apps that want to know
        # about outgoing messages so that they can do what
        # they will before the message is actually sent
        for phase in self.outgoing_phases:
            continue_sending = True

			# call outgoing phases in the opposite order of the
			# incoming phases so that, for example, the first app
			# called with an incoming message is the last app called
			# with an outgoing message
            for app in reversed(self.__sorted_apps()):
                self.debug('OUT' + ' ' + phase + ' ' + app.slug)
                try:
                    continue_sending = getattr(app, phase)(message)
                except Exception, e:
                    self.error("%s failed on %s: %r\n%s", app, phase, e, traceback.print_exc())
                if continue_sending is False:
                    self.info("App '%s' cancelled outgoing message", app.slug)
                    return False

        # now send the message out
        message.connection.backend.send(message)
        self.debug("SENT message '%s' to %s via %s" % (message.text,\
			message.connection.identity, message.connection.backend.slug))
        return True
<commit_msg>use app.config["type"] to log, for now<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import time, datetime, os
import threading
import traceback

import component
import log

class Router (component.Receiver):
    incoming_phases = ('filter', 'parse', 'handle', 'catch', 'cleanup')
    outgoing_phases = ('outgoing',)

    def __init__(self):
        component.Receiver.__init__(self)
        self.backends = []
        self.apps = []
        self.running = False
        self.logger = None

    def log(self, level, msg, *args):
        self.logger.write(self, level, msg, *args)

    def set_logger(self, level, file):
        self.logger = log.Logger(level, file)

    def build_component (self, class_template, conf):
        """Imports and instantiates an module, given a dict with
           the config key/value pairs to pass along."""
        # break the class name off the end of the module template
        # i.e. "apps.%s.app.App" -> ("apps.%s.app", "App")
        module_template, class_name = class_template.rsplit(".",1)

        # make a copy of the conf dict so we can delete from it
        conf = conf.copy()

        # resolve the component name into a real class
        module_name = module_template % (conf.pop("type"))
        module = __import__(module_name, {}, {}, [''])
        component_class = getattr(module, class_name)

        # create the component with an instance of this router
        # and keep hold of it here, so we can communicate both ways
        component = component_class(self)
        try:
            component._configure(**conf)
        except TypeError, e:
            # "__init__() got an unexpected keyword argument '...'"
            if "unexpected keyword" in e.message:
                missing_keyword = e.message.split("'")[1]
                raise Exception("Component '%s' does not support a '%s' option."
                        % (title, missing_keyword))
            else:
                raise
        return component

    def add_backend (self, conf):
        try:
            backend = self.build_component("rapidsms.backends.%s.Backend", conf)
            self.info("Added backend: %r" % conf)
            self.backends.append(backend)

        except:
            self.log_last_exception("Failed to add backend: %r" % conf)

    def get_backend (self, slug):
        '''gets a backend by slug, if it exists'''
        for backend in self.backends:
            if backend.slug == slug:
                return backend
        return None

    def add_app (self, conf):
        try:

            # find the App class via the config, since apps can be
            # loaded via arbitrary module strings now (apps.whatever vs
            # rapidsms.apps.whatever vs rapidsms.contrib.apps.whatever)
            app_module = __import__("%s.app" % conf["module"], {}, {}, ["App"])
            app_class = getattr(app_module, "App")
            app = app_class(self)

            # pass the configuration on to the app. note that
            # we're not watching for keyword argument any more,
            # since Component._configure wraps that nicely now
            app._configure(**dict(conf))
            self.apps.append(app)

            # dump the app config. should this be debug level?
            self.info("Added app: %r" % (conf))

        except:
            self.log_last_exception("Failed to add app: %r" % conf)

    def start_backend (self, backend):
        while self.running:
            try:
                backend.start()

                # if backend execution completed
                # normally (and did not raise),
                # allow the thread to terminate
                break

            except Exception, e:

                # an exception was raised in backend.start()
                # sleep for 5 seconds, then loop and restart it
                self.log_last_exception("Error in the %s backend" % backend.slug)

                # don't bother restarting the backend
                # if the router isn't running any more
                if not self.running:
                    break

                # TODO: where did the 5.0 constant come from?
                # we should probably be doing something more intelligent
                # here, rather than just hoping five seconds is enough
                time.sleep(5.0)
                self.info("Restarting the %s backend" % backend.slug)

    def start_all_apps (self):
        """Calls the _start_ method of each app registed via
           Router.add_app, logging any exceptions raised, but
           not allowing them to propagate. Returns True if all
           of the apps started without raising."""

        raised = False
        for app in self.apps:
            try:
                app.start()

            except Exception:
                self.log_last_exception("The %s app failed to start" % app.slug)
                raised = True

        # if any of the apps raised, we'll return
        # False, to warn that _something_ is wrong
        return not raised

    def start_all_backends (self):
        """Starts all backends registed via Router.add_backend,
           by calling self.start_backend in a new thread for each."""

        for backend in self.backends:
            worker = threading.Thread(
                target=self.start_backend,
                args=(backend,))

            worker.start()

            # attach the worker thread to the backend,
            # so we can check that it's still running
            backend.thread = worker

    def stop_all_backends (self):
        """Notifies all backends registered via Router.add_backend
           that they should stop. This method cannot guarantee that
           backends *will* stop in a timely manner."""

        for backend in self.backends:
            try:
                backend.stop()
                timeout = 5
                step = 0.1

                # wait up to five seconds for the backend's
                # worker thread to terminate, or log failure
                while(backend.thread.is_alive()):
                    if timeout <= 0:
                        raise RuntimeError, "The %s backend's worker thread did not terminate" % backend.slug

                    else:
                        time.sleep(step)
                        timeout -= step

            except Exception:
                self.log_last_exception("The %s backend failed to stop" % backend.slug)

    def start (self):
        self.running = True

        # dump some debug info for now
        self.info("BACKENDS: %r" % (self.backends))
        self.info("APPS: %r" % (self.apps))
        self.info("SERVING FOREVER...")

        self.start_all_backends()
        self.start_all_apps()

        # check for any pending messages
        try:
            fn = "/tmp/rapidsms-pending"
            f = file(fn)

            # trash the file, to prevent
            # these messages being re-sent
            msgs = f.readlines()
            os.unlink(fn)
            f.close()

            # iterate the pending messages,
            for pending_msg in msgs:
                be_name, identity, txt =\
                    pending_msg.strip().split(":")

                # find the backend named by the message,
                # reconstruct the object, and send it
                for backend in self.backends.values():
                    if backend.slug == be_slug:
                        msg = backend.message(identity, txt)
                        self.info("Sending pending message: %r" % msg)
                        msg.send()

        # something went bang. not sure what, and don't
        # particularly care. they'll be re-tried the
        # next time rapidsms starts up
        except:
            pass

        # wait until we're asked to stop
        while self.running:
            try:
                self.run()

            except KeyboardInterrupt:
                self.warning("Caught KeyboardInterrupt")
                break

            except SystemExit:
                self.warning("Caught SystemExit")
                break

        self.info("Stopping all backends...")
        self.stop_all_backends()
        self.running = False

    def stop (self):
        self.running = False

    def run(self):
        msg = self.next_message(timeout=1.0)
        if msg is not None:
            self.incoming(msg)

    def __sorted_apps(self):
        return sorted(self.apps, key=lambda a: a.priority())

    def incoming(self, message):
        #self.info("Incoming message via %s: %s ->'%s'" %\
        #    (message.connection.backend.slug, message.connection.identity, message.text))

        # loop through all of the apps and notify them of
        # the incoming message so that they all get a
        # chance to do what they will with it
        try:
            for phase in self.incoming_phases:
                for app in self.__sorted_apps():
                    self.debug('IN' + ' ' + phase + ' ' + app.config["type"])
                    responses = len(message.responses)
                    handled = False
                    try:
                        handled = getattr(app, phase)(message)
                    except Exception, e:
                        self.error("%s failed on %s: %r\n%s", app, phase, e, traceback.print_exc())

                    # during the "filter" phase, apps can return True
                    # to abort ALL further processing of this message
                    if phase == 'filter':
                        if handled is True:
                            self.warning('Message filtered by "%s" app', app.slug)
                            raise(StopIteration)

                    elif phase == 'handle' or phase == 'catch':
                        if handled is True:
                            self.debug("%s short-circuited %s phase", app.config["type"], phase)
                            break

                    elif responses < len(message.responses):
                        self.warning("App '%s' shouldn't send responses in %s()!",
                            app.config["type"], phase)

        # maybe raised within the loop, when
        # it's aborted during the filter phase
        except StopIteration:
            pass

        # now send the message's responses
        message.flush_responses()

        # we are no longer interested in
        # this message... but some crazy
        # synchronous backends might be!
        message.processed = True

    def outgoing(self, message):
        #self.info("Outgoing message via %s: %s <- '%s'" %\
        #    (message.connection.backend.slug, message.connection.identity, message.text))

        # first notify all of the apps that want to know
        # about outgoing messages so that they can do what
        # they will before the message is actually sent
        for phase in self.outgoing_phases:
            continue_sending = True

			# call outgoing phases in the opposite order of the
			# incoming phases so that, for example, the first app
			# called with an incoming message is the last app called
			# with an outgoing message
            for app in reversed(self.__sorted_apps()):
                self.debug('OUT' + ' ' + phase + ' ' + app.slug)
                try:
                    continue_sending = getattr(app, phase)(message)
                except Exception, e:
                    self.error("%s failed on %s: %r\n%s", app, phase, e, traceback.print_exc())
                if continue_sending is False:
                    self.info("App '%s' cancelled outgoing message", app.slug)
                    return False

        # now send the message out
        message.connection.backend.send(message)
        self.debug("SENT message '%s' to %s via %s" % (message.text,\
			message.connection.identity, message.connection.backend.slug))
        return True
<|endoftext|>


<commit_before>import hassapi as hass
import globals

#
# App to respond to buttons on an Aeotec Minimote
#
# Args:
#
# Minimote can send up to 8 scenes. Odd numbered scenes are short presses of the buttons, even are long presses
#
# Args:
#
# device - name of the device. This will be the ZWave name without an entity type, e.g. minimote_31
# scene_<id>_on - name of the entity to turn on when scene <id> is activated
# scene_<id>_off - name of the entity to turn off when scene <id> is activated. If the entity is a scene it will be turned on.
# scene_<id>_toggle - name of the entity to toggle when scene <id> is activated
#
# Each scene can have up to one of each type of action, or no actions - e.g. you can turn on one light and turn off another light for a particular scene if desired
#
# Release Notes
#
# Version 1.0:
#   Initial Version

class MiniMote(hass.Hass):
    def initialize(self):

        self.listen_event(
            self.zwave_event, "zwave.scene_activated", entity_id=self.args["device"]
        )

    def zwave_event(self, event_name, data, kwargs):
        # self.verbose_log("Event: {}, data = {}, args = {}".format(event_name, data, kwargs))
        scene = data["scene_id"]
        on = "scene_{}_on".format(scene)
        off = "scene_{}_off".format(scene)
        toggle = "scene_{}_toggle".format(scene)

        if on in self.args:
            self.log("Turning {} on".format(self.args[on]))
            self.turn_on(self.args[on])

        if off in self.args:
            self.log("Turning {} off".format(self.args[off]))
            self.turn_off(self.args[off])

        if toggle in self.args:
            self.log("Toggling {}".format(self.args[toggle]))
            self.toggle(self.args[toggle])
<commit_msg>fix flake8 issues for conf/example_apps/minimote.py<commit_after>import hassapi as hass

#
# App to respond to buttons on an Aeotec Minimote
#
# Args:
#
# Minimote can send up to 8 scenes. Odd numbered scenes are short presses of the buttons, even are long presses
#
# Args:
#
# device - name of the device. This will be the ZWave name without an entity type, e.g. minimote_31
# scene_<id>_on - name of the entity to turn on when scene <id> is activated
# scene_<id>_off - name of the entity to turn off when scene <id> is activated. If the entity is a scene it will be turned on.
# scene_<id>_toggle - name of the entity to toggle when scene <id> is activated
#
# Each scene can have up to one of each type of action, or no actions - e.g. you can turn on one light and turn off another light for a particular scene if desired
#
# Release Notes
#
# Version 1.0:
#   Initial Version

class MiniMote(hass.Hass):
    def initialize(self):

        self.listen_event(
            self.zwave_event, "zwave.scene_activated", entity_id=self.args["device"]
        )

    def zwave_event(self, event_name, data, kwargs):
        # self.verbose_log("Event: {}, data = {}, args = {}".format(event_name, data, kwargs))
        scene = data["scene_id"]
        on = "scene_{}_on".format(scene)
        off = "scene_{}_off".format(scene)
        toggle = "scene_{}_toggle".format(scene)

        if on in self.args:
            self.log("Turning {} on".format(self.args[on]))
            self.turn_on(self.args[on])

        if off in self.args:
            self.log("Turning {} off".format(self.args[off]))
            self.turn_off(self.args[off])

        if toggle in self.args:
            self.log("Toggling {}".format(self.args[toggle]))
            self.toggle(self.args[toggle])
<|endoftext|>


<commit_before>"""
WSGI config for conference project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conference.settings")

application = get_wsgi_application()
<commit_msg>Added whitenoise to WSGI for static assets<commit_after>"""
WSGI config for conference project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application
from whitenoise.django import DjangoWhiteNoise

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conference.settings")

application = get_wsgi_application()
application = DjangoWhiteNoise(application)
<|endoftext|>