<commit_before>import argparse
import os
import tensorflow as tf
import numpy as np
import gym
import json
import random
import sys

from helpers import discount_rewards, prepro
from agents.tools.wrappers import AutoReset, FrameHistory
from collections import deque

# Open AI gym Atari env: 0: 'NOOP', 2: 'UP', 3: 'DOWN'
ACTIONS = [0, 2, 3]

OBSERVATION_DIM = 80 * 80

MEMORY_CAPACITY = 100000
ROLLOUT_SIZE = 10000

def main(args):
    # MEMORY stores tuples:
    # (observation, label, reward)
    MEMORY = deque([], maxlen=MEMORY_CAPACITY)
    def gen():
        for m in list(MEMORY):
            yield m

    args_dict = vars(args)
    print('args: {}'.format(args_dict))

    with tf.Graph().as_default() as g:
        # rollout subgraph
        with tf.name_scope('rollout'):
            observations = tf.placeholder(shape=(None, OBSERVATION_DIM), dtype=tf.float32)

            weights_0 = tf.get_variable(
                name='weights_0',
                dtype=tf.float32,
                shape=(OBSERVATION_DIM, args.hidden_dim),
                initializer=tf.contrib.layers.xavier_initializer(uniform=False)
            )
            weights_1 = tf.get_variable(
                name='weights_1',
                dtype=tf.float32,
                shape=(args.hidden_dim, len(ACTIONS)),
                initializer=tf.contrib.layers.xavier_initializer(uniform=False)
            )

            hidden = tf.nn.relu(tf.matmul(observations, weights_0))
            logits = tf.matmul(hidden, weights_1)
            probs = tf.nn.softmax(logits=logits)

            logits_for_sampling = tf.reshape(logits, shape=(1, len(ACTIONS)))
            logits_for_sampling = tf.check_numerics(logits_for_sampling, 'logits_for_sampling error:')

            # Sample the action to be played during rollout.
            sample_action = tf.squeeze(tf.multinomial(logits=logits_for_sampling, num_samples=1))

        optimizer = tf.train.RMSPropOptimizer(
            learning_rate=args.learning_rate,
            decay=args.decay
        )

        # dataset subgraph for experience replay
        with tf.name_scope('dataset'):
            # the dataset reads from MEMORY
            ds = tf.data.Dataset.from_generator(gen, output_types=(tf.float32, tf.int32, tf.float32))
            ds = ds.shuffle(MEMORY_CAPACITY).repeat().batch(args.batch_size)
            iterator = ds.make_one_shot_iterator()

        # training subgraph
        with tf.name_scope('loss'):
            # the train_op includes getting a batch of data from the dataset, so we do not need to use a feed_dict when running the train_op.
            next_batch = iterator.get_next()
            train_observations, labels, processed_rewards = next_batch

            # the same weights as in the rollout subgraph
            train_hidden = tf.nn.relu(tf.matmul(train_observations, weights_0))
            train_logits = tf.matmul(train_hidden, weights_1)

            train_logits = tf.check_numerics(train_logits, 'train_logits error.')

            cross_entropies = tf.nn.sparse_softmax_cross_entropy_with_logits(
                logits=train_logits,
                labels=labels
            )

            loss = tf.reduce_sum(processed_rewards * cross_entropies)

        global_step = tf.train.get_or_create_global_step()

        train_op = optimizer.minimize(loss, global_step=global_step)

        init = tf.global_variables_initializer()
        saver = tf.train.Saver(max_to_keep=args.max_to_keep)

        with tf.name_scope('summaries'):
            rollout_reward = tf.placeholder(
                shape=(),
                dtype=tf.float32
            )

            # the weights to the hidden layer can be visualized
            for h in xrange(args.hidden_dim):
                slice_ = tf.slice(weights_0, [0, h], [-1, 1])
                image = tf.reshape(slice_, [1, 80, 80, 1])
                tf.summary.image('hidden_{:04d}'.format(h), image)

            # track the weights to find out how the logits became NaN
            for var in tf.trainable_variables():
                tf.summary.histogram(var.op.name, var)
                tf.summary.scalar('{}_max'.format(var.op.name), tf.reduce_max(var))
                tf.summary.scalar('{}_min'.format(var.op.name), tf.reduce_min(var))

            tf.summary.scalar('rollout_reward', rollout_reward)
            tf.summary.scalar('loss', loss)

            merged = tf.summary.merge_all()

    # for training
    inner_env = gym.make('Pong-v0')

    # tf.agents helper to more easily track consecutive pairs of frames
    env = FrameHistory(inner_env, past_indices=[0, 1], flatten=False)
    # tf.agents helper to automatically reset the environment
    env = AutoReset(env)

    with tf.Session(graph=g) as sess:
        if args.restore:
            restore_path = tf.train.latest_checkpoint(args.output_dir)
            print('Restoring from {}'.format(restore_path))
            saver.restore(sess, restore_path)
        else:
            sess.run(init)

        summary_path = os.path.join(args.output_dir, 'summary')
        summary_writer = tf.summary.FileWriter(summary_path, sess.graph)

        # lowest possible score after an episode as the
        # starting value of the running reward
        _rollout_reward = -21.0

        for i in range(args.n_epoch):
            print('>>>>>>> epoch {}'.format(i+1))

            print('>>> Rollout phase')
            epoch_memory = []
            episode_memory = []

            # The loop for actions/stepss
            _observation = np.zeros(OBSERVATION_DIM)
            while True:
                # sample one action with the given probability distribution
                _label = sess.run(sample_action, feed_dict={observations: [_observation]})

                _action = ACTIONS[_label]

                _pair_state, _reward, _done, _ = env.step(_action)

                if args.render:
                    env.render()

                # record experience
                episode_memory.append((_observation, _label, _reward))

                # Get processed frame delta for the next step
                pair_state = _pair_state

                current_state, previous_state = pair_state
                current_x = prepro(current_state)
                previous_x = prepro(previous_state)

                _observation = current_x - previous_x

                if _done:
                    obs, lbl, rwd = zip(*episode_memory)

                    # processed rewards
                    prwd = discount_rewards(rwd, args.gamma)
                    prwd -= np.mean(prwd)
                    prwd /= np.std(prwd)

                    # store the processed experience to memory
                    epoch_memory.extend(zip(obs, lbl, prwd))

                    # calculate the running rollout reward
                    _rollout_reward = 0.9 * _rollout_reward + 0.1 * sum(rwd)

                    episode_memory = []

                    if args.render:
                        _ = raw_input('episode done, press Enter to replay')
                        epoch_memory = []
                        continue

                if len(epoch_memory) >= ROLLOUT_SIZE:
                    break

            # add to the global memory
            MEMORY.extend(epoch_memory)

            print('>>> Train phase')
            print('rollout reward: {}'.format(_rollout_reward))

            feed_dict = {rollout_reward: _rollout_reward}

            # Here we train only once.
            _, _global_step = sess.run([train_op, global_step], feed_dict=feed_dict)

            if _global_step % args.save_checkpoint_steps == 0:

                print('Writing summary')

                summary = sess.run(merged, feed_dict=feed_dict)

                summary_writer.add_summary(summary, _global_step)

                save_path = os.path.join(args.output_dir, 'model.ckpt')
                save_path = saver.save(sess, save_path, global_step=_global_step)
                print('Model checkpoint saved: {}'.format(save_path))

if __name__ == '__main__':
    parser = argparse.ArgumentParser('pong trainer')
    parser.add_argument(
        '--n-epoch',
        type=int,
        default=6000)
    parser.add_argument(
        '--batch-size',
        type=int,
        default=10000)
    parser.add_argument(
        '--output-dir',
        type=str,
        default='/tmp/pong_output')
    parser.add_argument(
        '--job-dir',
        type=str,
        default='/tmp/pong_output')

    parser.add_argument(
        '--restore',
        default=False,
        action='store_true')
    parser.add_argument(
        '--render',
        default=False,
        action='store_true')
    parser.add_argument(
        '--save-checkpoint-steps',
        type=int,
        default=1)

    parser.add_argument(
        '--learning-rate',
        type=float,
        default=5e-3)
    parser.add_argument(
        '--decay',
        type=float,
        default=0.99)
    parser.add_argument(
        '--gamma',
        type=float,
        default=0.99)
    parser.add_argument(
        '--hidden-dim',
        type=int,
        default=200)

    args = parser.parse_args()

    # save all checkpoints
    args.max_to_keep = args.n_epoch / args.save_checkpoint_steps

    main(args)
<commit_msg>clean up<commit_after>import argparse
import os
import tensorflow as tf
import numpy as np
import gym
import json
import random
import sys

from helpers import discount_rewards, prepro
from agents.tools.wrappers import AutoReset, FrameHistory
from collections import deque

# Open AI gym Atari env: 0: 'NOOP', 2: 'UP', 3: 'DOWN'
ACTIONS = [0, 2, 3]

OBSERVATION_DIM = 80 * 80

MEMORY_CAPACITY = 100000
ROLLOUT_SIZE = 10000

def main(args):
    # MEMORY stores tuples:
    # (observation, label, reward)
    MEMORY = deque([], maxlen=MEMORY_CAPACITY)
    def gen():
        for m in list(MEMORY):
            yield m

    args_dict = vars(args)
    print('args: {}'.format(args_dict))

    with tf.Graph().as_default() as g:
        # rollout subgraph
        with tf.name_scope('rollout'):
            observations = tf.placeholder(shape=(None, OBSERVATION_DIM), dtype=tf.float32)

            weights_0 = tf.get_variable(
                name='weights_0',
                dtype=tf.float32,
                shape=(OBSERVATION_DIM, args.hidden_dim),
                initializer=tf.contrib.layers.xavier_initializer(uniform=False)
            )
            weights_1 = tf.get_variable(
                name='weights_1',
                dtype=tf.float32,
                shape=(args.hidden_dim, len(ACTIONS)),
                initializer=tf.contrib.layers.xavier_initializer(uniform=False)
            )

            hidden = tf.nn.relu(tf.matmul(observations, weights_0))
            logits = tf.matmul(hidden, weights_1)
            probs = tf.nn.softmax(logits=logits)

            logits_for_sampling = tf.reshape(logits, shape=(1, len(ACTIONS)))
            logits_for_sampling = tf.check_numerics(logits_for_sampling, 'logits_for_sampling error:')

            # Sample the action to be played during rollout.
            sample_action = tf.squeeze(tf.multinomial(logits=logits_for_sampling, num_samples=1))

        optimizer = tf.train.RMSPropOptimizer(
            learning_rate=args.learning_rate,
            decay=args.decay
        )

        # dataset subgraph for experience replay
        with tf.name_scope('dataset'):
            # the dataset reads from MEMORY
            ds = tf.data.Dataset.from_generator(gen, output_types=(tf.float32, tf.int32, tf.float32))
            ds = ds.shuffle(MEMORY_CAPACITY).repeat().batch(args.batch_size)
            iterator = ds.make_one_shot_iterator()

        # training subgraph
        with tf.name_scope('loss'):
            # the train_op includes getting a batch of data from the dataset, so we do not need to use a feed_dict when running the train_op.
            next_batch = iterator.get_next()
            train_observations, labels, processed_rewards = next_batch

            # the same weights as in the rollout subgraph
            train_hidden = tf.nn.relu(tf.matmul(train_observations, weights_0))
            train_logits = tf.matmul(train_hidden, weights_1)

            train_logits = tf.check_numerics(train_logits, 'train_logits error.')

            cross_entropies = tf.nn.sparse_softmax_cross_entropy_with_logits(
                logits=train_logits,
                labels=labels
            )

            loss = tf.reduce_sum(processed_rewards * cross_entropies)

        global_step = tf.train.get_or_create_global_step()

        train_op = optimizer.minimize(loss, global_step=global_step)

        init = tf.global_variables_initializer()
        saver = tf.train.Saver(max_to_keep=args.max_to_keep)

        with tf.name_scope('summaries'):
            rollout_reward = tf.placeholder(
                shape=(),
                dtype=tf.float32
            )

            # the weights to the hidden layer can be visualized
            for h in xrange(args.hidden_dim):
                slice_ = tf.slice(weights_0, [0, h], [-1, 1])
                image = tf.reshape(slice_, [1, 80, 80, 1])
                tf.summary.image('hidden_{:04d}'.format(h), image)

            # track the weights to find out how the logits became NaN
            for var in tf.trainable_variables():
                tf.summary.histogram(var.op.name, var)
                tf.summary.scalar('{}_max'.format(var.op.name), tf.reduce_max(var))
                tf.summary.scalar('{}_min'.format(var.op.name), tf.reduce_min(var))

            tf.summary.scalar('rollout_reward', rollout_reward)
            tf.summary.scalar('loss', loss)

            merged = tf.summary.merge_all()

    # for training
    inner_env = gym.make('Pong-v0')

    # tf.agents helper to more easily track consecutive pairs of frames
    env = FrameHistory(inner_env, past_indices=[0, 1], flatten=False)
    # tf.agents helper to automatically reset the environment
    env = AutoReset(env)

    with tf.Session(graph=g) as sess:
        if args.restore:
            restore_path = tf.train.latest_checkpoint(args.output_dir)
            print('Restoring from {}'.format(restore_path))
            saver.restore(sess, restore_path)
        else:
            sess.run(init)

        summary_path = os.path.join(args.output_dir, 'summary')
        summary_writer = tf.summary.FileWriter(summary_path, sess.graph)

        # lowest possible score after an episode as the
        # starting value of the running reward
        _rollout_reward = -21.0

        for i in range(args.n_epoch):
            print('>>>>>>> epoch {}'.format(i+1))

            print('>>> Rollout phase')
            epoch_memory = []
            episode_memory = []

            # The loop for actions/stepss
            _observation = np.zeros(OBSERVATION_DIM)
            while True:
                # sample one action with the given probability distribution
                _label = sess.run(sample_action, feed_dict={observations: [_observation]})

                _action = ACTIONS[_label]

                _pair_state, _reward, _done, _ = env.step(_action)

                if args.render:
                    env.render()

                # record experience
                episode_memory.append((_observation, _label, _reward))

                # Get processed frame delta for the next step
                pair_state = _pair_state

                current_state, previous_state = pair_state
                current_x = prepro(current_state)
                previous_x = prepro(previous_state)

                _observation = current_x - previous_x

                if _done:
                    obs, lbl, rwd = zip(*episode_memory)

                    # processed rewards
                    prwd = discount_rewards(rwd, args.gamma)
                    prwd -= np.mean(prwd)
                    prwd /= np.std(prwd)

                    # store the processed experience to memory
                    epoch_memory.extend(zip(obs, lbl, prwd))

                    # calculate the running rollout reward
                    _rollout_reward = 0.9 * _rollout_reward + 0.1 * sum(rwd)

                    episode_memory = []

                    if args.render:
                        _ = raw_input('episode done, press Enter to replay')
                        epoch_memory = []
                        continue

                if len(epoch_memory) >= ROLLOUT_SIZE:
                    break

            # add to the global memory
            MEMORY.extend(epoch_memory)

            print('>>> Train phase')
            print('rollout reward: {}'.format(_rollout_reward))

            # Here we train only once.
            _, _global_step = sess.run([train_op, global_step])

            if _global_step % args.save_checkpoint_steps == 0:

                print('Writing summary')

                feed_dict = {rollout_reward: _rollout_reward}
                summary = sess.run(merged, feed_dict=feed_dict)

                summary_writer.add_summary(summary, _global_step)

                save_path = os.path.join(args.output_dir, 'model.ckpt')
                save_path = saver.save(sess, save_path, global_step=_global_step)
                print('Model checkpoint saved: {}'.format(save_path))

if __name__ == '__main__':
    parser = argparse.ArgumentParser('pong trainer')
    parser.add_argument(
        '--n-epoch',
        type=int,
        default=6000)
    parser.add_argument(
        '--batch-size',
        type=int,
        default=10000)
    parser.add_argument(
        '--output-dir',
        type=str,
        default='/tmp/pong_output')
    parser.add_argument(
        '--job-dir',
        type=str,
        default='/tmp/pong_output')

    parser.add_argument(
        '--restore',
        default=False,
        action='store_true')
    parser.add_argument(
        '--render',
        default=False,
        action='store_true')
    parser.add_argument(
        '--save-checkpoint-steps',
        type=int,
        default=1)

    parser.add_argument(
        '--learning-rate',
        type=float,
        default=5e-3)
    parser.add_argument(
        '--decay',
        type=float,
        default=0.99)
    parser.add_argument(
        '--gamma',
        type=float,
        default=0.99)
    parser.add_argument(
        '--hidden-dim',
        type=int,
        default=200)

    args = parser.parse_args()

    # save all checkpoints
    args.max_to_keep = args.n_epoch / args.save_checkpoint_steps

    main(args)
<|endoftext|>


<commit_before>'''Term building transformations.'''

import aterm.factory
import aterm.types
import aterm.terms

from transf import base
from transf import exception
from transf import _operate
from transf import _helper

_factory = aterm.factory.Factory()

class Term(base.Transformation):

	def __init__(self, term):
		base.Transformation.__init__(self)
		if isinstance(term, basestring):
			self.term = _factory.parse(term)
		else:
			self.term = term

	def apply(self, term, ctx):
		return self.term

def Int(value):
	term = _factory.makeInt(value)
	return Term(term)

zero = Int(0)
one = Int(1)
two = Int(2)
three = Int(3)
four = Int(4)

def Real(value):
	term = _factory.makeReal(value)
	return Term(term)

def Str(value):
	term = _factory.makeStr(value)
	return Term(term)

_nil = _factory.makeNil()

def Nil():
	return Term(_nil)

nil = Nil()

class Cons(base.Transformation):

	def __init__(self, head, tail):
		base.Transformation.__init__(self)
		self.head = head
		self.tail = tail

	def apply(self, term, ctx):
		head = self.head.apply(term, ctx)
		tail = self.tail.apply(term, ctx)
		return term.factory.makeCons(head, tail)

def List(elms, tail = None):
	if tail is None:
		tail = nil
	return _operate.Nary(iter(elms), Cons, tail)

class Appl(base.Transformation):

	def __init__(self, name, args):
		base.Transformation.__init__(self)
		if isinstance(name, basestring):
			self.name = Str(name)
		else:
			self.name = name
		if isinstance(args, (tuple, list)):
			self.args = List(args)
		else:
			self.args = args

	def apply(self, term, ctx):
		name = self.name.apply(term, ctx)
		args = self.args.apply(term, ctx)
		return term.factory.makeAppl(name, args)

class Var(base.Transformation):

	def __init__(self, name):
		base.Transformation.__init__(self)
		self.name = name

	def apply(self, term, ctx):
		try:
			val = ctx[self.name]
		except KeyError:
			raise exception.Failure('undeclared variable', self.name)
		if isinstance(val, aterm.terms.Term):
			return val
		else:
			raise exception.Failure('undefined variable', self.name)

class Annos(base.Transformation):

	def __init__(self, annos):
		base.Transformation.__init__(self)
		self.annos = annos

	def apply(self, term, ctx):
		return term.setAnnotations(self.annos.apply(term, ctx))

class Pattern(base.Transformation):

	# TODO: Parse patterns into the above classes

	def __init__(self, pattern):
		base.Transformation.__init__(self)
		if isinstance(pattern, basestring):
			self.pattern = _factory.parse(pattern)
		else:
			self.pattern = pattern

	def apply(self, term, ctx):
		# FIXME: avoid the dict copy
		kargs = {}
		for name, value in ctx.iteritems():
			if isinstance(value, aterm.terms.Term):
				kargs[name] = value
		return self.pattern.make(term, **kargs)

_ = _helper.Factory(Int, Real, Str, List, Appl, Var, Pattern)
<commit_msg>Optimize build transfs.<commit_after>'''Term building transformations.'''

import aterm.factory
import aterm.types
import aterm.terms

from transf import base
from transf import exception
from transf import _operate
from transf import _helper

_factory = aterm.factory.Factory()

class _Term(base.Transformation):

	# NOTE: this class should not be derived to avoid
	# breaking optimizations

	def __init__(self, term):
		base.Transformation.__init__(self)
		self.term = term

	def apply(self, term, ctx):
		return self.term

def Term(term):
	if isinstance(term, basestring):
		term = _factory.parse(term)
	return _Term(term)

def Int(value):
	term = _factory.makeInt(value)
	return _Term(term)

zero = Int(0)
one = Int(1)
two = Int(2)
three = Int(3)
four = Int(4)

def Real(value):
	return _Term(_factory.makeReal(value))

def Str(value):
	return _Term(_factory.makeStr(value))

_nil = _factory.makeNil()

def Nil():
	return _Term(_nil)

nil = Nil()

class _Cons(base.Transformation):

	def __init__(self, head, tail):
		base.Transformation.__init__(self)
		self.head = head
		self.tail = tail

	def apply(self, term, ctx):
		head = self.head.apply(term, ctx)
		tail = self.tail.apply(term, ctx)
		return term.factory.makeCons(head, tail)

def Cons(head, tail):
	if isinstance(tail, (tuple, list)):
		tail = List(tail)
	if isinstance(head, _Term) and isinstance(tail, _Term):
		return _Term(_factory.makeCons(head.term, tail.term))
	return _Cons(head, tail)

def List(elms, tail = None):
	if tail is None:
		tail = nil
	return _operate.Nary(iter(elms), Cons, tail)

class _Appl(base.Transformation):

	def __init__(self, name, args):
		base.Transformation.__init__(self)
		self.name = name
		self.args = args

	def apply(self, term, ctx):
		name = self.name.apply(term, ctx)
		args = self.args.apply(term, ctx)
		return term.factory.makeAppl(name, args)

def Appl(name, args):
	if isinstance(name, basestring):
		name = Str(name)
	if isinstance(args, (tuple, list)):
		args = List(args)
	if isinstance(name, _Term) and isinstance(args, _Term):
		return _Term(_factory.makeAppl(name.term, args.term))
	return _Appl(name, args)

class Var(base.Transformation):

	def __init__(self, name):
		base.Transformation.__init__(self)
		self.name = name

	def apply(self, term, ctx):
		try:
			val = ctx[self.name]
		except KeyError:
			raise exception.Failure('undeclared variable', self.name)
		if isinstance(val, aterm.terms.Term):
			return val
		else:
			raise exception.Failure('undefined variable', self.name)

class Annos(base.Transformation):

	def __init__(self, annos):
		base.Transformation.__init__(self)
		self.annos = annos

	def apply(self, term, ctx):
		return term.setAnnotations(self.annos.apply(term, ctx))

class Pattern(base.Transformation):

	# TODO: Parse patterns into the above classes

	def __init__(self, pattern):
		base.Transformation.__init__(self)
		if isinstance(pattern, basestring):
			self.pattern = _factory.parse(pattern)
		else:
			self.pattern = pattern

	def apply(self, term, ctx):
		# FIXME: avoid the dict copy
		kargs = {}
		for name, value in ctx.iteritems():
			if isinstance(value, aterm.terms.Term):
				kargs[name] = value
		return self.pattern.make(term, **kargs)

_ = _helper.Factory(Int, Real, Str, List, Appl, Var, Pattern)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import argparse
import json
import logging
import os
import pkgutil
import sys

import requests

import angus.cloud
import angus.rest
import angus.version

__path__ = pkgutil.extend_path(__path__, __name__)

__version__ = angus.version.__version__
__updated__ = "2017-01-03"
__author__ = "Aurélien Moreau"
__copyright__ = "Copyright 2015-2017, Angus.ai"
__credits__ = ["Aurélien Moreau", "Gwennael Gate"]
__license__ = "Apache v2.0"
__maintainer__ = "Aurélien Moreau"
__status__ = "Production"

LOGGER = logging.getLogger('AngusSDK')

requests.packages.urllib3.disable_warnings()

def parse_cmd_line(argv):
    """Parse command line argument. See -h option.
    """
    formattter_class = argparse.RawDescriptionHelpFormatter

    parser = argparse.ArgumentParser(description='Angus SDK',
                                     formatter_class=formattter_class)

    parser.add_argument('-c', '--configuration', metavar='filename', type=str,
                        help='configuration file.')

    parser.add_argument('-id', '--clientid', metavar='clientid', type=str,
                        help='the client id.')

    parser.add_argument('-a', '--access', metavar='access', type=str,
                        help='the access token.')

    parser.add_argument('-v', '--verbose', dest='vlevel', action='count',
                        default=0,
                        help='increases log vervosity for each occurence.')

    parser.add_argument('-ca', '--capath', metavar='capath', type=str,
                        help='path to a trusted ca bundle.')

    parser.add_argument('-r', '--root', metavar='root', type=str,
                        help='the angus gate url')

    parser.add_argument('--version', action='version',
                        version="%(prog)s {}".format(__version__))

    args = parser.parse_args(argv)

    return args

def get_default_configuration(argv=""):
    """Get and parse the default configuration in ~/.angusdk/config.json
    """
    args = parse_cmd_line(argv)

    # Set the logger level
    logging.basicConfig()
    LOGGER.setLevel(max(3 - args.vlevel, 0) * 10)

    # Get the configuration file
    default_file = os.path.expanduser("~/.angusdk/config.json")
    if args.configuration is not None:
        if os.path.isfile(args.configuration):
            conf_file = args.configuration
        else:
            LOGGER.error("The configuration file '%s' does not exist",
                         args.configuration)
            sys.exit(-1)
    elif os.path.isfile(default_file):
        conf_file = default_file
    else:
        conf_file = None

    conf = angus.rest.Configuration()

    # Apply the configuration file if it exists
    if conf_file is not None:
        with open(conf_file, 'r') as conf:
            conf_data = json.loads(conf.read())
            if 'client_id'in conf_data and 'access_token' in conf_data:
                conf.set_credential(
                    conf_data['client_id'],
                    conf_data['access_token'])

            if 'ca_path' in conf_data:
                conf.set_ca_path(conf_data['ca_path'])

            if 'default_root' in conf_data:
                conf.default_root = conf_data['default_root']

    if args.clientid and args.access:
        conf.set_credential(args.clientid, args.access)

    if args.root:
        conf.default_root = args.root

    if args.capath:
        conf.set_ca_path(args.capath)

    return conf

def connect(url=None, conf=None, client_id=None,
            access_token=None, verify=True):
    """Create a resource root with the given configuration.
    """
    if None not in [client_id, access_token]:
        conf = angus.rest.Configuration()
        conf.set_credential(client_id, access_token)
        if not verify:
            conf.do_not_verify()

    return angus.cloud.Root(url, conf)
<commit_msg>Fix bug introduce by refactoring.<commit_after># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import argparse
import json
import logging
import os
import pkgutil
import sys

import requests

import angus.cloud
import angus.rest
import angus.version

__path__ = pkgutil.extend_path(__path__, __name__)

__version__ = angus.version.__version__
__updated__ = "2017-01-04"
__author__ = "Aurélien Moreau"
__copyright__ = "Copyright 2015-2017, Angus.ai"
__credits__ = ["Aurélien Moreau", "Gwennael Gate"]
__license__ = "Apache v2.0"
__maintainer__ = "Aurélien Moreau"
__status__ = "Production"

LOGGER = logging.getLogger('AngusSDK')

requests.packages.urllib3.disable_warnings()

def parse_cmd_line(argv):
    """Parse command line argument. See -h option.
    """
    formattter_class = argparse.RawDescriptionHelpFormatter

    parser = argparse.ArgumentParser(description='Angus SDK',
                                     formatter_class=formattter_class)

    parser.add_argument('-c', '--configuration', metavar='filename', type=str,
                        help='configuration file.')

    parser.add_argument('-id', '--clientid', metavar='clientid', type=str,
                        help='the client id.')

    parser.add_argument('-a', '--access', metavar='access', type=str,
                        help='the access token.')

    parser.add_argument('-v', '--verbose', dest='vlevel', action='count',
                        default=0,
                        help='increases log vervosity for each occurence.')

    parser.add_argument('-ca', '--capath', metavar='capath', type=str,
                        help='path to a trusted ca bundle.')

    parser.add_argument('-r', '--root', metavar='root', type=str,
                        help='the angus gate url')

    parser.add_argument('--version', action='version',
                        version="%(prog)s {}".format(__version__))

    args = parser.parse_args(argv)

    return args

def get_default_configuration(argv=""):
    """Get and parse the default configuration in ~/.angusdk/config.json
    """
    args = parse_cmd_line(argv)

    # Set the logger level
    logging.basicConfig()
    LOGGER.setLevel(max(3 - args.vlevel, 0) * 10)

    # Get the configuration file
    default_file = os.path.expanduser("~/.angusdk/config.json")
    if args.configuration is not None:
        if os.path.isfile(args.configuration):
            conf_file = args.configuration
        else:
            LOGGER.error("The configuration file '%s' does not exist",
                         args.configuration)
            sys.exit(-1)
    elif os.path.isfile(default_file):
        conf_file = default_file
    else:
        conf_file = None

    conf = angus.rest.Configuration()

    # Apply the configuration file if it exists
    if conf_file is not None:
        with open(conf_file, 'r') as fconf:
            conf_data = json.loads(fconf.read())
            if 'client_id'in conf_data and 'access_token' in conf_data:
                conf.set_credential(
                    conf_data['client_id'],
                    conf_data['access_token'])

            if 'ca_path' in conf_data:
                conf.set_ca_path(conf_data['ca_path'])

            if 'default_root' in conf_data:
                conf.default_root = conf_data['default_root']

    if args.clientid and args.access:
        conf.set_credential(args.clientid, args.access)

    if args.root:
        conf.default_root = args.root

    if args.capath:
        conf.set_ca_path(args.capath)

    return conf

def connect(url=None, conf=None, client_id=None,
            access_token=None, verify=True):
    """Create a resource root with the given configuration.
    """
    if None not in [client_id, access_token]:
        conf = angus.rest.Configuration()
        conf.set_credential(client_id, access_token)
        if not verify:
            conf.do_not_verify()

    return angus.cloud.Root(url, conf)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012-2018, Anima Istanbul
#
# This module is part of anima-tools and is released under the BSD 2
# License: http://www.opensource.org/licenses/BSD-2-Clause
"""Utilities for UI stuff
"""
import os
import shutil

from anima import logger
from anima.ui.lib import QtCore, QtGui, QtWidgets

def get_icon(icon_name):
    """Returns an icon from ui library
    """
    here = os.path.abspath(os.path.dirname(__file__))
    images_path = os.path.join(here, 'images')
    icon_full_path = os.path.join(images_path, icon_name)
    return QtGui.QIcon(icon_full_path)

def clear_thumbnail(gview):
    """Clears the thumbnail for the given QGraphicsView

    :param gview: The QGraphicsView instance
    """

    if not gview:
        return

    # clear the graphics scene in case there is no thumbnail
    scene = gview.scene()
    if not scene:
        scene = QtWidgets.QGraphicsScene(gview)
        gview.setScene(scene)

    scene.clear()

def update_gview_with_task_thumbnail(task, gview):
    """Updates the given QGraphicsView with the given Task thumbnail

    :param task: A
      :class:`~stalker.models.task.Task` instance

    :param gview: A QtGui.QGraphicsView instance
    """
    from stalker import Task

    if not isinstance(task, Task) or \
       not isinstance(gview, QtWidgets.QGraphicsView):
        # do nothing
        logger.debug('task is not a stalker.models.task.Task instance')
        return

    # get the thumbnail full path
    full_path = None
    if task.thumbnail:
        # use the cache system to get the thumbnail
        # try to get it as a normal file
        full_path = os.path.expandvars(
            task.thumbnail.full_path
        )
        if not os.path.exists(full_path):
            full_path = None
    else:
        logger.debug('there is no thumbnail')
        # try to get the thumbnail from parents
        for parent in reversed(task.parents):
            if parent.thumbnail:
                # try to get it as a normal file
                full_path = os.path.expandvars(
                    parent.thumbnail.full_path
                )
                if not os.path.exists(full_path):
                    full_path = None
                logger.debug('found parent thumbnail at: %s' % full_path)
                break

    if full_path:
        update_gview_with_image_file(
            full_path,
            gview
        )

def update_gview_with_image_file(image_full_path, gview):
    """updates the QGraphicsView with the given image
    """

    if not isinstance(gview, QtWidgets.QGraphicsView):
        return

    clear_thumbnail(gview)

    if image_full_path != "":
        image_full_path = os.path.normpath(image_full_path)
        image_format = os.path.splitext(image_full_path)[-1].replace('.', '').upper()
        logger.debug("creating pixmap from: %s" % image_full_path)

        # size = conf.thumbnail_size
        # width = size[0]
        # height = size[1]
        size = gview.size()
        width = size.width()
        height = size.height()
        logger.debug('width: %s' % width)
        logger.debug('height: %s' % height)

        if os.path.exists(image_full_path):
            pixmap = QtGui.QPixmap(image_full_path, format=image_format).scaled(
                width, height,
                QtCore.Qt.KeepAspectRatio,
                QtCore.Qt.SmoothTransformation
            )
            # pixmap = QtGui.QPixmap(image_full_path, format='JPG')

            scene = gview.scene()
            scene.addPixmap(pixmap)

def upload_thumbnail(task, thumbnail_full_path):
    """Uploads the given thumbnail for the given entity

    :param task: An instance of :class:`~stalker.models.entity.SimpleEntity`
      or a derivative.

    :param str thumbnail_full_path: A string which is showing the path
      of the thumbnail image
    """
    extension = os.path.splitext(thumbnail_full_path)[-1]

    # move the file to the task thumbnail folder
    # and mimic StalkerPyramids output format
    thumbnail_original_file_name = 'thumbnail%s' % extension
    thumbnail_final_full_path = os.path.join(
        task.absolute_path, 'Thumbnail', thumbnail_original_file_name
    )

    try:
        os.makedirs(os.path.dirname(thumbnail_final_full_path))
    except OSError:
        pass

    # # convert the thumbnail to jpg if it is a format that is not supported by
    # # browsers
    # ext_not_supported_by_browsers = ['.bmp', '.tga', '.tif', '.tiff', '.exr']
    # if extension in ext_not_supported_by_browsers:
    #     # use MediaManager to convert them
    #     from anima.utils import MediaManager
    #     mm = MediaManager()
    #     thumbnail_full_path = mm.generate_image_thumbnail(thumbnail_full_path)

    shutil.copy(thumbnail_full_path, thumbnail_final_full_path)

    from stalker import Link, Version, Repository

    thumbnail_os_independent_path = \
        Repository.to_os_independent_path(thumbnail_final_full_path)
    l_thumb = Link.query\
        .filter(Link.full_path == thumbnail_os_independent_path).first()

    if not l_thumb:
        l_thumb = Link(
            full_path=thumbnail_os_independent_path,
            original_filename=thumbnail_original_file_name
        )

    task.thumbnail = l_thumb

    # get a version of this Task
    from stalker.db.session import DBSession
    v = Version.query.filter(Version.task == task).first()
    if v:
        for naming_parent in v.naming_parents:
            if not naming_parent.thumbnail:
                naming_parent.thumbnail = l_thumb
                DBSession.add(naming_parent)

    DBSession.add(l_thumb)
    DBSession.commit()

def choose_thumbnail(parent):
    """shows a dialog for thumbnail upload
    """
    # get a file from a FileDialog
    thumbnail_full_path = QtWidgets.QFileDialog.getOpenFileName(
        parent, "Choose Thumbnail",
        os.path.expanduser("~"),
        # "Image Files (*.png *.jpg *.bmp *.tga *.tif *.tiff *.exr)"
        "Image Files (*.png *.jpg *.bmp)"
    )

    if isinstance(thumbnail_full_path, tuple):
        thumbnail_full_path = thumbnail_full_path[0]

    return thumbnail_full_path

def render_image_from_gview(gview, image_full_path):
    """renders the gview scene to an image at the given full path
    """
    assert isinstance(gview, QtWidgets.QGraphicsView)
    scene = gview.scene()
    # there should be only one item
    items = scene.items()
    if items:
        pixmapItem = items[0]
        pixmap = pixmapItem.pixmap()
        try:
            os.makedirs(os.path.split(image_full_path)[0])
        except OSError:
            # dir exists
            pass

        pixmap.save(
            image_full_path
        )

def load_font(font_filename):
    """loads extra fonts from the fonts folder
    """
    here = os.path.dirname(os.path.realpath(__file__))
    font_id = QtGui.QFontDatabase.addApplicationFont(
        os.path.join(here, 'fonts', font_filename)
    )
    loaded_font_families = QtGui.QFontDatabase.applicationFontFamilies(font_id)
    return loaded_font_families
<commit_msg>* [UI] Fixed ``anima.ui.utils.choose_thumbnail()`` helper function to also accept TIF files.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012-2018, Anima Istanbul
#
# This module is part of anima-tools and is released under the BSD 2
# License: http://www.opensource.org/licenses/BSD-2-Clause
"""Utilities for UI stuff
"""
import os
import shutil

from anima import logger
from anima.ui.lib import QtCore, QtGui, QtWidgets

def get_icon(icon_name):
    """Returns an icon from ui library
    """
    here = os.path.abspath(os.path.dirname(__file__))
    images_path = os.path.join(here, 'images')
    icon_full_path = os.path.join(images_path, icon_name)
    return QtGui.QIcon(icon_full_path)

def clear_thumbnail(gview):
    """Clears the thumbnail for the given QGraphicsView

    :param gview: The QGraphicsView instance
    """

    if not gview:
        return

    # clear the graphics scene in case there is no thumbnail
    scene = gview.scene()
    if not scene:
        scene = QtWidgets.QGraphicsScene(gview)
        gview.setScene(scene)

    scene.clear()

def update_gview_with_task_thumbnail(task, gview):
    """Updates the given QGraphicsView with the given Task thumbnail

    :param task: A
      :class:`~stalker.models.task.Task` instance

    :param gview: A QtGui.QGraphicsView instance
    """
    from stalker import Task

    if not isinstance(task, Task) or \
       not isinstance(gview, QtWidgets.QGraphicsView):
        # do nothing
        logger.debug('task is not a stalker.models.task.Task instance')
        return

    # get the thumbnail full path
    full_path = None
    if task.thumbnail:
        # use the cache system to get the thumbnail
        # try to get it as a normal file
        full_path = os.path.expandvars(
            task.thumbnail.full_path
        )
        if not os.path.exists(full_path):
            full_path = None
    else:
        logger.debug('there is no thumbnail')
        # try to get the thumbnail from parents
        for parent in reversed(task.parents):
            if parent.thumbnail:
                # try to get it as a normal file
                full_path = os.path.expandvars(
                    parent.thumbnail.full_path
                )
                if not os.path.exists(full_path):
                    full_path = None
                logger.debug('found parent thumbnail at: %s' % full_path)
                break

    if full_path:
        update_gview_with_image_file(
            full_path,
            gview
        )

def update_gview_with_image_file(image_full_path, gview):
    """updates the QGraphicsView with the given image
    """

    if not isinstance(gview, QtWidgets.QGraphicsView):
        return

    clear_thumbnail(gview)

    if image_full_path != "":
        image_full_path = os.path.normpath(image_full_path)
        image_format = os.path.splitext(image_full_path)[-1].replace('.', '').upper()
        logger.debug("creating pixmap from: %s" % image_full_path)

        # size = conf.thumbnail_size
        # width = size[0]
        # height = size[1]
        size = gview.size()
        width = size.width()
        height = size.height()
        logger.debug('width: %s' % width)
        logger.debug('height: %s' % height)

        if os.path.exists(image_full_path):
            pixmap = QtGui.QPixmap(image_full_path, format=image_format).scaled(
                width, height,
                QtCore.Qt.KeepAspectRatio,
                QtCore.Qt.SmoothTransformation
            )
            # pixmap = QtGui.QPixmap(image_full_path, format='JPG')

            scene = gview.scene()
            scene.addPixmap(pixmap)

def upload_thumbnail(task, thumbnail_full_path):
    """Uploads the given thumbnail for the given entity

    :param task: An instance of :class:`~stalker.models.entity.SimpleEntity`
      or a derivative.

    :param str thumbnail_full_path: A string which is showing the path
      of the thumbnail image
    """
    extension = os.path.splitext(thumbnail_full_path)[-1]

    # move the file to the task thumbnail folder
    # and mimic StalkerPyramids output format
    thumbnail_original_file_name = 'thumbnail%s' % extension
    thumbnail_final_full_path = os.path.join(
        task.absolute_path, 'Thumbnail', thumbnail_original_file_name
    )

    try:
        os.makedirs(os.path.dirname(thumbnail_final_full_path))
    except OSError:
        pass

    # # convert the thumbnail to jpg if it is a format that is not supported by
    # # browsers
    # ext_not_supported_by_browsers = ['.bmp', '.tga', '.tif', '.tiff', '.exr']
    # if extension in ext_not_supported_by_browsers:
    #     # use MediaManager to convert them
    #     from anima.utils import MediaManager
    #     mm = MediaManager()
    #     thumbnail_full_path = mm.generate_image_thumbnail(thumbnail_full_path)

    shutil.copy(thumbnail_full_path, thumbnail_final_full_path)

    from stalker import Link, Version, Repository

    thumbnail_os_independent_path = \
        Repository.to_os_independent_path(thumbnail_final_full_path)
    l_thumb = Link.query\
        .filter(Link.full_path == thumbnail_os_independent_path).first()

    if not l_thumb:
        l_thumb = Link(
            full_path=thumbnail_os_independent_path,
            original_filename=thumbnail_original_file_name
        )

    task.thumbnail = l_thumb

    # get a version of this Task
    from stalker.db.session import DBSession
    v = Version.query.filter(Version.task == task).first()
    if v:
        for naming_parent in v.naming_parents:
            if not naming_parent.thumbnail:
                naming_parent.thumbnail = l_thumb
                DBSession.add(naming_parent)

    DBSession.add(l_thumb)
    DBSession.commit()

def choose_thumbnail(parent):
    """shows a dialog for thumbnail upload
    """
    # get a file from a FileDialog
    thumbnail_full_path = QtWidgets.QFileDialog.getOpenFileName(
        parent, "Choose Thumbnail",
        os.path.expanduser("~"),
        # "Image Files (*.png *.jpg *.bmp *.tga *.tif *.tiff *.exr)"
        "Image Files (*.png *.jpg *.bmp *.tif *.tiff)"
    )

    if isinstance(thumbnail_full_path, tuple):
        thumbnail_full_path = thumbnail_full_path[0]

    return thumbnail_full_path

def render_image_from_gview(gview, image_full_path):
    """renders the gview scene to an image at the given full path
    """
    assert isinstance(gview, QtWidgets.QGraphicsView)
    scene = gview.scene()
    # there should be only one item
    items = scene.items()
    if items:
        pixmapItem = items[0]
        pixmap = pixmapItem.pixmap()
        try:
            os.makedirs(os.path.split(image_full_path)[0])
        except OSError:
            # dir exists
            pass

        pixmap.save(
            image_full_path
        )

def load_font(font_filename):
    """loads extra fonts from the fonts folder
    """
    here = os.path.dirname(os.path.realpath(__file__))
    font_id = QtGui.QFontDatabase.addApplicationFont(
        os.path.join(here, 'fonts', font_filename)
    )
    loaded_font_families = QtGui.QFontDatabase.applicationFontFamilies(font_id)
    return loaded_font_families
<|endoftext|>


<commit_before>from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.conf import settings
from django import http
from django.contrib.auth.models import User

from .mailgun import check_webhook_signature
from .tasks import send_announcement
from studygroups.models import Profile

from email.utils import parseaddr, formataddr
import logging

logger = logging.getLogger(__name__)

@csrf_exempt
@require_http_methods(['POST'])
def announce_webhook(request):
    """ This view is called by mailgun """
    # Verify mailgun call
    token = request.POST.get('token')
    timestamp = request.POST.get('timestamp')
    signature = request.POST.get('signature')
    authenticated = check_webhook_signature(settings.MAILGUN_API_KEY, token, timestamp, signature)
    if not authenticated:
        logger.warn('Authentication failed for announce_webhook')
        return http.HttpResponse(status=401) # TODO should this be 406 of just 200?

    # Get message details
    sender = request.POST.get('sender')
    from_ = request.POST.get('from')
    subject = request.POST.get('subject')
    body_text = request.POST.get('body-plain')
    body_html = request.POST.get('body-html')

    # Verify that email was sent from a staff user
    # parse sender to make sure it's not in the format John Silver <john@mail.net>
    user_name, user_email = parseaddr(sender)
    user = User.objects.filter(email=user_email).first()
    if not user or user.is_staff == False:
        logger.warn('Message sent to announce email from non-staff user')
        return http.HttpResponse(status=406)

    from_name = '{0} from P2PU'.format(user.first_name)
    from_ = formataddr((from_name, settings.ANNOUNCE_EMAIL))

    # Send announcement message to all users that opted in
    send_announcement.delay(from_, subject, body_text, body_html)

    return http.HttpResponse(status=200)

@csrf_exempt
def mailchimp_webhook(request, webhook_secret):
    """ change profile.communications_opt_in if subscriber has an account """

    if request.method == 'GET':
        return http.HttpResponse(status=200)

    if webhook_secret != settings.MAILCHIMP_WEBHOOK_SECRET:
        return http.HttpResponse(status=404)

    list_id = request.POST.get('data[id]')
    if list_id != settings.MAILCHIMP_LIST_ID:
        logger.warning('mailchimp webhook called with invalid list id')
        return http.HttpResponse(status=404)

    email = request.POST.get('data[email]')
    if User.objects.filter(email__iexact=email).count() == 0:
        return http.HttpResponse(status=200)

    profile = Profile.objects.get(user__email=email)
    if request.POST.get('type') == 'unsubscribe':
        profile.communication_opt_in = False
    elif request.POST.get('type') == 'subscribe':
        profile.communication_opt_in = True
    profile.save()
    return http.HttpResponse(status=200)
<commit_msg>Add logging to test mailchimp webhook<commit_after>from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.conf import settings
from django import http
from django.contrib.auth.models import User

from .mailgun import check_webhook_signature
from .tasks import send_announcement
from studygroups.models import Profile

from email.utils import parseaddr, formataddr
import logging

logger = logging.getLogger(__name__)

@csrf_exempt
@require_http_methods(['POST'])
def announce_webhook(request):
    """ This view is called by mailgun """
    # Verify mailgun call
    token = request.POST.get('token')
    timestamp = request.POST.get('timestamp')
    signature = request.POST.get('signature')
    authenticated = check_webhook_signature(settings.MAILGUN_API_KEY, token, timestamp, signature)
    if not authenticated:
        logger.warn('Authentication failed for announce_webhook')
        return http.HttpResponse(status=401) # TODO should this be 406 of just 200?

    # Get message details
    sender = request.POST.get('sender')
    from_ = request.POST.get('from')
    subject = request.POST.get('subject')
    body_text = request.POST.get('body-plain')
    body_html = request.POST.get('body-html')

    # Verify that email was sent from a staff user
    # parse sender to make sure it's not in the format John Silver <john@mail.net>
    user_name, user_email = parseaddr(sender)
    user = User.objects.filter(email=user_email).first()
    if not user or user.is_staff == False:
        logger.warn('Message sent to announce email from non-staff user')
        return http.HttpResponse(status=406)

    from_name = '{0} from P2PU'.format(user.first_name)
    from_ = formataddr((from_name, settings.ANNOUNCE_EMAIL))

    # Send announcement message to all users that opted in
    send_announcement.delay(from_, subject, body_text, body_html)

    return http.HttpResponse(status=200)

@csrf_exempt
def mailchimp_webhook(request, webhook_secret):
    """ change profile.communications_opt_in if subscriber has an account """

    if request.method == 'GET':
        return http.HttpResponse(status=200)

    import json
    logger.error(json.dumps(request.POST))

    if webhook_secret != settings.MAILCHIMP_WEBHOOK_SECRET:
        return http.HttpResponse(status=404)

    list_id = request.POST.get('data[id]')
    if list_id != settings.MAILCHIMP_LIST_ID:
        logger.warning('mailchimp webhook called with invalid list id')
        return http.HttpResponse(status=404)

    email = request.POST.get('data[email]')
    if User.objects.filter(email__iexact=email).count() == 0:
        return http.HttpResponse(status=200)

    profile = Profile.objects.get(user__email=email)
    if request.POST.get('type') == 'unsubscribe':
        profile.communication_opt_in = False
    elif request.POST.get('type') == 'subscribe':
        profile.communication_opt_in = True
    profile.save()
    return http.HttpResponse(status=200)
<|endoftext|>


<commit_before>import tweepy
import threading
import psycopg2
import re

def authorize():
    """
    Use OAuth to setup access token for the app.
    """
    auth = tweepy.OAuthHandler(
        'Xm4gPHD64LuLKJNam2aRbJhi9',
        'rZsFYrxsGfiq4TAQ2yKbtUBKeRhI7QnMLUxFGf0pI35DrZKDLH'
    )
    auth.set_access_token(
        '2489534185-wWhkpgFhDEodaaE73bL34JW9dlwG3ZtktQJA8xf',
        '5OewpVBuhQmyoh31EI4zOz2YJuFmycqPL1GiCGVlCDp4C'
    )

    return tweepy.API(auth)

def fetch_related_statuses(api, name, how_many_tweets=25):
    """
    Return a list of (user name, related tweet content).
    """
    pass

def write_data(tweets_list):
    pass

def fetch_user_statuses(api, target_screen_name=None, reference=0, how_many_tweets=50):
    """
    Return a list of (user name, content, tweet time(gmt)).
    """
    tweets = api.user_timeline(screen_name=target_screen_name, count=how_many_tweets)
    content = []
    for tweet in tweets:
        # If retweeted, a tweet begins with 'RT @name:'
        if tweet.text.startswith(u'RT @'):
            author = re.compile(r'RT @\S+:').match(tweet.text).group()[4:-1]
        else:
            author = tweet.user.name # Venue name
        content.append([reference, author, repr(tweet.text), tweet.created_at, 1])
    return content
<commit_msg>Modified fetch_user_status() to write twitter handle instead of display screen name to outputed information.<commit_after>import tweepy
import threading
import psycopg2
import re

def authorize():
    """
    Use OAuth to setup access token for the app.
    """
    auth = tweepy.OAuthHandler(
        'Xm4gPHD64LuLKJNam2aRbJhi9',
        'rZsFYrxsGfiq4TAQ2yKbtUBKeRhI7QnMLUxFGf0pI35DrZKDLH'
    )
    auth.set_access_token(
        '2489534185-wWhkpgFhDEodaaE73bL34JW9dlwG3ZtktQJA8xf',
        '5OewpVBuhQmyoh31EI4zOz2YJuFmycqPL1GiCGVlCDp4C'
    )

    return tweepy.API(auth)

def fetch_related_statuses(api, name, how_many_tweets=25):
    """
    Return a list of (user name, related tweet content).
    """
    pass

def write_data(tweets_list):
    pass

def fetch_user_statuses(api, target_screen_name=None, reference=0, how_many_tweets=50):
    """
    Return a list of (user name, content, tweet time(gmt)).
    """
    tweets = api.user_timeline(screen_name=target_screen_name, count=how_many_tweets)
    content = []
    for tweet in tweets:
        # If retweeted, a tweet begins with 'RT @name:'
        if tweet.text.startswith(u'RT @'):
            author = re.compile(r'RT @\S+:').match(tweet.text).group()[4:-1]
        else:
            author = target_screen_name  # Venue twitter handle
        content.append(
            [reference, author, repr(tweet.text), tweet.created_at, 1]
        )
    return content
<|endoftext|>


<commit_before>import tweepy
import threading
import psycopg2
import re

def authorize():
    """
    Use OAuth to setup access token for the app.
    """
    auth = tweepy.OAuthHandler(
        'Xm4gPHD64LuLKJNam2aRbJhi9',
        'rZsFYrxsGfiq4TAQ2yKbtUBKeRhI7QnMLUxFGf0pI35DrZKDLH'
    )
    auth.set_access_token(
        '2489534185-wWhkpgFhDEodaaE73bL34JW9dlwG3ZtktQJA8xf',
        '5OewpVBuhQmyoh31EI4zOz2YJuFmycqPL1GiCGVlCDp4C'
    )

    return tweepy.API(auth)

def fetch_related_statuses(api, name, how_many_tweets=25):
    """
    Return a list of (user name, related tweet content).
    """
    pass

def write_data(tweets_list):
    pass

def fetch_user_statuses(api, target_screen_name=None, reference=0, how_many_tweets=50):
    """
    Return a list of (user name, content, tweet time(gmt)).
    """
    tweets = api.user_timeline(screen_name=target_screen_name, count=how_many_tweets)
    content = []
    for tweet in tweets:
        # If retweeted, a tweet begins with 'RT @name:'
        if tweet.text.startswith(u'RT @'):
            author = re.compile(r'RT @\S+:').match(tweet.text).group()[4:-1]
        else:
            author = target_screen_name  # Venue twitter handle
        content.append(
            [reference, author, repr(tweet.text), tweet.created_at, 1]
        )
    return content
<commit_msg>Changed fetch_user_statuses to encode content to utf-8 before writing to database.<commit_after>import tweepy
import threading
import psycopg2
import re

def authorize():
    """
    Use OAuth to setup access token for the app.
    """
    auth = tweepy.OAuthHandler(
        'Xm4gPHD64LuLKJNam2aRbJhi9',
        'rZsFYrxsGfiq4TAQ2yKbtUBKeRhI7QnMLUxFGf0pI35DrZKDLH'
    )
    auth.set_access_token(
        '2489534185-wWhkpgFhDEodaaE73bL34JW9dlwG3ZtktQJA8xf',
        '5OewpVBuhQmyoh31EI4zOz2YJuFmycqPL1GiCGVlCDp4C'
    )

    return tweepy.API(auth)

def fetch_related_statuses(api, name, how_many_tweets=25):
    """
    Return a list of (user name, related tweet content).
    """
    pass

def write_data(tweets_list):
    with open('tweets.txt') as f:
        f.write(tweets_list)

def fetch_user_statuses(api, target_twitter_handle=None, reference=0, how_many_tweets=50):
    """
    Return a list of (user name, content, tweet time(gmt)).
    """
    tweets = api.user_timeline(screen_name=target_twitter_handle, count=how_many_tweets)
    content = []
    for tweet in tweets:
        # If retweeted, a tweet begins with 'RT @name:'
        if tweet.text.startswith(u'RT @'):
            author = re.compile(r'RT @\S+:').match(tweet.text).group()[4:-1]
        else:
            author = target_twitter_handle  # Venue twitter handle
        content.append(
            [reference, author, tweet.text.encode('utf-8'), tweet.created_at, 1]
        )
    return content
<|endoftext|>


<commit_before># Third-party
import astropy.units as u
from astropy.stats import median_absolute_deviation
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import numpy as np

from thejoker.plot import plot_rv_curves
from thejoker.sampler.likelihood import get_ivar

from .samples_analysis import unimodal_P, max_likelihood_sample

__all__ = ['plot_data_orbits', 'plot_mcmc_diagnostic', 'plot_two_panel']

# TODO: customize this in TheJoker?
_RV_LBL = 'RV [{0:latex_inline}]'

def plot_data_orbits(data, samples, n_orbits=128, jitter=None,
                     xlim_choice='data', ylim_fac=1., n_times=4096, title=None,
                     ax=None, highlight_P_extrema=True, plot_kwargs=None,
                     relative_to_t0=False):
    """
    Plot the APOGEE RV data vs. time and orbits computed from The Joker samples.

    Parameters
    ----------
    data : :class:`~twoface.data.APOGEERVData`
        The radial velocity data.
    samples : :class:`~thejoker.samples.JokerSamples`
        Posterior samples from The Joker.
    n_orbits : int, optional
        Number of orbits to plot over the data.
    jitter : :class:`~astropy.units.Quantity`, optional
        The jitter used to do the sampling. Only relevant if the jitter was
        fixed. Used to inflate the error bars for the data.
    xlim_choice : str, optional
        Multiple options for how to set the x-axis limits for the plot.
        ``xlim_choice = 'wide'`` sets the xlim to be twice the longest period
        sample.
        ``xlim_choice = 'tight'`` sets the xlim to be twice the time span of the
        data.

    """

    if jitter is not None:
        data = data.copy()

        ivar = get_ivar(data, jitter.to(data.rv.unit).value)
        data.ivar = ivar / data.rv.unit**2

    if ax is None:
        fig, ax = plt.subplots(1, 1, figsize=(12,6))
    else:
        fig = ax.figure

    w = np.ptp(data.t.mjd)
    if xlim_choice == 'tight': # twice the span of the data
        t_grid = np.linspace(data.t.mjd.min() - w*0.05,
                             data.t.mjd.max() + w*1.05,
                             n_times)

    elif xlim_choice == 'wide': # twice the longest period sample
        t_min = data.t.mjd.min()
        t_max = max(data.t.mjd.min() + 2*samples['P'].max().value,
                    data.t.mjd.max())
        t_grid = np.linspace(t_min - 0.05*w,
                             t_max + 0.05*w,
                             n_times)

    elif xlim_choice == 'data': # span of the data
        t_grid = np.linspace(data.t.mjd.min() - w*0.05,
                             data.t.mjd.max() + w*0.05,
                             n_times)

    else:
        raise ValueError('Invalid xlim_choice {0}. Can be "wide" or "tight".'
                         .format(xlim_choice))

    if plot_kwargs is None:
        plot_kwargs = dict()

    plot_kwargs.setdefault('color', 'tab:blue')
    # plot_kwargs.setdefault('alpha', 0.25)
    plot_kwargs.setdefault('linewidth', 0.5)
    plot_rv_curves(samples, t_grid, rv_unit=u.km/u.s, data=data, ax=ax,
                   n_plot=min(len(samples['P']), n_orbits),
                   plot_kwargs=plot_kwargs,
                   data_plot_kwargs=dict(zorder=5, elinewidth=1,),
                   relative_to_t0=relative_to_t0)

    if highlight_P_extrema:
        # Darken the shortest period sample
        dark_style = dict(color='#333333', alpha=0.5, linewidth=1, zorder=10)

        P_min_samples = samples[samples['P'].argmin()]
        plot_rv_curves(P_min_samples, t_grid, rv_unit=u.km/u.s, ax=ax,
                       n_plot=1, plot_kwargs=dark_style,
                       relative_to_t0=relative_to_t0)

        # Darken the longest period sample
        P_max_samples = samples[samples['P'].argmax()]
        plot_rv_curves(P_max_samples, t_grid, rv_unit=u.km/u.s, ax=ax,
                       n_plot=1, plot_kwargs=dark_style,
                       relative_to_t0=relative_to_t0)

    if relative_to_t0:
        ax.set_xlim(t_grid.min() - data.t0.mjd,
                    t_grid.max() - data.t0.mjd)
    else:
        ax.set_xlim(t_grid.min(), t_grid.max())

    _rv = data.rv.to(u.km/u.s).value
    h = np.ptp(_rv)
    ax.set_ylim(_rv.min()-ylim_fac*h, _rv.max()+ylim_fac*h)

    if title is not None:
        ax.set_title(title)

    return fig

def plot_mcmc_diagnostic(chain):
    """
    TODO:
    """

    names = [r'$\ln P$', r'$\sqrt{K}\,\cos M_0$', r'$\sqrt{K}\,\sin M_0$',
             r'$\sqrt{e}\,\cos \omega$', r'$\sqrt{e}\,\sin \omega$',
             r'$\ln s^2$', '$v_0$']

    ndim = chain.shape[-1]
    assert ndim == len(names)

    fig, axes = plt.subplots(ndim, 3, figsize=(12, 16), sharex=True)

    for k in range(ndim):
        axes[k, 0].set_ylabel(names[k])
        axes[k, 0].plot(chain[..., k].T, marker='',
                        drawstyle='steps-mid',
                        alpha=0.1, rasterized=True)
        axes[k, 1].plot(np.median(chain[..., k], axis=0),
                        marker='', drawstyle='steps-mid')

        std = 1.5 * median_absolute_deviation(chain[..., k], axis=0)
        axes[k, 2].plot(std, marker='', drawstyle='steps-mid')

    axes[0, 0].set_title('walkers')
    axes[0, 1].set_title('med(walkers)')
    axes[0, 2].set_title('1.5 MAD(walkers)')

    fig.tight_layout()
    return fig

def plot_two_panel(data, samples, axes=None, tight=True, title=None,
                   plot_data_orbits_kw=None, scatter_kw=None):
    """Make a two-panel plot with (1) the data and orbit samples, and (2) the
    orbit samples in period-eccentricity space.

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    samples : `~thejoker.JokerSamples`
    plot_data_orbits_kw : dict (optional)
    scatter_kw : dict (optional)

    Returns
    -------
    fig : `matplotlib.Figure`

    """
    if plot_data_orbits_kw is None:
        plot_data_orbits_kw = dict()

    if scatter_kw is None:
        scatter_kw = dict()

    if axes is None:
        fig = plt.figure(figsize=(12, 4.3))
        gs = GridSpec(1, 3)
        ax1 = fig.add_subplot(gs[:2])
        ax2 = fig.add_subplot(gs[2])
        axes = [ax1, ax2]

    else:
        ax1, ax2 = axes
        fig = ax1.figure

    # orbits
    plot_data_orbits_kw['xlim_choice'] = plot_data_orbits_kw.get(
        'xlim_choice', 'tight')
    fig = plot_data_orbits(data, samples, ax=ax1,
                           **plot_data_orbits_kw)
    if title is not None:
        ax1.set_title(title, fontsize=20)

    # orbit samples
    scatter_kw.setdefault('marker', '.')
    scatter_kw.setdefault('alpha', 0.5)
    scatter_kw.setdefault('linewidth', 0.)
    ax2.scatter(samples['P'], samples['e'], **scatter_kw)

    ax2.set_xscale('log', basex=2)
    ax2.set_xlim(0.5, 2**15)
    ax2.set_ylim(-0.02, 1.02)
    ax2.set_xlabel('period, $P$ [day]')
    ax2.set_ylabel('eccentricity, $e$')
    ax2.xaxis.set_ticks(2**np.arange(0, 15+1, 3))

    if tight:
        fig.tight_layout()

    return fig

def plot_phase_fold(data, sample, ax=None, label=True,
                    jitter_errorbar=True, orbit_style=None):
    """
    TODO:

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    sample : `~thejoker.JokerSamples`

    Returns
    -------
    fig : `matplotlib.Figure`
    """

    if ax is None:
        ax = plt.gca()
    fig = ax.figure

    if len(sample) > 1:
        sample = max_likelihood_sample(data, sample)
        # raise ValueError('Can only pass one sample.')

    if orbit_style is None:
        orbit_style = dict()
    orbit_style.setdefault('color', 'tab:blue')
    orbit_style.setdefault('zorder', -1)

    P = sample['P']
    M0 = sample['M0']
    orbit = sample.get_orbit(0)

    t0 = data.t0 + (P/(2*np.pi)*M0).to(u.day, u.dimensionless_angles())
    phase = data.phase(P=P, t0=t0)

    # plot the phase-folded data and orbit
    rv_unit = u.km/u.s
    ax.errorbar(phase, data.rv.to(rv_unit).value,
                data.stddev.to(rv_unit).value,
                linestyle='none', marker='o', color='k', markersize=5,
                zorder=10)

    if jitter_errorbar:
        ax.errorbar(phase, data.rv.to(rv_unit).value,
                    np.sqrt(data.stddev**2 +
                            sample['jitter']**2).to(rv_unit).value,
                    linestyle='none', marker='', elinewidth=0., color='#aaaaaa',
                    alpha=0.9, capsize=3, capthick=1, zorder=9)

    phase_grid = np.linspace(0, 1, 1024)
    ax.plot(phase_grid, orbit.radial_velocity(t0 + phase_grid*P),
            marker='', **orbit_style)

    if label:
        ax.set_xlabel(r'phase, $\frac{M-M_0}{2\pi}$')
        ax.set_ylabel(_RV_LBL.format(rv_unit))

    return fig

# TODO: add and update function above
def plot_phase_fold_residual(data, sample):
    """Plot the data phase-folded at the median period of the input samples,
    and the residuals as a function of phase.

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    sample : `~thejoker.JokerSamples`

    Returns
    -------
    fig : `matplotlib.Figure`
    """

    if not unimodal_P(sample, data):
        raise ValueError('multi-modal period distribution')

    if len(sample) > 1:
        raise ValueError('can only pass a single sample to phase-fold at.')

    # HACK: hard-set getting the median
    orbit = sample.get_orbit(0)
    M0 = sample['M0']
    P = sample['P']
    s = sample['jitter']
    t0 = data.t0 + (P/(2*np.pi)*M0).to(u.day, u.dimensionless_angles())
    phase = data.phase(P=P, t0=t0)

    # compute chi^2 of the orbit fit
    residual = data.rv - orbit.radial_velocity(data.t)
    err = np.sqrt(data.stddev**2 + s**2)
    # chisq = np.sum((residual**2 / err**2).decompose())

    fig, axes = plt.subplots(1, 2, figsize=(10, 5), sharex=True)

    # plot the phase-folded data and orbit
    rv_unit = u.km/u.s
    axes[0].errorbar(phase, data.rv.to(rv_unit).value,
                     data.stddev.to(rv_unit).value,
                     linestyle='none', marker='o', color='k', markersize=5)
    axes[0].errorbar(phase, data.rv.to(rv_unit).value, s.to(rv_unit).value,
                     linestyle='none', marker='', color='k',
                     alpha=0.4, zorder=-10)

    phase_grid = np.linspace(0, 1, 1024)
    axes[0].plot(phase_grid, orbit.radial_velocity(t0 + phase_grid*P),
                 marker='', zorder=-1, color='#aaaaaa')
    axes[0].set_xlabel('phase')
    axes[0].set_ylabel('radial velocity [{0:latex_inline}]'.format(rv_unit))
    # ax.set_title(r'$\chi^2 = {0:.2f}$'.format(chisq))

    # plot the residuals
    rv_unit = u.m/u.s
    axes[1].errorbar(phase, residual.to(rv_unit).value,
                     data.stddev.to(rv_unit).value,
                     linestyle='none', marker='o', color='k', markersize=5)
    axes[1].errorbar(phase, residual.to(rv_unit).value, s.to(rv_unit).value,
                     linestyle='none', marker='', color='k',
                     alpha=0.4, zorder=-10)

    axes[1].set_xlabel('phase')
    axes[1].set_ylabel('residual [{0:latex_inline}]'.format(rv_unit))
    lim = np.abs(axes[1].get_ylim()).max()
    axes[1].set_ylim(-lim, lim)
    axes[1].axhline(0, zorder=-100, color='tab:green', alpha=0.25)
    # axes[1].set_title(r'$s = ${:.0f}'.format(s.to(u.m/u.s)))

    fig.tight_layout()
    return fig
<commit_msg>default don't highlight<commit_after># Third-party
import astropy.units as u
from astropy.stats import median_absolute_deviation
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import numpy as np

from thejoker.plot import plot_rv_curves
from thejoker.sampler.likelihood import get_ivar

from .samples_analysis import unimodal_P, max_likelihood_sample

__all__ = ['plot_data_orbits', 'plot_mcmc_diagnostic', 'plot_two_panel']

# TODO: customize this in TheJoker?
_RV_LBL = 'RV [{0:latex_inline}]'

def plot_data_orbits(data, samples, n_orbits=128, jitter=None,
                     xlim_choice='data', ylim_fac=1., n_times=4096, title=None,
                     ax=None, highlight_P_extrema=True, plot_kwargs=None,
                     relative_to_t0=False):
    """
    Plot the APOGEE RV data vs. time and orbits computed from The Joker samples.

    Parameters
    ----------
    data : :class:`~twoface.data.APOGEERVData`
        The radial velocity data.
    samples : :class:`~thejoker.samples.JokerSamples`
        Posterior samples from The Joker.
    n_orbits : int, optional
        Number of orbits to plot over the data.
    jitter : :class:`~astropy.units.Quantity`, optional
        The jitter used to do the sampling. Only relevant if the jitter was
        fixed. Used to inflate the error bars for the data.
    xlim_choice : str, optional
        Multiple options for how to set the x-axis limits for the plot.
        ``xlim_choice = 'wide'`` sets the xlim to be twice the longest period
        sample.
        ``xlim_choice = 'tight'`` sets the xlim to be twice the time span of the
        data.

    """

    if jitter is not None:
        data = data.copy()

        ivar = get_ivar(data, jitter.to(data.rv.unit).value)
        data.ivar = ivar / data.rv.unit**2

    if ax is None:
        fig, ax = plt.subplots(1, 1, figsize=(12,6))
    else:
        fig = ax.figure

    w = np.ptp(data.t.mjd)
    if xlim_choice == 'tight': # twice the span of the data
        t_grid = np.linspace(data.t.mjd.min() - w*0.05,
                             data.t.mjd.max() + w*1.05,
                             n_times)

    elif xlim_choice == 'wide': # twice the longest period sample
        t_min = data.t.mjd.min()
        t_max = max(data.t.mjd.min() + 2*samples['P'].max().value,
                    data.t.mjd.max())
        t_grid = np.linspace(t_min - 0.05*w,
                             t_max + 0.05*w,
                             n_times)

    elif xlim_choice == 'data': # span of the data
        t_grid = np.linspace(data.t.mjd.min() - w*0.05,
                             data.t.mjd.max() + w*0.05,
                             n_times)

    else:
        raise ValueError('Invalid xlim_choice {0}. Can be "wide" or "tight".'
                         .format(xlim_choice))

    if plot_kwargs is None:
        plot_kwargs = dict()

    plot_kwargs.setdefault('color', 'tab:blue')
    # plot_kwargs.setdefault('alpha', 0.25)
    plot_kwargs.setdefault('linewidth', 0.5)
    plot_rv_curves(samples, t_grid, rv_unit=u.km/u.s, data=data, ax=ax,
                   n_plot=min(len(samples['P']), n_orbits),
                   plot_kwargs=plot_kwargs,
                   data_plot_kwargs=dict(zorder=5, elinewidth=1,),
                   relative_to_t0=relative_to_t0)

    if highlight_P_extrema:
        # Darken the shortest period sample
        dark_style = dict(color='#333333', alpha=0.5, linewidth=1, zorder=10)

        P_min_samples = samples[samples['P'].argmin()]
        plot_rv_curves(P_min_samples, t_grid, rv_unit=u.km/u.s, ax=ax,
                       n_plot=1, plot_kwargs=dark_style,
                       relative_to_t0=relative_to_t0)

        # Darken the longest period sample
        P_max_samples = samples[samples['P'].argmax()]
        plot_rv_curves(P_max_samples, t_grid, rv_unit=u.km/u.s, ax=ax,
                       n_plot=1, plot_kwargs=dark_style,
                       relative_to_t0=relative_to_t0)

    if relative_to_t0:
        ax.set_xlim(t_grid.min() - data.t0.mjd,
                    t_grid.max() - data.t0.mjd)
    else:
        ax.set_xlim(t_grid.min(), t_grid.max())

    _rv = data.rv.to(u.km/u.s).value
    h = np.ptp(_rv)
    ax.set_ylim(_rv.min()-ylim_fac*h, _rv.max()+ylim_fac*h)

    if title is not None:
        ax.set_title(title)

    return fig

def plot_mcmc_diagnostic(chain):
    """
    TODO:
    """

    names = [r'$\ln P$', r'$\sqrt{K}\,\cos M_0$', r'$\sqrt{K}\,\sin M_0$',
             r'$\sqrt{e}\,\cos \omega$', r'$\sqrt{e}\,\sin \omega$',
             r'$\ln s^2$', '$v_0$']

    ndim = chain.shape[-1]
    assert ndim == len(names)

    fig, axes = plt.subplots(ndim, 3, figsize=(12, 16), sharex=True)

    for k in range(ndim):
        axes[k, 0].set_ylabel(names[k])
        axes[k, 0].plot(chain[..., k].T, marker='',
                        drawstyle='steps-mid',
                        alpha=0.1, rasterized=True)
        axes[k, 1].plot(np.median(chain[..., k], axis=0),
                        marker='', drawstyle='steps-mid')

        std = 1.5 * median_absolute_deviation(chain[..., k], axis=0)
        axes[k, 2].plot(std, marker='', drawstyle='steps-mid')

    axes[0, 0].set_title('walkers')
    axes[0, 1].set_title('med(walkers)')
    axes[0, 2].set_title('1.5 MAD(walkers)')

    fig.tight_layout()
    return fig

def plot_two_panel(data, samples, axes=None, tight=True, title=None,
                   plot_data_orbits_kw=None, scatter_kw=None):
    """Make a two-panel plot with (1) the data and orbit samples, and (2) the
    orbit samples in period-eccentricity space.

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    samples : `~thejoker.JokerSamples`
    plot_data_orbits_kw : dict (optional)
    scatter_kw : dict (optional)

    Returns
    -------
    fig : `matplotlib.Figure`

    """
    if plot_data_orbits_kw is None:
        plot_data_orbits_kw = dict()

    if scatter_kw is None:
        scatter_kw = dict()

    if axes is None:
        fig = plt.figure(figsize=(12, 4.3))
        gs = GridSpec(1, 3)
        ax1 = fig.add_subplot(gs[:2])
        ax2 = fig.add_subplot(gs[2])
        axes = [ax1, ax2]

    else:
        ax1, ax2 = axes
        fig = ax1.figure

    # orbits
    plot_data_orbits_kw.setdefault('xlim_choice', 'tight')
    plot_data_orbits_kw.setdefault('highlight_P_extrema', False)
    fig = plot_data_orbits(data, samples, ax=ax1,
                           **plot_data_orbits_kw)
    if title is not None:
        ax1.set_title(title, fontsize=20)

    # orbit samples
    scatter_kw.setdefault('marker', '.')
    scatter_kw.setdefault('alpha', 0.5)
    scatter_kw.setdefault('linewidth', 0.)
    ax2.scatter(samples['P'], samples['e'], **scatter_kw)

    ax2.set_xscale('log', basex=10)
    ax2.set_xlim(0.5, 2**15)
    ax2.set_ylim(-0.02, 1.02)
    ax2.set_xlabel('period, $P$ [day]')
    ax2.set_ylabel('eccentricity, $e$')
    ax2.xaxis.set_ticks(10**np.arange(0, 4+1, 1))

    if tight:
        fig.tight_layout()

    return fig

def plot_phase_fold(data, sample, ax=None, label=True,
                    jitter_errorbar=True, orbit_style=None):
    """
    TODO:

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    sample : `~thejoker.JokerSamples`

    Returns
    -------
    fig : `matplotlib.Figure`
    """

    if ax is None:
        ax = plt.gca()
    fig = ax.figure

    if len(sample) > 1:
        sample = max_likelihood_sample(data, sample)
        # raise ValueError('Can only pass one sample.')

    if orbit_style is None:
        orbit_style = dict()
    orbit_style.setdefault('color', 'tab:blue')
    orbit_style.setdefault('zorder', -1)

    P = sample['P']
    M0 = sample['M0']
    orbit = sample.get_orbit(0)

    t0 = data.t0 + (P/(2*np.pi)*M0).to(u.day, u.dimensionless_angles())
    phase = data.phase(P=P, t0=t0)

    # plot the phase-folded data and orbit
    rv_unit = u.km/u.s
    ax.errorbar(phase, data.rv.to(rv_unit).value,
                data.stddev.to(rv_unit).value,
                linestyle='none', marker='o', color='k', markersize=5,
                zorder=10)

    if jitter_errorbar:
        ax.errorbar(phase, data.rv.to(rv_unit).value,
                    np.sqrt(data.stddev**2 +
                            sample['jitter']**2).to(rv_unit).value,
                    linestyle='none', marker='', elinewidth=0., color='#aaaaaa',
                    alpha=0.9, capsize=3, capthick=1, zorder=9)

    phase_grid = np.linspace(0, 1, 1024)
    ax.plot(phase_grid, orbit.radial_velocity(t0 + phase_grid*P),
            marker='', **orbit_style)

    if label:
        ax.set_xlabel(r'phase, $\frac{M-M_0}{2\pi}$')
        ax.set_ylabel(_RV_LBL.format(rv_unit))

    return fig

# TODO: add and update function above
def plot_phase_fold_residual(data, sample):
    """Plot the data phase-folded at the median period of the input samples,
    and the residuals as a function of phase.

    Parameters
    ----------
    star : `~twoface.db.AllStar`
    sample : `~thejoker.JokerSamples`

    Returns
    -------
    fig : `matplotlib.Figure`
    """

    if not unimodal_P(sample, data):
        raise ValueError('multi-modal period distribution')

    if len(sample) > 1:
        raise ValueError('can only pass a single sample to phase-fold at.')

    # HACK: hard-set getting the median
    orbit = sample.get_orbit(0)
    M0 = sample['M0']
    P = sample['P']
    s = sample['jitter']
    t0 = data.t0 + (P/(2*np.pi)*M0).to(u.day, u.dimensionless_angles())
    phase = data.phase(P=P, t0=t0)

    # compute chi^2 of the orbit fit
    residual = data.rv - orbit.radial_velocity(data.t)
    err = np.sqrt(data.stddev**2 + s**2)
    # chisq = np.sum((residual**2 / err**2).decompose())

    fig, axes = plt.subplots(1, 2, figsize=(10, 5), sharex=True)

    # plot the phase-folded data and orbit
    rv_unit = u.km/u.s
    axes[0].errorbar(phase, data.rv.to(rv_unit).value,
                     data.stddev.to(rv_unit).value,
                     linestyle='none', marker='o', color='k', markersize=5)
    axes[0].errorbar(phase, data.rv.to(rv_unit).value, s.to(rv_unit).value,
                     linestyle='none', marker='', color='k',
                     alpha=0.4, zorder=-10)

    phase_grid = np.linspace(0, 1, 1024)
    axes[0].plot(phase_grid, orbit.radial_velocity(t0 + phase_grid*P),
                 marker='', zorder=-1, color='#aaaaaa')
    axes[0].set_xlabel('phase')
    axes[0].set_ylabel('radial velocity [{0:latex_inline}]'.format(rv_unit))
    # ax.set_title(r'$\chi^2 = {0:.2f}$'.format(chisq))

    # plot the residuals
    rv_unit = u.m/u.s
    axes[1].errorbar(phase, residual.to(rv_unit).value,
                     data.stddev.to(rv_unit).value,
                     linestyle='none', marker='o', color='k', markersize=5)
    axes[1].errorbar(phase, residual.to(rv_unit).value, s.to(rv_unit).value,
                     linestyle='none', marker='', color='k',
                     alpha=0.4, zorder=-10)

    axes[1].set_xlabel('phase')
    axes[1].set_ylabel('residual [{0:latex_inline}]'.format(rv_unit))
    lim = np.abs(axes[1].get_ylim()).max()
    axes[1].set_ylim(-lim, lim)
    axes[1].axhline(0, zorder=-100, color='tab:green', alpha=0.25)
    # axes[1].set_title(r'$s = ${:.0f}'.format(s.to(u.m/u.s)))

    fig.tight_layout()
    return fig
<|endoftext|>


<commit_before>import os
import re
import sys
import warnings
from datetime import timezone

from tzlocal import utils

if sys.version_info >= (3, 9):
    from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
else:
    from backports.zoneinfo import ZoneInfo, ZoneInfoNotFoundError

_cache_tz = None

def _tz_from_env(tzenv):
    if tzenv[0] == ':':
        tzenv = tzenv[1:]

    # TZ specifies a file
    if os.path.isabs(tzenv) and os.path.exists(tzenv):
        with open(tzenv, 'rb') as tzfile:
            return ZoneInfo.from_file(tzfile, key='local')

    # TZ specifies a zoneinfo zone.
    try:
        tz = ZoneInfo(tzenv)
        # That worked, so we return this:
        return tz
    except ZoneInfoNotFoundError:
        raise ZoneInfoNotFoundError(
            "tzlocal() does not support non-zoneinfo timezones like %s. \n"
            "Please use a timezone in the form of Continent/City") from None

def _try_tz_from_env():
    tzenv = os.environ.get('TZ')
    if tzenv:
        try:
            return _tz_from_env(tzenv)
        except ZoneInfoNotFoundError:
            pass

def _get_localzone(_root='/'):
    """Tries to find the local timezone configuration.

    This method prefers finding the timezone name and passing that to pytz,
    over passing in the localtime file, as in the later case the zoneinfo
    name is unknown.

    The parameter _root makes the function look for files like /etc/localtime
    beneath the _root directory. This is primarily used by the tests.
    In normal usage you call the function without parameters."""

    tzenv = _try_tz_from_env()
    if tzenv:
        return tzenv

    # Are we under Termux on Android?
    if os.path.exists('/system/bin/getprop'):
        import subprocess
        androidtz = subprocess.check_output(['getprop', 'persist.sys.timezone']).strip().decode()
        return ZoneInfo(androidtz)

    # Now look for distribution specific configuration files
    # that contain the timezone name.
    for configfile in ('etc/timezone', 'var/db/zoneinfo'):
        tzpath = os.path.join(_root, configfile)
        try:
            with open(tzpath, 'rb') as tzfile:
                data = tzfile.read()

                # Issue #3 was that /etc/timezone was a zoneinfo file.
                # That's a misconfiguration, but we need to handle it gracefully:
                if data[:5] == b'TZif2':
                    continue

                etctz = data.strip().decode()
                if not etctz:
                    # Empty file, skip
                    continue
                for etctz in data.decode().splitlines():
                    # Get rid of host definitions and comments:
                    if ' ' in etctz:
                        etctz, dummy = etctz.split(' ', 1)
                    if '#' in etctz:
                        etctz, dummy = etctz.split('#', 1)
                    if not etctz:
                        continue
                    tz = ZoneInfo(etctz.replace(' ', '_'))
                    if _root == '/':
                        # We are using a file in etc to name the timezone.
                        # Verify that the timezone specified there is actually used:
                        utils.assert_tz_offset(tz)
                    return tz

        except IOError:
            # File doesn't exist or is a directory
            continue

    # CentOS has a ZONE setting in /etc/sysconfig/clock,
    # OpenSUSE has a TIMEZONE setting in /etc/sysconfig/clock and
    # Gentoo has a TIMEZONE setting in /etc/conf.d/clock
    # We look through these files for a timezone:

    zone_re = re.compile(r'\s*ZONE\s*=\s*\"')
    timezone_re = re.compile(r'\s*TIMEZONE\s*=\s*\"')
    end_re = re.compile('\"')

    for filename in ('etc/sysconfig/clock', 'etc/conf.d/clock'):
        tzpath = os.path.join(_root, filename)
        try:
            with open(tzpath, 'rt') as tzfile:
                data = tzfile.readlines()

            for line in data:
                # Look for the ZONE= setting.
                match = zone_re.match(line)
                if match is None:
                    # No ZONE= setting. Look for the TIMEZONE= setting.
                    match = timezone_re.match(line)
                if match is not None:
                    # Some setting existed
                    line = line[match.end():]
                    etctz = line[:end_re.search(line).start()]

                    # We found a timezone
                    tz = ZoneInfo(etctz.replace(' ', '_'))
                    if _root == '/':
                        # We are using a file in etc to name the timezone.
                        # Verify that the timezone specified there is actually used:
                        utils.assert_tz_offset(tz)
                    return tz

        except IOError:
            # File doesn't exist or is a directory
            continue

    # systemd distributions use symlinks that include the zone name,
    # see manpage of localtime(5) and timedatectl(1)
    tzpath = os.path.join(_root, 'etc/localtime')
    if os.path.exists(tzpath) and os.path.islink(tzpath):
        tzpath = os.path.realpath(tzpath)
        start = tzpath.find("/")+1
        while start != 0:
            tzpath = tzpath[start:]
            try:
                return ZoneInfo(tzpath)
            except ZoneInfoNotFoundError:
                pass
            start = tzpath.find("/")+1

    # No explicit setting existed. Use localtime
    for filename in ('etc/localtime', 'usr/local/etc/localtime'):
        tzpath = os.path.join(_root, filename)

        if not os.path.exists(tzpath):
            continue
        with open(tzpath, 'rb') as tzfile:
            return ZoneInfo.from_file(tzfile, key='local')

    warnings.warn('Can not find any timezone configuration, defaulting to UTC.')
    return timezone.utc

def get_localzone():
    """Get the computers configured local timezone, if any."""
    global _cache_tz
    if _cache_tz is None:
        _cache_tz = _get_localzone()

    return _cache_tz

def reload_localzone():
    """Reload the cached localzone. You need to call this if the timezone has changed."""
    global _cache_tz
    _cache_tz = _get_localzone()
    return _cache_tz
<commit_msg>Updated unix.py for additional error handling<commit_after>import os
import re
import sys
import warnings
from datetime import timezone

from tzlocal import utils

if sys.version_info >= (3, 9):
    from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
else:
    from backports.zoneinfo import ZoneInfo, ZoneInfoNotFoundError

_cache_tz = None

def _tz_from_env(tzenv):
    if tzenv[0] == ':':
        tzenv = tzenv[1:]

    # TZ specifies a file
    if os.path.isabs(tzenv) and os.path.exists(tzenv):
        with open(tzenv, 'rb') as tzfile:
            return ZoneInfo.from_file(tzfile, key='local')

    # TZ specifies a zoneinfo zone.
    try:
        tz = ZoneInfo(tzenv)
        # That worked, so we return this:
        return tz
    except ZoneInfoNotFoundError:
        raise ZoneInfoNotFoundError(
            "tzlocal() does not support non-zoneinfo timezones like %s. \n"
            "Please use a timezone in the form of Continent/City") from None

def _try_tz_from_env():
    tzenv = os.environ.get('TZ')
    if tzenv:
        try:
            return _tz_from_env(tzenv)
        except ZoneInfoNotFoundError:
            pass

def _get_localzone(_root='/'):
    """Tries to find the local timezone configuration.

    This method prefers finding the timezone name and passing that to pytz,
    over passing in the localtime file, as in the later case the zoneinfo
    name is unknown.

    The parameter _root makes the function look for files like /etc/localtime
    beneath the _root directory. This is primarily used by the tests.
    In normal usage you call the function without parameters."""

    tzenv = _try_tz_from_env()
    if tzenv:
        return tzenv

    # Are we under Termux on Android?
    if os.path.exists('/system/bin/getprop'):
        import subprocess
        androidtz = subprocess.check_output(['getprop', 'persist.sys.timezone']).strip().decode()
        return ZoneInfo(androidtz)

    # Now look for distribution specific configuration files
    # that contain the timezone name.
    for configfile in ('etc/timezone', 'var/db/zoneinfo'):
        tzpath = os.path.join(_root, configfile)
        try:
            with open(tzpath, 'rb') as tzfile:
                data = tzfile.read()

                # Issue #3 was that /etc/timezone was a zoneinfo file.
                # That's a misconfiguration, but we need to handle it gracefully:
                if data[:5] == b'TZif2':
                    continue

                etctz = data.strip().decode()
                if not etctz:
                    # Empty file, skip
                    continue
                for etctz in data.decode().splitlines():
                    # Get rid of host definitions and comments:
                    if ' ' in etctz:
                        etctz, dummy = etctz.split(' ', 1)
                    if '#' in etctz:
                        etctz, dummy = etctz.split('#', 1)
                    if not etctz:
                        continue
                    tz = ZoneInfo(etctz.replace(' ', '_'))
                    if _root == '/':
                        # We are using a file in etc to name the timezone.
                        # Verify that the timezone specified there is actually used:
                        utils.assert_tz_offset(tz)
                    return tz

        except IOError:
            # File doesn't exist or is a directory
            continue

    # CentOS has a ZONE setting in /etc/sysconfig/clock,
    # Centos 8 /etc/sysconfig/clock is binary and we should skip this check
    # OpenSUSE has a TIMEZONE setting in /etc/sysconfig/clock and
    # Gentoo has a TIMEZONE setting in /etc/conf.d/clock
    # We look through these files for a timezone:

    zone_re = re.compile(r'\s*ZONE\s*=\s*\"')
    timezone_re = re.compile(r'\s*TIMEZONE\s*=\s*\"')
    end_re = re.compile('\"')

    for filename in ('etc/sysconfig/clock', 'etc/conf.d/clock'):
        tzpath = os.path.join(_root, filename)
        try:
            with open(tzpath, 'rt') as tzfile:
                data = tzfile.readlines()

            for line in data:
                # Look for the ZONE= setting.
                match = zone_re.match(line)
                if match is None:
                    # No ZONE= setting. Look for the TIMEZONE= setting.
                    match = timezone_re.match(line)
                if match is not None:
                    # Some setting existed
                    line = line[match.end():]
                    etctz = line[:end_re.search(line).start()]

                    # We found a timezone
                    tz = ZoneInfo(etctz.replace(' ', '_'))
                    if _root == '/':
                        # We are using a file in etc to name the timezone.
                        # Verify that the timezone specified there is actually used:
                        utils.assert_tz_offset(tz)
                    return tz

        except Exception as e:
            # File doesn't exist or is a directory
            continue

    # systemd distributions use symlinks that include the zone name,
    # see manpage of localtime(5) and timedatectl(1)
    tzpath = os.path.join(_root, 'etc/localtime')
    if os.path.exists(tzpath) and os.path.islink(tzpath):
        tzpath = os.path.realpath(tzpath)
        start = tzpath.find("/")+1
        while start != 0:
            tzpath = tzpath[start:]
            try:
                return ZoneInfo(tzpath)
            except ZoneInfoNotFoundError:
                pass
            start = tzpath.find("/")+1

    # No explicit setting existed. Use localtime
    for filename in ('etc/localtime', 'usr/local/etc/localtime'):
        tzpath = os.path.join(_root, filename)

        if not os.path.exists(tzpath):
            continue
        with open(tzpath, 'rb') as tzfile:
            return ZoneInfo.from_file(tzfile, key='local')

    warnings.warn('Can not find any timezone configuration, defaulting to UTC.')
    return timezone.utc

def get_localzone():
    """Get the computers configured local timezone, if any."""
    global _cache_tz
    if _cache_tz is None:
        _cache_tz = _get_localzone()

    return _cache_tz

def reload_localzone():
    """Reload the cached localzone. You need to call this if the timezone has changed."""
    global _cache_tz
    _cache_tz = _get_localzone()
    return _cache_tz
<|endoftext|>