<commit_before>#!/bin/env python
from __future__ import absolute_import

__author__ = "Gina Haeussge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import errno
import subprocess
import sys
from distutils.dir_util import copy_tree
from octoprint.settings import settings

def _get_git_executables():
	GITS = ["git"]
	if sys.platform == "win32":
		GITS = ["git.cmd", "git.exe"]
	return GITS

def _git(args, cwd, hide_stderr=False, verbose=False, git_executable=None):
	if git_executable is not None:
		commands = [git_executable]
	else:
		commands = _get_git_executables()

	for c in commands:
		try:
			p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
			                     stderr=(subprocess.PIPE if hide_stderr
			                             else None))
			break
		except EnvironmentError:
			e = sys.exc_info()[1]
			if e.errno == errno.ENOENT:
				continue
			if verbose:
				print("unable to run %s" % args[0])
				print(e)
			return None, None
	else:
		if verbose:
			print("unable to find command, tried %s" % (commands,))
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= '3':
		stdout = stdout.decode()

	if p.returncode != 0:
		if verbose:
			print("unable to run %s (error)" % args[0])

	return p.returncode, stdout

def _python(args, cwd, python_executable, sudo=False):
	command = [python_executable] + args
	if sudo:
		command = ["sudo"] + command
	try:
		p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE,
		                     stderr=subprocess.PIPE)
	except:
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= "3":
		stdout = stdout.decode()

	return p.returncode, stdout

def _rescue_changes(git_executable, folder):
	print(">>> Running: git diff --shortstat")
	returncode, stdout = _git(["diff", "--shortstat"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git diff\" failed with returncode %d: %s" % (returncode, stdout))
	if stdout and stdout.strip():
		# we got changes in the working tree, maybe from the user, so we'll now rescue those into a patch
		import time
		import os
		timestamp = time.strftime("%Y%m%d%H%M")
		patch = os.path.join(folder, "%s-preupdate.patch" % timestamp)

		print(">>> Running: git diff and saving output to %s" % timestamp)
		returncode, stdout = _git(["diff"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, installation directory was dirty and state could not be persisted as a patch to %s" % patch)

		with open(patch, "wb") as f:
			f.write(stdout)

		return True

	return False

def update_source(git_executable, folder, target, force=False, branch=None):
	if _rescue_changes(git_executable, folder):
		print(">>> Running: git reset --hard")
		returncode, stdout = _git(["reset", "--hard"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git reset --hard\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git fetch")
	returncode, stdout = _git(["fetch"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git fetch\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if branch is not None and branch.strip() != "":
		print(">>> Running: git checkout {}".format(branch))
		returncode, stdout = _git(["checkout", branch], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git checkout\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git pull")
	returncode, stdout = _git(["pull"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git pull\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if force:
		reset_command = ["reset", "--hard"]
		reset_command += [target]

		print(">>> Running: git %s" % " ".join(reset_command))
		returncode, stdout = _git(reset_command, folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Error while updating, \"git %s\" failed with returncode %d: %s" % (" ".join(reset_command), returncode, stdout))
		print(stdout)

def install_source(python_executable, folder, user=False, sudo=False):
	print(">>> Running: python setup.py clean")
	returncode, stdout = _python(["setup.py", "clean"], folder, python_executable)
	if returncode != 0:
		print("\"python setup.py clean\" failed with returncode %d: %s" % (returncode, stdout))
		print("Continuing anyways")
	print(stdout)

	print(">>> Running: python setup.py install")
	args = ["setup.py", "install"]
	if user:
		args.append("--user")
	returncode, stdout = _python(args, folder, python_executable, sudo=sudo)
	if returncode != 0:
		raise RuntimeError("Could not update, \"python setup.py install\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	print(">>> Copying Firmware files to settings directory...")
	# folder where the installation settings files are located
	settings_folder = settings(init=True).getBaseFolder('base')
	try:
		# copies the files in the /etc directory
		copy_tree(folder + '/firmware', settings_folder + 'firmware')
	except Exception as ex:
		raise RuntimeError(
			"Could not update, copying the firmware files to respective settings directory failed with error: %s" % ex.message)
	finally:
		print("Firmware files installed.")

def parse_arguments():
	import argparse

	boolean_trues = ["true", "yes", "1"]
	boolean_falses = ["false", "no", "0"]

	parser = argparse.ArgumentParser(prog="update-beeweb.py")

	parser.add_argument("--git", action="store", type=str, dest="git_executable",
	                    help="Specify git executable to use")
	parser.add_argument("--python", action="store", type=str, dest="python_executable",
	                    help="Specify python executable to use")
	parser.add_argument("--force", action="store", type=lambda x: x in boolean_trues,
	                    dest="force", default=False,
	                    help="Set this to true to force the update to only the specified version (nothing newer, nothing older)")
	parser.add_argument("--sudo", action="store_true", dest="sudo",
	                    help="Install with sudo")
	parser.add_argument("--user", action="store_true", dest="user",
	                    help="Install to the user site directory instead of the general site directory")
	parser.add_argument("--branch", action="store", type=str, dest="branch", default=None,
	                    help="Specify the branch to make sure is checked out")
	parser.add_argument("folder", type=str,
	                    help="Specify the base folder of the OctoPrint installation to update")
	parser.add_argument("target", type=str,
	                    help="Specify the commit or tag to which to update")

	args = parser.parse_args()

	return args

def main():
	args = parse_arguments()

	git_executable = None
	if args.git_executable:
		git_executable = args.git_executable

	python_executable = sys.executable
	if args.python_executable:
		python_executable = args.python_executable

	folder = args.folder

	import os
	if not os.access(folder, os.W_OK):
		raise RuntimeError("Could not update, base folder is not writable")

	update_source(git_executable, folder, args.target, force=args.force, branch=args.branch)
	install_source(python_executable, folder, user=args.user, sudo=args.sudo)

if __name__ == "__main__":
	main()
<commit_msg>BEEWEB-114 Fix beeweb update script<commit_after>#!/bin/env python
from __future__ import absolute_import

__author__ = "Gina Haeussge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import errno
import subprocess
import sys
from distutils.dir_util import copy_tree
from octoprint.settings import settings

def _get_git_executables():
	GITS = ["git"]
	if sys.platform == "win32":
		GITS = ["git.cmd", "git.exe"]
	return GITS

def _git(args, cwd, hide_stderr=False, verbose=False, git_executable=None):
	if git_executable is not None:
		commands = [git_executable]
	else:
		commands = _get_git_executables()

	for c in commands:
		try:
			p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
			                     stderr=(subprocess.PIPE if hide_stderr
			                             else None))
			break
		except EnvironmentError:
			e = sys.exc_info()[1]
			if e.errno == errno.ENOENT:
				continue
			if verbose:
				print("unable to run %s" % args[0])
				print(e)
			return None, None
	else:
		if verbose:
			print("unable to find command, tried %s" % (commands,))
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= '3':
		stdout = stdout.decode()

	if p.returncode != 0:
		if verbose:
			print("unable to run %s (error)" % args[0])

	return p.returncode, stdout

def _python(args, cwd, python_executable, sudo=False):
	command = [python_executable] + args
	if sudo:
		command = ["sudo"] + command
	try:
		p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE,
		                     stderr=subprocess.PIPE)
	except:
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= "3":
		stdout = stdout.decode()

	return p.returncode, stdout

def _rescue_changes(git_executable, folder):
	print(">>> Running: git diff --shortstat")
	returncode, stdout = _git(["diff", "--shortstat"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git diff\" failed with returncode %d: %s" % (returncode, stdout))
	if stdout and stdout.strip():
		# we got changes in the working tree, maybe from the user, so we'll now rescue those into a patch
		import time
		import os
		timestamp = time.strftime("%Y%m%d%H%M")
		patch = os.path.join(folder, "%s-preupdate.patch" % timestamp)

		print(">>> Running: git diff and saving output to %s" % timestamp)
		returncode, stdout = _git(["diff"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, installation directory was dirty and state could not be persisted as a patch to %s" % patch)

		with open(patch, "wb") as f:
			f.write(stdout)

		return True

	return False

def update_source(git_executable, folder, target, force=False, branch=None):
	if _rescue_changes(git_executable, folder):
		print(">>> Running: git reset --hard")
		returncode, stdout = _git(["reset", "--hard"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git reset --hard\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git fetch")
	returncode, stdout = _git(["fetch"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git fetch\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if branch is not None and branch.strip() != "":
		print(">>> Running: git checkout {}".format(branch))
		returncode, stdout = _git(["checkout", branch], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git checkout\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git pull")
	returncode, stdout = _git(["pull"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git pull\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if force:
		reset_command = ["reset", "--hard"]
		reset_command += [target]

		print(">>> Running: git %s" % " ".join(reset_command))
		returncode, stdout = _git(reset_command, folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Error while updating, \"git %s\" failed with returncode %d: %s" % (" ".join(reset_command), returncode, stdout))
		print(stdout)

def install_source(python_executable, folder, user=False, sudo=False):
	print(">>> Running: python setup.py clean")
	returncode, stdout = _python(["setup.py", "clean"], folder, python_executable)
	if returncode != 0:
		print("\"python setup.py clean\" failed with returncode %d: %s" % (returncode, stdout))
		print("Continuing anyways")
	print(stdout)

	print(">>> Running: python setup.py install")
	args = ["setup.py", "install"]
	if user:
		args.append("--user")
	returncode, stdout = _python(args, folder, python_executable, sudo=sudo)
	if returncode != 0:
		raise RuntimeError("Could not update, \"python setup.py install\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	print(">>> Copying Firmware files to settings directory...")
	# folder where the installation settings files are located
	settings_folder = settings(init=True).getBaseFolder('base')
	try:
		# copies the files in the /etc directory
		copy_tree(folder + '/firmware', settings_folder + '/firmware')
	except Exception as ex:
		raise RuntimeError(
			"Could not update, copying the firmware files to respective settings directory failed with error: %s" % ex.message)
	finally:
		print("Firmware files installed.")

def parse_arguments():
	import argparse

	boolean_trues = ["true", "yes", "1"]
	boolean_falses = ["false", "no", "0"]

	parser = argparse.ArgumentParser(prog="update-beeweb.py")

	parser.add_argument("--git", action="store", type=str, dest="git_executable",
	                    help="Specify git executable to use")
	parser.add_argument("--python", action="store", type=str, dest="python_executable",
	                    help="Specify python executable to use")
	parser.add_argument("--force", action="store", type=lambda x: x in boolean_trues,
	                    dest="force", default=False,
	                    help="Set this to true to force the update to only the specified version (nothing newer, nothing older)")
	parser.add_argument("--sudo", action="store_true", dest="sudo",
	                    help="Install with sudo")
	parser.add_argument("--user", action="store_true", dest="user",
	                    help="Install to the user site directory instead of the general site directory")
	parser.add_argument("--branch", action="store", type=str, dest="branch", default=None,
	                    help="Specify the branch to make sure is checked out")
	parser.add_argument("folder", type=str,
	                    help="Specify the base folder of the OctoPrint installation to update")
	parser.add_argument("target", type=str,
	                    help="Specify the commit or tag to which to update")

	args = parser.parse_args()

	return args

def main():
	args = parse_arguments()

	git_executable = None
	if args.git_executable:
		git_executable = args.git_executable

	python_executable = sys.executable
	if args.python_executable:
		python_executable = args.python_executable

	folder = args.folder

	import os
	if not os.access(folder, os.W_OK):
		raise RuntimeError("Could not update, base folder is not writable")

	update_source(git_executable, folder, args.target, force=args.force, branch=args.branch)
	install_source(python_executable, folder, user=args.user, sudo=args.sudo)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>from os import path, makedirs, getcwd
from yaml import dump, load
import tensorflow as tf
import numpy as np

class Experiment(object):
    @classmethod
    def from_yml_file(cls, path):
        with open(path, 'r') as f:
            x = cls.from_yml(f.read())
        return x

    @classmethod
    def from_yml(cls, doc):
        return cls(**load(doc))

    def __init__(
        self,
        name,
        root=getcwd(),
        seed=1,
        tag=None,
        log_level=tf.logging.ERROR
    ):
        self.root = root
        self.name = name
        self.tag = str(seed) if tag is None else str(tag)
        self.seed = seed
        tf.logging.set_verbosity(log_level)
        self.reset_random_state()

    def reset_random_state(self):
        np.random.seed(self.seed)
        tf.set_random_seed(self.seed)
        return self

    def label(self): return self.name + '_' + self.tag
    def path(self): return path.join(self.root, self.label())
    def config_file(self): return path.join(self.path(), 'config.yml')
    def exists(self):
        return path.exists(self.path()) and path.exists(self.config_file())

    def ensure_present(self):
        tf.logging.info('Saving experiment files to "{}".'.format(self.path()))
        if not path.exists(self.path()): makedirs(self.path())
        with open(self.config_file(), 'w') as f:
            dump(self.config(), stream=f, default_flow_style=False)
        return self

    def write_config(self):
        return self.ensure_present()

    def config(self):
        return {
            'name': self.name,
            'root': self.root,
            'seed': self.seed,
            'tag': self.tag
        }

class TensorboardExperiment(Experiment):
    def ensure_present(self):
        super(TensorboardExperiment, self).ensure_present()
        tf.logging.info(
            'Run `tensorboard --logdir {}` to visualize results.'.format(
                self.path()
            )
        )
        return self
<commit_msg>Set the default python random seed by experiment reset method.<commit_after>from os import path, makedirs, getcwd
from yaml import dump, load
import tensorflow as tf
import numpy as np
import random

class Experiment(object):
    @classmethod
    def from_yml_file(cls, path):
        with open(path, 'r') as f:
            x = cls.from_yml(f.read())
        return x

    @classmethod
    def from_yml(cls, doc):
        return cls(**load(doc))

    def __init__(
        self,
        name,
        root=getcwd(),
        seed=1,
        tag=None,
        log_level=tf.logging.ERROR
    ):
        self.root = root
        self.name = name
        self.tag = str(seed) if tag is None else str(tag)
        self.seed = seed
        tf.logging.set_verbosity(log_level)
        self.reset_random_state()

    def reset_random_state(self):
        random.seed(self.seed)
        np.random.seed(self.seed)
        tf.set_random_seed(self.seed)
        return self

    def label(self): return self.name + '_' + self.tag
    def path(self): return path.join(self.root, self.label())
    def config_file(self): return path.join(self.path(), 'config.yml')
    def exists(self):
        return path.exists(self.path()) and path.exists(self.config_file())

    def ensure_present(self):
        tf.logging.info('Saving experiment files to "{}".'.format(self.path()))
        if not path.exists(self.path()): makedirs(self.path())
        with open(self.config_file(), 'w') as f:
            dump(self.config(), stream=f, default_flow_style=False)
        return self

    def write_config(self):
        return self.ensure_present()

    def config(self):
        return {
            'name': self.name,
            'root': self.root,
            'seed': self.seed,
            'tag': self.tag
        }

class TensorboardExperiment(Experiment):
    def ensure_present(self):
        super(TensorboardExperiment, self).ensure_present()
        tf.logging.info(
            'Run `tensorboard --logdir {}` to visualize results.'.format(
                self.path()
            )
        )
        return self
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
VERSION = (0, 0, 9, 'alpha')

# pragma: no cover
if VERSION[-1] != "final":
    __version__ = '.'.join(map(str, VERSION))
else:
    # pragma: no cover
    __version__ = '.'.join(map(str, VERSION[:-1]))
<commit_msg>Update to 0.0.10<commit_after># -*- coding: utf-8 -*-
VERSION = (0, 0, 10, 'alpha')

# pragma: no cover
if VERSION[-1] != "final":
    __version__ = '.'.join(map(str, VERSION))
else:
    # pragma: no cover
    __version__ = '.'.join(map(str, VERSION[:-1]))
<|endoftext|>


<commit_before>#!/bin/env python
from __future__ import absolute_import

__author__ = "Gina Haeussge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import errno
import subprocess
import sys
from distutils.dir_util import copy_tree
from octoprint.settings import settings

def _get_git_executables():
	GITS = ["git"]
	if sys.platform == "win32":
		GITS = ["git.cmd", "git.exe"]
	return GITS

def _git(args, cwd, hide_stderr=False, verbose=False, git_executable=None):
	if git_executable is not None:
		commands = [git_executable]
	else:
		commands = _get_git_executables()

	for c in commands:
		try:
			p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
			                     stderr=(subprocess.PIPE if hide_stderr
			                             else None))
			break
		except EnvironmentError:
			e = sys.exc_info()[1]
			if e.errno == errno.ENOENT:
				continue
			if verbose:
				print("unable to run %s" % args[0])
				print(e)
			return None, None
	else:
		if verbose:
			print("unable to find command, tried %s" % (commands,))
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= '3':
		stdout = stdout.decode()

	if p.returncode != 0:
		if verbose:
			print("unable to run %s (error)" % args[0])

	return p.returncode, stdout

def _python(args, cwd, python_executable, sudo=False):
	command = [python_executable] + args
	if sudo:
		command = ["sudo"] + command
	try:
		p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE,
		                     stderr=subprocess.PIPE)
	except:
		return None, None

	stdout = p.communicate()[0].strip()
	if sys.version >= "3":
		stdout = stdout.decode()

	return p.returncode, stdout

def _rescue_changes(git_executable, folder):
	print(">>> Running: git diff --shortstat")
	returncode, stdout = _git(["diff", "--shortstat"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git diff\" failed with returncode %d: %s" % (returncode, stdout))
	if stdout and stdout.strip():
		# we got changes in the working tree, maybe from the user, so we'll now rescue those into a patch
		import time
		import os
		timestamp = time.strftime("%Y%m%d%H%M")
		patch = os.path.join(folder, "%s-preupdate.patch" % timestamp)

		print(">>> Running: git diff and saving output to %s" % timestamp)
		returncode, stdout = _git(["diff"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, installation directory was dirty and state could not be persisted as a patch to %s" % patch)

		with open(patch, "wb") as f:
			f.write(stdout)

		return True

	return False

def update_source(git_executable, folder, target, force=False, branch=None):
	if _rescue_changes(git_executable, folder):
		print(">>> Running: git reset --hard")
		returncode, stdout = _git(["reset", "--hard"], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git reset --hard\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git fetch")
	returncode, stdout = _git(["fetch"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git fetch\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if branch is not None and branch.strip() != "":
		print(">>> Running: git checkout {}".format(branch))
		returncode, stdout = _git(["checkout", branch], folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Could not update, \"git checkout\" failed with returncode %d: %s" % (returncode, stdout))

	print(">>> Running: git pull")
	returncode, stdout = _git(["pull"], folder, git_executable=git_executable)
	if returncode != 0:
		raise RuntimeError("Could not update, \"git pull\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	if force:
		reset_command = ["reset", "--hard"]
		reset_command += [target]

		print(">>> Running: git %s" % " ".join(reset_command))
		returncode, stdout = _git(reset_command, folder, git_executable=git_executable)
		if returncode != 0:
			raise RuntimeError("Error while updating, \"git %s\" failed with returncode %d: %s" % (" ".join(reset_command), returncode, stdout))
		print(stdout)

def install_source(python_executable, folder, user=False, sudo=False):
	print(">>> Running: python setup.py clean")
	returncode, stdout = _python(["setup.py", "clean"], folder, python_executable)
	if returncode != 0:
		print("\"python setup.py clean\" failed with returncode %d: %s" % (returncode, stdout))
		print("Continuing anyways")
	print(stdout)

	print(">>> Running: python setup.py install")
	args = ["setup.py", "install"]
	if user:
		args.append("--user")
	returncode, stdout = _python(args, folder, python_executable, sudo=sudo)
	if returncode != 0:
		raise RuntimeError("Could not update, \"python setup.py install\" failed with returncode %d: %s" % (returncode, stdout))
	print(stdout)

	# Copies the firmware files to the settings directory
	print(">>> Copying Firmware files to settings directory...")
	# folder where the installation settings files are located
	settings_folder = settings(init=True).getBaseFolder('base')
	try:
		# copies the files in the /etc directory
		copy_tree(folder + '/firmware', settings_folder + '/firmware')
	except Exception as ex:
		raise RuntimeError(
			"Could not update, copying the firmware files to respective settings directory failed with error: %s" % ex.message)
	finally:
		print("Firmware files installed.")

	# If the update is running on Windows tries to remove cache files from the client app
	if sys.platform == "win32":
		print(">>> Removing client cache files...")
		try:
			import os
			from glob import glob
			from shutil import rmtree

			pattern = 'C:\\Users\\*\\AppData\\Roaming\\beesoft-nativefier*\\Cache\\*'

			for item in glob(pattern):
				if os.path.exists(item):
					try:
						os.remove(item)
					except Exception:
						continue

		except Exception as ex:
			raise RuntimeError('Could not remove the cache files from client app: %s' % ex.strerror)

def parse_arguments():
	import argparse

	boolean_trues = ["true", "yes", "1"]
	boolean_falses = ["false", "no", "0"]

	parser = argparse.ArgumentParser(prog="update-beeweb.py")

	parser.add_argument("--git", action="store", type=str, dest="git_executable",
	                    help="Specify git executable to use")
	parser.add_argument("--python", action="store", type=str, dest="python_executable",
	                    help="Specify python executable to use")
	parser.add_argument("--force", action="store", type=lambda x: x in boolean_trues,
	                    dest="force", default=False,
	                    help="Set this to true to force the update to only the specified version (nothing newer, nothing older)")
	parser.add_argument("--sudo", action="store_true", dest="sudo",
	                    help="Install with sudo")
	parser.add_argument("--user", action="store_true", dest="user",
	                    help="Install to the user site directory instead of the general site directory")
	parser.add_argument("--branch", action="store", type=str, dest="branch", default=None,
	                    help="Specify the branch to make sure is checked out")
	parser.add_argument("folder", type=str,
	                    help="Specify the base folder of the OctoPrint installation to update")
	parser.add_argument("target", type=str,
	                    help="Specify the commit or tag to which to update")

	args = parser.parse_args()

	return args

def main():
	args = parse_arguments()

	git_executable = None
	if args.git_executable:
		git_executable = args.git_executable

	python_executable = sys.executable
	if args.python_executable:
		python_executable = args.python_executable

	folder = args.folder

	import os
	if not os.access(folder, os.W_OK):
		raise RuntimeError("Could not update, base folder is not writable")

	update_source(git_executable, folder, args.target, force=args.force, branch=args.branch)
	install_source(python_executable, folder, user=args.user, sudo=args.sudo)

if __name__ == "__main__":
	main()
<commit_msg>BEEWEB-175 Improve client cache clearing for nativefier clients<commit_after>#!/bin/env python
from __future__ import absolute_import

__author__ = "Gina Haeussge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import errno
import subprocess
import sys
from distutils.dir_util import copy_tree
from octoprint.settings import settings

def _get_git_executables():
    GITS = ["git"]
    if sys.platform == "win32":
        GITS = ["git.cmd", "git.exe"]
    return GITS

def _git(args, cwd, hide_stderr=False, verbose=False, git_executable=None):
    if git_executable is not None:
        commands = [git_executable]
    else:
        commands = _get_git_executables()

    for c in commands:
        try:
            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
                                 stderr=(subprocess.PIPE if hide_stderr
                                         else None))
            break
        except EnvironmentError:
            e = sys.exc_info()[1]
            if e.errno == errno.ENOENT:
                continue
            if verbose:
                print("unable to run %s" % args[0])
                print(e)
            return None, None
    else:
        if verbose:
            print("unable to find command, tried %s" % (commands,))
        return None, None

    stdout = p.communicate()[0].strip()
    if sys.version >= '3':
        stdout = stdout.decode()

    if p.returncode != 0:
        if verbose:
            print("unable to run %s (error)" % args[0])

    return p.returncode, stdout

def _python(args, cwd, python_executable, sudo=False):
    command = [python_executable] + args
    if sudo:
        command = ["sudo"] + command
    try:
        p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
    except:
        return None, None

    stdout = p.communicate()[0].strip()
    if sys.version >= "3":
        stdout = stdout.decode()

    return p.returncode, stdout

def _rescue_changes(git_executable, folder):
    print(">>> Running: git diff --shortstat")
    returncode, stdout = _git(["diff", "--shortstat"], folder, git_executable=git_executable)
    if returncode != 0:
        raise RuntimeError("Could not update, \"git diff\" failed with returncode %d: %s" % (returncode, stdout))
    if stdout and stdout.strip():
        # we got changes in the working tree, maybe from the user, so we'll now rescue those into a patch
        import time
        import os
        timestamp = time.strftime("%Y%m%d%H%M")
        patch = os.path.join(folder, "%s-preupdate.patch" % timestamp)

        print(">>> Running: git diff and saving output to %s" % timestamp)
        returncode, stdout = _git(["diff"], folder, git_executable=git_executable)
        if returncode != 0:
            raise RuntimeError("Could not update, installation directory was dirty and state could not be persisted as a patch to %s" % patch)

        with open(patch, "wb") as f:
            f.write(stdout)

        return True

    return False

def update_source(git_executable, folder, target, force=False, branch=None):
    if _rescue_changes(git_executable, folder):
        print(">>> Running: git reset --hard")
        returncode, stdout = _git(["reset", "--hard"], folder, git_executable=git_executable)
        if returncode != 0:
            raise RuntimeError("Could not update, \"git reset --hard\" failed with returncode %d: %s" % (returncode, stdout))

    print(">>> Running: git fetch")
    returncode, stdout = _git(["fetch"], folder, git_executable=git_executable)
    if returncode != 0:
        raise RuntimeError("Could not update, \"git fetch\" failed with returncode %d: %s" % (returncode, stdout))
    print(stdout)

    if branch is not None and branch.strip() != "":
        print(">>> Running: git checkout {}".format(branch))
        returncode, stdout = _git(["checkout", branch], folder, git_executable=git_executable)
        if returncode != 0:
            raise RuntimeError("Could not update, \"git checkout\" failed with returncode %d: %s" % (returncode, stdout))

    print(">>> Running: git pull")
    returncode, stdout = _git(["pull"], folder, git_executable=git_executable)
    if returncode != 0:
        raise RuntimeError("Could not update, \"git pull\" failed with returncode %d: %s" % (returncode, stdout))
    print(stdout)

    if force:
        reset_command = ["reset", "--hard"]
        reset_command += [target]

        print(">>> Running: git %s" % " ".join(reset_command))
        returncode, stdout = _git(reset_command, folder, git_executable=git_executable)
        if returncode != 0:
            raise RuntimeError("Error while updating, \"git %s\" failed with returncode %d: %s" % (" ".join(reset_command), returncode, stdout))
        print(stdout)

def install_source(python_executable, folder, user=False, sudo=False):
    print(">>> Running: python setup.py clean")
    returncode, stdout = _python(["setup.py", "clean"], folder, python_executable)
    if returncode != 0:
        print("\"python setup.py clean\" failed with returncode %d: %s" % (returncode, stdout))
        print("Continuing anyways")
    print(stdout)

    print(">>> Running: python setup.py install")
    args = ["setup.py", "install"]
    if user:
        args.append("--user")
    returncode, stdout = _python(args, folder, python_executable, sudo=sudo)
    if returncode != 0:
        raise RuntimeError("Could not update, \"python setup.py install\" failed with returncode %d: %s" % (returncode, stdout))
    print(stdout)

    # Copies the firmware files to the settings directory
    print(">>> Copying Firmware files to settings directory...")
    # folder where the installation settings files are located
    settings_folder = settings(init=True).getBaseFolder('base')
    try:
        # copies the files in the /etc directory
        copy_tree(folder + '/firmware', settings_folder + '/firmware')
    except Exception as ex:
        raise RuntimeError(
            "Could not update, copying the firmware files to respective settings directory failed with error: %s" % ex.message)
    finally:
        print("Firmware files installed.")

    print(">>> Removing client cache files...")
    try:
        import os
        from glob import glob
        from shutil import rmtree

        # If the update is running on Windows tries to remove cache files from the client app
        if sys.platform == "win32":
            app_data_folder = os.getenv('APPDATA')
            pattern = app_data_folder + '\\beesoft-nativefier*\\Cache\\*'
        elif sys.platform == "darwin":
            from AppKit import NSSearchPathForDirectoriesInDomains
            # http://developer.apple.com/DOCUMENTATION/Cocoa/Reference/Foundation/Miscellaneous/Foundation_Functions/Reference/reference.html#//apple_ref/c/func/NSSearchPathForDirectoriesInDomains
            # NSApplicationSupportDirectory = 14
            # NSUserDomainMask = 1
            # True for expanding the tilde into a fully qualified path
            app_data_folder = NSSearchPathForDirectoriesInDomains(14, 1, True)[0]
            pattern = app_data_folder + '/beesoft-nativefier*/Cache/*'
        else:
            pattern = os.path.expanduser(os.path.join("~", "." + '/beesoft-nativefier*/Cache/*'))

        for item in glob(pattern):
            if os.path.exists(item):
                try:
                    os.remove(item)
                except Exception:
                    continue

    except Exception as ex:
        raise RuntimeError('Could not remove the cache files from client app: %s' % ex.strerror)

def parse_arguments():
    import argparse

    boolean_trues = ["true", "yes", "1"]
    boolean_falses = ["false", "no", "0"]

    parser = argparse.ArgumentParser(prog="update-beeweb.py")

    parser.add_argument("--git", action="store", type=str, dest="git_executable",
                        help="Specify git executable to use")
    parser.add_argument("--python", action="store", type=str, dest="python_executable",
                        help="Specify python executable to use")
    parser.add_argument("--force", action="store", type=lambda x: x in boolean_trues,
                        dest="force", default=False,
                        help="Set this to true to force the update to only the specified version (nothing newer, nothing older)")
    parser.add_argument("--sudo", action="store_true", dest="sudo",
                        help="Install with sudo")
    parser.add_argument("--user", action="store_true", dest="user",
                        help="Install to the user site directory instead of the general site directory")
    parser.add_argument("--branch", action="store", type=str, dest="branch", default=None,
                        help="Specify the branch to make sure is checked out")
    parser.add_argument("folder", type=str,
                        help="Specify the base folder of the OctoPrint installation to update")
    parser.add_argument("target", type=str,
                        help="Specify the commit or tag to which to update")

    args = parser.parse_args()

    return args

def main():
    args = parse_arguments()

    git_executable = None
    if args.git_executable:
        git_executable = args.git_executable

    python_executable = sys.executable
    if args.python_executable:
        python_executable = args.python_executable

    folder = args.folder

    import os
    if not os.access(folder, os.W_OK):
        raise RuntimeError("Could not update, base folder is not writable")

    update_source(git_executable, folder, args.target, force=args.force, branch=args.branch)
    install_source(python_executable, folder, user=args.user, sudo=args.sudo)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import datetime
from urllib2 import HTTPError
from dnsdb_query import DnsdbClient, QueryError
from cortexutils.analyzer import Analyzer

class DnsDbAnalyzer(Analyzer):

    def __init__(self):
        Analyzer.__init__(self)
        self.service = self.get_param(
            'config.service', None, 'Service parameter is missing')
        self.dnsdb_server = self.get_param(
            'config.server', None, 'Missing DNSDB server name')
        self.dnsdb_key = self.get_param(
            'config.key', None, 'Missing DNSDB API key')

    def execute_dnsdb_service(self, client):
        data = self.get_data()

        if self.service == 'domain_name' and self.data_type in ['domain', 'fqdn']:
            return client.query_rrset(data)
        elif self.service == 'ip_history' and self.data_type == 'ip':
            return client.query_rdata_ip(data)
        elif self.service == 'name_history' and self.data_type in ['domain', 'fqdn']:
            return client.query_rdata_name(data)
        else:
            self.error('Unknown DNSDB service or invalid data type')

    def update_date(self, field, row):
        if field in row:
            row[field] = datetime.datetime.utcfromtimestamp(
                row[field]).strftime('%Y%m%dT%H%M%S') + '+0000'
        return row

    def summary(self, raw):
        # taxonomy = {"level": "info", "namespace": "Farsight", "predicate": "DNSDB", "value": 0}
        taxonomies = []
        level = "info"
        namespace = "Farsight"
        predicate = "DNSDB"

        if "records" in raw:
            r = len(raw["records"])

            if r == 0 or r == 1:
                value = "{} record".format(r)
            else:
                value = "{} records".format(r)

            taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))

        return {'taxonomies': taxonomies}

    def run(self):
        try:
            client = DnsdbClient(self.dnsdb_server, self.dnsdb_key)
            self.report({
                "records": map(lambda r: self.update_date('time_first', self.update_date('time_last', r)),
                               self.execute_dnsdb_service(client))
            })
        except HTTPError, e:
            if e.code != 404:
                self.unexpectedError(e)
            else:
                self.report({"records": []})

if __name__ == '__main__':
    DnsDbAnalyzer().run()
<commit_msg>Add DNSDB API parameters (#319)<commit_after>#!/usr/bin/env python
# encoding: utf-8
import datetime
from urllib2 import HTTPError
from dnsdb_query import DnsdbClient, QueryError
from cortexutils.analyzer import Analyzer

class DnsDbAnalyzer(Analyzer):

    def __init__(self):
        Analyzer.__init__(self)
        self.service = self.get_param(
            'config.service', None, 'Service parameter is missing')
        self.dnsdb_server = self.get_param(
            'config.server', None, 'Missing DNSDB server name')
        self.dnsdb_key = self.get_param(
            'config.key', None, 'Missing DNSDB API key')

    def execute_dnsdb_service(self, client):
        data = self.get_data()
        rrtype = self.getParam('parameters.rrtype', None, None)
        bailiwick = self.getParam('parameters.bailiwick', None, None)
        before = self.getParam('parameters.before', None, None)
        after = self.getParam('parameters.after', None, None)

        if self.service == 'domain_name' and self.data_type in ['domain', 'fqdn']:
            return client.query_rrset(data, rrtype=rrtype, bailiwick=bailiwick, before=before, after=after)
        elif self.service == 'ip_history' and self.data_type == 'ip':
            return client.query_rdata_ip(data, before=before, after=after)
        elif self.service == 'name_history' and self.data_type in ['domain', 'fqdn']:
            return client.query_rdata_name(data, rrtype=rrtype, before=before, after=after)
        else:
            self.error('Unknown DNSDB service or invalid data type')

    def update_date(self, field, row):
        if field in row:
            row[field] = datetime.datetime.utcfromtimestamp(
                row[field]).strftime('%Y%m%dT%H%M%S') + '+0000'
        return row

    def summary(self, raw):
        # taxonomy = {"level": "info", "namespace": "Farsight", "predicate": "DNSDB", "value": 0}
        taxonomies = []
        level = "info"
        namespace = "Farsight"
        predicate = "DNSDB"

        if "records" in raw:
            r = len(raw["records"])

            if r == 0 or r == 1:
                value = "{} record".format(r)
            else:
                value = "{} records".format(r)

            taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))

        return {'taxonomies': taxonomies}

    def run(self):
        try:
            client = DnsdbClient(self.dnsdb_server, self.dnsdb_key)
            self.report({
                "records": map(lambda r: self.update_date('time_first', self.update_date('time_last', r)),
                               self.execute_dnsdb_service(client))
            })
        except HTTPError, e:
            if e.code != 404:
                self.unexpectedError(e)
            else:
                self.report({"records": []})

if __name__ == '__main__':
    DnsDbAnalyzer().run()
<|endoftext|>


<commit_before>import logging
from typing import Any, Dict, Optional, Union

import stripe
from django.conf import settings
from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
from django.utils.timezone import now as timezone_now
from django.utils.translation import gettext as _

from corporate.lib.stripe import (
    STRIPE_PUBLISHABLE_KEY,
    cents_to_dollar_string,
    do_change_plan_status,
    do_replace_payment_source,
    downgrade_at_the_end_of_billing_cycle,
    downgrade_now_without_creating_additional_invoices,
    get_latest_seat_count,
    make_end_of_cycle_updates_if_needed,
    renewal_amount,
    start_of_next_billing_cycle,
    stripe_get_customer,
    update_license_ledger_for_manual_plan,
    validate_licenses,
)
from corporate.models import (
    CustomerPlan,
    get_current_plan_by_customer,
    get_current_plan_by_realm,
    get_customer_by_realm,
)
from zerver.decorator import require_billing_access, zulip_login_required
from zerver.lib.exceptions import JsonableError
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_success
from zerver.lib.validator import check_int, check_int_in
from zerver.models import UserProfile

billing_logger = logging.getLogger("corporate.stripe")

# Should only be called if the customer is being charged automatically
def payment_method_string(stripe_customer: stripe.Customer) -> str:
    stripe_source: Optional[Union[stripe.Card, stripe.Source]] = stripe_customer.default_source
    # In case of e.g. an expired card
    if stripe_source is None:  # nocoverage
        return _("No payment method on file")
    if stripe_source.object == "card":
        assert isinstance(stripe_source, stripe.Card)
        return _("{brand} ending in {last4}").format(
            brand=stripe_source.brand,
            last4=stripe_source.last4,
        )
    # There might be one-off stuff we do for a particular customer that
    # would land them here. E.g. by default we don't support ACH for
    # automatic payments, but in theory we could add it for a customer via
    # the Stripe dashboard.
    return _("Unknown payment method. Please contact {email}.").format(
        email=settings.ZULIP_ADMINISTRATOR,
    )  # nocoverage

@zulip_login_required
def billing_home(request: HttpRequest) -> HttpResponse:
    user = request.user
    assert user.is_authenticated

    customer = get_customer_by_realm(user.realm)
    context: Dict[str, Any] = {
        "admin_access": user.has_billing_access,
        "has_active_plan": False,
    }

    if user.realm.plan_type == user.realm.STANDARD_FREE:
        context["is_sponsored"] = True
        return render(request, "corporate/billing.html", context=context)

    if customer is None:
        from corporate.views.upgrade import initial_upgrade

        return HttpResponseRedirect(reverse(initial_upgrade))

    if customer.sponsorship_pending:
        context["sponsorship_pending"] = True
        return render(request, "corporate/billing.html", context=context)

    if not CustomerPlan.objects.filter(customer=customer).exists():
        from corporate.views.upgrade import initial_upgrade

        return HttpResponseRedirect(reverse(initial_upgrade))

    if not user.has_billing_access:
        return render(request, "corporate/billing.html", context=context)

    plan = get_current_plan_by_customer(customer)
    if plan is not None:
        now = timezone_now()
        new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, now)
        if last_ledger_entry is not None:
            if new_plan is not None:  # nocoverage
                plan = new_plan
            assert plan is not None  # for mypy
            downgrade_at_end_of_cycle = plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE
            switch_to_annual_at_end_of_cycle = (
                plan.status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE
            )
            licenses = last_ledger_entry.licenses
            licenses_at_next_renewal = last_ledger_entry.licenses_at_next_renewal
            seat_count = get_latest_seat_count(user.realm)

            # Should do this in javascript, using the user's timezone
            renewal_date = "{dt:%B} {dt.day}, {dt.year}".format(
                dt=start_of_next_billing_cycle(plan, now)
            )
            renewal_cents = renewal_amount(plan, now)
            charge_automatically = plan.charge_automatically
            assert customer.stripe_customer_id is not None  # for mypy
            stripe_customer = stripe_get_customer(customer.stripe_customer_id)
            if charge_automatically:
                payment_method = payment_method_string(stripe_customer)
            else:
                payment_method = "Billed by invoice"

            context.update(
                plan_name=plan.name,
                has_active_plan=True,
                free_trial=plan.is_free_trial(),
                downgrade_at_end_of_cycle=downgrade_at_end_of_cycle,
                automanage_licenses=plan.automanage_licenses,
                switch_to_annual_at_end_of_cycle=switch_to_annual_at_end_of_cycle,
                licenses=licenses,
                licenses_at_next_renewal=licenses_at_next_renewal,
                seat_count=seat_count,
                renewal_date=renewal_date,
                renewal_amount=cents_to_dollar_string(renewal_cents),
                payment_method=payment_method,
                charge_automatically=charge_automatically,
                publishable_key=STRIPE_PUBLISHABLE_KEY,
                stripe_email=stripe_customer.email,
                CustomerPlan=CustomerPlan,
                onboarding=request.GET.get("onboarding") is not None,
            )

    return render(request, "corporate/billing.html", context=context)

@require_billing_access
@has_request_variables
def update_plan(
    request: HttpRequest,
    user: UserProfile,
    status: Optional[int] = REQ(
        "status",
        json_validator=check_int_in(
            [
                CustomerPlan.ACTIVE,
                CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE,
                CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE,
                CustomerPlan.ENDED,
            ]
        ),
        default=None,
    ),
    licenses: Optional[int] = REQ("licenses", json_validator=check_int, default=None),
    licenses_at_next_renewal: Optional[int] = REQ(
        "licenses_at_next_renewal", json_validator=check_int, default=None
    ),
) -> HttpResponse:
    plan = get_current_plan_by_realm(user.realm)
    assert plan is not None  # for mypy

    new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, timezone_now())
    if new_plan is not None:
        raise JsonableError(
            _("Unable to update the plan. The plan has been expired and replaced with a new plan.")
        )

    if last_ledger_entry is None:
        raise JsonableError(_("Unable to update the plan. The plan has ended."))

    if status is not None:
        if status == CustomerPlan.ACTIVE:
            assert plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE
            do_change_plan_status(plan, status)
        elif status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE:
            assert plan.status == CustomerPlan.ACTIVE
            downgrade_at_the_end_of_billing_cycle(user.realm)
        elif status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE:
            assert plan.billing_schedule == CustomerPlan.MONTHLY
            assert plan.status == CustomerPlan.ACTIVE
            assert plan.fixed_price is None
            do_change_plan_status(plan, status)
        elif status == CustomerPlan.ENDED:
            assert plan.is_free_trial()
            downgrade_now_without_creating_additional_invoices(user.realm)
        return json_success()

    if licenses is not None:
        if plan.automanage_licenses:
            raise JsonableError(
                _(
                    "Unable to update licenses manually. Your plan is on automatic license management."
                )
            )
        if last_ledger_entry.licenses == licenses:
            raise JsonableError(
                _(
                    "Your plan is already on {licenses} licenses in the current billing period."
                ).format(licenses=licenses)
            )
        if last_ledger_entry.licenses > licenses:
            raise JsonableError(
                _("You cannot decrease the licenses in the current billing period.").format(
                    licenses=licenses
                )
            )
        validate_licenses(plan.charge_automatically, licenses, get_latest_seat_count(user.realm))
        update_license_ledger_for_manual_plan(plan, timezone_now(), licenses=licenses)
        return json_success()

    if licenses_at_next_renewal is not None:
        if plan.automanage_licenses:
            raise JsonableError(
                _(
                    "Unable to update licenses manually. Your plan is on automatic license management."
                )
            )
        if last_ledger_entry.licenses_at_next_renewal == licenses_at_next_renewal:
            raise JsonableError(
                _(
                    "Your plan is already scheduled to renew with {licenses_at_next_renewal} licenses."
                ).format(licenses_at_next_renewal=licenses_at_next_renewal)
            )
        validate_licenses(
            plan.charge_automatically,
            licenses_at_next_renewal,
            get_latest_seat_count(user.realm),
        )
        update_license_ledger_for_manual_plan(
            plan, timezone_now(), licenses_at_next_renewal=licenses_at_next_renewal
        )
        return json_success()

    raise JsonableError(_("Nothing to change."))

@require_billing_access
@has_request_variables
def replace_payment_source(
    request: HttpRequest,
    user: UserProfile,
    stripe_token: str = REQ(),
) -> HttpResponse:
    do_replace_payment_source(user, stripe_token, pay_invoices=True)
    return json_success()
<commit_msg>billing: Refactor billing_home to use REQ.<commit_after>import logging
from typing import Any, Dict, Optional, Union

import stripe
from django.conf import settings
from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
from django.utils.timezone import now as timezone_now
from django.utils.translation import gettext as _

from corporate.lib.stripe import (
    STRIPE_PUBLISHABLE_KEY,
    cents_to_dollar_string,
    do_change_plan_status,
    do_replace_payment_source,
    downgrade_at_the_end_of_billing_cycle,
    downgrade_now_without_creating_additional_invoices,
    get_latest_seat_count,
    make_end_of_cycle_updates_if_needed,
    renewal_amount,
    start_of_next_billing_cycle,
    stripe_get_customer,
    update_license_ledger_for_manual_plan,
    validate_licenses,
)
from corporate.models import (
    CustomerPlan,
    get_current_plan_by_customer,
    get_current_plan_by_realm,
    get_customer_by_realm,
)
from zerver.decorator import require_billing_access, zulip_login_required
from zerver.lib.exceptions import JsonableError
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_success
from zerver.lib.validator import check_bool, check_int, check_int_in
from zerver.models import UserProfile

billing_logger = logging.getLogger("corporate.stripe")

# Should only be called if the customer is being charged automatically
def payment_method_string(stripe_customer: stripe.Customer) -> str:
    stripe_source: Optional[Union[stripe.Card, stripe.Source]] = stripe_customer.default_source
    # In case of e.g. an expired card
    if stripe_source is None:  # nocoverage
        return _("No payment method on file")
    if stripe_source.object == "card":
        assert isinstance(stripe_source, stripe.Card)
        return _("{brand} ending in {last4}").format(
            brand=stripe_source.brand,
            last4=stripe_source.last4,
        )
    # There might be one-off stuff we do for a particular customer that
    # would land them here. E.g. by default we don't support ACH for
    # automatic payments, but in theory we could add it for a customer via
    # the Stripe dashboard.
    return _("Unknown payment method. Please contact {email}.").format(
        email=settings.ZULIP_ADMINISTRATOR,
    )  # nocoverage

@zulip_login_required
@has_request_variables
def billing_home(
    request: HttpRequest, onboarding: bool = REQ(default=False, json_validator=check_bool)
) -> HttpResponse:
    user = request.user
    assert user.is_authenticated

    customer = get_customer_by_realm(user.realm)
    context: Dict[str, Any] = {
        "admin_access": user.has_billing_access,
        "has_active_plan": False,
    }

    if user.realm.plan_type == user.realm.STANDARD_FREE:
        context["is_sponsored"] = True
        return render(request, "corporate/billing.html", context=context)

    if customer is None:
        from corporate.views.upgrade import initial_upgrade

        return HttpResponseRedirect(reverse(initial_upgrade))

    if customer.sponsorship_pending:
        context["sponsorship_pending"] = True
        return render(request, "corporate/billing.html", context=context)

    if not CustomerPlan.objects.filter(customer=customer).exists():
        from corporate.views.upgrade import initial_upgrade

        return HttpResponseRedirect(reverse(initial_upgrade))

    if not user.has_billing_access:
        return render(request, "corporate/billing.html", context=context)

    plan = get_current_plan_by_customer(customer)
    if plan is not None:
        now = timezone_now()
        new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, now)
        if last_ledger_entry is not None:
            if new_plan is not None:  # nocoverage
                plan = new_plan
            assert plan is not None  # for mypy
            downgrade_at_end_of_cycle = plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE
            switch_to_annual_at_end_of_cycle = (
                plan.status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE
            )
            licenses = last_ledger_entry.licenses
            licenses_at_next_renewal = last_ledger_entry.licenses_at_next_renewal
            seat_count = get_latest_seat_count(user.realm)

            # Should do this in javascript, using the user's timezone
            renewal_date = "{dt:%B} {dt.day}, {dt.year}".format(
                dt=start_of_next_billing_cycle(plan, now)
            )
            renewal_cents = renewal_amount(plan, now)
            charge_automatically = plan.charge_automatically
            assert customer.stripe_customer_id is not None  # for mypy
            stripe_customer = stripe_get_customer(customer.stripe_customer_id)
            if charge_automatically:
                payment_method = payment_method_string(stripe_customer)
            else:
                payment_method = "Billed by invoice"

            context.update(
                plan_name=plan.name,
                has_active_plan=True,
                free_trial=plan.is_free_trial(),
                downgrade_at_end_of_cycle=downgrade_at_end_of_cycle,
                automanage_licenses=plan.automanage_licenses,
                switch_to_annual_at_end_of_cycle=switch_to_annual_at_end_of_cycle,
                licenses=licenses,
                licenses_at_next_renewal=licenses_at_next_renewal,
                seat_count=seat_count,
                renewal_date=renewal_date,
                renewal_amount=cents_to_dollar_string(renewal_cents),
                payment_method=payment_method,
                charge_automatically=charge_automatically,
                publishable_key=STRIPE_PUBLISHABLE_KEY,
                stripe_email=stripe_customer.email,
                CustomerPlan=CustomerPlan,
                onboarding=onboarding,
            )

    return render(request, "corporate/billing.html", context=context)

@require_billing_access
@has_request_variables
def update_plan(
    request: HttpRequest,
    user: UserProfile,
    status: Optional[int] = REQ(
        "status",
        json_validator=check_int_in(
            [
                CustomerPlan.ACTIVE,
                CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE,
                CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE,
                CustomerPlan.ENDED,
            ]
        ),
        default=None,
    ),
    licenses: Optional[int] = REQ("licenses", json_validator=check_int, default=None),
    licenses_at_next_renewal: Optional[int] = REQ(
        "licenses_at_next_renewal", json_validator=check_int, default=None
    ),
) -> HttpResponse:
    plan = get_current_plan_by_realm(user.realm)
    assert plan is not None  # for mypy

    new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, timezone_now())
    if new_plan is not None:
        raise JsonableError(
            _("Unable to update the plan. The plan has been expired and replaced with a new plan.")
        )

    if last_ledger_entry is None:
        raise JsonableError(_("Unable to update the plan. The plan has ended."))

    if status is not None:
        if status == CustomerPlan.ACTIVE:
            assert plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE
            do_change_plan_status(plan, status)
        elif status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE:
            assert plan.status == CustomerPlan.ACTIVE
            downgrade_at_the_end_of_billing_cycle(user.realm)
        elif status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE:
            assert plan.billing_schedule == CustomerPlan.MONTHLY
            assert plan.status == CustomerPlan.ACTIVE
            assert plan.fixed_price is None
            do_change_plan_status(plan, status)
        elif status == CustomerPlan.ENDED:
            assert plan.is_free_trial()
            downgrade_now_without_creating_additional_invoices(user.realm)
        return json_success()

    if licenses is not None:
        if plan.automanage_licenses:
            raise JsonableError(
                _(
                    "Unable to update licenses manually. Your plan is on automatic license management."
                )
            )
        if last_ledger_entry.licenses == licenses:
            raise JsonableError(
                _(
                    "Your plan is already on {licenses} licenses in the current billing period."
                ).format(licenses=licenses)
            )
        if last_ledger_entry.licenses > licenses:
            raise JsonableError(
                _("You cannot decrease the licenses in the current billing period.").format(
                    licenses=licenses
                )
            )
        validate_licenses(plan.charge_automatically, licenses, get_latest_seat_count(user.realm))
        update_license_ledger_for_manual_plan(plan, timezone_now(), licenses=licenses)
        return json_success()

    if licenses_at_next_renewal is not None:
        if plan.automanage_licenses:
            raise JsonableError(
                _(
                    "Unable to update licenses manually. Your plan is on automatic license management."
                )
            )
        if last_ledger_entry.licenses_at_next_renewal == licenses_at_next_renewal:
            raise JsonableError(
                _(
                    "Your plan is already scheduled to renew with {licenses_at_next_renewal} licenses."
                ).format(licenses_at_next_renewal=licenses_at_next_renewal)
            )
        validate_licenses(
            plan.charge_automatically,
            licenses_at_next_renewal,
            get_latest_seat_count(user.realm),
        )
        update_license_ledger_for_manual_plan(
            plan, timezone_now(), licenses_at_next_renewal=licenses_at_next_renewal
        )
        return json_success()

    raise JsonableError(_("Nothing to change."))

@require_billing_access
@has_request_variables
def replace_payment_source(
    request: HttpRequest,
    user: UserProfile,
    stripe_token: str = REQ(),
) -> HttpResponse:
    do_replace_payment_source(user, stripe_token, pay_invoices=True)
    return json_success()
<|endoftext|>


<commit_before># encoding: utf-8
"""
Tools for the pan-M31 PHAT patch fitting exercise.

2015-05-28 - Created by Jonathan Sick
"""

import os
from collections import OrderedDict

from matplotlib.path import Path

import numpy as np
from astropy.table import Table
from astropy.io import fits
import astropy
from astropy.coordinates import Distance, Angle, SkyCoord
import astropy.units as u

from m31hst import phat_v2_phot_path, phat_brick_path
from m31hst.phatast import PhatAstTable

from starfisher.pipeline import PipelineBase
from starfisher.pipeline import PlaneBase
from starfisher.pipeline import ExtinctionBase
from starfisher.pipeline import DatasetBase
from starfisher.pipeline import CrowdingBase
from starfisher.dust import ExtinctionDistribution
from starfisher import ExtantCrowdingTable

# from starfisher.pipeline import (
#     PipelineBase, IsochroneSetBase, DatasetBase, LockBase,
#     CrowdingBase, ExtinctionBase)

from androcmd.planes import make_f475w_f160w, make_lewis_ms

from androcmd.phatpipeline import ExtendedSolarIsocs, ExtendedSolarLockfile
from androcmd.dust import mw_Av, phat_rel_extinction, LewisDustLaw

class PatchFitPlanes(PlaneBase):
    """Color plane set for the PHAT color baseline comparison test."""
    def __init__(self, **kwargs):
        self._planes = OrderedDict([
            ('oir_all', make_f475w_f160w()),
            ('lewis', make_lewis_ms()),
        ])
        super(PatchFitPlanes, self).__init__(**kwargs)

    @property
    def planes(self):
        return self._planes

class LewisPatchDust(ExtinctionBase):
    """Mixin for a uniform dust distribution fitted from Lewis et al 15 Fig 17.

    The maximum extinction is estimated from a Draine et al 2015 dust map.
    Requires that the brick be known.
    """
    def __init__(self, **kwargs):
        # self.brick = kwargs.pop('brick', 23)
        super(LewisPatchDust, self).__init__(**kwargs)

    def build_extinction(self):
        """Young and old dust at equal here."""
        lewis = LewisDustLaw()
        max_av = lewis.estimate_mean_extinction(self.poly)
        av = np.random.uniform(low=mw_Av(),
                               high=max_av,
                               size=1000)

        self.young_av = ExtinctionDistribution()
        self.young_av.set_samples(av)

        self.old_av = ExtinctionDistribution()
        self.old_av.set_samples(av)

        self.rel_extinction = phat_rel_extinction()

class AutoPhatCrowding(CrowdingBase):
    """Use crowding from the PHAT AST fields, automatically selecting
    the AST field that best matches the location of the patch being
    considered.
    """
    def __init__(self, **kwargs):
        super(AutoPhatCrowding, self).__init__(**kwargs)

    def build_crowding(self):
        crowd_path = os.path.join(self.synth_dir, "crowding.dat")
        full_crowd_path = os.path.join(os.getenv('STARFISH'), crowd_path)

        patch_coord = SkyCoord(ra=self.ra0 * u.degree,
                               dec=self.dec0 * u.degree)

        tbl = PhatAstTable()
        centers = np.array([f['center'] for f in tbl.fields])

        ast_coords = SkyCoord(ra=centers[:, 0] * u.degree,
                              dec=centers[:, 1] * u.degree,
                              frame='icrs')
        dists = ast_coords.separation(patch_coord)
        best_ast_field = np.argmin(dists)
        # FIXME actually want to compute different in galactocentric radii?

        tbl.write_crowdfile_for_field(full_crowd_path,
                                      best_ast_field,
                                      bands=self.bands)
        self.crowd = ExtantCrowdingTable(crowd_path)

    def mask_planes(self):
        """Mask each CMD plane based on the incomplete or empty regions of
        the PHAT artificial star testing projected into the Hess plane.

        This hook is called automatically by the base pipeline before
        synth is run.
        """
        # FIXME note that AST field 0 *is always* used
        print "Using PhatCrowding.mask_planes"
        ast = PhatAstTable()
        for key, plane in self.planes.iteritems():
            band = plane.y_mag  # FIXME assumes CMD; only 1 y axis mag.
            hess, x_grid, y_grid = ast.completeness_hess(
                0, band,
                plane.x_mag, plane.y_mag,
                plane.xlim, plane.ylim, 0.5)
            yidx, xidx = np.where(hess < 0.5)  # mask less than 50% complete
            for yi, xi in zip(yidx, xidx):
                plane.mask_region((x_grid[xi], x_grid[xi + 1]),
                                  (y_grid[yi], y_grid[yi + 1]))
            yidx, xidx = np.where(~np.isfinite(hess))  # mask empty AST
            for yi, xi in zip(yidx, xidx):
                plane.mask_region((x_grid[xi], x_grid[xi + 1]),
                                  (y_grid[yi], y_grid[yi + 1]))

class ThreeZPipeline(PatchFitPlanes, ExtendedSolarIsocs,
                     ExtendedSolarLockfile, LewisPatchDust, AutoPhatCrowding,
                     PipelineBase):
    """Pipeline for patch fitting with three metallicity tracks."""
    def __init__(self, **kwargs):
        # Get patch attributes
        self.patch = kwargs.pop('patch')
        self.poly = kwargs.pop('poly')
        self.brick = kwargs.pop('brick')
        self.ra0 = kwargs.pop('ra0')
        self.dec0 = kwargs.pop('dec0')
        self.area = kwargs.pop('area')
        super(ThreeZPipeline, self).__init__(**kwargs)

class PatchCatalog(DatasetBase):
    """Mixin for PHAT photometry data.

    Photometry is lazy loaded to it is efficient to rebuild the pipeline
    object.
    """
    def __init__(self, **kwargs):
        self._phat_data = None
        self.patch = kwargs.pop('patch')
        self.poly = kwargs.pop('poly')
        self.brick = kwargs.pop('brick')
        self.ra0 = kwargs.pop('ra0')
        self.dec0 = kwargs.pop('dec0')
        self.area = kwargs.pop('area')
        self.area_proj = kwargs.pop('area_proj')
        print "uncaught PatchCatalog keywords", kwargs
        super(PatchCatalog, self).__init__(**kwargs)

    def _load_phat_data(self):
        self._phat_data = Table.read(phat_v2_phot_path(self.brick),
                                     format='fits')
        phat_bands = ('F475W', 'F814W', 'F275W', 'F336W', 'F110W', 'F160W')
        # Normalize bandpass names
        for band in phat_bands:
            old_name = "_".join((band.lower(), 'vega'))
            self._phat_data.rename_column(old_name, band)

        # Spatially select stars in the patch footprint
        self._phat_data = self._spatial_selection(self._phat_data)

    def _spatial_selection(self, dataset):
        data_ra = dataset['ra']
        data_dec = dataset['dec']
        xy = np.vstack((data_ra, data_dec)).T
        print "xy.shape", xy.shape

        polygon = Path(self.poly, closed=False)
        inside = polygon.contains_points(xy)
        return dataset[inside == True]  # noqa

    def get_phot(self, band):
        if self._phat_data is None:
            self._load_phat_data()  # lazy loading

        if not isinstance(band, basestring):
            band1, band2 = band
            return self._phat_data[band1] - self._phat_data[band2]
        else:
            return self._phat_data[band]

    def _select_gst(self, x_band, y_band):
        mags = []
        if not isinstance(x_band, basestring):
            mags.extend(x_band)
        else:
            mags.append(x_band)

        if not isinstance(y_band, basestring):
            mags.extend(y_band)
        else:
            mags.append(y_band)
        mags = list(set(mags))

        gsts = []
        for band in mags:
            key = '{0}_gst'.format(band.lower())
            gsts.append(self._phat_data[key])
        gsts_array = np.vstack(gsts).T
        gsts = np.all(gsts_array, axis=1)
        return np.where(gsts == True)[0]  # NOQA

    def write_phot(self, x_band, y_band, data_root, suffix):
        """Only good (GST=1 in all relevant bands) photometry is written."""
        if self._phat_data is None:
            self._load_phat_data()  # lazy loading

        x = self.get_phot(x_band)
        y = self.get_phot(y_band)
        gst_sel = self._select_gst(x_band, y_band)

        phot_dtype = np.dtype([('x', np.float), ('y', np.float)])
        photdata = np.empty(len(gst_sel), dtype=phot_dtype)
        photdata['x'][:] = x[gst_sel]
        photdata['y'][:] = y[gst_sel]

        path = data_root + suffix
        full_path = os.path.join(os.getenv('STARFISH'), path)
        fit_dir = os.path.dirname(full_path)
        if not os.path.exists(fit_dir):
            os.makedirs(fit_dir)
        np.savetxt(full_path, photdata, delimiter=' ', fmt='%.4f')

    @property
    def polygon(self):
        """Polygon bounding box around the dataset."""
        return self.poly

def build_patches(brick, proj_size=100):
    """Build patches from a brick.

    brick : int
        PHAT brick number.
    proj_size : float
        Size of a patch, in projected side length, in parsecs.
    """
    patches = []

    # Get the patch FITS file
    path = phat_brick_path(brick, 'F814W')
    with fits.open(path) as f:
        header = fits.getheader(f, 0)
        wcs = astropy.wcs.WCS(header)

    # degree per pixel
    pixel_scale = np.mean(
        np.sqrt(astropy.wcs.utils.proj_plane_pixel_scales(wcs) ** 2.))

    theta = np.rad2deg(np.arctan(proj_size / (785. * 10. ** 3.)))

    n_pix_side_min = theta / pixel_scale

    # Number of boxes, in each dimension so each box is *at least*
    # proj_size on each size.
    nx = np.floor(header['NAXIS1'] / n_pix_side_min)
    ny = np.floor(header['NAXIS2'] / n_pix_side_min)

    x_edges = np.linspace(0, header['NAXIS1'], num=nx + 1,
                          endpoint=True, dtype=int)
    y_edges = np.linspace(0, header['NAXIS2'], num=ny + 1,
                          endpoint=True, dtype=int)

    patch_num = 1
    for i in xrange(nx):
        for j in xrange(ny):
            x1 = x_edges[i]
            x2 = x_edges[i + 1]
            y1 = y_edges[i]
            y2 = y_edges[i + 1]
            xy_verts = np.array([[x1, y1],
                                 [x1, y2],
                                 [x2, y2],
                                 [x2, y1]])
            radec = wcs.all_pix2world(xy_verts, 0)
            # projected arcsec^2
            area_proj = (y2 - y1) * (x2 - x1) * (pixel_scale * 3600.) ** 2.
            # area in pc^2, de-projected
            pc_per_arcsec = 785. * 10. ** 3. * np.tan(np.deg2rad(1. / 3600))
            area = area_proj * (pc_per_arcsec) ** 2. / np.cos(77 * np.pi / 180.)  # NOQA
            ra0 = radec[:, 0].mean()
            dec0 = radec[:, 1].mean()
            r_kpc, phi = compute_patch_gal_coords(ra0, dec0)
            patch = {'patch': '{0:02d}_{1:03d}'.format(brick, patch_num),
                     'brick': brick,
                     'poly': radec.tolist(),
                     'ra0': ra0,
                     'dec0': dec0,
                     'r_kpc': r_kpc,
                     'phi': phi,
                     'area_proj': area_proj,
                     'area': area}
            patches.append(patch)
            patch_num += 1

    return patches

def compute_patch_gal_coords(ra, dec):
    """Compute the galactocentric coordinates (radius in kpc; PA in degrees)
    for a patch given its central coordinate.
    """
    from wedge.galaxycoords import galaxy_coords
    d0 = Distance(785, unit=u.kpc)
    pa0 = Angle('37d42m54s')
    incl0 = Angle('77.5d')
    coord0 = SkyCoord('00h42m44.33s +41d16m07.5s')
    coord = SkyCoord(ra * u.degree, dec * u.degree)
    r, phi, r_sky = galaxy_coords(coord, coord0, pa0, incl0, d0)
    return r.kpc, phi.degree
<commit_msg>Copy galaxy_coords function into codebase<commit_after># encoding: utf-8
"""
Tools for the pan-M31 PHAT patch fitting exercise.

2015-05-28 - Created by Jonathan Sick
"""

import os
from collections import OrderedDict

from matplotlib.path import Path

import numpy as np
from astropy.table import Table
from astropy.io import fits
import astropy
from astropy.coordinates import Distance, Angle, SkyCoord
import astropy.units as u

from m31hst import phat_v2_phot_path, phat_brick_path
from m31hst.phatast import PhatAstTable

from starfisher.pipeline import PipelineBase
from starfisher.pipeline import PlaneBase
from starfisher.pipeline import ExtinctionBase
from starfisher.pipeline import DatasetBase
from starfisher.pipeline import CrowdingBase
from starfisher.dust import ExtinctionDistribution
from starfisher import ExtantCrowdingTable

# from starfisher.pipeline import (
#     PipelineBase, IsochroneSetBase, DatasetBase, LockBase,
#     CrowdingBase, ExtinctionBase)

from androcmd.planes import make_f475w_f160w, make_lewis_ms

from androcmd.phatpipeline import ExtendedSolarIsocs, ExtendedSolarLockfile
from androcmd.dust import mw_Av, phat_rel_extinction, LewisDustLaw

class PatchFitPlanes(PlaneBase):
    """Color plane set for the PHAT color baseline comparison test."""
    def __init__(self, **kwargs):
        self._planes = OrderedDict([
            ('oir_all', make_f475w_f160w()),
            ('lewis', make_lewis_ms()),
        ])
        super(PatchFitPlanes, self).__init__(**kwargs)

    @property
    def planes(self):
        return self._planes

class LewisPatchDust(ExtinctionBase):
    """Mixin for a uniform dust distribution fitted from Lewis et al 15 Fig 17.

    The maximum extinction is estimated from a Draine et al 2015 dust map.
    Requires that the brick be known.
    """
    def __init__(self, **kwargs):
        # self.brick = kwargs.pop('brick', 23)
        super(LewisPatchDust, self).__init__(**kwargs)

    def build_extinction(self):
        """Young and old dust at equal here."""
        lewis = LewisDustLaw()
        max_av = lewis.estimate_mean_extinction(self.poly)
        av = np.random.uniform(low=mw_Av(),
                               high=max_av,
                               size=1000)

        self.young_av = ExtinctionDistribution()
        self.young_av.set_samples(av)

        self.old_av = ExtinctionDistribution()
        self.old_av.set_samples(av)

        self.rel_extinction = phat_rel_extinction()

class AutoPhatCrowding(CrowdingBase):
    """Use crowding from the PHAT AST fields, automatically selecting
    the AST field that best matches the location of the patch being
    considered.
    """
    def __init__(self, **kwargs):
        super(AutoPhatCrowding, self).__init__(**kwargs)

    def build_crowding(self):
        crowd_path = os.path.join(self.synth_dir, "crowding.dat")
        full_crowd_path = os.path.join(os.getenv('STARFISH'), crowd_path)

        patch_coord = SkyCoord(ra=self.ra0 * u.degree,
                               dec=self.dec0 * u.degree)

        tbl = PhatAstTable()
        centers = np.array([f['center'] for f in tbl.fields])

        ast_coords = SkyCoord(ra=centers[:, 0] * u.degree,
                              dec=centers[:, 1] * u.degree,
                              frame='icrs')
        dists = ast_coords.separation(patch_coord)
        best_ast_field = np.argmin(dists)
        # FIXME actually want to compute different in galactocentric radii?

        tbl.write_crowdfile_for_field(full_crowd_path,
                                      best_ast_field,
                                      bands=self.bands)
        self.crowd = ExtantCrowdingTable(crowd_path)

    def mask_planes(self):
        """Mask each CMD plane based on the incomplete or empty regions of
        the PHAT artificial star testing projected into the Hess plane.

        This hook is called automatically by the base pipeline before
        synth is run.
        """
        # FIXME note that AST field 0 *is always* used
        print "Using PhatCrowding.mask_planes"
        ast = PhatAstTable()
        for key, plane in self.planes.iteritems():
            band = plane.y_mag  # FIXME assumes CMD; only 1 y axis mag.
            hess, x_grid, y_grid = ast.completeness_hess(
                0, band,
                plane.x_mag, plane.y_mag,
                plane.xlim, plane.ylim, 0.5)
            yidx, xidx = np.where(hess < 0.5)  # mask less than 50% complete
            for yi, xi in zip(yidx, xidx):
                plane.mask_region((x_grid[xi], x_grid[xi + 1]),
                                  (y_grid[yi], y_grid[yi + 1]))
            yidx, xidx = np.where(~np.isfinite(hess))  # mask empty AST
            for yi, xi in zip(yidx, xidx):
                plane.mask_region((x_grid[xi], x_grid[xi + 1]),
                                  (y_grid[yi], y_grid[yi + 1]))

class ThreeZPipeline(PatchFitPlanes, ExtendedSolarIsocs,
                     ExtendedSolarLockfile, LewisPatchDust, AutoPhatCrowding,
                     PipelineBase):
    """Pipeline for patch fitting with three metallicity tracks."""
    def __init__(self, **kwargs):
        # Get patch attributes
        self.patch = kwargs.pop('patch')
        self.poly = kwargs.pop('poly')
        self.brick = kwargs.pop('brick')
        self.ra0 = kwargs.pop('ra0')
        self.dec0 = kwargs.pop('dec0')
        self.area = kwargs.pop('area')
        super(ThreeZPipeline, self).__init__(**kwargs)

class PatchCatalog(DatasetBase):
    """Mixin for PHAT photometry data.

    Photometry is lazy loaded to it is efficient to rebuild the pipeline
    object.
    """
    def __init__(self, **kwargs):
        self._phat_data = None
        self.patch = kwargs.pop('patch')
        self.poly = kwargs.pop('poly')
        self.brick = kwargs.pop('brick')
        self.ra0 = kwargs.pop('ra0')
        self.dec0 = kwargs.pop('dec0')
        self.area = kwargs.pop('area')
        self.area_proj = kwargs.pop('area_proj')
        print "uncaught PatchCatalog keywords", kwargs
        super(PatchCatalog, self).__init__(**kwargs)

    def _load_phat_data(self):
        self._phat_data = Table.read(phat_v2_phot_path(self.brick),
                                     format='fits')
        phat_bands = ('F475W', 'F814W', 'F275W', 'F336W', 'F110W', 'F160W')
        # Normalize bandpass names
        for band in phat_bands:
            old_name = "_".join((band.lower(), 'vega'))
            self._phat_data.rename_column(old_name, band)

        # Spatially select stars in the patch footprint
        self._phat_data = self._spatial_selection(self._phat_data)

    def _spatial_selection(self, dataset):
        data_ra = dataset['ra']
        data_dec = dataset['dec']
        xy = np.vstack((data_ra, data_dec)).T
        print "xy.shape", xy.shape

        polygon = Path(self.poly, closed=False)
        inside = polygon.contains_points(xy)
        return dataset[inside == True]  # noqa

    def get_phot(self, band):
        if self._phat_data is None:
            self._load_phat_data()  # lazy loading

        if not isinstance(band, basestring):
            band1, band2 = band
            return self._phat_data[band1] - self._phat_data[band2]
        else:
            return self._phat_data[band]

    def _select_gst(self, x_band, y_band):
        mags = []
        if not isinstance(x_band, basestring):
            mags.extend(x_band)
        else:
            mags.append(x_band)

        if not isinstance(y_band, basestring):
            mags.extend(y_band)
        else:
            mags.append(y_band)
        mags = list(set(mags))

        gsts = []
        for band in mags:
            key = '{0}_gst'.format(band.lower())
            gsts.append(self._phat_data[key])
        gsts_array = np.vstack(gsts).T
        gsts = np.all(gsts_array, axis=1)
        return np.where(gsts == True)[0]  # NOQA

    def write_phot(self, x_band, y_band, data_root, suffix):
        """Only good (GST=1 in all relevant bands) photometry is written."""
        if self._phat_data is None:
            self._load_phat_data()  # lazy loading

        x = self.get_phot(x_band)
        y = self.get_phot(y_band)
        gst_sel = self._select_gst(x_band, y_band)

        phot_dtype = np.dtype([('x', np.float), ('y', np.float)])
        photdata = np.empty(len(gst_sel), dtype=phot_dtype)
        photdata['x'][:] = x[gst_sel]
        photdata['y'][:] = y[gst_sel]

        path = data_root + suffix
        full_path = os.path.join(os.getenv('STARFISH'), path)
        fit_dir = os.path.dirname(full_path)
        if not os.path.exists(fit_dir):
            os.makedirs(fit_dir)
        np.savetxt(full_path, photdata, delimiter=' ', fmt='%.4f')

    @property
    def polygon(self):
        """Polygon bounding box around the dataset."""
        return self.poly

def build_patches(brick, proj_size=100):
    """Build patches from a brick.

    brick : int
        PHAT brick number.
    proj_size : float
        Size of a patch, in projected side length, in parsecs.
    """
    patches = []

    # Get the patch FITS file
    path = phat_brick_path(brick, 'F814W')
    with fits.open(path) as f:
        header = fits.getheader(f, 0)
        wcs = astropy.wcs.WCS(header)

    # degree per pixel
    pixel_scale = np.mean(
        np.sqrt(astropy.wcs.utils.proj_plane_pixel_scales(wcs) ** 2.))

    theta = np.rad2deg(np.arctan(proj_size / (785. * 10. ** 3.)))

    n_pix_side_min = theta / pixel_scale

    # Number of boxes, in each dimension so each box is *at least*
    # proj_size on each size.
    nx = np.floor(header['NAXIS1'] / n_pix_side_min)
    ny = np.floor(header['NAXIS2'] / n_pix_side_min)

    x_edges = np.linspace(0, header['NAXIS1'], num=nx + 1,
                          endpoint=True, dtype=int)
    y_edges = np.linspace(0, header['NAXIS2'], num=ny + 1,
                          endpoint=True, dtype=int)

    patch_num = 1
    for i in xrange(nx):
        for j in xrange(ny):
            x1 = x_edges[i]
            x2 = x_edges[i + 1]
            y1 = y_edges[i]
            y2 = y_edges[i + 1]
            xy_verts = np.array([[x1, y1],
                                 [x1, y2],
                                 [x2, y2],
                                 [x2, y1]])
            radec = wcs.all_pix2world(xy_verts, 0)
            # projected arcsec^2
            area_proj = (y2 - y1) * (x2 - x1) * (pixel_scale * 3600.) ** 2.
            # area in pc^2, de-projected
            pc_per_arcsec = 785. * 10. ** 3. * np.tan(np.deg2rad(1. / 3600))
            area = area_proj * (pc_per_arcsec) ** 2. / np.cos(77 * np.pi / 180.)  # NOQA
            ra0 = radec[:, 0].mean()
            dec0 = radec[:, 1].mean()
            r_kpc, phi = compute_patch_gal_coords(ra0, dec0)
            patch = {'patch': '{0:02d}_{1:03d}'.format(brick, patch_num),
                     'brick': brick,
                     'poly': radec.tolist(),
                     'ra0': ra0,
                     'dec0': dec0,
                     'r_kpc': r_kpc,
                     'phi': phi,
                     'area_proj': area_proj,
                     'area': area}
            patches.append(patch)
            patch_num += 1

    return patches

def compute_patch_gal_coords(ra, dec):
    """Compute the galactocentric coordinates (radius in kpc; PA in degrees)
    for a patch given its central coordinate.
    """
    d0 = Distance(785, unit=u.kpc)
    pa0 = Angle('37d42m54s')
    incl0 = Angle('77.5d')
    coord0 = SkyCoord('00h42m44.33s +41d16m07.5s')
    coord = SkyCoord(ra * u.degree, dec * u.degree)
    r, phi, r_sky = galaxy_coords(coord, coord0, pa0, incl0, d0)
    return r.kpc, phi.degree

def galaxy_coords(coord, glx_ctr, glx_PA, glx_incl, glx_dist):
    """Computes deprojected galactocentric distance.
    Inspired by: http://idl-moustakas.googlecode.com/svn-history/
        r560/trunk/impro/hiiregions/im_hiiregion_deproject.pro
    Parameters
    ----------
    coord : :class:`astropy.coordinates.SkyCoord`
        Coordinate of points to compute galactocentric distance for.
        Can be either a single coordinate, or array of coordinates.
    glx_ctr : :class:`astropy.coordinates.SkyCoord`
        Galaxy center.
    glx_PA : :class:`astropy.coordinates.Angle`
        Position angle of galaxy disk.
    glx_incl : :class:`astropy.coordinates.Angle`
        Inclination angle of the galaxy disk.
    glx_dist : :class:`astropy.coordinates.Distance`
        Distance to galaxy.
    Returns
    -------
    obj_dist : class:`astropy.coordinates.Distance`
        Galactocentric distance(s) for coordinate point(s).
    """
    # distance from coord to glx centre
    sky_radius = glx_ctr.separation(coord)
    avg_dec = 0.5 * (glx_ctr.dec + coord.dec).radian
    x = (glx_ctr.ra - coord.ra) * np.cos(avg_dec)
    y = glx_ctr.dec - coord.dec
    # azimuthal angle from coord to glx  -- not completely happy with this
    phi = glx_PA - Angle('90d') \
        + Angle(np.arctan2(y.arcsec, x.arcsec), unit=u.rad)

    # convert to coordinates in rotated frame, where y-axis is galaxy major
    # ax; have to convert to arcmin b/c can't do sqrt(x^2+y^2) when x and y
    # are angles
    xp = (sky_radius * np.cos(phi.radian)).arcmin
    yp = (sky_radius * np.sin(phi.radian)).arcmin

    # de-project
    ypp = yp / np.cos(glx_incl.radian)
    obj_radius = np.sqrt(xp ** 2 + ypp ** 2)  # in arcmin
    obj_dist = Distance(Angle(obj_radius, unit=u.arcmin).radian * glx_dist,
                        unit=glx_dist.unit)

    # Computing PA in disk
    # negative sign needed to get correct orientation from major axis
    obj_phi = Angle(np.arctan2(ypp, -xp), unit=u.rad)
    # FIXME only for scalars
    if obj_phi < 0.:
        obj_phi = Angle(2. * np.pi, unit=u.rad) + obj_phi

    return obj_dist, obj_phi, sky_radius.arcsec
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class Period:
    def __init__(self, name):
        self.name = name

class Company:
    def __init__(self, partner_id):
        self.partner_id = partner_id

class Partner:
    def __init__(self, name, nif):
        self.name = name
        self.vat = nif

class Tax:
    def __init__(self, amount):
        self.amount = amount

class InvoiceLineTaxes:
    def __init__(self, name, base, tax_amount, tax_id):
        self.name = name
        self.base = base  # base imponible
        self.tax_amount = tax_amount
        self.tax_id = tax_id

class Invoice:

    def __init__(self, description, number, type, partner_id, company_id,
                 amount_total, period_id, date_invoice, tax_line, sii_sent,
                 rectificative_type):
        self.name = description
        self.number = number
        self.type = type
        self.partner_id = partner_id
        self.company_id = company_id
        self.period_id = period_id
        self.amount_total = amount_total
        self.date_invoice = date_invoice
        self.tax_line = tax_line
        self.sii_sent = sii_sent
        self.rectificative_type = rectificative_type

class DataGenerator:
    def __init__(self):
        self.sii_sent = False
        self.period = Period(name='03/2016')
        self.tax_ibi = Tax(amount=0.15)
        self.tax_iva_21 = Tax(amount=0.21)
        self.tax_iva_4 = Tax(amount=0.04)
        self.tax_line = [
            InvoiceLineTaxes(
                name='IVA 21%', base=1000, tax_amount=210,
                tax_id=self.tax_iva_21
            ),
            InvoiceLineTaxes(
                name='IVA 4%', base=1000, tax_amount=40, tax_id=self.tax_iva_4
            ),
            InvoiceLineTaxes(
                name='IBI 15%', base=1000, tax_amount=150, tax_id=self.tax_ibi
            )
        ]
        self.partner_invoice = Partner(
            name=unicode('Francisco García', 'utf-8'), nif='12345678T'
        )
        self.partner_company = Partner(
            name=unicode('Compañía Eléctrica S.A.', 'utf-8'), nif='55555555T'
        )
        self.company = Company(partner_id=self.partner_company)

        self.invoice_number = 'F012345'
        self.date_invoice = '2016-12-31'
        self.amount_total = 15

    def get_in_invoice(self):
        invoice = Invoice(
            type='in_invoice',
            description='Factura recibida',
            rectificative_type='N',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_out_invoice(self):
        invoice = Invoice(
            type='out_invoice',
            description='Factura emitida',
            rectificative_type='N',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_in_refund_invoice(self):
        invoice = Invoice(
            type='in_refund',
            description='Factura rectificativa recibida',
            rectificative_type='R',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_out_refund_invoice(self):
        invoice = Invoice(
            type='out_refund',
            description='Factura rectificativa emitida',
            rectificative_type='R',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice
<commit_msg>Add FiscalPosition class for testing data<commit_after># -*- coding: utf-8 -*-

class Period:
    def __init__(self, name):
        self.name = name

class Company:
    def __init__(self, partner_id):
        self.partner_id = partner_id

class Partner:
    def __init__(self, name, nif):
        self.name = name
        self.vat = nif

class FiscalPosition:
    def __init__(self, cre_in_invoice, cre_out_invoice):
        self.sii_in_clave_regimen_especial = cre_in_invoice
        self.sii_out_clave_regimen_especial = cre_out_invoice

class Tax:
    def __init__(self, amount):
        self.amount = amount

class InvoiceLineTaxes:
    def __init__(self, name, base, tax_amount, tax_id):
        self.name = name
        self.base = base  # base imponible
        self.tax_amount = tax_amount
        self.tax_id = tax_id

class Invoice:

    def __init__(self, description, number, type, partner_id, company_id,
                 amount_total, period_id, date_invoice, tax_line, sii_sent,
                 rectificative_type):
        self.name = description
        self.number = number
        self.type = type
        self.partner_id = partner_id
        self.company_id = company_id
        self.period_id = period_id
        self.amount_total = amount_total
        self.date_invoice = date_invoice
        self.tax_line = tax_line
        self.sii_sent = sii_sent
        self.rectificative_type = rectificative_type

class DataGenerator:
    def __init__(self):
        self.sii_sent = False
        self.period = Period(name='03/2016')
        self.tax_ibi = Tax(amount=0.15)
        self.tax_iva_21 = Tax(amount=0.21)
        self.tax_iva_4 = Tax(amount=0.04)
        self.tax_line = [
            InvoiceLineTaxes(
                name='IVA 21%', base=1000, tax_amount=210,
                tax_id=self.tax_iva_21
            ),
            InvoiceLineTaxes(
                name='IVA 4%', base=1000, tax_amount=40, tax_id=self.tax_iva_4
            ),
            InvoiceLineTaxes(
                name='IBI 15%', base=1000, tax_amount=150, tax_id=self.tax_ibi
            )
        ]
        self.partner_invoice = Partner(
            name=unicode('Francisco García', 'utf-8'), nif='12345678T'
        )
        self.partner_company = Partner(
            name=unicode('Compañía Eléctrica S.A.', 'utf-8'), nif='55555555T'
        )
        self.company = Company(partner_id=self.partner_company)

        self.invoice_number = 'F012345'
        self.date_invoice = '2016-12-31'
        self.amount_total = 15

    def get_in_invoice(self):
        invoice = Invoice(
            type='in_invoice',
            description='Factura recibida',
            rectificative_type='N',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_out_invoice(self):
        invoice = Invoice(
            type='out_invoice',
            description='Factura emitida',
            rectificative_type='N',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_in_refund_invoice(self):
        invoice = Invoice(
            type='in_refund',
            description='Factura rectificativa recibida',
            rectificative_type='R',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice

    def get_out_refund_invoice(self):
        invoice = Invoice(
            type='out_refund',
            description='Factura rectificativa emitida',
            rectificative_type='R',
            number=self.invoice_number,
            partner_id=self.partner_invoice,
            company_id=self.company,
            amount_total=self.amount_total,
            period_id=self.period,
            date_invoice=self.date_invoice,
            tax_line=self.tax_line,
            sii_sent=self.sii_sent
        )
        return invoice
<|endoftext|>