<commit_before>import gscommon as gs
import gsshell
import sublime
import threading
import os
import gsq
import time
import hashlib
import base64
import Queue

DOMAIN = 'MarGo9'

# customization of, or user-owned gocode and margo will no longer be supported
# so we'll hardcode the relevant paths and refer to them directly instead of relying on PATH
MARGO9_SRC = gs.dist_path('margo9')
GOCODE_SRC = gs.dist_path('something_borrowed/gocode')
MARGO9_BIN = gs.home_path('bin', 'gosublime.margo9.exe')
GOCODE_BIN = gs.home_path('bin', 'gosublime.gocode.exe')

def _sb(s):
	bdir = gs.home_path('bin')
	if s.startswith(bdir):
		s = '~bin%s' % (s[len(bdir):])
	return s

def _tp(s):
	return (_sb(s), ('ok' if os.path.exists(s) else 'missing'))

def _so(out, err, start, end):
	out = out.strip()
	err = err.strip()
	ok = not out and not err
	if ok:
		out = 'ok %0.3fs' % (end - start)
	else:
		out = '%s\n%s' % (out, err)
	return (out.strip(), ok)

def _run(cmd, cwd=None, shell=False):
	nv = {
		'GOBIN': '',
		'GOPATH': gs.dist_path('something_borrowed'),
	}
	return gsshell.run(cmd, shell=shell, cwd=cwd, env=nv)

def maybe_install():
	if not os.path.exists(MARGO9_BIN) or not os.path.exists(GOCODE_BIN):
		install('', True)

def install(aso_tokens, force_install):
	init_start = time.time()

	try:
		os.makedirs(gs.home_path('bin'))
	except:
		pass

	if not force_install and aso_tokens == _gen_tokens():
		m_out = 'no'
		g_out = 'no'
	else:
		start = time.time()
		m_out, err, _ = _run(['go', 'build', '-o', MARGO9_BIN], cwd=MARGO9_SRC)
		m_out, m_ok = _so(m_out, err, start, time.time())

		# on windows the file cannot be replaced if it's running so close gocode first.
		# in theory, mg9 has the same issue but since it's attached to a st2 instance,
		# the only way to close it is to close st2 (which we're presumably already doing)
		start = time.time()
		if os.path.exists(GOCODE_BIN):
			_run([GOCODE_BIN, 'close'])

		g_out, err, _ = _run(['go', 'build', '-o', GOCODE_BIN], cwd=GOCODE_SRC)
		g_out, g_ok = _so(g_out, err, start, time.time())

		if m_ok and g_ok:
			def f():
				gs.aso().set('mg9_install_tokens', _gen_tokens())
				gs.save_aso()

			sublime.set_timeout(f, 0)

	a = (
		'GoSublime init (%0.3fs)' % (time.time() - init_start),
		'| install margo9: %s' % m_out,
		'| install gocode: %s' % g_out,
		'|           ~bin: %s' % gs.home_path('bin'),
		'|         margo9: %s (%s)' % _tp(MARGO9_BIN),
		'|         gocode: %s (%s)' % _tp(GOCODE_BIN),
	)
	gs.println(*a)

def _fasthash(fn):
	try:
		with open(fn) as f:
			chunk = f.read(1024*8)
		st = os.stat(fn)
		return '%s:%d' % (hashlib.sha1(chunk).hexdigest(), st.st_size)
	except Exception:
		pass
	return ''

def _read(fn):
	s = ''
	try:
		with open(fn) as f:
			s = f.read()
	except Exception:
		pass
	return s

def _token(head, bin):
	head = _read(gs.dist_path(head))
	token = '%s %s' % (head.strip(), _fasthash(bin))
	return token

def _gen_tokens():
	return '%s\n%s' % (_token('margo9.head', MARGO9_BIN), _token('something_borrowed/gocode.head', GOCODE_BIN))

def do_init():
	aso_tokens = gs.aso().get('mg9_install_tokens', '')
	f = lambda: install(aso_tokens, False)
	gsq.do('GoSublime', f, msg='Installing MarGo9 and Gocode', set_status=True)

def _gocode(args, env={}, input=None):
	home = gs.home_path()
	# gocode should store its settings here
	nv = {
		'XDG_CONFIG_HOME': home,
	}
	nv.update(env)

	# until mg9 is in active use we'll fallback to existing gocode
	bin = GOCODE_BIN if os.path.exists(GOCODE_BIN) else 'gocode'
	cmd = gs.lst(bin, args)
	return gsshell.run(cmd, input=input, env=nv, cwd=home)

def gocode(args, env={}, input=None):
	last_propose = gs.attr('gocode.last_propose_builtins', False)
	propose = gs.setting('complete_builtins', False)
	if last_propose != propose:
		gs.set_attr('gocode.last_propose_builtins', propose)
		_gocode(['set', 'propose-builtins', 'true' if propose else 'false'])

	last_gopath = gs.attr('gocode.last_gopath')
	gopath = gs.getenv('GOPATH')
	if gopath and gopath != last_gopath:
		out, _, _ = gsshell.run(cmd=['go', 'env', 'GOOS', 'GOARCH'])
		vars = out.strip().split()
		if len(vars) == 2:
			gs.set_attr('gocode.last_gopath', gopath)
			libpath = []
			osarch = '_'.join(vars)
			for p in gopath.split(os.pathsep):
				if p:
					libpath.append(os.path.join(p, 'pkg', osarch))
			_gocode(['set', 'lib-path', os.pathsep.join(libpath)])

	return _gocode(args, env=env, input=input)

def do(method, arg, shell=False):
	maybe_install()

	header, _ = gs.json_encode({'method': method, 'token': 'mg9.call'})
	body, _ = gs.json_encode(arg)
	s = '%s %s' % (header, body)
	s = 'base64:%s' % base64.b64encode(s)
	out, err, _ = gsshell.run([MARGO9_BIN, '-do', s], stderr=None, shell=shell)
	res = {'error': err}

	if out:
		try:
			for ln in out.split('\n'):
				ln = ln.strip()
				if ln:
					r, err = gs.json_decode(ln, {})
					if err:
						res = {'error': 'Invalid response %s' % err}
					else:
						if r.get('token') == 'mg9.call':
							res = r.get('data') or {}
							if gs.is_a({}, res) and r.get('error'):
								r['error'] = res['error']
							return res
						res = {'error': 'Unexpected response %s' % r}
		except Exception:
			res = {'error': gs.traceback()}

	return res
<commit_msg>* impl mg9 ipc<commit_after>import gscommon as gs
import gsshell
import sublime
import threading
import os
import gsq
import time
import hashlib
import base64
import Queue
import uuid

DOMAIN = 'MarGo9'

# customization of, or user-owned gocode and margo will no longer be supported
# so we'll hardcode the relevant paths and refer to them directly instead of relying on PATH
MARGO9_SRC = gs.dist_path('margo9')
GOCODE_SRC = gs.dist_path('something_borrowed/gocode')
MARGO9_BIN = gs.home_path('bin', 'gosublime.margo9.exe')
GOCODE_BIN = gs.home_path('bin', 'gosublime.gocode.exe')

if not gs.checked(DOMAIN, '_vars'):
	_send_q = Queue.Queue()
	_recv_q = Queue.Queue()
	_stash = {}

def _sb(s):
	bdir = gs.home_path('bin')
	if s.startswith(bdir):
		s = '~bin%s' % (s[len(bdir):])
	return s

def _tp(s):
	return (_sb(s), ('ok' if os.path.exists(s) else 'missing'))

def _so(out, err, start, end):
	out = out.strip()
	err = err.strip()
	ok = not out and not err
	if ok:
		out = 'ok %0.3fs' % (end - start)
	else:
		out = '%s\n%s' % (out, err)
	return (out.strip(), ok)

def _run(cmd, cwd=None, shell=False):
	nv = {
		'GOBIN': '',
		'GOPATH': gs.dist_path('something_borrowed'),
	}
	return gsshell.run(cmd, shell=shell, cwd=cwd, env=nv)

def maybe_install():
	if not os.path.exists(MARGO9_BIN) or not os.path.exists(GOCODE_BIN):
		install('', True)

def install(aso_tokens, force_install):
	init_start = time.time()

	try:
		os.makedirs(gs.home_path('bin'))
	except:
		pass

	if not force_install and aso_tokens == _gen_tokens():
		m_out = 'no'
		g_out = 'no'
	else:
		start = time.time()
		m_out, err, _ = _run(['go', 'build', '-o', MARGO9_BIN], cwd=MARGO9_SRC)
		m_out, m_ok = _so(m_out, err, start, time.time())

		# on windows the file cannot be replaced if it's running so close gocode first.
		# in theory, mg9 has the same issue but since it's attached to a st2 instance,
		# the only way to close it is to close st2 (which we're presumably already doing)
		start = time.time()
		if os.path.exists(GOCODE_BIN):
			_run([GOCODE_BIN, 'close'])

		g_out, err, _ = _run(['go', 'build', '-o', GOCODE_BIN], cwd=GOCODE_SRC)
		g_out, g_ok = _so(g_out, err, start, time.time())

		if m_ok and g_ok:
			def f():
				gs.aso().set('mg9_install_tokens', _gen_tokens())
				gs.save_aso()

			sublime.set_timeout(f, 0)

	a = (
		'GoSublime init (%0.3fs)' % (time.time() - init_start),
		'| install margo9: %s' % m_out,
		'| install gocode: %s' % g_out,
		'|           ~bin: %s' % gs.home_path('bin'),
		'|         margo9: %s (%s)' % _tp(MARGO9_BIN),
		'|         gocode: %s (%s)' % _tp(GOCODE_BIN),
	)
	gs.println(*a)

def _fasthash(fn):
	try:
		with open(fn) as f:
			chunk = f.read(1024*8)
		st = os.stat(fn)
		return '%s:%d' % (hashlib.sha1(chunk).hexdigest(), st.st_size)
	except Exception:
		pass
	return ''

def _read(fn):
	s = ''
	try:
		with open(fn) as f:
			s = f.read()
	except Exception:
		pass
	return s

def _token(head, bin):
	head = _read(gs.dist_path(head))
	token = '%s %s' % (head.strip(), _fasthash(bin))
	return token

def _gen_tokens():
	return '%s\n%s' % (_token('margo9.head', MARGO9_BIN), _token('something_borrowed/gocode.head', GOCODE_BIN))

def do_init():
	aso_tokens = gs.aso().get('mg9_install_tokens', '')
	f = lambda: install(aso_tokens, False)
	gsq.do('GoSublime', f, msg='Installing MarGo9 and Gocode', set_status=True)

def _gocode(args, env={}, input=None):
	home = gs.home_path()
	# gocode should store its settings here
	nv = {
		'XDG_CONFIG_HOME': home,
	}
	nv.update(env)

	# until mg9 is in active use we'll fallback to existing gocode
	bin = GOCODE_BIN if os.path.exists(GOCODE_BIN) else 'gocode'
	cmd = gs.lst(bin, args)
	return gsshell.run(cmd, input=input, env=nv, cwd=home)

def gocode(args, env={}, input=None):
	last_propose = gs.attr('gocode.last_propose_builtins', False)
	propose = gs.setting('complete_builtins', False)
	if last_propose != propose:
		gs.set_attr('gocode.last_propose_builtins', propose)
		_gocode(['set', 'propose-builtins', 'true' if propose else 'false'])

	last_gopath = gs.attr('gocode.last_gopath')
	gopath = gs.getenv('GOPATH')
	if gopath and gopath != last_gopath:
		out, _, _ = gsshell.run(cmd=['go', 'env', 'GOOS', 'GOARCH'])
		vars = out.strip().split()
		if len(vars) == 2:
			gs.set_attr('gocode.last_gopath', gopath)
			libpath = []
			osarch = '_'.join(vars)
			for p in gopath.split(os.pathsep):
				if p:
					libpath.append(os.path.join(p, 'pkg', osarch))
			_gocode(['set', 'lib-path', os.pathsep.join(libpath)])

	return _gocode(args, env=env, input=input)

def acall(method, arg, cb):
	if not gs.checked(DOMAIN, 'launch _send'):
		gsq.launch(DOMAIN, _send)

	_send_q.put((method, arg, cb))

def bcall(method, arg, shell=False):
	maybe_install()

	header, _ = gs.json_encode({'method': method, 'token': 'mg9.call'})
	body, _ = gs.json_encode(arg)
	s = '%s %s' % (header, body)
	s = 'base64:%s' % base64.b64encode(s)
	out, err, _ = gsshell.run([MARGO9_BIN, '-do', s], stderr=None, shell=shell)
	res = {'error': err}

	if out:
		try:
			for ln in out.split('\n'):
				ln = ln.strip()
				if ln:
					r, err = gs.json_decode(ln, {})
					if err:
						res = {'error': 'Invalid response %s' % err}
					else:
						if r.get('token') == 'mg9.call':
							res = r.get('data') or {}
							if gs.is_a({}, res) and r.get('error'):
								r['error'] = res['error']
							return res
						res = {'error': 'Unexpected response %s' % r}
		except Exception:
			res = {'error': gs.traceback()}

	return res

def _recv():
	while True:
		try:
			ln = _recv_q.get()
			try:
				ln = ln.strip()
				if ln:
					r, _ = gs.json_decode(ln, {})
					token = r.get('token', '')
					f = _stash.get(token)
					if f:
						del _stash[token]
						f(r.get('data', {}), r.get('error', ''))
			except Exception:
				gs.println(gs.traceback())
		except Exception:
			gs.println(gs.traceback())
			break

def _send():
	while True:
		try:
			try:
				method, arg, cb = _send_q.get()

				proc = gs.attr('mg9.proc')
				if not proc or proc.poll() is not None:
					if proc:
						try:
							proc.kill()
							proc.stdout.close()
						except:
							pass

					maybe_install()

					if not gs.checked(DOMAIN, 'launch _recv'):
						gsq.launch(DOMAIN, _recv)

					# idea the env should be setup before-hand with a bcall
					# so we won't run this through the shell
					proc, _, err = gsshell.proc([MARGO9_BIN], shell=False, stderr=None, bufsize=1)
					gs.set_attr('mg9.proc', proc)

					if not proc:
						gs.notice(DOMAIN, 'Cannot start MarGo9: %s' % err)
						continue

					gsq.launch(DOMAIN, lambda: _read_stdout(proc))

				token = 'mg9.autoken.%s' % uuid.uuid4()
				_stash[token] = cb

				header, _ = gs.json_encode({'method': method, 'token': token})
				body, _ = gs.json_encode(arg)
				ln = '%s %s\n' % (header, body)
				proc.stdin.write(ln)
			except Exception:
				gs.println(gs.traceback())
		except Exception:
			gs.println(gs.traceback())
			break

def _read_stdout(proc):
	try:
		while True:
			ln = proc.stdout.readline()
			if not ln:
				break

			_recv_q.put(ln)
	except Exception:
		gs.println(gs.traceback())

		proc.stdout.close()
		proc.wait()
		proc = None

if not gs.checked(DOMAIN, 'do_init'):
	if gs.settings_obj().get('test_mg9_enabled') is True:
		sublime.set_timeout(do_init, 0)

<|endoftext|>


<commit_before>#!/usr/bin/python

import requests
import bs4

class Club:
	"""A MLS Club"""
	def __init__(self, name, abbreviation, subreddit):
		self.name = name
		self.abbreviation = abbreviation
		self.subreddit = subreddit

CHI = Club('Chicago Fire', 'CHI', '/r/chicagofire')
COL = Club('Colorado Rapids', 'COL', '/r/rapids')
CLB = Club('Columbus Crew SC', 'CLB', '/r/thremassive')
DC = Club('D.C. United', 'DC', '/r/dcunited')
FCD = Club('FC Dallas', 'DAL', '/r/fcdallas')
HOU = Club('Houston Dynamo', 'HOU', '/r/dynamo')
LAG = Club('LA Galaxy', 'LA', '/r/lagalaxy')
MTL = Club('Montreal Impact', 'MTL', '/r/montrealimpact')
NE = Club('New England Revolution', 'NE', '/r/newenglandrevolution')
NYC = Club('New York City FC', 'NYC', '/r/nycfc')
NYRB = Club('New York Red Bulls', 'NYRB', '/r/rbny')
OCSC = Club('Orlando City SC', 'OCSC', '/r/oclions')
PHI = Club('Philadelphia Union', 'PHI', '/r/phillyunion')
POR = Club('Portland Timbers', 'POR', '/r/timbers')
RSL = Club('Real Salt Lake', 'RSL', '/r/realsaltlake')
SJ = Club('San Jose Earthquakes', 'SJ', '/r/sjearthquakes')
SEA = Club('Seattle Sounders FC', 'SEA', '/r/soundersfc')
SKC = Club('Sporting Kansas City', 'SKC', '/r/sportingkc')
TOR = Club('Toronto FC', 'TFC', '/r/tfc')
VAN = Club('Vancouver Whitecaps FC', 'VAN', '/r/whitecapsfc')

ALL_CLUBS = {
	CHI, COL, CLB, DC, FCD, HOU, LAG, MTL, NE, NYC, NYRB, OCSC, PHI, POR, RSL,
	SJ, SEA, SKC, TOR, VAN
}

# Setup our variables for URLs and data to parse
standings_url = requests.get('http://www.mlssoccer.com/standings')
standing_soup = bs4.BeautifulSoup(standings_url.text)

# Western Conference is the 2nd table
standings = standing_soup.select('.stats-table')[1]
standings_data = standings.find('tbody').find_all('tr')

def strip_text(text):
	return text.get_text().strip();

def link_club(_club):
	abbr = all_clubs[_club][0]
	sub = all_clubs[_club][1]
	club = '[%s](%s)' % (abbr, sub)
	return club

def format_club(_club):
	club = strip_text(_club)
	club = link_club(club)
	return club

# Print the standings to stdout
for row in standings_data:
	cells = row.find_all('td')
	rank = strip_text(cells[0])
	club = format_club(cells[1].find('a')) # find 'a' to ignore playoff indicator
	points = '**%s**' % strip_text(cells[2])
	gp = strip_text(cells[3])
	gd = strip_text(cells[10])
	gf = strip_text(cells[8])
	print '%s|%s|%s|%s|%s|%s' % (rank, club, points, gp, gd, gf)
<commit_msg>Standings: Data for both Eastern and Western Conferences<commit_after>#!/usr/bin/python

import requests
import bs4

class Club:
	"""A MLS Club"""
	def __init__(self, name, abbreviation, subreddit):
		self.name = name
		self.abbreviation = abbreviation
		self.subreddit = subreddit

CHI = Club('Chicago Fire', 'CHI', '/r/chicagofire')
COL = Club('Colorado Rapids', 'COL', '/r/rapids')
CLB = Club('Columbus Crew SC', 'CLB', '/r/thremassive')
DC = Club('D.C. United', 'DC', '/r/dcunited')
FCD = Club('FC Dallas', 'DAL', '/r/fcdallas')
HOU = Club('Houston Dynamo', 'HOU', '/r/dynamo')
LAG = Club('LA Galaxy', 'LA', '/r/lagalaxy')
MTL = Club('Montreal Impact', 'MTL', '/r/montrealimpact')
NE = Club('New England Revolution', 'NE', '/r/newenglandrevolution')
NYC = Club('New York City FC', 'NYC', '/r/nycfc')
NYRB = Club('New York Red Bulls', 'NYRB', '/r/rbny')
OCSC = Club('Orlando City SC', 'OCSC', '/r/oclions')
PHI = Club('Philadelphia Union', 'PHI', '/r/phillyunion')
POR = Club('Portland Timbers', 'POR', '/r/timbers')
RSL = Club('Real Salt Lake', 'RSL', '/r/realsaltlake')
SJ = Club('San Jose Earthquakes', 'SJ', '/r/sjearthquakes')
SEA = Club('Seattle Sounders FC', 'SEA', '/r/soundersfc')
SKC = Club('Sporting Kansas City', 'SKC', '/r/sportingkc')
TOR = Club('Toronto FC', 'TFC', '/r/tfc')
VAN = Club('Vancouver Whitecaps FC', 'VAN', '/r/whitecapsfc')

ALL_CLUBS = {
	CHI, COL, CLB, DC, FCD, HOU, LAG, MTL, NE, NYC, NYRB, OCSC, PHI, POR, RSL,
	SJ, SEA, SKC, TOR, VAN
}

# Setup our variables for URLs and data to parse
standings_url = requests.get('http://www.mlssoccer.com/standings')
standing_soup = bs4.BeautifulSoup(standings_url.text)

# East Conference is the 1st table
eastern_table = standing_soup.select('.stats-table')[0]
eastern_data = eastern_table.find('tbody').find_all('tr')

# Western Conference is the 2nd table
western_table = standing_soup.select('.stats-table')[1]
western_data = western_table.find('tbody').find_all('tr')

def strip_text(text):
	return text.get_text().strip();

def link_club(_club):
	abbr = all_clubs[_club][0]
	sub = all_clubs[_club][1]
	club = '[%s](%s)' % (abbr, sub)
	return club

def format_club(_club):
	club = strip_text(_club)
	club = link_club(club)
	return club

# Print the standings to stdout
for row in standings_data:
	cells = row.find_all('td')
	rank = strip_text(cells[0])
	club = format_club(cells[1].find('a')) # find 'a' to ignore playoff indicator
	points = '**%s**' % strip_text(cells[2])
	gp = strip_text(cells[3])
	gd = strip_text(cells[10])
	gf = strip_text(cells[8])
	print '%s|%s|%s|%s|%s|%s' % (rank, club, points, gp, gd, gf)
<|endoftext|>


<commit_before>from __future__ import print_function

from tapetool.helpers import read_wav, db, rms
from tapetool.analysis import it_timeslice
from tapetool.filters import thd_for_1k

fs, data = read_wav("05-1905.wav")

reflevel = data[16*fs:26*fs]
thd_rl = 100. * rms(thd_for_1k(fs, reflevel)) / rms(reflevel)
print("THD at reference level:", thd_rl)

ramp = data[58*fs:68*fs]
sig = list(it_timeslice(fs, ramp, 0.1))
thd = list(it_timeslice(fs, thd_for_1k(fs, ramp), 0.1))

rl = db(rms(reflevel))
out = open('mol.dat', 'w')
for a, b in zip(sig, thd):
    t, l1 = a
    _, l2 = b
    x = db(l1) - rl
    k = 100. * l2 / l1
    print(*(t, ) + tuple(x) + tuple(k), file=out)
<commit_msg>changed MOL to use timeslice<commit_after>from __future__ import print_function

from tapetool.helpers import read_wav, db, rms
from tapetool.analysis import timeslice
from tapetool.filters import thd_for_1k

fs, data = read_wav("05-1905.wav")

reflevel = data[16*fs:26*fs]
thd_rl = 100. * rms(thd_for_1k(fs, reflevel)) / rms(reflevel)
print("THD at reference level:", thd_rl)

ramp = data[58*fs:68*fs]
sig_x, sig_y = timeslice(fs, ramp, 0.05)
_, thd_y = timeslice(fs, thd_for_1k(fs, ramp), 0.05)

rl = db(rms(reflevel))
with open('mol.dat', 'w') as out:
    for t, a, b in zip(sig_x, sig_y, thd_y):
        print(t, db(a) - rl[0], 100. * b / a, file=out)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import numpy as np

from keras.engine import Layer
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D
from keras import backend as K

img_rows, img_cols = 96, 96

class InformationDropout(Layer):
  """docstring for InformationDropout"""
  def __init__(self):
    self.supports_masking = True
    super(InformationDropout, self).__init__()

  def build(self, input_shape):
    ## TODO: このフェーズでパラメーターの登録をやる．下みたいなノリで
    # self.gamma = self.add_weight(shape,
    #                                  initializer=self.gamma_init,
    #                                  regularizer=self.gamma_regularizer,
    #                                  name='{}_gamma'.format(self.name))
    pass

  def call(self, x, mask=None):
    ## TODO: このフェーズで入力されるxをどのように変換するかについてを記述する
    # if 0. < self.p < 1.:
    #     noise_shape = self._get_noise_shape(x)
    #     x = K.in_train_phase(K.dropout(x, self.p, noise_shape), x)
    # return x

    return x

  def get_config(self):
    ## TODO: このフェーズでパラメーターを辞書として登録する？
    # config = {'p': self.p}
    config = {}
    base_config = super(InformationDropout, self).get_config()
    return dict(list(base_config.items()) + list(config.items()))

def get_model():
  ## TODO: dropoutで書いているところをinformation dropoutで置き換える

  kernel_size = (3, 3)
  nb_filters = [32, 64, 96, 192, 10]

  if K.image_dim_ordering() == 'th':
    input_shape = (1, img_rows, img_cols)
  else:
    input_shape = (img_rows, img_cols, 1)

  model = Sequential()

  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          input_shape=input_shape))
  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], 1, 1,
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[4], 1, 1,
                          border_mode='same',
                          activation='relu'))
  ## TODO: spatial averageというのが何なのかよくわからん
  model.add(Flatten())
  model.add(Activation('softmax'))

  return model

  # model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
  #                         border_mode='valid',
  #                         input_shape=input_shape))
  # model.add(Activation('relu'))
  # model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
  # model.add(Activation('relu'))
  # model.add(MaxPooling2D(pool_size=pool_size))
  # model.add(Dropout(0.25))

  # model.add(Flatten())
  # model.add(Dense(128))
  # model.add(Activation('relu'))
  # model.add(Dropout(0.5))
  # model.add(Dense(nb_classes))
  # model.add(Activation('softmax'))
  # return model

if __name__ == '__main__':
  # information_dropout = InformationDropout()
  # print(information_dropout.get_config())
  model = get_model()
  print(model.summary())
<commit_msg>call function of InformationDropout<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import numpy as np

from keras.engine import Layer
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D
from keras import backend as K

img_rows, img_cols = 96, 96

class InformationDropout(Layer):
  """docstring for InformationDropout"""
  def __init__(self):
    self.supports_masking = True
    super(InformationDropout, self).__init__()

  def build(self, input_shape):
    ## TODO: このフェーズでパラメーターの登録をやる．下みたいなノリで
    # self.gamma = self.add_weight(shape,
    #                                  initializer=self.gamma_init,
    #                                  regularizer=self.gamma_regularizer,
    #                                  name='{}_gamma'.format(self.name))
    pass

  def call(self, inputs, mask=None):
    ## TODO: このフェーズで入力されるxをどのように変換するかについてを記述する
    # if 0. < self.p < 1.:
    #     noise_shape = self._get_noise_shape(x)
    #     x = K.in_train_phase(K.dropout(x, self.p, noise_shape), x)
    # return x
    x, logalpha = inputs
    noise_x = x * K.random_normal(shape=K.shape(x), mean=0.0,
                                  std=K.exp(logalpha))
    # return K.in_train_phase(noise_x, x)
    return noise_x

  def get_config(self):
    ## TODO: このフェーズでパラメーターを辞書として登録する？
    # config = {'p': self.p}
    config = {}
    base_config = super(InformationDropout, self).get_config()
    return dict(list(base_config.items()) + list(config.items()))

def get_model():
  ## TODO: dropoutで書いているところをinformation dropoutで置き換える

  kernel_size = (3, 3)
  nb_filters = [32, 64, 96, 192, 10]

  if K.image_dim_ordering() == 'th':
    input_shape = (1, img_rows, img_cols)
  else:
    input_shape = (img_rows, img_cols, 1)

  model = Sequential()

  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          input_shape=input_shape))
  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[0], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[1], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[2], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu',
                          subsample=(2, 2)))
  model.add(Dropout(0.5))

  model.add(Convolution2D(nb_filters[3], kernel_size[0], kernel_size[1],
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[3], 1, 1,
                          border_mode='same',
                          activation='relu'))
  model.add(Convolution2D(nb_filters[4], 1, 1,
                          border_mode='same',
                          activation='relu'))
  ## TODO: spatial averageというのが何なのかよくわからん
  model.add(Flatten())
  model.add(Activation('softmax'))

  return model

  # model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
  #                         border_mode='valid',
  #                         input_shape=input_shape))
  # model.add(Activation('relu'))
  # model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
  # model.add(Activation('relu'))
  # model.add(MaxPooling2D(pool_size=pool_size))
  # model.add(Dropout(0.25))

  # model.add(Flatten())
  # model.add(Dense(128))
  # model.add(Activation('relu'))
  # model.add(Dropout(0.5))
  # model.add(Dense(nb_classes))
  # model.add(Activation('softmax'))
  # return model

if __name__ == '__main__':
  # information_dropout = InformationDropout()
  # print(information_dropout.get_config())
  model = get_model()
  print(model.summary())
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division

import numpy as np
from numpy.lib import scimath as sm
import scipy.optimize as opt

import astropy.cosmology
from astropy import units as u

def arcsec(z):
    """Compute the inverse sec of the complex number z."""
    val1 = 1j / z
    val2 = sm.sqrt(1 - 1/z**2)
    val = 1j * np.log(val2 + val1)
    return 0.5 * np.pi + val

class NFW(object):
    """Compute properties of an NFW halo.

    This class implements the Navarro-Frenk-White [1]_ halo density
    profile in dependence on halo properties such as mass and
    concentration, and cosmology.

    Parameters:
    -----------
    size : float or astropy.quantity.Quantity
        Either the halo radius or mass (default), which is assumed is
        specified by `size_type` Float inputs are assumed to be either
        in Mpc (radius) or solar masses (mass).
    c : float
        Halo concentration parameter.
    z : float
        Halo redshift.
    size_type : {"radius", "mass"}, optional
        Specifies whether `size` is the halo radius or mass.
    overdensity : float, optional
        The overdensity factor above the critical or mean density of
        the Universe at which mass or radius are computed (the default
        is 200).
    overdensity_type : {"critical", "mean"}, optional
         Specifies whether overdensities are computed with respect to
         the critical or mean density of the Universe (default is
         "critical").
    cosmology : astropy.cosmology, optional
         The cosmological background model of the NFW halo. Defaults
         to `None`, in which case the current cosmology at the time a
         method is called is used. See Notes below for details.

    Attributes:
    -----------
    var_cosmology
    overdensity_type : {"critical", "mean"}
        The type of overdensity.
    overdensity : float
        The overdensity with respect to the mean/critical density.
    cosmology : astropy.cosmology
        The background cosmology for this halo.
    rho_c : astropy.quantity.Quantity
        The critical density at halo redshift `z`.
    r_Delta : astropy.quantity.Quantity
        The halo radius at the current `overdensity`.
    r_s : astropy.quantity.Quantity
        The halo scale radius.
    c : float
        NFW concentration parameter.
    z : float
        Halo redshift.
    delta_c : float
        Characteristic overdensity.

    Returns:
    --------
    object
        An instance describing the NFW halo.

    Notes:
    ------
    Quantities like the halo concentration, mass, critical density
    etc. are defined with respect to a certain background cosmology
    and for characteristic overdensities with respect to this
    cosmology and halo redshift. Updating the class attributes that
    define these dependencies will update all attributes that are
    affected by this update. For example, if the overdensity factor is
    changed from 200 to 500, the concentration parameter `c` will
    change its value from c200 = rs / r200 to c = rs / r500. Note that
    the scale radius does not depend on cosmolgy or overdensity
    choice.

    Note specifically, that if no cosmology is passed at
    instantiation, the current cosmology will always be used. If the
    current astropy cosmology changes after instantiation, the NFW
    instance will follow this change. If you want to explicitly keep
    the NFW instance fixed at the current cosmology *at instantiation*
    you should pass
    `cosmology=astropy.cosmology.default_cosmology.get()` to
    __init__().

    References:
    -----------
    [1] Navarro, Julio F.  Frenk, Carlos S.  White, Simon D. M., "A
    Universal Density Profile from Hierarchical Clustering", The
    Astrophysical Journal vol. 490, pp. 493-508, 1997

    """

    def __init__(self, size, c, z, size_type="mass",
                 overdensity=200, overdensity_type="critical",
                 cosmology=None):

        if overdensity_type not in ['critical', 'mean']:
            raise ValueError("overdensity_type must be one of 'mean', "
                             "'critical'")
        self._overdensity_type = overdensity_type

        if size_type not in ['mass', 'radius']:
            raise ValueError("size_type must be one of 'mass', 'radius'")
        self._size_type = size_type

        self._c = float(c)
        self._z = float(z)
        self._overdensity = overdensity
        if cosmology is not None:
            self._cosmology = cosmology
            self._var_cosmology = False
        else:
            self._cosmology = astropy.cosmology.default_cosmology.get()
            self._var_cosmology = True

        if size_type == "mass":
            self._size = u.Quantity(size, u.solMass)
        else:
            self._size = u.Quantity(size, u.Mpc)

        self._rho_c = None
        self._r_s = None
        self._r_Delta = None
        self._update_new_cosmology()

        return

    def _update_required(self):
        """
        Check whether the instance needs updating due to a new cosmology
        """
        if not self._var_cosmology:
            return False
        if self._cosmology == astropy.cosmology.default_cosmology.get():
            return False
        return True

    def _update_new_cosmology(self):
        if self._var_cosmology:
            self._cosmology = astropy.cosmology.default_cosmology.get()
        self._update_rho_c()
        self._update_r_Delta()
        self._update_r_s()
        return

    def _update_rho_c(self):
        self._rho_c = self._cosmology.critical_density(self.z)
        self._rho_c = self._rho_c.to(u.solMass / u.Mpc**3)
        return

    def _update_r_Delta(self):
        if self._size_type == "mass":
            if self._overdensity_type == 'critical':
                rho = self._rho_c
            else:
                rho = self._rho_c * self._cosmology.Om(self.z)

            self._r_Delta = (3 * self._size / (4*np.pi)
                             * 1 / (self._overdensity*rho))**(1/3)
        else:
            self._r_Delta = self._size
        return

    def _update_r_s(self):
        self._r_s = self.r_Delta / self.c
        return

    @property
    def var_cosmology(self):
        """Does the cosmology change with the current cosmology?

        Returns:
        --------
        var_cosmology : bool
            True if the computations are computed for the current
            astropy cosmology, False if the cosmology specified at
            instantiation is used.
        """
        return self._var_cosmology

    @property
    def overdensity_type(self):
        return self._overdensity_type

    @property
    def overdensity(self):
        return self._overdensity

    @property
    def cosmology(self):
        """The cosmology used by this halo."""
        if self._update_required():
            self._update_new_cosmology()
        return self._cosmology

    @property
    def rho_c(self):
        """Critical density at halo redshift
        """
        if self._update_required():
            self._update_new_cosmology()
        return self._rho_c

    @property
    def r_Delta(self):
        if self._update_required():
            self._update_new_cosmology()
        return self._r_Delta

    @property
    def r_s(self):
        """Scale radius
        """
        if self._update_required():
            self._update_new_cosmology()
        return self._r_s

    @property
    def c(self):
        """Halo concentration"""
        return self._c

    @property
    def z(self):
        """Halo redshift"""
        return self._z

    @property
    def delta_c(self):
        """Characteristic overdensity
        """
        return self._overdensity/3 * self.c**3 / (np.log(1 + self.c)
                                                  - self.c/(1. + self.c))

    def concentration(self, overdensity=None, overdensity_type=None):
        """Compute halo concentration at an overdensity.

        Parameters:
        -----------
        overdensity : float, optional
            The overdensity with respect to the mean/critical density
            at which the halo concentration is computed. Defaults to
            `None`, in which case the value of `overdensity` instance
            attribute is used.
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the `overdensity` factor is with respect
            to the critical or mean density of the Universe. Defaults
            to `None`, in which case the value of the
            `overdensity_type` instance attribute is used.

        Returns:
        --------
        c : float
            Halo concentration parameter.

        Notes:
        ------
        If both `overdensity` and `overdensity_type` are `None`, the
        return value is identical to the precomputed value of the
        instance attribute `c`.
        """
        if overdensity is None and overdensity_type is None:
            return self.c
        overdensity = overdensity
        overdensity_type = overdensity_type
        return self.radius_Delta(overdensity, overdensity_type) / self.r_s

    def __str__(self):
        prop_str = "NFW halo with concentration %.2g at redshift %.2f:\n\n" \
                   % (self.c, self.z,)
        for delta in (200, 500, 2500):
            prop_str += "M_%d = %.2e M_sun\tr_%d = %.2g Mpc\n" % \
                        (delta, self.mass_Delta(delta).value, delta,
                         self.radius_Delta(delta).value)
        return prop_str

    def __repr__(self):
        return self.__str__()

    def _mean_density_zero(self, r, Delta, overdensity_type=None):
        # make sure all functions calling this one set overdensity_type
        # appropriately
        assert (overdensity_type is not None)
        if overdensity_type == 'critical':
            rho = self.rho_c
        else:
            rho = self.rho_c * self.cosmology.Om(self.z)
        return (self.mean_density(r) - Delta * rho).value

    def radius_Delta(self, overdensity, overdensity_type=None):
        """Compute the radius which contains a given overdensity.

        Paramters:
        ----------
        overdensity : float
            Overdensity factor with respect to the critical/mean density
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the overdensity factor is with respect
            to the critical or mean density of the Universe. Default
            is `None`, in which case the value of the attribute
            `overdensity_type` is used.

        Returns:
        --------
        radius : astropy.quantity.Quantity
            Radius inside which the average halo density is
            `overdensity` times the critical/mean density of the
            Universe.
        """
        if overdensity_type is None:
            overdensity_type = self._overdensity_type
        x0 = opt.brentq(self._mean_density_zero, 1e-6, 10,
                        args=(overdensity, overdensity_type))
        return x0 * u.Mpc

    def mass_Delta(self, overdensity, overdensity_type=None):
        """Compute the mass inside radius which contains a given overdensity.

        Parameters:
        -----------
        overdensity : float
            Overdensity factor with respect to the critical/mean density
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the overdensity factor is with respect
            to the critical or mean density of the Universe. Default
            is `None`, in which case the value of the attribute
            `overdensity_type` is used.

        Returns:
        --------
        mass : astropy.quantity.Quantity
            Halo mass inside the radius which contains an overdensity
            of factor `overdensity` with respect to the critical/mean
            density.
        """
        if overdensity_type is None:
            overdensity_type = self._overdensity_type
        r = self.radius_Delta(overdensity, overdensity_type)
        return self.mass(r)

    def density(self, r):
        """Compute the density radius r from the center of the halo.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        rho : astropy.quantity.Quantity
            Density of the NFW halo at a distance `r` from the halo center.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return self.rho_c * self.delta_c / (x * (1 + x)**2)

    def mean_density(self, r):
        """Compute the mean density inside a radius.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        rho_bar : astropy.quantity.Quantity
            Mean density of the NFW halo inside the radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return 3 * (1 / x)**3 * self.delta_c * self.rho_c \
            * (np.log((1 + x)) - x / (1 + x))

    def mass(self, r):
        """Compute the mass inside radius r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        mass : astropy.quantity.Quantity
            Mass contained within the radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return 4 * np.pi * self.delta_c * self.rho_c * self.r_s**3 \
            * (np.log((1 + x)) - x / (1 + x))

    def projected_mass(self, r):
        """Compute the projected mass of the NFW profile inside a cylinder.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Radius of the cylinder. If the argument is float, Mpc are assumed.

        Returns:
        --------
        m_proj: astropy.quantity.Quantity
            Projected mass in the cylinder of radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = (r / self.r_s).value
        fc = np.log(1 + self.c) - self.c / (1 + self.c)
        f = (arcsec(x) / sm.sqrt(x**2 - 1)).real
        m_proj = self.mass_Delta(self._overdensity) / fc * (np.log(x / 2) + f)
        return m_proj

    def sigma(self, r):
        """Compute the surface mass density distance r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Projected distance from the halo center. If the argument is
            float, Mpc are assumed.

        Returns:
        --------
        sigma : astropy.quantity.Quantity
            Surface mass density of the halo at projected distance `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        val1 = 1 / (x**2 - 1)
        val2 = (arcsec(x) / (sm.sqrt(x**2 - 1))**3).real
        return 2 * self.r_s * self.rho_c * self.delta_c * (val1 - val2)

    def delta_sigma(self, r):
        """Compute the surface mass density minus its mean at radius r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Projected distance from the halo cener. If the argument is
            float, Mpc are assumed.

        Returns:
        --------
        delta_sigma : astropy.quantity.Quantity
            Surface mass density at distance `r` minus the mean
            surface mass density inside the same radius.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        fac = 2 * self.r_s * self.rho_c * self.delta_c
        val1 = 1 / (1 - x**2)
        num = ((3 * x**2) - 2) * arcsec(x)
        div = x**2 * (sm.sqrt(x**2 - 1))**3
        val2 = (num / div).real
        val3 = 2 * np.log(x / 2) / x**2
        return fac * (val1+val2+val3)
<commit_msg>remove __str__ and __repr__, weren't well thought through<commit_after>#!/usr/bin/env python

from __future__ import division

import numpy as np
from numpy.lib import scimath as sm
import scipy.optimize as opt

import astropy.cosmology
from astropy import units as u

def arcsec(z):
    """Compute the inverse sec of the complex number z."""
    val1 = 1j / z
    val2 = sm.sqrt(1 - 1/z**2)
    val = 1j * np.log(val2 + val1)
    return 0.5 * np.pi + val

class NFW(object):
    """Compute properties of an NFW halo.

    This class implements the Navarro-Frenk-White [1]_ halo density
    profile in dependence on halo properties such as mass and
    concentration, and cosmology.

    Parameters:
    -----------
    size : float or astropy.quantity.Quantity
        Either the halo radius or mass (default), which is assumed is
        specified by `size_type` Float inputs are assumed to be either
        in Mpc (radius) or solar masses (mass).
    c : float
        Halo concentration parameter.
    z : float
        Halo redshift.
    size_type : {"radius", "mass"}, optional
        Specifies whether `size` is the halo radius or mass.
    overdensity : float, optional
        The overdensity factor above the critical or mean density of
        the Universe at which mass or radius are computed (the default
        is 200).
    overdensity_type : {"critical", "mean"}, optional
         Specifies whether overdensities are computed with respect to
         the critical or mean density of the Universe (default is
         "critical").
    cosmology : astropy.cosmology, optional
         The cosmological background model of the NFW halo. Defaults
         to `None`, in which case the current cosmology at the time a
         method is called is used. See Notes below for details.

    Attributes:
    -----------
    var_cosmology
    overdensity_type : {"critical", "mean"}
        The type of overdensity.
    overdensity : float
        The overdensity with respect to the mean/critical density.
    cosmology : astropy.cosmology
        The background cosmology for this halo.
    rho_c : astropy.quantity.Quantity
        The critical density at halo redshift `z`.
    r_Delta : astropy.quantity.Quantity
        The halo radius at the current `overdensity`.
    r_s : astropy.quantity.Quantity
        The halo scale radius.
    c : float
        NFW concentration parameter.
    z : float
        Halo redshift.
    delta_c : float
        Characteristic overdensity.

    Returns:
    --------
    object
        An instance describing the NFW halo.

    Notes:
    ------
    Quantities like the halo concentration, mass, critical density
    etc. are defined with respect to a certain background cosmology
    and for characteristic overdensities with respect to this
    cosmology and halo redshift. Updating the class attributes that
    define these dependencies will update all attributes that are
    affected by this update. For example, if the overdensity factor is
    changed from 200 to 500, the concentration parameter `c` will
    change its value from c200 = rs / r200 to c = rs / r500. Note that
    the scale radius does not depend on cosmolgy or overdensity
    choice.

    Note specifically, that if no cosmology is passed at
    instantiation, the current cosmology will always be used. If the
    current astropy cosmology changes after instantiation, the NFW
    instance will follow this change. If you want to explicitly keep
    the NFW instance fixed at the current cosmology *at instantiation*
    you should pass
    `cosmology=astropy.cosmology.default_cosmology.get()` to
    __init__().

    References:
    -----------
    [1] Navarro, Julio F.  Frenk, Carlos S.  White, Simon D. M., "A
    Universal Density Profile from Hierarchical Clustering", The
    Astrophysical Journal vol. 490, pp. 493-508, 1997

    """

    def __init__(self, size, c, z, size_type="mass",
                 overdensity=200, overdensity_type="critical",
                 cosmology=None):

        if overdensity_type not in ['critical', 'mean']:
            raise ValueError("overdensity_type must be one of 'mean', "
                             "'critical'")
        self._overdensity_type = overdensity_type

        if size_type not in ['mass', 'radius']:
            raise ValueError("size_type must be one of 'mass', 'radius'")
        self._size_type = size_type

        self._c = float(c)
        self._z = float(z)
        self._overdensity = overdensity
        if cosmology is not None:
            self._cosmology = cosmology
            self._var_cosmology = False
        else:
            self._cosmology = astropy.cosmology.default_cosmology.get()
            self._var_cosmology = True

        if size_type == "mass":
            self._size = u.Quantity(size, u.solMass)
        else:
            self._size = u.Quantity(size, u.Mpc)

        self._rho_c = None
        self._r_s = None
        self._r_Delta = None
        self._update_new_cosmology()

        return

    def _update_required(self):
        """
        Check whether the instance needs updating due to a new cosmology
        """
        if not self._var_cosmology:
            return False
        if self._cosmology == astropy.cosmology.default_cosmology.get():
            return False
        return True

    def _update_new_cosmology(self):
        if self._var_cosmology:
            self._cosmology = astropy.cosmology.default_cosmology.get()
        self._update_rho_c()
        self._update_r_Delta()
        self._update_r_s()
        return

    def _update_rho_c(self):
        self._rho_c = self._cosmology.critical_density(self.z)
        self._rho_c = self._rho_c.to(u.solMass / u.Mpc**3)
        return

    def _update_r_Delta(self):
        if self._size_type == "mass":
            if self._overdensity_type == 'critical':
                rho = self._rho_c
            else:
                rho = self._rho_c * self._cosmology.Om(self.z)

            self._r_Delta = (3 * self._size / (4*np.pi)
                             * 1 / (self._overdensity*rho))**(1/3)
        else:
            self._r_Delta = self._size
        return

    def _update_r_s(self):
        self._r_s = self.r_Delta / self.c
        return

    @property
    def var_cosmology(self):
        """Does the cosmology change with the current cosmology?

        Returns:
        --------
        var_cosmology : bool
            True if the computations are computed for the current
            astropy cosmology, False if the cosmology specified at
            instantiation is used.
        """
        return self._var_cosmology

    @property
    def overdensity_type(self):
        return self._overdensity_type

    @property
    def overdensity(self):
        return self._overdensity

    @property
    def cosmology(self):
        """The cosmology used by this halo."""
        if self._update_required():
            self._update_new_cosmology()
        return self._cosmology

    @property
    def rho_c(self):
        """Critical density at halo redshift
        """
        if self._update_required():
            self._update_new_cosmology()
        return self._rho_c

    @property
    def r_Delta(self):
        if self._update_required():
            self._update_new_cosmology()
        return self._r_Delta

    @property
    def r_s(self):
        """Scale radius
        """
        if self._update_required():
            self._update_new_cosmology()
        return self._r_s

    @property
    def c(self):
        """Halo concentration"""
        return self._c

    @property
    def z(self):
        """Halo redshift"""
        return self._z

    @property
    def delta_c(self):
        """Characteristic overdensity
        """
        return self._overdensity/3 * self.c**3 / (np.log(1 + self.c)
                                                  - self.c/(1. + self.c))

    def concentration(self, overdensity=None, overdensity_type=None):
        """Compute halo concentration at an overdensity.

        Parameters:
        -----------
        overdensity : float, optional
            The overdensity with respect to the mean/critical density
            at which the halo concentration is computed. Defaults to
            `None`, in which case the value of `overdensity` instance
            attribute is used.
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the `overdensity` factor is with respect
            to the critical or mean density of the Universe. Defaults
            to `None`, in which case the value of the
            `overdensity_type` instance attribute is used.

        Returns:
        --------
        c : float
            Halo concentration parameter.

        Notes:
        ------
        If both `overdensity` and `overdensity_type` are `None`, the
        return value is identical to the precomputed value of the
        instance attribute `c`.
        """
        if overdensity is None and overdensity_type is None:
            return self.c
        overdensity = overdensity
        overdensity_type = overdensity_type
        return self.radius_Delta(overdensity, overdensity_type) / self.r_s

    def _mean_density_zero(self, r, Delta, overdensity_type=None):
        # make sure all functions calling this one set overdensity_type
        # appropriately
        assert (overdensity_type is not None)
        if overdensity_type == 'critical':
            rho = self.rho_c
        else:
            rho = self.rho_c * self.cosmology.Om(self.z)
        return (self.mean_density(r) - Delta * rho).value

    def radius_Delta(self, overdensity, overdensity_type=None):
        """Compute the radius which contains a given overdensity.

        Paramters:
        ----------
        overdensity : float
            Overdensity factor with respect to the critical/mean density
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the overdensity factor is with respect
            to the critical or mean density of the Universe. Default
            is `None`, in which case the value of the attribute
            `overdensity_type` is used.

        Returns:
        --------
        radius : astropy.quantity.Quantity
            Radius inside which the average halo density is
            `overdensity` times the critical/mean density of the
            Universe.
        """
        if overdensity_type is None:
            overdensity_type = self._overdensity_type
        x0 = opt.brentq(self._mean_density_zero, 1e-6, 10,
                        args=(overdensity, overdensity_type))
        return x0 * u.Mpc

    def mass_Delta(self, overdensity, overdensity_type=None):
        """Compute the mass inside radius which contains a given overdensity.

        Parameters:
        -----------
        overdensity : float
            Overdensity factor with respect to the critical/mean density
        overdensity_type : {"critical", "mean"}, optional
            Specifies whether the overdensity factor is with respect
            to the critical or mean density of the Universe. Default
            is `None`, in which case the value of the attribute
            `overdensity_type` is used.

        Returns:
        --------
        mass : astropy.quantity.Quantity
            Halo mass inside the radius which contains an overdensity
            of factor `overdensity` with respect to the critical/mean
            density.
        """
        if overdensity_type is None:
            overdensity_type = self._overdensity_type
        r = self.radius_Delta(overdensity, overdensity_type)
        return self.mass(r)

    def density(self, r):
        """Compute the density radius r from the center of the halo.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        rho : astropy.quantity.Quantity
            Density of the NFW halo at a distance `r` from the halo center.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return self.rho_c * self.delta_c / (x * (1 + x)**2)

    def mean_density(self, r):
        """Compute the mean density inside a radius.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        rho_bar : astropy.quantity.Quantity
            Mean density of the NFW halo inside the radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return 3 * (1 / x)**3 * self.delta_c * self.rho_c \
            * (np.log((1 + x)) - x / (1 + x))

    def mass(self, r):
        """Compute the mass inside radius r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Distance from the halo center. If the argument is float,
            Mpc are assumed.

        Returns:
        --------
        mass : astropy.quantity.Quantity
            Mass contained within the radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        return 4 * np.pi * self.delta_c * self.rho_c * self.r_s**3 \
            * (np.log((1 + x)) - x / (1 + x))

    def projected_mass(self, r):
        """Compute the projected mass of the NFW profile inside a cylinder.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Radius of the cylinder. If the argument is float, Mpc are assumed.

        Returns:
        --------
        m_proj: astropy.quantity.Quantity
            Projected mass in the cylinder of radius `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = (r / self.r_s).value
        fc = np.log(1 + self.c) - self.c / (1 + self.c)
        f = (arcsec(x) / sm.sqrt(x**2 - 1)).real
        m_proj = self.mass_Delta(self._overdensity) / fc * (np.log(x / 2) + f)
        return m_proj

    def sigma(self, r):
        """Compute the surface mass density distance r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Projected distance from the halo center. If the argument is
            float, Mpc are assumed.

        Returns:
        --------
        sigma : astropy.quantity.Quantity
            Surface mass density of the halo at projected distance `r`.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        val1 = 1 / (x**2 - 1)
        val2 = (arcsec(x) / (sm.sqrt(x**2 - 1))**3).real
        return 2 * self.r_s * self.rho_c * self.delta_c * (val1 - val2)

    def delta_sigma(self, r):
        """Compute the surface mass density minus its mean at radius r.

        Parameters:
        -----------
        r : float or astropy.quantity.Quantity
            Projected distance from the halo cener. If the argument is
            float, Mpc are assumed.

        Returns:
        --------
        delta_sigma : astropy.quantity.Quantity
            Surface mass density at distance `r` minus the mean
            surface mass density inside the same radius.
        """
        r = u.Quantity(r, u.Mpc)
        x = r / self.r_s
        fac = 2 * self.r_s * self.rho_c * self.delta_c
        val1 = 1 / (1 - x**2)
        num = ((3 * x**2) - 2) * arcsec(x)
        div = x**2 * (sm.sqrt(x**2 - 1))**3
        val2 = (num / div).real
        val3 = 2 * np.log(x / 2) / x**2
        return fac * (val1+val2+val3)
<|endoftext|>


<commit_before># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox

LOCAL_DEPS = ('../core/',)

@nox.session
@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
def unit_tests(session, python_version):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run('py.test', '--quiet',
        '--cov=google.cloud.datastore', '--cov=tests.unit', '--cov-append',
        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
        'tests/unit',
    )

@nox.session
@nox.parametrize('python_version', ['2.7', '3.6'])
def system_tests(session, python_version):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        return

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/system')

@nox.session
def doctests(session):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        return

    # Doctests run against Python 3.6 only.
    # It is difficult to make doctests run against both Python 2 and Python 3
    # because they test string output equivalence, which is difficult to
    # make match (e.g. unicode literals starting with "u").
    session.interpreter = 'python3.6'

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', 'sphinx', *LOCAL_DEPS)
    session.install('../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/doctests.py')

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', 'pylint', 'gcp-devrel-py-tools', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google/cloud/datastore')
    session.run(
        'gcp-devrel-py-tools', 'run-pylint',
        '--config', 'pylint.config.py',
        '--library-filesets', 'google',
        '--test-filesets', 'tests',
        # Temporarily allow this to fail.
        success_codes=range(0, 100))

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'
    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<commit_msg>Skipping system tests when credentials env. var is unset. (#3475)<commit_after># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox

LOCAL_DEPS = ('../core/',)

@nox.session
@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
def unit_tests(session, python_version):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run('py.test', '--quiet',
        '--cov=google.cloud.datastore', '--cov=tests.unit', '--cov-append',
        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
        'tests/unit',
    )

@nox.session
@nox.parametrize('python_version', ['2.7', '3.6'])
def system_tests(session, python_version):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        session.skip('Credentials must be set via environment variable.')

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/system')

@nox.session
def doctests(session):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        session.skip('Credentials must be set via environment variable.')

    # Doctests run against Python 3.6 only.
    # It is difficult to make doctests run against both Python 2 and Python 3
    # because they test string output equivalence, which is difficult to
    # make match (e.g. unicode literals starting with "u").
    session.interpreter = 'python3.6'

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', 'sphinx', *LOCAL_DEPS)
    session.install('../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/doctests.py')

@nox.session
def lint(session):
    """Run linters.

    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', 'pylint', 'gcp-devrel-py-tools', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google/cloud/datastore')
    session.run(
        'gcp-devrel-py-tools', 'run-pylint',
        '--config', 'pylint.config.py',
        '--library-filesets', 'google',
        '--test-filesets', 'tests',
        # Temporarily allow this to fail.
        success_codes=range(0, 100))

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'
    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<|endoftext|>


<commit_before># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox

LOCAL_DEPS = ('../core/',)

@nox.session
@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
def unit_tests(session, python_version):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run('py.test', '--quiet',
        '--cov=google.cloud.bigquery', '--cov=tests.unit', '--cov-append',
        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
        'tests/unit',
    )

@nox.session
@nox.parametrize('python_version', ['2.7', '3.6'])
def system_tests(session, python_version):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        return

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../storage/', '../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/system.py')

@nox.session
def lint(session):
    """Run flake8.

    Returns a failure if flake8 finds linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google/cloud/bigquery')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<commit_msg>Adding check that **all** setup.py README's are valid RST. (#3318)<commit_after># Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

import nox

LOCAL_DEPS = ('../core/',)

@nox.session
@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
def unit_tests(session, python_version):
    """Run the unit test suite."""

    # Run unit tests against all supported versions of Python.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package in-place.
    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
    session.install('-e', '.')

    # Run py.test against the unit tests.
    session.run('py.test', '--quiet',
        '--cov=google.cloud.bigquery', '--cov=tests.unit', '--cov-append',
        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
        'tests/unit',
    )

@nox.session
@nox.parametrize('python_version', ['2.7', '3.6'])
def system_tests(session, python_version):
    """Run the system test suite."""

    # Sanity check: Only run system tests if the environment variable is set.
    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
        return

    # Run the system tests against latest Python 2 and Python 3 only.
    session.interpreter = 'python{}'.format(python_version)

    # Install all test dependencies, then install this package into the
    # virutalenv's dist-packages.
    session.install('mock', 'pytest', *LOCAL_DEPS)
    session.install('../storage/', '../test_utils/')
    session.install('.')

    # Run py.test against the system tests.
    session.run('py.test', '--quiet', 'tests/system.py')

@nox.session
def lint(session):
    """Run flake8.

    Returns a failure if flake8 finds linting errors or sufficiently
    serious code quality issues.
    """
    session.interpreter = 'python3.6'
    session.install('flake8', *LOCAL_DEPS)
    session.install('.')
    session.run('flake8', 'google/cloud/bigquery')

@nox.session
def lint_setup_py(session):
    """Verify that setup.py is valid (including RST check)."""
    session.interpreter = 'python3.6'
    session.install('docutils', 'Pygments')
    session.run(
        'python', 'setup.py', 'check', '--restructuredtext', '--strict')

@nox.session
def cover(session):
    """Run the final coverage report.

    This outputs the coverage report aggregating coverage from the unit
    test runs (not system test runs), and then erases coverage data.
    """
    session.interpreter = 'python3.6'
    session.install('coverage', 'pytest-cov')
    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
    session.run('coverage', 'erase')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8
#
# Author: R.F. Smith <rsmith@xs4all.nl>
# Created: 2014-02-10 21:43:11 +0100
# Modified: $Date$
#
# To the extent possible under law, R.F. Smith has waived all copyright and
# related or neighboring rights to old.py. This work is published
# from the Netherlands. See http://creativecommons.org/publicdomain/zero/1.0/

"""Renames a directory by prefixing the name with 'old-'."""

from __future__ import print_function, division  # 2 and 3 compatibility.

import sys
import os

__version__ = '$Revision$'[11:-2]

def main(argv):
    """Entry point for this script.

    :param argv: command line arguments.
    """
    if len(argv) is not 2:
        binary = os.path.basename(argv[0])
        print("{} ver. {}".format(binary, __version__), file=sys.stderr)
        print("Usage: {} directory".format(binary), file=sys.stderr)
        sys.exit(1)
    dirname = argv[1]
    if not os.path.isdir(dirname):
        print("'{}' is not a directory. Exiting".format(dirname))
        sys.exit(2)
    if dirname.startswith('.'):
        newname = ''.join(['old-dot', dirname[1:]])
    else:
        newname = ''.join(['old-', dirname])
    if os.path.exists(newname):
        print("'{}' already exists. Exiting".format(newname))
        sys.exit(3)
    os.rename(dirname, newname)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Fixed punctuation in old.py<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8
#
# Author: R.F. Smith <rsmith@xs4all.nl>
# Created: 2014-02-10 21:43:11 +0100
# Modified: $Date$
#
# To the extent possible under law, R.F. Smith has waived all copyright and
# related or neighboring rights to old.py. This work is published
# from the Netherlands. See http://creativecommons.org/publicdomain/zero/1.0/

"""Renames a directory by prefixing the name with 'old-'."""

from __future__ import print_function, division  # 2 and 3 compatibility.

import sys
import os

__version__ = '$Revision$'[11:-2]

def main(argv):
    """Entry point for this script.

    :param argv: command line arguments.
    """
    if len(argv) is not 2:
        binary = os.path.basename(argv[0])
        print("{} ver. {}".format(binary, __version__), file=sys.stderr)
        print("Usage: {} directory".format(binary), file=sys.stderr)
        sys.exit(1)
    dirname = argv[1]
    if not os.path.isdir(dirname):
        print("'{}' is not a directory. Exiting.".format(dirname))
        sys.exit(2)
    if dirname.startswith('.'):
        newname = ''.join(['old-dot', dirname[1:]])
    else:
        newname = ''.join(['old-', dirname])
    if os.path.exists(newname):
        print("'{}' already exists. Exiting.".format(newname))
        sys.exit(3)
    os.rename(dirname, newname)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>from sqlalchemy import create_engine, exists
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, TEXT, DateTime, Float
from sqlalchemy.orm import sessionmaker

Base = declarative_base()

class Dwelling(Base):
    __tablename__ = 'dwelling'

    id                 = Column(Integer, primary_key=True)
    address_line_1     = Column(String)
    address_line_2     = Column(String)
    area_size          = Column(String)
    contact_number     = Column(String)
    county             = Column(String)
    daft_link          = Column(String)
    dwelling_type      = Column(String)
    facilities         = Column(TEXT)
    formalised_address = Column(TEXT)
    listing_image      = Column(String)
    num_bathrooms      = Column(Integer)
    num_bedrooms       = Column(Integer)
    price              = Column(String)
    price_change       = Column(String)
    price_number       = Column(Integer)
    price_month        = Column(Integer)
    timestamp          = Column(DateTime)
    town               = Column(String)
    viewings           = Column(String)
    features           = Column(String)
    agent              = Column(String)
    agent_url          = Column(String)
    posted_since       = Column(String)
    lat                = Column(Float)
    long               = Column(Float)
    hash               = Column(String)

class Dataset(object):
    def __init__(self, **kwargs):
        self.conection_string = kwargs['con_str']
        self.eng      = self._create_eng()
        self.session = None

        Base.metadata.bind = self.eng
        Base.metadata.create_all()

    def open_session(self):
        self.session = self._create_session()

    def close_session(self):
        if self.session:
            self.session.close()

    def commit(self):
        self.session.commit()

    def insert(self, obj):
        self.session.add(obj)

    def _create_eng(self):
        return create_engine(self.conection_string)

    def _create_session(self):
        return sessionmaker(bind=self.eng)()

    def exists(self, h):
        return self.session.query(
            exists().where(Dwelling.hash == h)
        ).scalar()

class DataController(object):
    def __init__(self, con_str):
        self.ds = Dataset(con_str=con_str)

    def __enter__(self):
        self.ds.open_session()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.ds.close_session()

    def insert(self, doc):
        if self.ds.exists(doc.get('hash')):
            return
        self.ds.insert(Dwelling(
            address_line_1=doc.get('address_line_1'),
            address_line_2=doc.get('address_line_2'),
            area_size=doc.get('area_size'),
            contact_number=doc.get('contact_number'),
            county=doc.get('county'),
            daft_link=doc.get('daft_link'),
            dwelling_type=doc.get('dwelling_type'),
            facilities=','.join(doc.get('facilities')),
            formalised_address=doc.get('formalised_address'),
            listing_image=doc.get('listing_image'),
            num_bathrooms=doc.get('num_bathrooms'),
            num_bedrooms=doc.get('num_bedrooms'),
            price=doc.get('price'),
            price_change=doc.get('price_change'),
            price_number=doc.get('price_number'),
            price_month=doc.get('price_month'),
            timestamp=doc.get('timestamp'),
            town=doc.get('town'),
            viewings=','.join(doc.get('viewings')),
            features=doc.get('features'),
            agent=doc.get('agent'),
            agent_url=doc.get('agent_url'),
            posted_since=doc.get('posted_since'),
            hash=doc.get('hash'),
        ))
        self.ds.commit()<commit_msg>fix: check empty list<commit_after>from sqlalchemy import create_engine, exists
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, TEXT, DateTime, Float
from sqlalchemy.orm import sessionmaker

Base = declarative_base()

class Dwelling(Base):
    __tablename__ = 'dwelling'

    id                 = Column(Integer, primary_key=True)
    address_line_1     = Column(String)
    address_line_2     = Column(String)
    area_size          = Column(String)
    contact_number     = Column(String)
    county             = Column(String)
    daft_link          = Column(String)
    dwelling_type      = Column(String)
    facilities         = Column(TEXT)
    formalised_address = Column(TEXT)
    listing_image      = Column(String)
    num_bathrooms      = Column(Integer)
    num_bedrooms       = Column(Integer)
    price              = Column(String)
    price_change       = Column(String)
    price_number       = Column(Integer)
    price_month        = Column(Integer)
    timestamp          = Column(DateTime)
    town               = Column(String)
    viewings           = Column(String)
    features           = Column(String)
    agent              = Column(String)
    agent_url          = Column(String)
    posted_since       = Column(String)
    lat                = Column(Float)
    long               = Column(Float)
    hash               = Column(String)

class Dataset(object):
    def __init__(self, **kwargs):
        self.conection_string = kwargs['con_str']
        self.eng      = self._create_eng()
        self.session = None

        Base.metadata.bind = self.eng
        Base.metadata.create_all()

    def open_session(self):
        self.session = self._create_session()

    def close_session(self):
        if self.session:
            self.session.close()

    def commit(self):
        self.session.commit()

    def insert(self, obj):
        self.session.add(obj)

    def _create_eng(self):
        return create_engine(self.conection_string)

    def _create_session(self):
        return sessionmaker(bind=self.eng)()

    def exists(self, h):
        return self.session.query(
            exists().where(Dwelling.hash == h)
        ).scalar()

class DataController(object):
    def __init__(self, con_str):
        self.ds = Dataset(con_str=con_str)

    def __enter__(self):
        self.ds.open_session()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.ds.close_session()

    def insert(self, doc):
        if self.ds.exists(doc.get('hash')):
            return
        self.ds.insert(Dwelling(
            address_line_1=doc.get('address_line_1'),
            address_line_2=doc.get('address_line_2'),
            area_size=doc.get('area_size'),
            contact_number=doc.get('contact_number'),
            county=doc.get('county'),
            daft_link=doc.get('daft_link'),
            dwelling_type=doc.get('dwelling_type'),
            facilities=','.join(doc.get('facilities', [])),
            formalised_address=doc.get('formalised_address'),
            listing_image=doc.get('listing_image'),
            num_bathrooms=doc.get('num_bathrooms'),
            num_bedrooms=doc.get('num_bedrooms'),
            price=doc.get('price'),
            price_change=doc.get('price_change'),
            price_number=doc.get('price_number'),
            price_month=doc.get('price_month'),
            timestamp=doc.get('timestamp'),
            town=doc.get('town'),
            viewings=','.join(doc.get('viewings')),
            features=doc.get('features'),
            agent=doc.get('agent'),
            agent_url=doc.get('agent_url'),
            posted_since=doc.get('posted_since'),
            hash=doc.get('hash'),
        ))
        self.ds.commit()<|endoftext|>


<commit_before>import argparse

from control import PGM
from problem import Problem

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("model", help="python file with model description")
    parser.add_argument("output", help="directory to save outputs")
    args = parser.parse_args()

    problem = Problem(default_settings_filename='defaults.py')
    problem.load_model(args.model)

    control = PGM(problem, figname = args.output)
    control.run()

if __name__ == '__main__':
    main()
<commit_msg>Add path_mp4 to pgm.py to produce video<commit_after>import argparse

from control import PGM
from problem import Problem

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("model", help="python file with model description")
    parser.add_argument("output", help="directory to save outputs")
    parser.add_argument("path_mp4", help="directory with set of png images produced by pgm to create mp4 video")
    args = parser.parse_args()

    if args.path_mp4:
        create_video(args.path_mp4)

    problem = Problem(default_settings_filename='defaults.py')
    problem.load_model(args.model)

    control = PGM(problem, figname = args.output)
    control.run()

def create_video(path_to_images):
    import os
    os.system(f'cd {path_to_images}')
    os.system('ffmpeg -framerate 25 -pattern_type glob -i "*.png"  -c:v libx264 -profile:v high -crf 20 -pix_fmt yuv420p output.mp4')
    os.system('cd -')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># map

## procedural
names = ['Mary', 'Isla', 'Sam']

for i in range(len(names)):
    names[i] = hash(names[i])

print names

## functional
names = ['Mary', 'Isla', 'Sam']

print map(lambda x: hash(x), names)

# reduce

## procedural
people = [{'name': 'Mary', 'height': 160},
          {'name': 'Isla', 'height': 80},
          {'name': 'Sam'}]

height_total = 0
height_count = 0
for person in people:
    if 'height' in person:
        height_total += person['height']
        height_count += 1

if height_count > 0:
    average_height = height_total / height_count

    print average_height
    # => 120

## functional

people_with_height = filter(lambda p: 'height' in p, people)
height_count = len(people_with_height)
height_total = reduce(lambda a, p: a + p['height'], people_with_height, 0)

if height_count > 0:
    average_height = height_total / height_count

    print average_height<commit_msg>added race examples<commit_after># These examples are made following this article:
# https://maryrosecook.com/blog/post/a-practical-introduction-to-functional-programming

# map

## procedural
names = ['Mary', 'Isla', 'Sam']

for i in range(len(names)):
    names[i] = hash(names[i])

print names

## functional
names = ['Mary', 'Isla', 'Sam']

# transforms the names list into an hashes list
print map(lambda x: hash(x), names)

# reduce

## procedural
people = [{'name': 'Mary', 'height': 160},
          {'name': 'Isla', 'height': 80},
          {'name': 'Sam'}]

height_total = 0
height_count = 0
for person in people:
    if 'height' in person:
        height_total += person['height']
        height_count += 1

if height_count > 0:
    average_height = height_total / height_count

    print average_height
    # => 120

## functional
# filters for each person having the height
people_with_height = filter(lambda p: 'height' in p, people)
height_count = len(people_with_height)
# sums the height of each person
height_total = reduce(lambda a, p: a + p['height'], people_with_height, 0)

if height_count > 0:
    average_height = height_total / height_count

    print average_height

## functional solution #2
# transforms the height-filtered people list into a heights list
heights = map(lambda p: p['height'], filter(lambda p: 'height' in p, people))
height_count = len(heights)

if height_count > 0:
    # sum the heights
    average_height = reduce(lambda a, h: a + h, heights, 0) / height_count

    print average_height

# procedural
from random import random

print 'procedural race'

time = 5
car_positions = [1, 1, 1]

while time:
    # decrease time
    time -= 1

    print ''
    for i in range(len(car_positions)):
        # move car
        if random() > 0.3:
            car_positions[i] += 1

        # draw car
        print '-' * car_positions[i]

# functional
def draw_car(position):
    print '-' * position

def move_car(position):
    if random() > 0.3:
        position += 1

    return position

def race(time, positions):
    if time > 0:
        print ''

        positions = map(move_car, positions)
        map(draw_car, positions)
        time -= 1

        race(time, positions)

print 'functional race'
time = 5
car_positions = [1, 1, 1]
race(time, car_positions)

# functional solution #2
def draw(state):
    print ''
    print '\n'.join(map(lambda p: '-' * p, state['car_positions']))

def move(car_position):
    return car_position + 1 if random() > 0.3 else car_position

def race_step(state):
    return {'time': state['time'] - 1, 'car_positions': map(move, state['car_positions'])}

def race(state):
    draw(state)
    if state['time']:
        race(race_step(state))

print 'functional race #2'
time = 5
car_positions = [1, 1, 1]
race({'time': 5, 'car_positions': [1,1,1]})<|endoftext|>


<commit_before>from slack import Slack
from datetime import datetime
from elasticsearch import Elasticsearch
import settings
import time
import json
import logging.handlers

slack = Slack(settings.SLACK_TOKEN, settings.BOT_NAME)
logger = logging.getLogger('slack-logger')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_handler = logging.handlers.TimedRotatingFileHandler(
    './logs/slack-log', when='midnight', interval=1, encoding="UTF-8")
file_handler.suffix = "%Y-%m-%d"
stream_handler = logging.StreamHandler()
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
es_client = Elasticsearch()

def es_create(index: str, doc_type: str, body: dict) -> bool:
    res = es_client.create(index=index, doc_type=doc_type, body=body)
    return res.get('created') or False

def es_search(index: str, doc_type: str, body: dict) -> list:
    query = {'query': {'term': {}}}
    query['query']['term'].update(body)
    res = es_client.search(index=index, doc_type=doc_type, body=query)
    if res.get('hits') and res['hits'].get('hits'):
        return [doc['_source']['content'] for doc in res['hits']['hits']]
    return []

def handle_message(d: dict):
    channame = slack.channels.get(d['channel'])
    if not channame:
        slack.refresh_channels()
        channame = slack.channels.get(d['channel']) or "?"
    username = slack.users.get(d['user'])
    if not username:
        slack.refresh_users()
        username = slack.users.get(d['user']) or "?"
    date_ts = datetime.fromtimestamp(float(d['ts']))
    log_to_json(channame, username, d['text'], date_ts)
    res = handle_command(d['text'])
    if res:
        slack.post_formatted_message(d['channel'], res)
    return ""

def handle_command(text: str) -> dict:
    if text.startswith("!"):
        if text.split()[0] == "!logsearch":
            args = text.split()[1:]
            if len(args) < 2 or len(args) > 3:
                return {'text': "Usage:\n!search <key> <value> [<max>]\nkey: channel, user, text, time"}
            res = es_search(settings.ES_INDEX, settings.ES_TYPE, {args[0]: args[1]})
            if res:
                c = len(res)
                if len(args) == 3:
                    if c > len(args[2]):
                        res = res[:args[3]]
                else:
                    if c > 5:
                        res = res[:5]
                outdict = dict()
                outdict['pretext'] = "%s개가 검색됨" % c
                outdict['text'] = '\n'.join(["%s %s@%s: %s" %
                                             (doc['time'], doc['user'], doc['channel'], doc['text'])
                                             for doc in res])
                return outdict
        elif text.split()[0] == "!loghelp":
            return {'text': "Usage:\n!search <key> <value>\nkey: channel, user, text, time"}
    return dict()

def log_to_json(channel: str, user: str, text: str, date_ts: datetime):
    outdict = dict()
    outdict['channel'] = channel
    outdict['user'] = user
    outdict['text'] = text
    outdict['time'] = date_ts.strftime('%Y-%m-%d %H:%M:%S')
    logger.info(json.dumps(outdict, ensure_ascii=False))
    es_create(settings.ES_INDEX, settings.ES_TYPE, outdict)

def main():
    print("Logger Started!")
    while True:
        raw_text = slack.read()
        if not raw_text:
            continue
        text = json.loads(raw_text)
        if text.get('type') == 'message' and 'subtype' not in text and text.get('text'):
            handle_message(json.loads(raw_text))
        time.sleep(0.01)

if __name__ == '__main__':
    main()
<commit_msg>elasticsearch-py 대신 requests 사용<commit_after>from slack import Slack
from datetime import datetime
import requests
import settings
import time
import json
import logging.handlers

slack = Slack(settings.SLACK_TOKEN, settings.BOT_NAME)
logger = logging.getLogger('slack-logger')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_handler = logging.handlers.TimedRotatingFileHandler(
    './logs/slack-log', when='midnight', interval=1, encoding="UTF-8")
file_handler.suffix = "%Y-%m-%d"
stream_handler = logging.StreamHandler()
logger.addHandler(file_handler)
logger.addHandler(stream_handler)

def es_create(index: str, doc_type: str, body: dict) -> bool:
    query = json.dumps(body)
    res = requests.post('http://localhost:9200/%s/%s/' % (index, doc_type), data=query)
    return json.loads(res.text).get('created') or False

def es_search(index: str, doc_type: str, body: dict) -> list:
    query = {'query': {'term': {}}}
    query['query']['term'].update(body)
    res = requests.post('http://localhost:9200/%s/%s/_search' % (index, doc_type), data=query)
    jsoned = json.loads(res.text)
    if jsoned.get('hits') and jsoned['hits'].get('hits'):
        return [doc['_source']['content'] for doc in jsoned['hits']['hits']]
    return []

def handle_message(d: dict):
    channame = slack.channels.get(d['channel'])
    if not channame:
        slack.refresh_channels()
        channame = slack.channels.get(d['channel']) or "?"
    username = slack.users.get(d['user'])
    if not username:
        slack.refresh_users()
        username = slack.users.get(d['user']) or "?"
    date_ts = datetime.fromtimestamp(float(d['ts']))
    log_to_json(channame, username, d['text'], date_ts)
    res = handle_command(d['text'])
    if res:
        slack.post_formatted_message(d['channel'], res)
    return ""

def handle_command(text: str) -> dict:
    if text.startswith("!"):
        if text.split()[0] == "!logsearch":
            args = text.split()[1:]
            if len(args) < 2 or len(args) > 3:
                return {'text': "Usage:\n!search <key> <value> [<max>]\nkey: channel, user, text, time"}
            res = es_search(settings.ES_INDEX, settings.ES_TYPE, {args[0]: args[1]})
            if res:
                c = len(res)
                if len(args) == 3:
                    if c > len(args[2]):
                        res = res[:args[3]]
                else:
                    if c > 5:
                        res = res[:5]
                outdict = dict()
                outdict['pretext'] = "%s개가 검색됨" % c
                outdict['text'] = '\n'.join(["%s %s@%s: %s" %
                                             (doc['time'], doc['user'], doc['channel'], doc['text'])
                                             for doc in res])
                return outdict
        elif text.split()[0] == "!loghelp":
            return {'text': "Usage:\n!search <key> <value>\nkey: channel, user, text, time"}
    return dict()

def log_to_json(channel: str, user: str, text: str, date_ts: datetime):
    outdict = dict()
    outdict['channel'] = channel
    outdict['user'] = user
    outdict['text'] = text
    outdict['time'] = date_ts.strftime('%Y-%m-%d %H:%M:%S')
    logger.info(json.dumps(outdict, ensure_ascii=False))
    es_create(settings.ES_INDEX, settings.ES_TYPE, outdict)

def main():
    print("Logger Started!")
    while True:
        raw_text = slack.read()
        if not raw_text:
            continue
        text = json.loads(raw_text)
        if text.get('type') == 'message' and 'subtype' not in text and text.get('text'):
            handle_message(json.loads(raw_text))
        time.sleep(0.01)

if __name__ == '__main__':
    main()
<|endoftext|>