<commit_before># encoding: utf-8
"""
aigp.py

Created by Thomas Mangin on 2013-09-24.
Copyright (c) 2009-2013 Exa Networks. All rights reserved.
"""

from struct import pack,unpack

from exabgp.bgp.message.update.attribute.id import AttributeID
from exabgp.bgp.message.update.attribute import Flag,Attribute

# =================================================================== AIGP (26)

# 0                   1                   2                   3
# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
# |     Type      |         Length                |               |
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+               |
# ~                                                               ~
# |                           Value                               |
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+..........................

# Length: Two octets encoding the length in octets of the TLV,
# including the type and length fields.

class TLV (object):
	def __init__(self,type,value):
		self.type = type
		self.value = value

def _TLVFactory (data):
	while data:
		t = ord(data[0])
		l = unpack('!H',data[1:3])[0]
		v,data = data[3:l],data[l:]
		yield TLV(t,v)

def TLVFactory (data):
	return list(_TLVFactory(data))

def pack_tlv (tlvs):
	return ''.join('%s%s%s' % (chr(tlv.type),pack('!H',len(tlv.value)+3),tlv.value) for tlv in tlvs)

# def __init__ (self,value):
# 	self.aigp = unpack('!Q',TLVFactory(value)[0].value)[0]
# 	self.packed = self._attribute(value)

class AIGP (Attribute):
	ID = AttributeID.AIGP
	FLAG = Flag.OPTIONAL
	MULTIPLE = False
	TYPES = [1,]

	def __init__ (self,aigp,packed=None):
		self.aigp = aigp
		if packed:
			self.packed = packed
		else:
			self.packed = self._attribute(aigp)

	def pack (self,asn4=None):
		return self.packed

	def __str__ (self):
		return '0x' + ''.join('%02x' % ord(_) for _ in self.aigp[-8:])

	@classmethod
	def unpack (cls,data):
		cls(unpack('!Q',data[:8] & 0x000000FFFFFFFFFF),data[:8])
<commit_msg>tidy (unused tlv)<commit_after># encoding: utf-8
"""
aigp.py

Created by Thomas Mangin on 2013-09-24.
Copyright (c) 2009-2013 Exa Networks. All rights reserved.
"""

from struct import pack,unpack

from exabgp.bgp.message.update.attribute.id import AttributeID
from exabgp.bgp.message.update.attribute import Flag,Attribute

# ========================================================================== TLV

# 0                   1                   2                   3
# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
# |     Type      |         Length                |               |
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+               |
# ~                                                               ~
# |                           Value                               |
# +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+..........................

# Length: Two octets encoding the length in octets of the TLV,
# including the type and length fields.

class TLV (object):
	def __init__(self,type,value):
		self.type = type
		self.value = value

class TLVS (list):
	@staticmethod
	def unpack (data):
		def loop (data):
			while data:
				t = ord(data[0])
				l = unpack('!H',data[1:3])[0]
				v,data = data[3:l],data[l:]
				yield TLV(t,v)
		return TLVS(list(loop(data)))

	def pack (self):
		return ''.join('%s%s%s' % (chr(tlv.type),pack('!H',len(tlv.value)+3),tlv.value) for tlv in self)

# ==================================================================== AIGP (26)
#

class AIGP (Attribute):
	ID = AttributeID.AIGP
	FLAG = Flag.OPTIONAL
	MULTIPLE = False
	TYPES = [1,]

	def __init__ (self,aigp,packed=None):
		self.aigp = aigp
		if packed:
			self.packed = packed
		else:
			self.packed = self._attribute(aigp)

	def pack (self,asn4=None):
		return self.packed

	def __str__ (self):
		return '0x' + ''.join('%02x' % ord(_) for _ in self.aigp[-8:])

	@classmethod
	def unpack (cls,data):
		cls(unpack('!Q',data[:8] & 0x000000FFFFFFFFFF),data[:8])
<|endoftext|>


<commit_before>''' Extractor hierarchy. '''

from .base import ExtractorResult, merge_results
from .api import IndicoAPITextExtractor, IndicoAPIImageExtractor, ClarifaiAPIExtractor
from .audio import STFTAudioExtractor, MeanAmplitudeExtractor
from .google import (GoogleVisionAPIFaceExtractor,
                     GoogleVisionAPILabelExtractor,
                     GoogleVisionAPIPropertyExtractor)
from .image import (BrightnessExtractor, SaliencyExtractor, SharpnessExtractor,
                    VibranceExtractor)
from .models import TensorFlowInceptionV3Extractor
from .text import (ComplexTextExtractor, DictionaryExtractor,
                   PredefinedDictionaryExtractor, LengthExtractor,
                   NumUniqueWordsExtractor, PartOfSpeechExtractor)
from .video import (DenseOpticalFlowExtractor)

__all__ = [
    'ExtractorResult',
    'IndicoAPIExtractor',
    'ClarifaiAPIExtractor',
    'STFTAudioExtractor',
    'MeanAmplitudeExtractor',
    'GoogleVisionAPIFaceExtractor',
    'GoogleVisionAPILabelExtractor',
    'GoogleVisionAPIPropertyExtractor',
    'BrightnessExtractor',
    'SaliencyExtractor',
    'SharpnessExtractor',
    'VibranceExtractor',
    'TensorFlowInceptionV3Extractor',
    'ComplexTextExtractor',
    'DictionaryExtractor',
    'PredefinedDictionaryExtractor',
    'LengthExtractor',
    'NumUniqueWordsExtractor',
    'PartOfSpeechExtractor',
    'DenseOpticalFlowExtractor',
    'merge_results'
]
<commit_msg>Added to __all__<commit_after>''' Extractor hierarchy. '''

from .base import ExtractorResult, merge_results
from .api import IndicoAPITextExtractor, IndicoAPIImageExtractor, ClarifaiAPIExtractor
from .audio import STFTAudioExtractor, MeanAmplitudeExtractor
from .google import (GoogleVisionAPIFaceExtractor,
                     GoogleVisionAPILabelExtractor,
                     GoogleVisionAPIPropertyExtractor)
from .image import (BrightnessExtractor, SaliencyExtractor, SharpnessExtractor,
                    VibranceExtractor)
from .models import TensorFlowInceptionV3Extractor
from .text import (ComplexTextExtractor, DictionaryExtractor,
                   PredefinedDictionaryExtractor, LengthExtractor,
                   NumUniqueWordsExtractor, PartOfSpeechExtractor)
from .video import (DenseOpticalFlowExtractor)

__all__ = [
    'ExtractorResult',
    'IndicoAPITextExtractor',
    'IndicoAPIImageExtractor',
    'ClarifaiAPIExtractor',
    'STFTAudioExtractor',
    'MeanAmplitudeExtractor',
    'GoogleVisionAPIFaceExtractor',
    'GoogleVisionAPILabelExtractor',
    'GoogleVisionAPIPropertyExtractor',
    'BrightnessExtractor',
    'SaliencyExtractor',
    'SharpnessExtractor',
    'VibranceExtractor',
    'TensorFlowInceptionV3Extractor',
    'ComplexTextExtractor',
    'DictionaryExtractor',
    'PredefinedDictionaryExtractor',
    'LengthExtractor',
    'NumUniqueWordsExtractor',
    'PartOfSpeechExtractor',
    'DenseOpticalFlowExtractor',
    'merge_results'
]
<|endoftext|>


<commit_before>#! /usr/bin/env python

"""
Tool to find wront contour order between different masters, and
other interpolatability (or lack thereof) issues.
"""

from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *

from fontTools.pens.basePen import BasePen
from symfont import GlyphStatistics
import itertools

class PerContourOrComponentPen(BasePen):
	def __init__(self, Pen, glyphset=None):
		BasePen.__init__(self, glyphset)
		self._glyphset = glyphset
		self._Pen = Pen
		self.value = []
	def _moveTo(self, p0):
		self._newItem()
		self.value[-1].moveTo(p0)
	def _lineTo(self, p1):
		self.value[-1].lineTo(p1)
	def _qCurveToOne(self, p1, p2):
		self.value[-1].qCurveTo(p1, p2)
	def _curveToOne(self, p1, p2, p3):
		self.value[-1].curveTo(p1, p2, p3)
	def _closePath(self):
		self.value[-1].closePath()
	def _endPath(self):
		self.value[-1].endPath()
	def addComponent(self, glyphName, transformation):
		self._newItem()
		self.value[-1].addComponent(glyphName, transformation)

	def _newItem(self):
		self.value.append(self._Pen(glyphset=self._glyphset))

class RecordingPen(BasePen):
	def __init__(self, glyphset):
		BasePen.__init__(self, glyphset)
		self._glyphset = glyphset
		self.value = []
	def _moveTo(self, p0):
		self.value.append(('moveTo', (p0,)))
	def _lineTo(self, p1):
		self.value.append(('lineTo', (p1,)))
	def _qCurveToOne(self, p1, p2):
		self.value.append(('qCurveTo', (p1,p2)))
	def _curveToOne(self, p1, p2, p3):
		self.value.append(('curveTo', (p1,p2,p3)))
	def _closePath(self):
		self.value.append(('closePath', ()))
	def _endPath(self):
		self.value.append(('endPath', ()))
	# Humm, adding the following method slows things down some 20%.
	# We don't have as much control as we like currently.
	#def addComponent(self, glyphName, transformation):
	#	self.value.append(('addComponent', (glyphName, transformation)))

	def draw(self, pen):
		for operator,operands in self.value:
			getattr(pen, operator)(*operands)

def _vdiff(v0, v1):
	return tuple(b-a for a,b in zip(v0,v1))
def _vlen(vec):
	v = 0
	for x in vec:
		v += x*x
	return v

def _matching_cost(G, matching):
	return sum(G[i][j] for i,j in enumerate(matching))

def min_cost_perfect_bipartite_matching(G):
	n = len(G)
	if n <= 8:
		# brute-force
		permutations = itertools.permutations(range(n))
		best = list(next(permutations))
		best_cost = _matching_cost(G, best)
		for p in permutations:
			cost = _matching_cost(G, p)
			if cost < best_cost:
				best, best_cost = list(p), cost
		return best, best_cost
	else:

		# Set up current matching and inverse
		matching = list(range(n)) # identity matching
		matching_cost = _matching_cost(G, matching)
		reverse = list(matching)

		return matching, matching_cost
		# TODO implement real matching here

		# Set up cover
		cover0 = [max(c for c in row) for row in G]
		cover1 = [0] * n
		cover_weight = sum(cover0)

		while cover_weight < matching_cost:
			break
			NotImplemented

		return matching, matching_cost

def test(glyphsets, glyphs=None, names=None):

	if names is None:
		names = glyphsets
	if glyphs is None:
		glyphs = glyphsets[0].keys()

	hist = []
	for glyph_name in glyphs:
		#print()
		#print(glyph_name)

		try:
			allVectors = []
			for glyphset in glyphsets:
				#print('.', end='')
				#print()
				glyph = glyphset[glyph_name]

				perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset)
				glyph.draw(perContourPen)
				contourPens = perContourPen.value
				del perContourPen

				contourVectors = []
				allVectors.append(contourVectors)
				for contour in contourPens:
					stats = GlyphStatistics(contour, glyphset=glyphset)
					vector = (
						int(stats.Perimeter * .125),
						int(abs(stats.Area) ** .5 * .5),
						int(stats.MeanX),
						int(stats.MeanY),
						int(stats.StdDevX * 2),
						int(stats.StdDevY * 2),
						int(stats.Covariance/(stats.StdDevX*stats.StdDevY)**.5),
					)
					contourVectors.append(vector)
					#print(vector)

			# Check each master against the next one in the list.
			for i,(m0,m1) in enumerate(zip(allVectors[:-1],allVectors[1:])):
				if len(m0) != len(m1):
					print('%s: %s+%s: Glyphs not compatible!!!!!' % (glyph_name, names[i], names[i+1]))
					continue
				if not m0:
					continue
				costs = [[_vlen(_vdiff(v0,v1)) for v1 in m1] for v0 in m0]
				matching, matching_cost = min_cost_perfect_bipartite_matching(costs)
				if matching != list(range(len(m0))):
					print('%s: %s+%s: Glyph has wrong contour/component order: %s' % (glyph_name, names[i], names[i+1], matching)) #, m0, m1)
					break
				upem = 2048
				item_cost = int(round((matching_cost / len(m0) / len(m0[0])) ** .5 / upem * 100))
				hist.append(item_cost)
				threshold = 7
				if item_cost >= threshold:
					print('%s: %s+%s: Glyph has very high cost: %d%%' % (glyph_name, names[i], names[i+1], item_cost))

		except ValueError as e:
			print('%s: math error %s; skipping glyph' % (glyph_name, e))
			#raise
	#for x in hist:
	#	print(x)

def main(args):
	filenames = args
	glyphs = None
	#glyphs = ['uni08DB', 'uniFD76']
	#glyphs = ['uni08DE', 'uni0034']
	#glyphs = ['uni08DE', 'uni0034', 'uni0751', 'uni0753', 'uni0754', 'uni08A4', 'uni08A4.fina', 'uni08A5.fina']

	from os.path import basename
	names = [basename(filename).rsplit('.', 1)[0] for filename in filenames]

	from fontTools.ttLib import TTFont
	fonts = [TTFont(filename) for filename in filenames]

	glyphsets = [font.getGlyphSet() for font in fonts]
	test(glyphsets, glyphs=glyphs, names=names)

if __name__ == '__main__':
	import sys
	main(sys.argv[1:])
<commit_msg>[Snippets/interpolatable] Use Hungarian algorithm from munkres or scipy when available<commit_after>#! /usr/bin/env python

"""
Tool to find wront contour order between different masters, and
other interpolatability (or lack thereof) issues.
"""

from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *

from fontTools.pens.basePen import BasePen
from symfont import GlyphStatistics
import itertools

class PerContourOrComponentPen(BasePen):
	def __init__(self, Pen, glyphset=None):
		BasePen.__init__(self, glyphset)
		self._glyphset = glyphset
		self._Pen = Pen
		self.value = []
	def _moveTo(self, p0):
		self._newItem()
		self.value[-1].moveTo(p0)
	def _lineTo(self, p1):
		self.value[-1].lineTo(p1)
	def _qCurveToOne(self, p1, p2):
		self.value[-1].qCurveTo(p1, p2)
	def _curveToOne(self, p1, p2, p3):
		self.value[-1].curveTo(p1, p2, p3)
	def _closePath(self):
		self.value[-1].closePath()
	def _endPath(self):
		self.value[-1].endPath()
	def addComponent(self, glyphName, transformation):
		self._newItem()
		self.value[-1].addComponent(glyphName, transformation)

	def _newItem(self):
		self.value.append(self._Pen(glyphset=self._glyphset))

class RecordingPen(BasePen):
	def __init__(self, glyphset):
		BasePen.__init__(self, glyphset)
		self._glyphset = glyphset
		self.value = []
	def _moveTo(self, p0):
		self.value.append(('moveTo', (p0,)))
	def _lineTo(self, p1):
		self.value.append(('lineTo', (p1,)))
	def _qCurveToOne(self, p1, p2):
		self.value.append(('qCurveTo', (p1,p2)))
	def _curveToOne(self, p1, p2, p3):
		self.value.append(('curveTo', (p1,p2,p3)))
	def _closePath(self):
		self.value.append(('closePath', ()))
	def _endPath(self):
		self.value.append(('endPath', ()))
	# Humm, adding the following method slows things down some 20%.
	# We don't have as much control as we like currently.
	#def addComponent(self, glyphName, transformation):
	#	self.value.append(('addComponent', (glyphName, transformation)))

	def draw(self, pen):
		for operator,operands in self.value:
			getattr(pen, operator)(*operands)

def _vdiff(v0, v1):
	return tuple(b-a for a,b in zip(v0,v1))
def _vlen(vec):
	v = 0
	for x in vec:
		v += x*x
	return v

def _matching_cost(G, matching):
	return sum(G[i][j] for i,j in enumerate(matching))

def min_cost_perfect_bipartite_matching(G):
	n = len(G)
	try:
		from scipy.optimize import linear_sum_assignment
		rows, cols = linear_sum_assignment(G)
		# This branch untested
		assert rows == list(range(n))
		return cols, _matching_cost(G, cols)
	except ImportError:
		pass

	try:
		from munkres import Munkres
		cols = [None] * n
		for row,col in Munkres().compute(G):
			cols[row] = col
		return cols, _matching_cost(G, cols)
	except ImportError:
		pass

	if n > 6:
		raise Exception("Install Python module 'munkres' or 'scipy >= 0.17.0'")

	# Otherwise just brute-force
	permutations = itertools.permutations(range(n))
	best = list(next(permutations))
	best_cost = _matching_cost(G, best)
	for p in permutations:
		cost = _matching_cost(G, p)
		if cost < best_cost:
			best, best_cost = list(p), cost
	return best, best_cost

def test(glyphsets, glyphs=None, names=None):

	if names is None:
		names = glyphsets
	if glyphs is None:
		glyphs = glyphsets[0].keys()

	hist = []
	for glyph_name in glyphs:
		#print()
		#print(glyph_name)

		try:
			allVectors = []
			for glyphset in glyphsets:
				#print('.', end='')
				#print()
				glyph = glyphset[glyph_name]

				perContourPen = PerContourOrComponentPen(RecordingPen, glyphset=glyphset)
				glyph.draw(perContourPen)
				contourPens = perContourPen.value
				del perContourPen

				contourVectors = []
				allVectors.append(contourVectors)
				for contour in contourPens:
					stats = GlyphStatistics(contour, glyphset=glyphset)
					vector = (
						int(stats.Perimeter * .125),
						int(abs(stats.Area) ** .5 * .5),
						int(stats.MeanX),
						int(stats.MeanY),
						int(stats.StdDevX * 2),
						int(stats.StdDevY * 2),
						int(stats.Covariance/(stats.StdDevX*stats.StdDevY)**.5),
					)
					contourVectors.append(vector)
					#print(vector)

			# Check each master against the next one in the list.
			for i,(m0,m1) in enumerate(zip(allVectors[:-1],allVectors[1:])):
				if len(m0) != len(m1):
					print('%s: %s+%s: Glyphs not compatible!!!!!' % (glyph_name, names[i], names[i+1]))
					continue
				if not m0:
					continue
				costs = [[_vlen(_vdiff(v0,v1)) for v1 in m1] for v0 in m0]
				matching, matching_cost = min_cost_perfect_bipartite_matching(costs)
				if matching != list(range(len(m0))):
					print('%s: %s+%s: Glyph has wrong contour/component order: %s' % (glyph_name, names[i], names[i+1], matching)) #, m0, m1)
					break
				upem = 2048
				item_cost = int(round((matching_cost / len(m0) / len(m0[0])) ** .5 / upem * 100))
				hist.append(item_cost)
				threshold = 7
				if item_cost >= threshold:
					print('%s: %s+%s: Glyph has very high cost: %d%%' % (glyph_name, names[i], names[i+1], item_cost))

		except ValueError as e:
			print('%s: math error %s; skipping glyph' % (glyph_name, e))
			#raise
	#for x in hist:
	#	print(x)

def main(args):
	filenames = args
	glyphs = None
	#glyphs = ['uni08DB', 'uniFD76']
	#glyphs = ['uni08DE', 'uni0034']
	#glyphs = ['uni08DE', 'uni0034', 'uni0751', 'uni0753', 'uni0754', 'uni08A4', 'uni08A4.fina', 'uni08A5.fina']

	from os.path import basename
	names = [basename(filename).rsplit('.', 1)[0] for filename in filenames]

	from fontTools.ttLib import TTFont
	fonts = [TTFont(filename) for filename in filenames]

	glyphsets = [font.getGlyphSet() for font in fonts]
	test(glyphsets, glyphs=glyphs, names=names)

if __name__ == '__main__':
	import sys
	main(sys.argv[1:])
<|endoftext|>


<commit_before>from servicenow import Utils

ttl_cache=0

class Base(object):
    __table__ = None

    def __init__(self, Connection):
        self.Connection = Connection

    @Utils.cached(ttl=ttl_cache)
    def list_by_query(self, query, **kwargs):
        return self.format(self.Connection._list_by_query(self.__table__, query, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def list(self, meta, **kwargs):
        return self.format(self.Connection._list(self.__table__, meta, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_all(self, meta, **kwargs):
        return self.format(self.Connection._get(self.__table__, meta, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_all_by_query(self, query, **kwargs):
        return self.format(self.Connection._get_by_query(self.__table__, query, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_one(self, meta, **kwargs):
        response = self.fetch_all(meta, **kwargs)
        if 'records' in response:
            if len(response['records']) > 0:
                return response['records'][0]
        else:
            if len(response) > 0:
                return response[0]
        return {}

    def create(self, data, **kwargs):
        return self.format(self.Connection._post(
            self.__table__, data, **kwargs))

    def create_multiple(self, data, **kwargs):
        return self.format(self.Connection._post_multiple(
            self.__table__, data, **kwargs))

    def update(self, where, data, **kwargs):
        return self.format(self.Connection._update(
            self.__table__, where, data, **kwargs))

    def delete(self, id, **kwargs):
        return self.format(self.Connection._delete(
            self.__table__, id, **kwargs))

    def delete_multiple(self, query, **kwargs):
        return self.format(self.Connection._delete_multiple(
            self.__table__, query, **kwargs))

    def format(self, response):
        return self.Connection._format(response)

    def last_updated(self, minutes, meta={}, **kwargs):
        metaon = {'sys_updated_on':
                      'Last {0} minutes@javascript:gs.minutesAgoStart({1})@'
                      'javascript:gs.minutesAgoEnd(0)'.format(minutes, minutes)}
        return self.format(self.Connection._get(
            self.__table__, meta, metaon=metaon, **kwargs))

class Call(Base):
    __table__ = 'u_new_call.do'

class Change(Base):
    __table__ = 'change_request.do'

class Group(Base):
    __table__ = 'sys_user_group.do'

class Incident(Base):
    __table__ = 'incident.do'

class Journal(Base):
    __table__ = 'sys_journal_field.do'

class Problem(Base):
    __table__ = 'problem.do'

class Request(Base):
    __table__ = 'u_request.do'

class Server(Base):
    __table__ = 'cmdb_ci_server.do'

class Ticket(Base):
    __table__ = 'u_service_desk.do'

class Task(Base):
    __table__ = 'task_ci_list.do'

class User(Base):
    __table__ = 'sys_user.do'

class Customer(Base):
    __table__ = 'core_company.do'

class Router(Base):
    __table__ = 'cmdb_ci_ip_router.do'

class Switch(Base):
    __table__ = 'cmdb_ci_ip_switch.do'

class Cluster(Base):
    __table__ = 'cmdb_ci_cluster.do'

class Firewall(Base):
    __table__ = 'u_firewall.do'

class VPN(Base):
    __table__ = 'cmdb_ci_vpn.do'

class ConfigurationItem(Base):
    __table__ = 'cmdb_ci.do'
<commit_msg>adds Location table<commit_after>from servicenow import Utils

ttl_cache=0

class Base(object):
    __table__ = None

    def __init__(self, Connection):
        self.Connection = Connection

    @Utils.cached(ttl=ttl_cache)
    def list_by_query(self, query, **kwargs):
        return self.format(self.Connection._list_by_query(self.__table__, query, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def list(self, meta, **kwargs):
        return self.format(self.Connection._list(self.__table__, meta, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_all(self, meta, **kwargs):
        return self.format(self.Connection._get(self.__table__, meta, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_all_by_query(self, query, **kwargs):
        return self.format(self.Connection._get_by_query(self.__table__, query, **kwargs))

    @Utils.cached(ttl=ttl_cache)
    def fetch_one(self, meta, **kwargs):
        response = self.fetch_all(meta, **kwargs)
        if 'records' in response:
            if len(response['records']) > 0:
                return response['records'][0]
        else:
            if len(response) > 0:
                return response[0]
        return {}

    def create(self, data, **kwargs):
        return self.format(self.Connection._post(
            self.__table__, data, **kwargs))

    def create_multiple(self, data, **kwargs):
        return self.format(self.Connection._post_multiple(
            self.__table__, data, **kwargs))

    def update(self, where, data, **kwargs):
        return self.format(self.Connection._update(
            self.__table__, where, data, **kwargs))

    def delete(self, id, **kwargs):
        return self.format(self.Connection._delete(
            self.__table__, id, **kwargs))

    def delete_multiple(self, query, **kwargs):
        return self.format(self.Connection._delete_multiple(
            self.__table__, query, **kwargs))

    def format(self, response):
        return self.Connection._format(response)

    def last_updated(self, minutes, meta={}, **kwargs):
        metaon = {'sys_updated_on':
                      'Last {0} minutes@javascript:gs.minutesAgoStart({1})@'
                      'javascript:gs.minutesAgoEnd(0)'.format(minutes, minutes)}
        return self.format(self.Connection._get(
            self.__table__, meta, metaon=metaon, **kwargs))

class Call(Base):
    __table__ = 'u_new_call.do'

class Change(Base):
    __table__ = 'change_request.do'

class Group(Base):
    __table__ = 'sys_user_group.do'

class Incident(Base):
    __table__ = 'incident.do'

class Journal(Base):
    __table__ = 'sys_journal_field.do'

class Problem(Base):
    __table__ = 'problem.do'

class Request(Base):
    __table__ = 'u_request.do'

class Server(Base):
    __table__ = 'cmdb_ci_server.do'

class Ticket(Base):
    __table__ = 'u_service_desk.do'

class Task(Base):
    __table__ = 'task_ci_list.do'

class User(Base):
    __table__ = 'sys_user.do'

class Customer(Base):
    __table__ = 'core_company.do'

class Router(Base):
    __table__ = 'cmdb_ci_ip_router.do'

class Switch(Base):
    __table__ = 'cmdb_ci_ip_switch.do'

class Cluster(Base):
    __table__ = 'cmdb_ci_cluster.do'

class Firewall(Base):
    __table__ = 'u_firewall.do'

class VPN(Base):
    __table__ = 'cmdb_ci_vpn.do'

class ConfigurationItem(Base):
    __table__ = 'cmdb_ci.do'

class Location(Base):
    __table__ = 'cmn_location.do'

<|endoftext|>


<commit_before>import datetime
import logging
import sys
import time
import unittest

from huey import RedisHuey
from huey.api import Huey
from huey.consumer import Consumer
from huey.registry import registry
from huey.storage import BaseStorage
from huey.storage import RedisStorage

def b(s):
    if sys.version_info[0] == 3:
        return s.encode('utf-8')
    return s

class DummyHuey(Huey):
    def get_storage(self, **kwargs):
        return BaseStorage()

class BrokenRedisStorage(RedisStorage):
    def dequeue(self):
        raise ValueError('broken redis dequeue')

broken_redis_storage = BrokenRedisStorage()

class BrokenHuey(Huey):
    def get_storage(self):
        return broken_redis_storage

dummy_huey = DummyHuey()
test_huey = RedisHuey('testing', blocking=False, read_timeout=0.1)

# Logger used by the consumer.
logger = logging.getLogger('huey.consumer')

# Create a log handler that will track messages generated by the consumer.
class CaptureLogs(logging.Handler):
    def __init__(self, *args, **kwargs):
        self.messages = []
        logging.Handler.__init__(self, *args, **kwargs)

    def emit(self, record):
        self.messages.append(record.getMessage())

    def __enter__(self):
        logger.addHandler(self)
        logger.setLevel(logging.INFO)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        logger.removeHandler(self)

class BaseTestCase(unittest.TestCase):
    pass

class HueyTestCase(BaseTestCase):
    def setUp(self):
        self.huey = self.get_huey()
        self.consumer = self.get_consumer(workers=2, scheduler_interval=10)

        self.events = iter(self.huey.storage)

        self._periodic_tasks = registry._periodic_tasks
        registry._periodic_tasks = self.get_periodic_tasks()

        self._sleep = time.sleep
        time.sleep = lambda x: None

    def tearDown(self):
        if self.consumer is not None:
            self.consumer.stop()
        self.huey.flush()
        registry._periodic_tasks = self._periodic_tasks
        time.sleep = self._sleep

    def get_huey(self):
        return test_huey

    def get_consumer(self, **kwargs):
        return Consumer(self.huey, **kwargs)

    def get_periodic_tasks(self):
        return []

    def assertTaskEvents(self, *states):
        for (status, task) in states:
            event_data = next(self.events)
            self.assertEqual(event_data['status'], status)
            self.assertEqual(event_data['id'], task.task_id)

    def assertLogs(self, capture, expected):
        self.assertEqual(len(capture.messages), len(expected))
        for (log, msg) in zip(capture.messages, expected):
            self.assertTrue(log.startswith(msg))

    def worker(self, task, ts=None):
        worker = self.consumer._create_worker()
        ts = ts or datetime.datetime.utcnow()
        worker.handle_task(task, ts)
        return worker

    def scheduler(self, ts=None, periodic=False):
        scheduler = self.consumer._create_scheduler()
        ts = ts or datetime.datetime.utcnow()
        if periodic:
            scheduler._counter = scheduler._q
        scheduler.loop(ts)
        return scheduler
<commit_msg>Add NullHandler() to avoid spurious output in tests.<commit_after>import datetime
import logging
import sys
import time
import unittest

from huey import RedisHuey
from huey.api import Huey
from huey.consumer import Consumer
from huey.registry import registry
from huey.storage import BaseStorage
from huey.storage import RedisStorage

def b(s):
    if sys.version_info[0] == 3:
        return s.encode('utf-8')
    return s

class DummyHuey(Huey):
    def get_storage(self, **kwargs):
        return BaseStorage()

class BrokenRedisStorage(RedisStorage):
    def dequeue(self):
        raise ValueError('broken redis dequeue')

broken_redis_storage = BrokenRedisStorage()

class BrokenHuey(Huey):
    def get_storage(self):
        return broken_redis_storage

dummy_huey = DummyHuey()
test_huey = RedisHuey('testing', blocking=False, read_timeout=0.1)

# Logger used by the consumer.
logger = logging.getLogger('huey.consumer')
logger.addHandler(logging.NullHandler())

# Create a log handler that will track messages generated by the consumer.
class CaptureLogs(logging.Handler):
    def __init__(self, *args, **kwargs):
        self.messages = []
        logging.Handler.__init__(self, *args, **kwargs)

    def emit(self, record):
        self.messages.append(record.getMessage())

    def __enter__(self):
        logger.addHandler(self)
        logger.setLevel(logging.INFO)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        logger.removeHandler(self)

class BaseTestCase(unittest.TestCase):
    pass

class HueyTestCase(BaseTestCase):
    def setUp(self):
        self.huey = self.get_huey()
        self.consumer = self.get_consumer(workers=2, scheduler_interval=10)

        self.events = iter(self.huey.storage)

        self._periodic_tasks = registry._periodic_tasks
        registry._periodic_tasks = self.get_periodic_tasks()

        self._sleep = time.sleep
        time.sleep = lambda x: None

    def tearDown(self):
        if self.consumer is not None:
            self.consumer.stop()
        self.huey.flush()
        registry._periodic_tasks = self._periodic_tasks
        time.sleep = self._sleep

    def get_huey(self):
        return test_huey

    def get_consumer(self, **kwargs):
        return Consumer(self.huey, **kwargs)

    def get_periodic_tasks(self):
        return []

    def assertTaskEvents(self, *states):
        for (status, task) in states:
            event_data = next(self.events)
            self.assertEqual(event_data['status'], status)
            self.assertEqual(event_data['id'], task.task_id)

    def assertLogs(self, capture, expected):
        self.assertEqual(len(capture.messages), len(expected))
        for (log, msg) in zip(capture.messages, expected):
            self.assertTrue(log.startswith(msg))

    def worker(self, task, ts=None):
        worker = self.consumer._create_worker()
        ts = ts or datetime.datetime.utcnow()
        worker.handle_task(task, ts)
        return worker

    def scheduler(self, ts=None, periodic=False):
        scheduler = self.consumer._create_scheduler()
        ts = ts or datetime.datetime.utcnow()
        if periodic:
            scheduler._counter = scheduler._q
        scheduler.loop(ts)
        return scheduler
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.views.generic.detail import SingleObjectMixin
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
from django.utils.translation import ugettext_lazy as _
from django import forms
from django.views.generic import FormView
from django.shortcuts import redirect
from django.http import HttpResponseBadRequest

from devilry.apps.core.models import Delivery
from devilry.apps.core.models import StaticFeedback
from devilry_gradingsystem.models import FeedbackDraft
from devilry_gradingsystem.widgets.editmarkdown import EditMarkdownLayoutObject
from devilry_gradingsystem.widgets.editfeedbackbuttonbar import EditFeedbackButtonBar

class FeedbackEditorSingleDeliveryObjectMixin(SingleObjectMixin):
    """
    Mixin that provides:

    - A delivery object.
    - Authorization for access to the delivery object.

    If your are creating a grading system plugin, you should NOT USE THIS
    directly. Use :class:`.FeedbackEditorMixin` instead.
    """
    model = Delivery
    pk_url_kwarg = 'deliveryid'
    context_object_name = 'delivery'

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).dispatch(*args, **kwargs)

    def _setup_common_data(self):
        self.object = self.get_object()
        self.delivery = self.object
        self.last_draft = None
        if self.delivery.devilry_gradingsystem_feedbackdraft_set.count() > 0:
            self.last_draft = self.delivery.devilry_gradingsystem_feedbackdraft_set.all()[0]

    def get(self, *args, **kwargs):
        self._setup_common_data()
        assignment = self.delivery.deadline.assignment_group.assignment
        if not assignment.has_valid_grading_setup():
            return redirect('devilry_examiner_singledeliveryview', deliveryid=self.delivery.id)
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).get(*args, **kwargs)

    def post(self, *args, **kwargs):
        self._setup_common_data()
        assignment = self.delivery.deadline.assignment_group.assignment
        if not assignment.has_valid_grading_setup():
            return HttpResponseBadRequest('Grading system is not set up correctly')
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).post(*args, **kwargs)

    def get_queryset(self):
        """
        Ensure we only match deliveries where the current user has access
        as an examiner.
        """
        return Delivery.objects.filter_examiner_has_access(self.request.user)\
            .select_related( # Use select_related to lookup all the related objects in the query
                'deadline',
                'deadline__assignment_group',
                'deadline__assignment_group__parentnode', # Assignment
                'deadline__assignment_group__parentnode__parentnode', # Period
                'deadline__assignment_group__parentnode__parentnode__parentnode') # Subject

    def get_context_data(self, **kwargs):
        context = super(FeedbackEditorSingleDeliveryObjectMixin, self).get_context_data(**kwargs)
        delivery = self.object
        assignment = delivery.deadline.assignment_group.assignment
        context['valid_grading_system_setup'] = assignment.has_valid_grading_setup()
        return context

class FeedbackEditorMixin(FeedbackEditorSingleDeliveryObjectMixin):
    """
    Base mixin class for all feedback editor views.
    """

    def get_success_url(self):
        return reverse('devilry_examiner_singledeliveryview',
            kwargs={'deliveryid': self.delivery.id})

    def create_feedbackdraft(self, points, feedbacktext_raw, feedbacktext_html, publish=False):
        draft = FeedbackDraft(
            delivery=self.delivery,
            points=points,
            feedbacktext_raw=feedbacktext_raw,
            feedbacktext_html=feedbacktext_html,
            saved_by=self.request.user
        )
        if publish:
            draft.published = True
            draft.staticfeedback = draft.to_staticfeedback()
            draft.staticfeedback.full_clean()
            draft.staticfeedback.save()
        draft.save()
        return draft

class FeedbackEditorFormBase(forms.Form):
    def __init__(self, *args, **kwargs):
        self.last_draft = kwargs.pop('last_draft')
        super(FeedbackEditorFormBase, self).__init__(*args, **kwargs)
        self._add_feedbacktext_field()

    def _add_feedbacktext_field(self):
        if self.last_draft:
            feedbacktext_editor = self.last_draft.feedbacktext_editor
        else:
            feedbacktext_editor = FeedbackDraft.DEFAULT_FEEDBACKTEXT_EDITOR
        self.fields['feedbacktext'] = forms.CharField(
            label=_('Feedback text'),
            required=False)

    def get_feedbacktext_layout_elements(self):
        return [EditMarkdownLayoutObject()]

    def get_submitbuttons_layout_elements(self):
        return [EditFeedbackButtonBar()]

    def add_common_layout_elements(self):
        for element in self.get_feedbacktext_layout_elements():
            self.helper.layout.append(element)
        for element in self.get_submitbuttons_layout_elements():
            self.helper.layout.append(element)

class FeedbackEditorFormView(FeedbackEditorMixin, FormView):
    def get_form_kwargs(self):
        kwargs = super(FeedbackEditorFormView, self).get_form_kwargs()
        kwargs['last_draft'] = self.last_draft
        return kwargs

    def get_success_url(self):
        publish = 'submit_publish' in self.request.POST
        if publish:
            return super(FeedbackEditorFormView, self).get_success_url()
        else:
            return self.request.path

    def get_points_from_form(self, form):
        raise NotImplementedError()

    def get_create_feedbackdraft_kwargs(self, form, publish):
        return {
           'feedbacktext_raw': form.cleaned_data['feedbacktext'],
           'feedbacktext_html': form.cleaned_data['feedbacktext'],
           'publish': publish,
           'points': self.get_points_from_form(form)
        }

    def save_pluginspecific_state(self, form):
        """
        Save extra state that is specific to this plugin. I.E: Input from
        users that has no corresponding field in FeedbackDraft, and has to be
        stored in the data models for the plugin.
        """
        pass

    def form_valid(self, form):
        publish = 'submit_publish' in self.request.POST
        preview = 'submit_preview' in self.request.POST
        self.save_pluginspecific_state(form)
        draft = self.create_feedbackdraft(**self.get_create_feedbackdraft_kwargs(form, publish))
        if preview:
            return redirect('devilry_gradingsystem_feedbackdraft_preview',
                deliveryid=self.delivery.id,
                draftid=draft.id)
        else:
            return super(FeedbackEditorFormView, self).form_valid(form)

    def get_initial_from_last_draft(self):
        return {
            'feedbacktext': self.last_draft.feedbacktext_raw
        }

    def get_initial(self):
        initial = {}
        if self.last_draft:
            initial = self.get_initial_from_last_draft()
        return initial<commit_msg>devilry_gradingsystem.views.feedbackeditorbase: Convert feedbacktext to markdown.<commit_after>from django.core.urlresolvers import reverse
from django.views.generic.detail import SingleObjectMixin
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
from django.utils.translation import ugettext_lazy as _
from django import forms
from django.views.generic import FormView
from django.shortcuts import redirect
from django.http import HttpResponseBadRequest

from devilry.apps.markup.parse_markdown import markdown_full
from devilry.apps.core.models import Delivery
from devilry.apps.core.models import StaticFeedback
from devilry_gradingsystem.models import FeedbackDraft
from devilry_gradingsystem.widgets.editmarkdown import EditMarkdownLayoutObject
from devilry_gradingsystem.widgets.editfeedbackbuttonbar import EditFeedbackButtonBar

class FeedbackEditorSingleDeliveryObjectMixin(SingleObjectMixin):
    """
    Mixin that provides:

    - A delivery object.
    - Authorization for access to the delivery object.

    If your are creating a grading system plugin, you should NOT USE THIS
    directly. Use :class:`.FeedbackEditorMixin` instead.
    """
    model = Delivery
    pk_url_kwarg = 'deliveryid'
    context_object_name = 'delivery'

    @method_decorator(login_required)
    def dispatch(self, *args, **kwargs):
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).dispatch(*args, **kwargs)

    def _setup_common_data(self):
        self.object = self.get_object()
        self.delivery = self.object
        self.last_draft = None
        if self.delivery.devilry_gradingsystem_feedbackdraft_set.count() > 0:
            self.last_draft = self.delivery.devilry_gradingsystem_feedbackdraft_set.all()[0]

    def get(self, *args, **kwargs):
        self._setup_common_data()
        assignment = self.delivery.deadline.assignment_group.assignment
        if not assignment.has_valid_grading_setup():
            return redirect('devilry_examiner_singledeliveryview', deliveryid=self.delivery.id)
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).get(*args, **kwargs)

    def post(self, *args, **kwargs):
        self._setup_common_data()
        assignment = self.delivery.deadline.assignment_group.assignment
        if not assignment.has_valid_grading_setup():
            return HttpResponseBadRequest('Grading system is not set up correctly')
        return super(FeedbackEditorSingleDeliveryObjectMixin, self).post(*args, **kwargs)

    def get_queryset(self):
        """
        Ensure we only match deliveries where the current user has access
        as an examiner.
        """
        return Delivery.objects.filter_examiner_has_access(self.request.user)\
            .select_related( # Use select_related to lookup all the related objects in the query
                'deadline',
                'deadline__assignment_group',
                'deadline__assignment_group__parentnode', # Assignment
                'deadline__assignment_group__parentnode__parentnode', # Period
                'deadline__assignment_group__parentnode__parentnode__parentnode') # Subject

    def get_context_data(self, **kwargs):
        context = super(FeedbackEditorSingleDeliveryObjectMixin, self).get_context_data(**kwargs)
        delivery = self.object
        assignment = delivery.deadline.assignment_group.assignment
        context['valid_grading_system_setup'] = assignment.has_valid_grading_setup()
        return context

class FeedbackEditorMixin(FeedbackEditorSingleDeliveryObjectMixin):
    """
    Base mixin class for all feedback editor views.
    """

    def get_success_url(self):
        return reverse('devilry_examiner_singledeliveryview',
            kwargs={'deliveryid': self.delivery.id})

    def create_feedbackdraft(self, points, feedbacktext_raw, feedbacktext_html, publish=False):
        draft = FeedbackDraft(
            delivery=self.delivery,
            points=points,
            feedbacktext_raw=feedbacktext_raw,
            feedbacktext_html=feedbacktext_html,
            saved_by=self.request.user
        )
        if publish:
            draft.published = True
            draft.staticfeedback = draft.to_staticfeedback()
            draft.staticfeedback.full_clean()
            draft.staticfeedback.save()
        draft.save()
        return draft

class FeedbackEditorFormBase(forms.Form):
    def __init__(self, *args, **kwargs):
        self.last_draft = kwargs.pop('last_draft')
        super(FeedbackEditorFormBase, self).__init__(*args, **kwargs)
        self._add_feedbacktext_field()

    def _add_feedbacktext_field(self):
        if self.last_draft:
            feedbacktext_editor = self.last_draft.feedbacktext_editor
        else:
            feedbacktext_editor = FeedbackDraft.DEFAULT_FEEDBACKTEXT_EDITOR
        self.fields['feedbacktext'] = forms.CharField(
            label=_('Feedback text'),
            required=False)

    def get_feedbacktext_layout_elements(self):
        return [EditMarkdownLayoutObject()]

    def get_submitbuttons_layout_elements(self):
        return [EditFeedbackButtonBar()]

    def add_common_layout_elements(self):
        for element in self.get_feedbacktext_layout_elements():
            self.helper.layout.append(element)
        for element in self.get_submitbuttons_layout_elements():
            self.helper.layout.append(element)

class FeedbackEditorFormView(FeedbackEditorMixin, FormView):
    def get_form_kwargs(self):
        kwargs = super(FeedbackEditorFormView, self).get_form_kwargs()
        kwargs['last_draft'] = self.last_draft
        return kwargs

    def get_success_url(self):
        publish = 'submit_publish' in self.request.POST
        if publish:
            return super(FeedbackEditorFormView, self).get_success_url()
        else:
            return self.request.path

    def get_points_from_form(self, form):
        raise NotImplementedError()

    def get_create_feedbackdraft_kwargs(self, form, publish):
        return {
           'feedbacktext_raw': form.cleaned_data['feedbacktext'],
           'feedbacktext_html': markdown_full(form.cleaned_data['feedbacktext']),
           'publish': publish,
           'points': self.get_points_from_form(form)
        }

    def save_pluginspecific_state(self, form):
        """
        Save extra state that is specific to this plugin. I.E: Input from
        users that has no corresponding field in FeedbackDraft, and has to be
        stored in the data models for the plugin.
        """
        pass

    def form_valid(self, form):
        publish = 'submit_publish' in self.request.POST
        preview = 'submit_preview' in self.request.POST
        self.save_pluginspecific_state(form)
        draft = self.create_feedbackdraft(**self.get_create_feedbackdraft_kwargs(form, publish))
        if preview:
            return redirect('devilry_gradingsystem_feedbackdraft_preview',
                deliveryid=self.delivery.id,
                draftid=draft.id)
        else:
            return super(FeedbackEditorFormView, self).form_valid(form)

    def get_initial_from_last_draft(self):
        return {
            'feedbacktext': self.last_draft.feedbacktext_raw
        }

    def get_initial(self):
        initial = {}
        if self.last_draft:
            initial = self.get_initial_from_last_draft()
        return initial<|endoftext|>


<commit_before>'''
Processed BCSD-orig 20-year average country-level data by model

Aggregated to impact regions using area weights. Produced from BCSD-originals
using the GCP climate toolbox at https://github.com/ClimateImpactLab/pipelines
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)

__author__ = 'Michael Delgado'
__contact__ = 'mdelgado@rhg.com'
__version__ = '0.1.0'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{scenario}/day/atmos/{variable}',
    'r1i1p1/v1.0/{variable}_day_BCSD_{scenario}_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/temps/web/gcp/climate/{scenario}/{agglev}/{variable}',
    '{variable}_{agglev}_{aggwt}_{model}_{pername}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    repo='https://github.com/ClimateImpactLab/pipelines',
    file='/climate/jobs/impactlab-website/bcsd-orig.py',
    execute='climate.jobs.impactlab_website.bcsd_orig.main',
    project='gcp',
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='20yr')

def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')

def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')

def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    # dict(variable='tasmax', transformation=tasmax_over_95F),
    # dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(scenario='historical', pername='1986', years=list(range(1986, 2006))),
    dict(scenario='rcp85', pername='2020', years=list(range(2020, 2040))),
    dict(scenario='rcp85', pername='2040', years=list(range(2040, 2060))),
    dict(scenario='rcp85', pername='2080', years=list(range(2080, 2100)))]

MODELS = list(map(lambda x: dict(model=x), [
    'ACCESS1-0',
    'bcc-csm1-1',
    'BNU-ESM',
    'CanESM2',
    'CCSM4',
    'CESM1-BGC',
    'CNRM-CM5',
    'CSIRO-Mk3-6-0',
    'GFDL-CM3',
    'GFDL-ESM2G',
    'GFDL-ESM2M',
    'IPSL-CM5A-LR',
    'IPSL-CM5A-MR',
    'MIROC-ESM-CHEM',
    'MIROC-ESM',
    'MIROC5',
    'MPI-ESM-LR',
    'MPI-ESM-MR',
    'MRI-CGCM3',
    'inmcm4',
    'NorESM1-M']))

AGGREGATIONS = [{'agglev': 'ISO', 'aggwt': 'areawt'}]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS, AGGREGATIONS)

def run_job(variable, transformation, years, **kwargs):

    aggwt, agglev = kwargs.get('aggwt', 'popwt'), kwargs.get('agglev', 'hierid')

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update({
        'variable': variable,
        'transformation': transformation,
        'years': '{}-{}'.format(sorted(years)[0], sorted(years)[-1]),
        'aggwt': aggwt,
        'agglev': agglev})

    metadata.update(**kwargs)

    # Get transformed data
    ds = xr.concat([
        (load_climate_data(
            BCSD_orig_files.format(year=y, **metadata),
                variable)
            .pipe(transformation))
        for y in years],
        dim=pd.Index(years, name='year')).mean(dim='year')

    # Reshape to regions
    ds = weighted_aggregate_grid_to_regions(
            ds, variable, aggwt, agglev)

    # Update netCDF metadata
    ds.attrs.update(**metadata)

    # Write output
    fp = WRITE_PATH.format(**metadata)
    if not os.path.isdir(os.path.dirname(fp)):
        os.makedirs(os.path.dirname(fp))
    ds.to_netcdf(fp)

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('uploader')
logger.setLevel('INFO')

def main():

    njobs = reduce(lambda x, y: x*y, map(len, ITERATION_COMPONENTS))

    for i, job_components in enumerate(
            itertools.product(*ITERATION_COMPONENTS)):

        job = {}
        for job_component in job_components:
            job.update(job_component)

        logger.info('beginning job {} of {}'.format(i, njobs))

        try:
            run_job(**job)
        except Exception, e:
            logger.error(
                'Error encountered in job {} of {}:\n\nJob spec:\n{}\n\n'
                    .format(i, njobs, job),
                exc_info=e)

if __name__ == '__main__':
    main()
<commit_msg>switch output directory<commit_after>'''
Processed BCSD-orig 20-year average country-level data by model

Aggregated to impact regions using area weights. Produced from BCSD-originals
using the GCP climate toolbox at https://github.com/ClimateImpactLab/pipelines
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)

__author__ = 'Michael Delgado'
__contact__ = 'mdelgado@rhg.com'
__version__ = '0.1.0'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{scenario}/day/atmos/{variable}',
    'r1i1p1/v1.0/{variable}_day_BCSD_{scenario}_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/diagnostics/web/gcp/climate/{scenario}/{agglev}/{variable}',
    '{variable}_{agglev}_{aggwt}_{model}_{pername}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    repo='https://github.com/ClimateImpactLab/pipelines',
    file='/climate/jobs/impactlab-website/bcsd-orig.py',
    execute='climate.jobs.impactlab_website.bcsd_orig.main',
    project='gcp',
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='20yr')

def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')

def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')

def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    # dict(variable='tasmax', transformation=tasmax_over_95F),
    # dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(scenario='historical', pername='1986', years=list(range(1986, 2006))),
    dict(scenario='rcp85', pername='2020', years=list(range(2020, 2040))),
    dict(scenario='rcp85', pername='2040', years=list(range(2040, 2060))),
    dict(scenario='rcp85', pername='2080', years=list(range(2080, 2100)))]

MODELS = list(map(lambda x: dict(model=x), [
    'ACCESS1-0',
    'bcc-csm1-1',
    'BNU-ESM',
    'CanESM2',
    'CCSM4',
    'CESM1-BGC',
    'CNRM-CM5',
    'CSIRO-Mk3-6-0',
    'GFDL-CM3',
    'GFDL-ESM2G',
    'GFDL-ESM2M',
    'IPSL-CM5A-LR',
    'IPSL-CM5A-MR',
    'MIROC-ESM-CHEM',
    'MIROC-ESM',
    'MIROC5',
    'MPI-ESM-LR',
    'MPI-ESM-MR',
    'MRI-CGCM3',
    'inmcm4',
    'NorESM1-M']))

AGGREGATIONS = [{'agglev': 'ISO', 'aggwt': 'areawt'}]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS, AGGREGATIONS)

def run_job(variable, transformation, years, **kwargs):

    aggwt, agglev = kwargs.get('aggwt', 'popwt'), kwargs.get('agglev', 'hierid')

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update({
        'variable': variable,
        'transformation': transformation,
        'years': '{}-{}'.format(sorted(years)[0], sorted(years)[-1]),
        'aggwt': aggwt,
        'agglev': agglev})

    metadata.update(**kwargs)

    # Get transformed data
    ds = xr.concat([
        (load_climate_data(
            BCSD_orig_files.format(year=y, **metadata),
                variable)
            .pipe(transformation))
        for y in years],
        dim=pd.Index(years, name='year')).mean(dim='year')

    # Reshape to regions
    ds = weighted_aggregate_grid_to_regions(
            ds, variable, aggwt, agglev)

    # Update netCDF metadata
    ds.attrs.update(**metadata)

    # Write output
    fp = WRITE_PATH.format(**metadata)
    if not os.path.isdir(os.path.dirname(fp)):
        os.makedirs(os.path.dirname(fp))
    ds.to_netcdf(fp)

FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('uploader')
logger.setLevel('INFO')

def main():

    njobs = reduce(lambda x, y: x*y, map(len, ITERATION_COMPONENTS))

    for i, job_components in enumerate(
            itertools.product(*ITERATION_COMPONENTS)):

        job = {}
        for job_component in job_components:
            job.update(job_component)

        logger.info('beginning job {} of {}'.format(i, njobs))

        try:
            run_job(**job)
        except Exception, e:
            logger.error(
                'Error encountered in job {} of {}:\n\nJob spec:\n{}\n\n'
                    .format(i, njobs, job),
                exc_info=e)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>class SharedVariables:
    database = None

    movie_col = ['Id', 'Name', 'Rating']
    projection_col = ['Id', 'Movie_ID', 'Type', 'Projections Date Time',
                      'Available Spots']

    options = ['show movies', 'show movie projection <movieid> '
               '[<projection date>]', 'make reservation', 'reset database',
               'add movie', 'add projection', 'update movie',
               'update projection', 'update reservation', 'delete movie',
               'delete projection', 'update reservation', 'help', 'exit']

    available_seat = '.'
    taken_seat = 'x'

    number_of_rows = 10
    number_of_cols = 10

    end_of_day = "23:59:59"

    @classmethod
    def get_seats():
        seats = []
        for i in range(SharedVariables.number_of_rows):
            seats.append(list(SharedVariables.number_of_cols *
                              SharedVariables.available_seat))
        return seats
<commit_msg>Update help options<commit_after>class SharedVariables:
    database = None

    movie_col = ['Id', 'Name', 'Rating']
    projection_col = ['Id', 'Movie_ID', 'Type', 'Projections Date Time',
                      'Available Spots']

    options = ['show movies', 'show movie projection <movieid> '
               '[<projection date>]', 'make reservation', 'add movie',
               'add projection', 'update movie', 'update projection',
               'delete movie', 'delete projection', 'update reservation',
               'reset database' 'help', 'exit']

    available_seat = '.'
    taken_seat = 'x'

    number_of_rows = 10
    number_of_cols = 10

    end_of_day = "23:59:59"

    @classmethod
    def get_seats():
        seats = []
        for i in range(SharedVariables.number_of_rows):
            seats.append(list(SharedVariables.number_of_cols *
                              SharedVariables.available_seat))
        return seats
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

from autograd import numpy as ag_np
from autograd import value_and_grad
from autograd.core import primitive

import numpy as np
import functools
import pickle
import h5py

import keras.backend as K
import keras.callbacks as cbks
import keras.models

from pyannote.audio.callback import LoggingCallback
from pyannote.audio.callback import BaseLogger
from pyannote.audio.callback import Debugging

# populate CUSTOM_OBJECTS with user-defined optimizers and layers
import pyannote.audio.optimizers
import pyannote.audio.embedding.models
from pyannote.audio.embedding.losses import precomputed_gradient_loss
from pyannote.audio.keras_utils import CUSTOM_OBJECTS

from pyannote.generators.batch import batchify

EPSILON = 1e-6

@primitive
def arccos(x):
    return np.arccos(np.clip(x, -1., 1))
def arccos_vjp(g, ans, vs, gvs, x):
    return -g / np.sqrt(np.maximum(1. - x**2, EPSILON))
arccos.defvjp(arccos_vjp)

def value_and_multigrad(fun, argnums=[0]):
    """Takes gradients wrt multiple arguments simultaneously."""
    def combined_arg_fun(multi_arg, *args, **kwargs):
        extra_args_list = list(args)
        for argnum_ix, arg_ix in enumerate(argnums):
            extra_args_list[arg_ix] = multi_arg[argnum_ix]
        return fun(*extra_args_list, **kwargs)
    gradfun = value_and_grad(combined_arg_fun, argnum=0)
    def gradfun_rearranged(*args, **kwargs):
        multi_arg = tuple([args[i] for i in argnums])
        return gradfun(multi_arg, *args, **kwargs)
    return gradfun_rearranged

class MixinDistanceAutograd:
    """Differentiable distances between pairs of embeddings"""

    @staticmethod
    def l2_normalize(embedding):
        norm = ag_np.sqrt(ag_np.sum(embedding ** 2, axis=-1))
        return (embedding.T / norm).T

    @staticmethod
    def get_metric_max(metric):
        """Maximum distance between two (L2-normalized) embeddings

        Parameters
        ----------
        metric : {'sqeuclidean', 'euclidean', 'cosine', 'angular'}
            Metric name.

        Returns
        -------
        max : float
            Maximum possible distance between two embeddings
        """

        return {'sqeuclidean': 4.,
                'euclidean': 2.,
                'cosine': 2.,
                'angular': np.pi}[metric]

    @staticmethod
    def sqeuclidean(embedding, other_embedding=None):
        """Compute squared euclidean distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Squared euclidean distance
        """

        n_samples, n_dimensions = embedding.shape
        if other_embedding is None:
            other_embedding = embedding

        return ag_np.stack(
            ag_np.sum((embedding[i] - other_embedding) ** 2, axis=1)
            for i in range(n_samples))

    @staticmethod
    def euclidean(embedding, other_embedding=None):
        """Compute euclidean distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Euclidean distance
        """
        return ag_np.sqrt(self.sqeuclidean(
            embedding, other_embedding=other_embedding))

    @staticmethod
    def cosine(embedding, other_embedding=None):
        """Compute cosine distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional
            L2-normalized embeddings

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Cosine distance
        """

        n_samples, n_dimensions = embedding.shape
        if other_embedding is None:
            other_embedding = embedding

        return 1. - ag_np.stack(
            ag_np.sum(embedding[i] * other_embedding, axis=1)
            for i in range(n_samples))

    @staticmethod
    def angular(embedding, other_embedding=None):
        """Compute angular distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional
            L2-normalized embeddings

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Angular distance
        """

        n_samples, _ = embedding.shape

        if other_embedding is None:
            other_embedding = embedding

        return arccos(ag_np.stack(
            ag_np.sum(embedding[i] * other_embedding, axis=1)
            for i in range(n_samples)))

class SequenceEmbeddingAutograd(MixinDistanceAutograd, cbks.Callback):
    """Base class for sequence embedding

    Parameters
    ----------
    metric : {'sqeuclidean', 'euclidean', 'cosine', 'angular'}, optional
        Defaults to 'sqeuclidean'.
    gradient_factor : float, optional
        Multiply gradient by this number. Defaults to 1.
    batch_size : int, optional
        Internal batch size. Defaults to 32.
    """

    def __init__(self, metric='cosine', gradient_factor=1., batch_size=32):
        super(SequenceEmbeddingAutograd, self).__init__()
        self.metric = metric
        self.gradient_factor = gradient_factor
        self.batch_size = batch_size

        self.metric_ = getattr(self, metric)
        self.metric_max_ = self.get_metric_max(metric)

        self.float_autograd_ = ag_np.array(0.).dtype.name
        self.float_backend_ = K.floatx()

    @classmethod
    def restart(cls, log_dir, epoch):

        weights_h5 = LoggingCallback.WEIGHTS_H5.format(log_dir=log_dir,
                                                       epoch=epoch)

        # TODO update this code once keras > 2.0.4 is released
        try:
            embedding = keras.models.load_model(
                weights_h5, custom_objects=CUSTOM_OBJECTS,
                compile=True)
        except TypeError as e:
            embedding = keras.models.load_model(
                weights_h5, custom_objects=CUSTOM_OBJECTS)

        embedding.epoch = epoch

        return embedding

    def embed(self, embedding, X, internal=False):
        """Apply embedding on sequences

        Parameters
        ----------
        embedding : keras.Model
            Current state of embedding network
        X : (n_sequences, n_samples, n_features) numpy array
            Batch of input sequences
        internal : bool, optional
            Set to True to return internal representation

        Returns
        -------
        fX : (n_sequences, ...) numpy array
            Batch of embeddings.

        """

        if internal:

            embed = K.function(
                [embedding.get_layer(name='input').input, K.learning_phase()],
                [embedding.get_layer(name='internal').output])

            # split large batch in smaller batches if needed
            if len(X) > self.batch_size:
                batch_generator = batchify(X, {'type': 'ndarray'},
                                           batch_size=self.batch_size,
                                           incomplete=True)
                fX = np.stack(embed([x, 0])[0] for x in batch_generator)
            else:
                fX = embed([X, 0])[0]

        else:
            fX = embedding.predict(X, batch_size=self.batch_size)

        return fX.astype(self.float_autograd_)

    def fit(self, init_embedding, batch_generator, batches_per_epoch,
            n_classes=None, epochs=1000, log_dir=None, optimizer='rmsprop'):
        """

        Parameters
        ----------
        init_embedding : callable or keras.Model
            If callable, takes 'input_shape' as input and returns a Keras model
            that takes a sequence as input, and returns the embedding as output.
            If keras.Model, it must already be compiled.
        batch_generator : (infinite) iterable
            Generates (dict) batches such as batch['X'] is a numpy array of
            shape (variable_batch_size, n_samples, n_features).
        batches_per_epoch : int
            Number of batches per epoch.
        n_classes : int
        log_dir : str
        optimizer: str, optional
            Keras optimizer. Defaults to 'rmsprop'.
        """

        # consume batch generator once
        batch = next(batch_generator)

        if isinstance(init_embedding, keras.models.Model):
            embedding = init_embedding

            init_epoch = embedding.epoch
            restart = True

        else:

            # infer batch size and input shape
            batch_size, n_samples, n_features = batch['X'].shape

            # initialize embedding
            input_shape = (n_samples, n_features)
            embedding = init_embedding(input_shape)
            embedding.compile(optimizer=optimizer,
                              loss=precomputed_gradient_loss)

            init_epoch = -1
            restart = False

        callbacks = [self]
        callbacks.append(Debugging())
        callbacks.append(BaseLogger())
        callbacks.append(cbks.ProgbarLogger(count_mode='steps'))
        if log_dir is not None:
            callbacks.append(LoggingCallback(log_dir, restart=restart))
        self.history = cbks.History()
        callbacks.append(self.history)

        callbacks = cbks.CallbackList(callbacks)
        callbacks.set_model(embedding)
        callbacks.set_params({
            'epochs': epochs,
            'steps': batches_per_epoch,
            'verbose': True,
            'metrics': ['loss'],
        })

        callbacks.on_train_begin(logs={'n_classes': n_classes,
                                       'log_dir': log_dir,
                                       'restart': restart,
                                       'epoch': init_epoch})

        for epoch in range(init_epoch + 1, epochs):

            epoch_logs = {'log_dir': log_dir, 'restart': restart}
            callbacks.on_epoch_begin(epoch, logs=epoch_logs)

            for batch_index in range(batches_per_epoch):

                batch_size, _, _ = batch['X'].shape
                batch_logs = {'batch': batch_index,
                              'size': batch_size,
                              'epoch': epoch,
                              'log_dir': log_dir}

                callbacks.on_batch_begin(batch_index, logs=batch_logs)

                # compute loss and its gradient
                logs = self.loss_and_grad(batch, embedding)
                batch_logs.update(logs)

                # can anyone tell me why this usually works better
                # when gradient_factor is large?
                gradient = self.gradient_factor * batch_logs['gradient']

                X = batch['X'].astype(self.float_backend_)
                y = gradient.astype(self.float_backend_)

                # split large batch in smaller batches if needed
                if len(X) > self.batch_size:
                    signature = ({'type': 'ndarray'}, {'type': 'ndarray'})
                    sub_batch_generator = batchify(zip(X, y), signature,
                                                   batch_size=self.batch_size,
                                                   incomplete=True)
                else:
                    sub_batch_generator = [(X, y)]

                # backprop
                for X, y in sub_batch_generator:
                    embedding.train_on_batch(X, y)

                callbacks.on_batch_end(batch_index, logs=batch_logs)

                # next batch
                batch = next(batch_generator)

            callbacks.on_epoch_end(epoch, logs=epoch_logs)

        callbacks.on_train_end()

        return self.history
<commit_msg>fix: make an iterator from X<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

from autograd import numpy as ag_np
from autograd import value_and_grad
from autograd.core import primitive

import numpy as np
import functools
import pickle
import h5py

import keras.backend as K
import keras.callbacks as cbks
import keras.models

from pyannote.audio.callback import LoggingCallback
from pyannote.audio.callback import BaseLogger
from pyannote.audio.callback import Debugging

# populate CUSTOM_OBJECTS with user-defined optimizers and layers
import pyannote.audio.optimizers
import pyannote.audio.embedding.models
from pyannote.audio.embedding.losses import precomputed_gradient_loss
from pyannote.audio.keras_utils import CUSTOM_OBJECTS

from pyannote.generators.batch import batchify

EPSILON = 1e-6

@primitive
def arccos(x):
    return np.arccos(np.clip(x, -1., 1))
def arccos_vjp(g, ans, vs, gvs, x):
    return -g / np.sqrt(np.maximum(1. - x**2, EPSILON))
arccos.defvjp(arccos_vjp)

def value_and_multigrad(fun, argnums=[0]):
    """Takes gradients wrt multiple arguments simultaneously."""
    def combined_arg_fun(multi_arg, *args, **kwargs):
        extra_args_list = list(args)
        for argnum_ix, arg_ix in enumerate(argnums):
            extra_args_list[arg_ix] = multi_arg[argnum_ix]
        return fun(*extra_args_list, **kwargs)
    gradfun = value_and_grad(combined_arg_fun, argnum=0)
    def gradfun_rearranged(*args, **kwargs):
        multi_arg = tuple([args[i] for i in argnums])
        return gradfun(multi_arg, *args, **kwargs)
    return gradfun_rearranged

class MixinDistanceAutograd:
    """Differentiable distances between pairs of embeddings"""

    @staticmethod
    def l2_normalize(embedding):
        norm = ag_np.sqrt(ag_np.sum(embedding ** 2, axis=-1))
        return (embedding.T / norm).T

    @staticmethod
    def get_metric_max(metric):
        """Maximum distance between two (L2-normalized) embeddings

        Parameters
        ----------
        metric : {'sqeuclidean', 'euclidean', 'cosine', 'angular'}
            Metric name.

        Returns
        -------
        max : float
            Maximum possible distance between two embeddings
        """

        return {'sqeuclidean': 4.,
                'euclidean': 2.,
                'cosine': 2.,
                'angular': np.pi}[metric]

    @staticmethod
    def sqeuclidean(embedding, other_embedding=None):
        """Compute squared euclidean distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Squared euclidean distance
        """

        n_samples, n_dimensions = embedding.shape
        if other_embedding is None:
            other_embedding = embedding

        return ag_np.stack(
            ag_np.sum((embedding[i] - other_embedding) ** 2, axis=1)
            for i in range(n_samples))

    @staticmethod
    def euclidean(embedding, other_embedding=None):
        """Compute euclidean distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Euclidean distance
        """
        return ag_np.sqrt(self.sqeuclidean(
            embedding, other_embedding=other_embedding))

    @staticmethod
    def cosine(embedding, other_embedding=None):
        """Compute cosine distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional
            L2-normalized embeddings

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Cosine distance
        """

        n_samples, n_dimensions = embedding.shape
        if other_embedding is None:
            other_embedding = embedding

        return 1. - ag_np.stack(
            ag_np.sum(embedding[i] * other_embedding, axis=1)
            for i in range(n_samples))

    @staticmethod
    def angular(embedding, other_embedding=None):
        """Compute angular distance

        Parameters
        ----------
        embedding : (n_samples, n_dimension) numpy array
        other_embedding : (n_other_samples, n_dimension, ) numpy array, optional
            L2-normalized embeddings

        Returns
        -------
        distance : (n_samples, n_other_samples) numpy array or float
            Angular distance
        """

        n_samples, _ = embedding.shape

        if other_embedding is None:
            other_embedding = embedding

        return arccos(ag_np.stack(
            ag_np.sum(embedding[i] * other_embedding, axis=1)
            for i in range(n_samples)))

class SequenceEmbeddingAutograd(MixinDistanceAutograd, cbks.Callback):
    """Base class for sequence embedding

    Parameters
    ----------
    metric : {'sqeuclidean', 'euclidean', 'cosine', 'angular'}, optional
        Defaults to 'sqeuclidean'.
    gradient_factor : float, optional
        Multiply gradient by this number. Defaults to 1.
    batch_size : int, optional
        Internal batch size. Defaults to 32.
    """

    def __init__(self, metric='cosine', gradient_factor=1., batch_size=32):
        super(SequenceEmbeddingAutograd, self).__init__()
        self.metric = metric
        self.gradient_factor = gradient_factor
        self.batch_size = batch_size

        self.metric_ = getattr(self, metric)
        self.metric_max_ = self.get_metric_max(metric)

        self.float_autograd_ = ag_np.array(0.).dtype.name
        self.float_backend_ = K.floatx()

    @classmethod
    def restart(cls, log_dir, epoch):

        weights_h5 = LoggingCallback.WEIGHTS_H5.format(log_dir=log_dir,
                                                       epoch=epoch)

        # TODO update this code once keras > 2.0.4 is released
        try:
            embedding = keras.models.load_model(
                weights_h5, custom_objects=CUSTOM_OBJECTS,
                compile=True)
        except TypeError as e:
            embedding = keras.models.load_model(
                weights_h5, custom_objects=CUSTOM_OBJECTS)

        embedding.epoch = epoch

        return embedding

    def embed(self, embedding, X, internal=False):
        """Apply embedding on sequences

        Parameters
        ----------
        embedding : keras.Model
            Current state of embedding network
        X : (n_sequences, n_samples, n_features) numpy array
            Batch of input sequences
        internal : bool, optional
            Set to True to return internal representation

        Returns
        -------
        fX : (n_sequences, ...) numpy array
            Batch of embeddings.

        """

        if internal:

            embed = K.function(
                [embedding.get_layer(name='input').input, K.learning_phase()],
                [embedding.get_layer(name='internal').output])

            # split large batch in smaller batches if needed
            if len(X) > self.batch_size:
                batch_generator = batchify(iter(X), {'type': 'ndarray'},
                                           batch_size=self.batch_size,
                                           incomplete=True)
                fX = np.stack(embed([x, 0])[0] for x in batch_generator)
            else:
                fX = embed([X, 0])[0]

        else:
            fX = embedding.predict(X, batch_size=self.batch_size)

        return fX.astype(self.float_autograd_)

    def fit(self, init_embedding, batch_generator, batches_per_epoch,
            n_classes=None, epochs=1000, log_dir=None, optimizer='rmsprop'):
        """

        Parameters
        ----------
        init_embedding : callable or keras.Model
            If callable, takes 'input_shape' as input and returns a Keras model
            that takes a sequence as input, and returns the embedding as output.
            If keras.Model, it must already be compiled.
        batch_generator : (infinite) iterable
            Generates (dict) batches such as batch['X'] is a numpy array of
            shape (variable_batch_size, n_samples, n_features).
        batches_per_epoch : int
            Number of batches per epoch.
        n_classes : int
        log_dir : str
        optimizer: str, optional
            Keras optimizer. Defaults to 'rmsprop'.
        """

        # consume batch generator once
        batch = next(batch_generator)

        if isinstance(init_embedding, keras.models.Model):
            embedding = init_embedding

            init_epoch = embedding.epoch
            restart = True

        else:

            # infer batch size and input shape
            batch_size, n_samples, n_features = batch['X'].shape

            # initialize embedding
            input_shape = (n_samples, n_features)
            embedding = init_embedding(input_shape)
            embedding.compile(optimizer=optimizer,
                              loss=precomputed_gradient_loss)

            init_epoch = -1
            restart = False

        callbacks = [self]
        callbacks.append(Debugging())
        callbacks.append(BaseLogger())
        callbacks.append(cbks.ProgbarLogger(count_mode='steps'))
        if log_dir is not None:
            callbacks.append(LoggingCallback(log_dir, restart=restart))
        self.history = cbks.History()
        callbacks.append(self.history)

        callbacks = cbks.CallbackList(callbacks)
        callbacks.set_model(embedding)
        callbacks.set_params({
            'epochs': epochs,
            'steps': batches_per_epoch,
            'verbose': True,
            'metrics': ['loss'],
        })

        callbacks.on_train_begin(logs={'n_classes': n_classes,
                                       'log_dir': log_dir,
                                       'restart': restart,
                                       'epoch': init_epoch})

        for epoch in range(init_epoch + 1, epochs):

            epoch_logs = {'log_dir': log_dir, 'restart': restart}
            callbacks.on_epoch_begin(epoch, logs=epoch_logs)

            for batch_index in range(batches_per_epoch):

                batch_size, _, _ = batch['X'].shape
                batch_logs = {'batch': batch_index,
                              'size': batch_size,
                              'epoch': epoch,
                              'log_dir': log_dir}

                callbacks.on_batch_begin(batch_index, logs=batch_logs)

                # compute loss and its gradient
                logs = self.loss_and_grad(batch, embedding)
                batch_logs.update(logs)

                # can anyone tell me why this usually works better
                # when gradient_factor is large?
                gradient = self.gradient_factor * batch_logs['gradient']

                X = batch['X'].astype(self.float_backend_)
                y = gradient.astype(self.float_backend_)

                # split large batch in smaller batches if needed
                if len(X) > self.batch_size:
                    signature = ({'type': 'ndarray'}, {'type': 'ndarray'})
                    sub_batch_generator = batchify(zip(X, y), signature,
                                                   batch_size=self.batch_size,
                                                   incomplete=True)
                else:
                    sub_batch_generator = [(X, y)]

                # backprop
                for X, y in sub_batch_generator:
                    embedding.train_on_batch(X, y)

                callbacks.on_batch_end(batch_index, logs=batch_logs)

                # next batch
                batch = next(batch_generator)

            callbacks.on_epoch_end(epoch, logs=epoch_logs)

        callbacks.on_train_end()

        return self.history
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _

from django_cradmin.apps.cradmin_generic_token_with_metadata.models import GenericTokenWithMetadata
from django_cradmin.viewhelpers import objecttable
from .mixins import QuerysetForRoleMixin

class EmailColumn(objecttable.MultiActionColumn):
    modelfield = 'id'

    def get_header(self):
        return _('Email')

    def get_buttons(self, obj):
        return [
            objecttable.Button(
                label=_('Delete'),
                url=self.reverse_appurl('delete', args=[obj.id]),
                buttonclass="btn btn-danger btn-sm"),
        ]

    def render_value(self, obj):
        return obj.metadata['email']

    def is_sortable(self):
        return False

class CreatedDatetimeColumn(objecttable.DatetimeColumn):
    modelfield = 'created_datetime'

    def get_default_ordering(self):
        return None

class ExpirationDatetimeColumn(objecttable.DatetimeColumn):
    modelfield = 'expiration_datetime'

class Overview(QuerysetForRoleMixin, objecttable.ObjectTableView):
    model = GenericTokenWithMetadata
    columns = [
        EmailColumn,
        CreatedDatetimeColumn,
        ExpirationDatetimeColumn
    ]
    searchfields = ['metadata_json']

    def get_no_items_message(self):
        return _('No pending invites. Click the button above to invite new administrators for the site.')

    def get_pagetitle(self):
        return _('Invite admins')

    def get_buttons(self):
        app = self.request.cradmin_app
        return [
            objecttable.Button(_('Send private invite'), url=app.reverse_appurl('send')),
        ]
<commit_msg>demo...inviteadmins: Updated for changes to objecttable.Button.<commit_after>from __future__ import unicode_literals
from django.utils.translation import ugettext_lazy as _

from django_cradmin.apps.cradmin_generic_token_with_metadata.models import GenericTokenWithMetadata
from django_cradmin.viewhelpers import objecttable
from .mixins import QuerysetForRoleMixin

class EmailColumn(objecttable.MultiActionColumn):
    modelfield = 'id'

    def get_header(self):
        return _('Email')

    def get_buttons(self, obj):
        return [
            objecttable.Button(
                label=_('Delete'),
                url=self.reverse_appurl('delete', args=[obj.id]),
                buttonclass="btn btn-danger btn-sm"),
        ]

    def render_value(self, obj):
        return obj.metadata['email']

    def is_sortable(self):
        return False

class CreatedDatetimeColumn(objecttable.DatetimeColumn):
    modelfield = 'created_datetime'

    def get_default_ordering(self):
        return None

class ExpirationDatetimeColumn(objecttable.DatetimeColumn):
    modelfield = 'expiration_datetime'

class Overview(QuerysetForRoleMixin, objecttable.ObjectTableView):
    model = GenericTokenWithMetadata
    columns = [
        EmailColumn,
        CreatedDatetimeColumn,
        ExpirationDatetimeColumn
    ]
    searchfields = ['metadata_json']

    def get_no_items_message(self):
        return _('No pending invites. Click the button above to invite new administrators for the site.')

    def get_pagetitle(self):
        return _('Invite admins')

    def get_buttons(self):
        app = self.request.cradmin_app
        return [
            objecttable.Button(label=_('Send private invite'),
                               url=app.reverse_appurl('send'),
                               buttonclass='btn btn-primary'),
        ]
<|endoftext|>