<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from sys import argv
from codecs import open as codecs_open
from decimal import Decimal
from datetime import datetime
from re import compile as re_compile

import itertools

TRANSACTION_TYPES = {
    u'PRZELEW UZNANIOWY': re_compile(
        r'(?P<type>PRZELEW UZNANIOWY) '
        r'(\(NADANO (?P<send_date>[\d ]+-[\d ]+-[\d ]+)\) )?'
        r'(?P<title>.+?)(  +|$)'
        r'(?P<name>.+)?'
    ),
    u'PRZELEW OBCIĄŻENIOWY': re_compile(
        r'(?P<type>PRZELEW OBCIĄŻENIOWY) '
        r'(\(NADANO (?P<send_date>[\d ]+-[\d ]+-[\d ]+)\) )?'
        r'(?P<title>.+?)(  +|$)'
        r'(?P<name>.*)?'
    ),
    u'WPŁATA GOTÓWKOWA': re_compile(
        r'(?P<type>WPŁATA GOTÓWKOWA)'
        r'(?P<name>.+?)  +'
        r'(?P<title>.+)'
    ),
    u'': re_compile(
        r'(?P<type>)'
        r'(?P<name>)'
        r'(?P<title>)'
    ),
}

DATE_FORMAT = '%d-%m-%Y'

def parse_lines(lines):
    return [
        parse_line(line.rstrip('\n'), line_num)
        for line_num, line in enumerate(lines, start=1)
    ]

def parse_line(line, line_num=None):
    date, cost, name, main, iban, ref_id, op_code = line.split(u';', 6)
    main = main.strip()
    if not name:
        data = parse_main(main, line_num)
    else:
        name = name.strip()
        data = dict(
            main_line=u'{} {}'.format(name, main),
            title=main,
            name=name,
        )
    data.update(
        date=datetime.strptime(date.strip(), DATE_FORMAT).date(),
        cost=Decimal(cost),
        line_num=line_num,
        iban=iban,
        ref_id=ref_id,
    )
    return data

def get_data(main, line_num=None):
    for transaction_type, regex in TRANSACTION_TYPES.items():
        if not main.startswith(transaction_type):
            continue
        match = regex.match(main)
        if match is not None:
            return match.groupdict()
    else:
        raise ValueError(repr(main), line_num)

def parse_main(main, line_num=None):
    data = get_data(main, line_num)

    return dict(
        title=data['title'],
        name=data.get('name', ''),
        main_line=main,
    )

def open_and_parse(arg):
    with codecs_open(arg, encoding='utf-8') as f:
        return parse_lines(f.readlines())

if __name__ == "__main__":
    data = list(itertools.chain.from_iterable(
        open_and_parse(arg) for arg in argv[1:]
    ))
    data.sort(key=lambda obj: obj['date'])
    s = 0
    for x in data:
        s += x['cost']
        line = u'\t'.join([
            str(x['date']), x['title'], x['name'], str(x['cost'])
        ])
        print(line.encode('utf-8'))
    print('SUM', str(s))
<commit_msg>add more dates for csv file<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from sys import argv
from codecs import open as codecs_open
from decimal import Decimal
from datetime import datetime
from re import compile as re_compile

import itertools

TRANSACTION_TYPES = {
    u'PRZELEW UZNANIOWY': re_compile(
        r'(?P<type>PRZELEW UZNANIOWY) '
        r'(\(NADANO (?P<send_date>[\d ]+-[\d ]+-[\d ]+)\) )?'
        r'(?P<title>.+?)(  +|$)'
        r'(?P<name>.+)?'
    ),
    u'PRZELEW OBCIĄŻENIOWY': re_compile(
        r'(?P<type>PRZELEW OBCIĄŻENIOWY) '
        r'(\(NADANO (?P<send_date>[\d ]+-[\d ]+-[\d ]+)\) )?'
        r'(?P<title>.+?)(  +|$)'
        r'(?P<name>.*)?'
    ),
    u'WPŁATA GOTÓWKOWA': re_compile(
        r'(?P<type>WPŁATA GOTÓWKOWA)'
        r'(?P<name>.+?)  +'
        r'(?P<title>.+)'
    ),
    u'': re_compile(
        r'(?P<type>)'
        r'(?P<name>)'
        r'(?P<title>)'
    ),
}

DATE_FORMATS = (
    '%d-%m-%Y',
    '%Y-%m-%d',
)

def to_date(raw):
    raw = raw.strip()
    for date_format in DATE_FORMATS:
        try:
            dt = datetime.strptime(raw, DATE_FORMAT)
        except ValueError:
            continue
        else:
            return dt.date()
    raise ValueError(f"unknown date format: {raw}")

def parse_lines(lines):
    return [
        parse_line(line.rstrip('\n'), line_num)
        for line_num, line in enumerate(lines, start=1)
    ]

def parse_line(line, line_num=None):
    date, cost, name, main, iban, ref_id, op_code = line.split(u';', 6)
    main = main.strip()
    if not name:
        data = parse_main(main, line_num)
    else:
        name = name.strip()
        data = dict(
            main_line=u'{} {}'.format(name, main),
            title=main,
            name=name,
        )
    data.update(
        date=to_date(date),
        cost=Decimal(cost),
        line_num=line_num,
        iban=iban,
        ref_id=ref_id,
    )
    return data

def get_data(main, line_num=None):
    for transaction_type, regex in TRANSACTION_TYPES.items():
        if not main.startswith(transaction_type):
            continue
        match = regex.match(main)
        if match is not None:
            return match.groupdict()
    else:
        raise ValueError(repr(main), line_num)

def parse_main(main, line_num=None):
    data = get_data(main, line_num)

    return dict(
        title=data['title'],
        name=data.get('name', ''),
        main_line=main,
    )

def open_and_parse(arg):
    with codecs_open(arg, encoding='utf-8') as f:
        return parse_lines(f.readlines())

if __name__ == "__main__":
    data = list(itertools.chain.from_iterable(
        open_and_parse(arg) for arg in argv[1:]
    ))
    data.sort(key=lambda obj: obj['date'])
    s = 0
    for x in data:
        s += x['cost']
        line = u'\t'.join([
            str(x['date']), x['title'], x['name'], str(x['cost'])
        ])
        print(line.encode('utf-8'))
    print('SUM', str(s))
<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.db import models
from django.utils import simplejson
from django.utils.encoding import smart_unicode

class JSONField(models.TextField):
    """Simple JSON field that stores python structures as JSON strings
    on database.
    """

    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        """
        Convert the input JSON value into python structures, raises
        django.core.exceptions.ValidationError if the data can't be converted.
        """
        if self.blank and not value:
            return None
        if isinstance(value, basestring):
            try:
                return simplejson.loads(value)
            except Exception, e:
                raise ValidationError(str(e))
        else:
            return value

    def validate(self, value, model_instance):
        """Check value is a valid JSON string, raise ValidationError on
        error."""
        super(JSONField, self).validate(value, model_instance)
        try:
            return simplejson.loads(value)
        except Exception, e:
            raise ValidationError(str(e))

    def get_prep_value(self, value):
        """Convert value to JSON string before save"""
        try:
            return simplejson.dumps(value)
        except Exception, e:
            raise ValidationError(str(e))

    def value_to_string(self, obj):
        """Return value from object converted to string properly"""
        return smart_unicode(self.get_prep_value(self._get_val_from_obj(obj)))

try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^social_auth\.fields\.JSONField"])
except:
    pass
<commit_msg>Fix JSONField rendering on admin section. Closes gh-176<commit_after>from django.core.exceptions import ValidationError
from django.db import models
from django.utils import simplejson
from django.utils.encoding import smart_unicode

class JSONField(models.TextField):
    """Simple JSON field that stores python structures as JSON strings
    on database.
    """
    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        """
        Convert the input JSON value into python structures, raises
        django.core.exceptions.ValidationError if the data can't be converted.
        """
        if self.blank and not value:
            return None
        if isinstance(value, basestring):
            try:
                return simplejson.loads(value)
            except Exception, e:
                raise ValidationError(e.message)
        else:
            return value

    def validate(self, value, model_instance):
        """Check value is a valid JSON string, raise ValidationError on
        error."""
        if isinstance(value, basestring):
            super(JSONField, self).validate(value, model_instance)
            try:
                simplejson.loads(value)
            except Exception, e:
                raise ValidationError(e.message)

    def get_prep_value(self, value):
        """Convert value to JSON string before save"""
        try:
            return simplejson.dumps(value)
        except Exception, e:
            raise ValidationError(e.message)

    def value_to_string(self, obj):
        """Return value from object converted to string properly"""
        return smart_unicode(self.get_prep_value(self._get_val_from_obj(obj)))

    def value_from_object(self, obj):
        """Return value dumped to string."""
        return self.get_prep_value(self._get_val_from_obj(obj))

try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^social_auth\.fields\.JSONField"])
except:
    pass
<|endoftext|>


<commit_before>from fontTools.misc.py23 import bytesjoin
from fontTools.misc import sstruct
from fontTools.misc.textTools import safeEval
from . import DefaultTable
import array
import itertools
import logging
import struct
import sys
import fontTools.ttLib.tables.TupleVariation as tv

log = logging.getLogger(__name__)
TupleVariation = tv.TupleVariation

# https://www.microsoft.com/typography/otspec/gvar.htm
# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm
#
# Apple's documentation of 'gvar':
# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gvar.html
#
# FreeType2 source code for parsing 'gvar':
# http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c

GVAR_HEADER_FORMAT = """
	> # big endian
	version:			H
	reserved:			H
	axisCount:			H
	sharedTupleCount:		H
	offsetToSharedTuples:		I
	glyphCount:			H
	flags:				H
	offsetToGlyphVariationData:	I
"""

GVAR_HEADER_SIZE = sstruct.calcsize(GVAR_HEADER_FORMAT)

class table__g_v_a_r(DefaultTable.DefaultTable):
	dependencies = ["fvar", "glyf"]

	def __init__(self, tag=None):
		DefaultTable.DefaultTable.__init__(self, tag)
		self.version, self.reserved = 1, 0
		self.variations = {}

	def compile(self, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		sharedTuples =  tv.compileSharedTuples(
			axisTags, itertools.chain(*self.variations.values()))
		sharedTupleIndices = {coord:i for i, coord in enumerate(sharedTuples)}
		sharedTupleSize = sum([len(c) for c in sharedTuples])
		compiledGlyphs = self.compileGlyphs_(
			ttFont, axisTags, sharedTupleIndices)
		offset = 0
		offsets = []
		for glyph in compiledGlyphs:
			offsets.append(offset)
			offset += len(glyph)
		offsets.append(offset)
		compiledOffsets, tableFormat = self.compileOffsets_(offsets)

		header = {}
		header["version"] = self.version
		header["reserved"] = self.reserved
		header["axisCount"] = len(axisTags)
		header["sharedTupleCount"] = len(sharedTuples)
		header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets)
		header["glyphCount"] = len(compiledGlyphs)
		header["flags"] = tableFormat
		header["offsetToGlyphVariationData"] = header["offsetToSharedTuples"] + sharedTupleSize
		compiledHeader = sstruct.pack(GVAR_HEADER_FORMAT, header)

		result = [compiledHeader, compiledOffsets]
		result.extend(sharedTuples)
		result.extend(compiledGlyphs)
		return bytesjoin(result)

	def compileGlyphs_(self, ttFont, axisTags, sharedCoordIndices):
		result = []
		for glyphName in ttFont.getGlyphOrder():
			glyph = ttFont["glyf"][glyphName]
			pointCount = self.getNumPoints_(glyph)
			variations = self.variations.get(glyphName, [])
			result.append(compileGlyph_(variations, pointCount,
			                            axisTags, sharedCoordIndices))
		return result

	def decompile(self, data, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		glyphs = ttFont.getGlyphOrder()
		sstruct.unpack(GVAR_HEADER_FORMAT, data[0:GVAR_HEADER_SIZE], self)
		assert len(glyphs) == self.glyphCount
		assert len(axisTags) == self.axisCount
		offsets = self.decompileOffsets_(data[GVAR_HEADER_SIZE:], tableFormat=(self.flags & 1), glyphCount=self.glyphCount)
		sharedCoords = tv.decompileSharedTuples(
			axisTags, self.sharedTupleCount, data, self.offsetToSharedTuples)
		self.variations = {}
		offsetToData = self.offsetToGlyphVariationData
		for i in range(self.glyphCount):
			glyphName = glyphs[i]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			gvarData = data[offsetToData + offsets[i] : offsetToData + offsets[i + 1]]
			try:
				self.variations[glyphName] = decompileGlyph_(
					numPointsInGlyph, sharedCoords, axisTags, gvarData)
			except Exception:
				log.error(
					"Failed to decompile deltas for glyph '%s' (%d points)",
					glyphName, numPointsInGlyph,
				)
				raise

	@staticmethod
	def decompileOffsets_(data, tableFormat, glyphCount):
		if tableFormat == 0:
			# Short format: array of UInt16
			offsets = array.array("H")
			offsetsSize = (glyphCount + 1) * 2
		else:
			# Long format: array of UInt32
			offsets = array.array("I")
			offsetsSize = (glyphCount + 1) * 4
		offsets.frombytes(data[0 : offsetsSize])
		if sys.byteorder != "big": offsets.byteswap()

		# In the short format, offsets need to be multiplied by 2.
		# This is not documented in Apple's TrueType specification,
		# but can be inferred from the FreeType implementation, and
		# we could verify it with two sample GX fonts.
		if tableFormat == 0:
			offsets = [off * 2 for off in offsets]

		return offsets

	@staticmethod
	def compileOffsets_(offsets):
		"""Packs a list of offsets into a 'gvar' offset table.

		Returns a pair (bytestring, tableFormat). Bytestring is the
		packed offset table. Format indicates whether the table
		uses short (tableFormat=0) or long (tableFormat=1) integers.
		The returned tableFormat should get packed into the flags field
		of the 'gvar' header.
		"""
		assert len(offsets) >= 2
		for i in range(1, len(offsets)):
			assert offsets[i - 1] <= offsets[i]
		if max(offsets) <= 0xffff * 2:
			packed = array.array("H", [n >> 1 for n in offsets])
			tableFormat = 0
		else:
			packed = array.array("I", offsets)
			tableFormat = 1
		if sys.byteorder != "big": packed.byteswap()
		return (packed.tobytes(), tableFormat)

	def toXML(self, writer, ttFont):
		writer.simpletag("version", value=self.version)
		writer.newline()
		writer.simpletag("reserved", value=self.reserved)
		writer.newline()
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		for glyphName in ttFont.getGlyphNames():
			variations = self.variations.get(glyphName)
			if not variations:
				continue
			writer.begintag("glyphVariations", glyph=glyphName)
			writer.newline()
			for gvar in variations:
				gvar.toXML(writer, axisTags)
			writer.endtag("glyphVariations")
			writer.newline()

	def fromXML(self, name, attrs, content, ttFont):
		if name == "version":
			self.version = safeEval(attrs["value"])
		elif name == "reserved":
			self.reserved = safeEval(attrs["value"])
		elif name == "glyphVariations":
			if not hasattr(self, "variations"):
				self.variations = {}
			glyphName = attrs["glyph"]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			glyphVariations = []
			for element in content:
				if isinstance(element, tuple):
					name, attrs, content = element
					if name == "tuple":
						gvar = TupleVariation({}, [None] * numPointsInGlyph)
						glyphVariations.append(gvar)
						for tupleElement in content:
							if isinstance(tupleElement, tuple):
								tupleName, tupleAttrs, tupleContent = tupleElement
								gvar.fromXML(tupleName, tupleAttrs, tupleContent)
			self.variations[glyphName] = glyphVariations

	@staticmethod
	def getNumPoints_(glyph):
		NUM_PHANTOM_POINTS = 4
		if glyph.isComposite():
			return len(glyph.components) + NUM_PHANTOM_POINTS
		else:
			# Empty glyphs (eg. space, nonmarkingreturn) have no "coordinates" attribute.
			return len(getattr(glyph, "coordinates", [])) + NUM_PHANTOM_POINTS

def compileGlyph_(variations, pointCount, axisTags, sharedCoordIndices):
	tupleVariationCount, tuples, data = tv.compileTupleVariationStore(
		variations, pointCount, axisTags, sharedCoordIndices)
	if tupleVariationCount == 0:
		return b""
	result = (
		struct.pack(">HH", tupleVariationCount, 4 + len(tuples)) + tuples + data
	)
	if len(result) % 2 != 0:
		result = result + b"\0"  # padding
	return result

def decompileGlyph_(pointCount, sharedTuples, axisTags, data):
	if len(data) < 4:
		return []
	tupleVariationCount, offsetToData = struct.unpack(">HH", data[:4])
	dataPos = offsetToData
	return tv.decompileTupleVariationStore(
		"gvar", axisTags,
		tupleVariationCount, pointCount,
		sharedTuples, data, 4, offsetToData
	)
<commit_msg>[gvar] Remove use of bytesjoin()<commit_after>from fontTools.misc import sstruct
from fontTools.misc.textTools import safeEval
from . import DefaultTable
import array
import itertools
import logging
import struct
import sys
import fontTools.ttLib.tables.TupleVariation as tv

log = logging.getLogger(__name__)
TupleVariation = tv.TupleVariation

# https://www.microsoft.com/typography/otspec/gvar.htm
# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm
#
# Apple's documentation of 'gvar':
# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gvar.html
#
# FreeType2 source code for parsing 'gvar':
# http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c

GVAR_HEADER_FORMAT = """
	> # big endian
	version:			H
	reserved:			H
	axisCount:			H
	sharedTupleCount:		H
	offsetToSharedTuples:		I
	glyphCount:			H
	flags:				H
	offsetToGlyphVariationData:	I
"""

GVAR_HEADER_SIZE = sstruct.calcsize(GVAR_HEADER_FORMAT)

class table__g_v_a_r(DefaultTable.DefaultTable):
	dependencies = ["fvar", "glyf"]

	def __init__(self, tag=None):
		DefaultTable.DefaultTable.__init__(self, tag)
		self.version, self.reserved = 1, 0
		self.variations = {}

	def compile(self, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		sharedTuples =  tv.compileSharedTuples(
			axisTags, itertools.chain(*self.variations.values()))
		sharedTupleIndices = {coord:i for i, coord in enumerate(sharedTuples)}
		sharedTupleSize = sum([len(c) for c in sharedTuples])
		compiledGlyphs = self.compileGlyphs_(
			ttFont, axisTags, sharedTupleIndices)
		offset = 0
		offsets = []
		for glyph in compiledGlyphs:
			offsets.append(offset)
			offset += len(glyph)
		offsets.append(offset)
		compiledOffsets, tableFormat = self.compileOffsets_(offsets)

		header = {}
		header["version"] = self.version
		header["reserved"] = self.reserved
		header["axisCount"] = len(axisTags)
		header["sharedTupleCount"] = len(sharedTuples)
		header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets)
		header["glyphCount"] = len(compiledGlyphs)
		header["flags"] = tableFormat
		header["offsetToGlyphVariationData"] = header["offsetToSharedTuples"] + sharedTupleSize
		compiledHeader = sstruct.pack(GVAR_HEADER_FORMAT, header)

		result = [compiledHeader, compiledOffsets]
		result.extend(sharedTuples)
		result.extend(compiledGlyphs)
		return b''.join(result)

	def compileGlyphs_(self, ttFont, axisTags, sharedCoordIndices):
		result = []
		for glyphName in ttFont.getGlyphOrder():
			glyph = ttFont["glyf"][glyphName]
			pointCount = self.getNumPoints_(glyph)
			variations = self.variations.get(glyphName, [])
			result.append(compileGlyph_(variations, pointCount,
			                            axisTags, sharedCoordIndices))
		return result

	def decompile(self, data, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		glyphs = ttFont.getGlyphOrder()
		sstruct.unpack(GVAR_HEADER_FORMAT, data[0:GVAR_HEADER_SIZE], self)
		assert len(glyphs) == self.glyphCount
		assert len(axisTags) == self.axisCount
		offsets = self.decompileOffsets_(data[GVAR_HEADER_SIZE:], tableFormat=(self.flags & 1), glyphCount=self.glyphCount)
		sharedCoords = tv.decompileSharedTuples(
			axisTags, self.sharedTupleCount, data, self.offsetToSharedTuples)
		self.variations = {}
		offsetToData = self.offsetToGlyphVariationData
		for i in range(self.glyphCount):
			glyphName = glyphs[i]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			gvarData = data[offsetToData + offsets[i] : offsetToData + offsets[i + 1]]
			try:
				self.variations[glyphName] = decompileGlyph_(
					numPointsInGlyph, sharedCoords, axisTags, gvarData)
			except Exception:
				log.error(
					"Failed to decompile deltas for glyph '%s' (%d points)",
					glyphName, numPointsInGlyph,
				)
				raise

	@staticmethod
	def decompileOffsets_(data, tableFormat, glyphCount):
		if tableFormat == 0:
			# Short format: array of UInt16
			offsets = array.array("H")
			offsetsSize = (glyphCount + 1) * 2
		else:
			# Long format: array of UInt32
			offsets = array.array("I")
			offsetsSize = (glyphCount + 1) * 4
		offsets.frombytes(data[0 : offsetsSize])
		if sys.byteorder != "big": offsets.byteswap()

		# In the short format, offsets need to be multiplied by 2.
		# This is not documented in Apple's TrueType specification,
		# but can be inferred from the FreeType implementation, and
		# we could verify it with two sample GX fonts.
		if tableFormat == 0:
			offsets = [off * 2 for off in offsets]

		return offsets

	@staticmethod
	def compileOffsets_(offsets):
		"""Packs a list of offsets into a 'gvar' offset table.

		Returns a pair (bytestring, tableFormat). Bytestring is the
		packed offset table. Format indicates whether the table
		uses short (tableFormat=0) or long (tableFormat=1) integers.
		The returned tableFormat should get packed into the flags field
		of the 'gvar' header.
		"""
		assert len(offsets) >= 2
		for i in range(1, len(offsets)):
			assert offsets[i - 1] <= offsets[i]
		if max(offsets) <= 0xffff * 2:
			packed = array.array("H", [n >> 1 for n in offsets])
			tableFormat = 0
		else:
			packed = array.array("I", offsets)
			tableFormat = 1
		if sys.byteorder != "big": packed.byteswap()
		return (packed.tobytes(), tableFormat)

	def toXML(self, writer, ttFont):
		writer.simpletag("version", value=self.version)
		writer.newline()
		writer.simpletag("reserved", value=self.reserved)
		writer.newline()
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		for glyphName in ttFont.getGlyphNames():
			variations = self.variations.get(glyphName)
			if not variations:
				continue
			writer.begintag("glyphVariations", glyph=glyphName)
			writer.newline()
			for gvar in variations:
				gvar.toXML(writer, axisTags)
			writer.endtag("glyphVariations")
			writer.newline()

	def fromXML(self, name, attrs, content, ttFont):
		if name == "version":
			self.version = safeEval(attrs["value"])
		elif name == "reserved":
			self.reserved = safeEval(attrs["value"])
		elif name == "glyphVariations":
			if not hasattr(self, "variations"):
				self.variations = {}
			glyphName = attrs["glyph"]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			glyphVariations = []
			for element in content:
				if isinstance(element, tuple):
					name, attrs, content = element
					if name == "tuple":
						gvar = TupleVariation({}, [None] * numPointsInGlyph)
						glyphVariations.append(gvar)
						for tupleElement in content:
							if isinstance(tupleElement, tuple):
								tupleName, tupleAttrs, tupleContent = tupleElement
								gvar.fromXML(tupleName, tupleAttrs, tupleContent)
			self.variations[glyphName] = glyphVariations

	@staticmethod
	def getNumPoints_(glyph):
		NUM_PHANTOM_POINTS = 4
		if glyph.isComposite():
			return len(glyph.components) + NUM_PHANTOM_POINTS
		else:
			# Empty glyphs (eg. space, nonmarkingreturn) have no "coordinates" attribute.
			return len(getattr(glyph, "coordinates", [])) + NUM_PHANTOM_POINTS

def compileGlyph_(variations, pointCount, axisTags, sharedCoordIndices):
	tupleVariationCount, tuples, data = tv.compileTupleVariationStore(
		variations, pointCount, axisTags, sharedCoordIndices)
	if tupleVariationCount == 0:
		return b""
	result = (
		struct.pack(">HH", tupleVariationCount, 4 + len(tuples)) + tuples + data
	)
	if len(result) % 2 != 0:
		result = result + b"\0"  # padding
	return result

def decompileGlyph_(pointCount, sharedTuples, axisTags, data):
	if len(data) < 4:
		return []
	tupleVariationCount, offsetToData = struct.unpack(">HH", data[:4])
	dataPos = offsetToData
	return tv.decompileTupleVariationStore(
		"gvar", axisTags,
		tupleVariationCount, pointCount,
		sharedTuples, data, 4, offsetToData
	)
<|endoftext|>


<commit_before>from zeit.content.cp.interfaces import IAutomaticTeaserBlock
from zeit.solr import query as lq
from zope.cachedescriptors.property import Lazy as cachedproperty
import grokcore.component as grok
import logging
import zeit.content.cp.interfaces
import zeit.find.search
import zope.component
import zope.interface

log = logging.getLogger(__name__)

class AutomaticArea(zeit.cms.content.xmlsupport.Persistent):

    zope.component.adapts(zeit.content.cp.interfaces.IArea)
    zope.interface.implements(zeit.content.cp.interfaces.IRenderedArea)

    def __init__(self, context):
        self.context = context
        self.xml = self.context.xml
        self.__parent__ = self.context

    # Convenience: Delegate IArea to our context, so we can be used like one.
    def __getattr__(self, name):
        # There's no interface for xmlsupport.Persistent which could tell us
        # that this attribute needs special treatment.
        if name == '__parent__':
            return super(AutomaticArea, self).__getattr__(name)
        if name in zeit.content.cp.interfaces.IArea:
            return getattr(self.context, name)
        raise AttributeError(name)

    def values(self):
        if not self.automatic:
            return self.context.values()

        try:
            content = self._content_query()
        except LookupError:
            log.warning('%s found no IContentQuery type %s',
                        self.context, self.automatic_type)
            return self.context.values()

        result = []
        for block in self.context.values():
            if not IAutomaticTeaserBlock.providedBy(block):
                result.append(block)
                continue
            # This assumes that the *first* block always has a leader layout,
            # since otherwise the first result that may_be_leader might be
            # given to a non-leader block.
            if block.layout.id in ['leader', 'zon-large']:
                teaser = pop_filter(content, is_lead_candidate)
                if teaser is None:
                    teaser = pop_filter(content)
                    block.change_layout(
                        zeit.content.cp.layout.get_layout('buttons'))
            else:
                teaser = pop_filter(content)
            if teaser is None:
                continue
            block.insert(0, teaser)
            result.append(block)

        return result

    @cachedproperty
    def _content_query(self):
        return zope.component.getAdapter(
            self, zeit.content.cp.interfaces.IContentQuery,
            name=self.automatic_type)

    def select_modules(self, *interfaces):
        for module in self.values():
            if any([x.providedBy(module) for x in interfaces]):
                yield module

def pop_filter(items, predicate=None):
    """Remove the first object from the list for which predicate returns True;
    no predicate means no filtering.
    """
    for i, item in enumerate(items):
        if predicate is None or predicate(item):
            items.pop(i)
            return item

def is_lead_candidate(content):
    metadata = zeit.cms.content.interfaces.ICommonMetadata(content, None)
    if metadata is None:
        return False
    return metadata.lead_candidate

class ContentQuery(grok.Adapter):

    grok.context(zeit.content.cp.interfaces.IRenderedArea)
    grok.implements(zeit.content.cp.interfaces.IContentQuery)
    grok.baseclass()

    def __init__(self, context):
        self.context = context
        self.start = 0
        self.rows = context.count

    def __call__(self):
        return []

    @cachedproperty
    def existing_teasers(self):
        cp = zeit.content.cp.interfaces.ICenterPage(self.context)
        result = set()
        result.update(cp.teasered_content_above(self.context))
        result.update(cp.manual_content_below(self.context))
        return result

class SolrContentQuery(ContentQuery):

    grok.context(zeit.content.cp.interfaces.IRenderedArea)
    grok.implements(zeit.content.cp.interfaces.IContentQuery)
    grok.name('query')

    FIELDS = ' '.join(zeit.find.search.DEFAULT_RESULT_FIELDS)

    def __init__(self, context):
        super(SolrContentQuery, self).__init__(context)
        self.query_string = self.context.raw_query
        self.order = self.context.raw_order

    def __call__(self):
        return self._query_solr(self.query_string, self.order)

    def _query_solr(self, query, sort_order):
        result = []
        try:
            solr = zope.component.getUtility(zeit.solr.interfaces.ISolr)
            for item in solr.search(
                    query, sort=sort_order,
                    start=self.start,
                    rows=self.rows,
                    fl=self.FIELDS,
                    fq=self.filter_query):
                content = zeit.cms.interfaces.ICMSContent(
                    item['uniqueId'], None)
                if content is not None:
                    result.append(content)
        except:
            log.warning('Error during solr query %r for %s',
                        query, self.context.uniqueId, exc_info=True)
        return result

    @property
    def filter_query(self):
        if not self.context.hide_dupes or not self.existing_teasers:
            return lq.any_value()
        return lq.not_(lq.or_(*[lq._field('uniqueId', '"%s"' % x.uniqueId)
                                for x in self.existing_teasers]))

class ChannelContentQuery(SolrContentQuery):

    grok.name('channel')

    SOLR_FIELD = {
        'Channel': 'channels',
        'Keyword': 'keywords',
    }

    def __init__(self, context):
        super(SolrContentQuery, self).__init__(context)
        self.query_string = self._build_query()
        self.order = self.context.query_order

    def _build_query(self):
        query = zeit.find.search.query(filter_terms=[
            zeit.solr.query.field_raw('published', 'published*')])
        conditions = []
        for type_, channel, subchannel in self.context.query:
            if subchannel:
                value = '%s*%s' % (channel, subchannel)
            else:
                # XXX Unclear whether this will work as desired for keywords.
                value = '%s*' % channel
            conditions.append(zeit.solr.query.field_raw(
                self.SOLR_FIELD[type_], value))
        if conditions:
            query = zeit.solr.query.and_(
                query, zeit.solr.query.or_(*conditions))
        return query

class CenterpageContentQuery(ContentQuery):

    grok.name('centerpage')

    def __call__(self):
        teasered = zeit.content.cp.interfaces.ITeaseredContent(
            self.context.referenced_cp, iter([]))
        result = []
        for content in teasered:
            if self.context.hide_dupes and content in self.existing_teasers:
                continue
            result.append(content)
            if len(result) >= self.rows:
                break
        return result
<commit_msg>ZON-3227: Read start and count from area, this is an extension point for zeit.web to do pagination<commit_after>from zeit.content.cp.interfaces import IAutomaticTeaserBlock
from zeit.solr import query as lq
from zope.cachedescriptors.property import Lazy as cachedproperty
import grokcore.component as grok
import logging
import zeit.content.cp.interfaces
import zeit.find.search
import zope.component
import zope.interface

log = logging.getLogger(__name__)

class AutomaticArea(zeit.cms.content.xmlsupport.Persistent):

    zope.component.adapts(zeit.content.cp.interfaces.IArea)
    zope.interface.implements(zeit.content.cp.interfaces.IRenderedArea)

    start = 0  # Extension point for zeit.web to do pagination

    def __init__(self, context):
        self.context = context
        self.xml = self.context.xml
        self.__parent__ = self.context

    # Convenience: Delegate IArea to our context, so we can be used like one.
    def __getattr__(self, name):
        # There's no interface for xmlsupport.Persistent which could tell us
        # that this attribute needs special treatment.
        if name == '__parent__':
            return super(AutomaticArea, self).__getattr__(name)
        if name in zeit.content.cp.interfaces.IArea:
            return getattr(self.context, name)
        raise AttributeError(name)

    def values(self):
        if not self.automatic:
            return self.context.values()

        try:
            content = self._content_query()
        except LookupError:
            log.warning('%s found no IContentQuery type %s',
                        self.context, self.automatic_type)
            return self.context.values()

        result = []
        for block in self.context.values():
            if not IAutomaticTeaserBlock.providedBy(block):
                result.append(block)
                continue
            # This assumes that the *first* block always has a leader layout,
            # since otherwise the first result that may_be_leader might be
            # given to a non-leader block.
            if block.layout.id in ['leader', 'zon-large']:
                teaser = pop_filter(content, is_lead_candidate)
                if teaser is None:
                    teaser = pop_filter(content)
                    block.change_layout(
                        zeit.content.cp.layout.get_layout('buttons'))
            else:
                teaser = pop_filter(content)
            if teaser is None:
                continue
            block.insert(0, teaser)
            result.append(block)

        return result

    @cachedproperty
    def _content_query(self):
        return zope.component.getAdapter(
            self, zeit.content.cp.interfaces.IContentQuery,
            name=self.automatic_type)

    def select_modules(self, *interfaces):
        for module in self.values():
            if any([x.providedBy(module) for x in interfaces]):
                yield module

def pop_filter(items, predicate=None):
    """Remove the first object from the list for which predicate returns True;
    no predicate means no filtering.
    """
    for i, item in enumerate(items):
        if predicate is None or predicate(item):
            items.pop(i)
            return item

def is_lead_candidate(content):
    metadata = zeit.cms.content.interfaces.ICommonMetadata(content, None)
    if metadata is None:
        return False
    return metadata.lead_candidate

class ContentQuery(grok.Adapter):

    grok.context(zeit.content.cp.interfaces.IRenderedArea)
    grok.implements(zeit.content.cp.interfaces.IContentQuery)
    grok.baseclass()

    def __init__(self, context):
        self.context = context

    def __call__(self):
        return []

    @property
    def start(self):
        return self.context.start

    @property
    def rows(self):
        return self.context.count

    @cachedproperty
    def existing_teasers(self):
        cp = zeit.content.cp.interfaces.ICenterPage(self.context)
        result = set()
        result.update(cp.teasered_content_above(self.context))
        result.update(cp.manual_content_below(self.context))
        return result

class SolrContentQuery(ContentQuery):

    grok.context(zeit.content.cp.interfaces.IRenderedArea)
    grok.implements(zeit.content.cp.interfaces.IContentQuery)
    grok.name('query')

    FIELDS = ' '.join(zeit.find.search.DEFAULT_RESULT_FIELDS)

    def __init__(self, context):
        super(SolrContentQuery, self).__init__(context)
        self.query_string = self.context.raw_query
        self.order = self.context.raw_order

    def __call__(self):
        return self._query_solr(self.query_string, self.order)

    def _query_solr(self, query, sort_order):
        result = []
        try:
            solr = zope.component.getUtility(zeit.solr.interfaces.ISolr)
            for item in solr.search(
                    query, sort=sort_order,
                    start=self.start,
                    rows=self.rows,
                    fl=self.FIELDS,
                    fq=self.filter_query):
                content = zeit.cms.interfaces.ICMSContent(
                    item['uniqueId'], None)
                if content is not None:
                    result.append(content)
        except:
            log.warning('Error during solr query %r for %s',
                        query, self.context.uniqueId, exc_info=True)
        return result

    @property
    def filter_query(self):
        if not self.context.hide_dupes or not self.existing_teasers:
            return lq.any_value()
        return lq.not_(lq.or_(*[lq._field('uniqueId', '"%s"' % x.uniqueId)
                                for x in self.existing_teasers]))

class ChannelContentQuery(SolrContentQuery):

    grok.name('channel')

    SOLR_FIELD = {
        'Channel': 'channels',
        'Keyword': 'keywords',
    }

    def __init__(self, context):
        super(SolrContentQuery, self).__init__(context)
        self.query_string = self._build_query()
        self.order = self.context.query_order

    def _build_query(self):
        query = zeit.find.search.query(filter_terms=[
            zeit.solr.query.field_raw('published', 'published*')])
        conditions = []
        for type_, channel, subchannel in self.context.query:
            if subchannel:
                value = '%s*%s' % (channel, subchannel)
            else:
                # XXX Unclear whether this will work as desired for keywords.
                value = '%s*' % channel
            conditions.append(zeit.solr.query.field_raw(
                self.SOLR_FIELD[type_], value))
        if conditions:
            query = zeit.solr.query.and_(
                query, zeit.solr.query.or_(*conditions))
        return query

class CenterpageContentQuery(ContentQuery):

    grok.name('centerpage')

    def __call__(self):
        teasered = zeit.content.cp.interfaces.ITeaseredContent(
            self.context.referenced_cp, iter([]))
        result = []
        for content in teasered:
            if self.context.hide_dupes and content in self.existing_teasers:
                continue
            result.append(content)
            if len(result) >= self.rows:
                break
        return result
<|endoftext|>


<commit_before>'''

    @brief  Solve the benchmark "differentially heated cavity" natural convection problem using finite elements.

    @detail

        Solve the natural convection test problem from

            @article
            {danaila2014newton,
              title={A Newton method with adaptive finite elements for solving phase-change problems with natural convection},
              author={Danaila, Ionut and Moglan, Raluca and Hecht, Fr{\'e}d{\'e}ric and Le Masson, St{\'e}phane},
              journal={Journal of Computational Physics},
              volume={274},
              pages={826--840},
              year={2014},
              publisher={Academic Press}
            }

        Match the notation in danaila2014newton as best as possible.

    @author Alexander G. Zimmerman <zimmerman@aices.rwth-aachen.de>

'''

from fenics import \
    UnitSquareMesh, FiniteElement, VectorElement, MixedElement, \
    FunctionSpace, VectorFunctionSpace, \
    Function, TrialFunction, TestFunctions, split, \
    DirichletBC, Constant, Expression, \
    dx, \
    dot, inner, grad, nabla_grad, sym, div, \
    errornorm, norm, \
    File, \
    Progress, set_log_level, PROGRESS, \
    project, interpolate, \
    solve, parameters, info, derivative, \
    LinearVariationalProblem, LinearVariationalSolver, NonlinearVariationalProblem, NonlinearVariationalSolver, \
    SubDomain, EdgeFunction, near, adapt

def arguments():
    """Returns tuple containing dictionary of calling function's
       named arguments and a list of calling function's unnamed
       positional arguments.
    """
    from inspect import getargvalues, stack
    posname, kwname, args = getargvalues(stack()[1][0])[-3:]
    posargs = args.pop(posname, [])
    args.update(args.pop(kwname, []))
    return args, posargs

def run(
    output_dir = 'output_nsb_pcm', \
    Ra = 1.e6, \
    Pr = 0.71, \
    K = 1., \
    g = (0., -1.), \
    mu = 1., \
    mesh = UnitSquareMesh(20, 20, "crossed"), \
    s_u = ('0.', '0.'), \
    s_p = '0.', \
    s_theta = '0.', \
    initial_values_expression = ( \
        '0.', \
        '0.', \
        '0.', \
        '0.5*near(x[0],  0.) -0.5*near(x[0],  1.)'), \
    bc_expressions = [ \
        [0, ('0.', '0.'), 3, 'near(x[0],  0.) | near(x[0],  1.) | near(x[1], 0.) | near(x[1],  1.)'], \
        [2, '0.', 2, 'near(x[0],  0.)'], \
        [2, '0.', 2, 'near(x[0],  1.)']], \
    final_time = 1., \
    time_step_size = 1.e-3, \
    adaptive_time = True, \
    gamma = 1.e-7, \
    pressure_degree = 1, \
    temperature_degree = 1, \
    linearize = True, \
    newton_relative_tolerance = 1.e-9, \
    max_newton_iterations = 10, \
    stop_when_steady = True, \
    steady_relative_tolerance = 1.e-8, \
    debug_b_factor = 1., \
    debug_c_factor = 1., \
    exact_solution_expression = [] \
    ):

    #
    print("Running nsb_pcm with the following arguments:")

    print(arguments())

    # @todo Try 3D
    dim = 2

    '''
    Per Equation 8 from danaila2014newton, we implement an equation scaled with v_ref = nu_liquid/H => t_ref = nu_liquid/H^2 => Re = 1.
    '''
    Re = 1.

    # Compute derived parameters
    velocity_degree = pressure_degree + 1

    # Define function spaces for the system
    VxV = VectorFunctionSpace(mesh, 'P', velocity_degree)

    Q = FunctionSpace(mesh, 'P', pressure_degree) # @todo mixing up test function space

    V = FunctionSpace(mesh, 'P', temperature_degree)

    '''
    MixedFunctionSpace used to be available but is now deprecated.
    The way that fenics separates function spaces and elements is confusing.
    To create the mixed space, I'm using the approach from https://fenicsproject.org/qa/11983/mixedfunctionspace-in-2016-2-0
    '''
    VxV_ele = VectorElement('P', mesh.ufl_cell(), velocity_degree)

    '''
    @todo How can we use the space $Q = \left{q \in L^2(\Omega) | \int{q = 0}\right}$ ?

    All Navier-Stokes FEniCS examples I've found simply use P2P1. danaila2014newton says that
    they are using the "classical Hilbert spaces" for velocity and pressure, but then they write
    down the space Q with less restrictions than H^1_0.

    '''
    Q_ele = FiniteElement('P', mesh.ufl_cell(), pressure_degree)

    V_ele = FiniteElement('P', mesh.ufl_cell(), temperature_degree)

    W_ele = MixedElement([VxV_ele, Q_ele, V_ele])

    W = FunctionSpace(mesh, W_ele)

    # Define function and test functions
    w = Function(W)

    v, q, phi = TestFunctions(W)

    # Split solution function to access variables separately
    u, p, theta = split(w)

    # Set source term expressions
    s_u = Expression(s_u, element=VxV_ele, mu=mu, gamma=gamma)

    s_p = Expression(s_p, element=Q_ele, mu=mu, gamma=gamma)

    s_theta = Expression(s_theta, element=V_ele)

    # Specify the initial values
    w_n = interpolate(Expression(initial_values_expression, element=W_ele), W)

    u_n, p_n, theta_n = split(w_n)

    # Define expressions needed for variational form
    Ra = Constant(Ra)

    Pr = Constant(Pr)

    Re = Constant(Re)

    K = Constant(K)

    mu = Constant(mu)

    g = Constant(g)

    gamma = Constant(gamma)

    # Define variational form
    def f_B(_theta):

        return _theta*Ra/(Pr*Re*Re)*g

    def a(_mu, _u, _v):

        def D(_u):

            return sym(grad(_u))

        return 2.*_mu*inner(D(_u), D(_v))

    def b(_u, _q):

        return -div(_u)*_q*debug_b_factor

    def c(_w, _z, _v):

        return dot(dot(_w, nabla_grad(_z)), _v)*debug_c_factor # @todo Is this use of nabla_grad correct?

    # Specify boundary conditions
    bc = []

    for subspace, expression, degree, coordinates, method in bc_expressions:

        bc.append(DirichletBC(W.sub(subspace), Expression(expression, degree=degree), coordinates, method=method))

    # Implement the nonlinear variational form
    if not linearize:

        def nonlinear_variational_form(dt):

            dt = Constant(dt)

            F = (\
                    b(u, q) - gamma*p*q - s_p*q \
                    + dot(u, v)/dt + c(u, u, v) + a(mu, u, v) + b(v, p) - dot(u_n, v)/dt + dot(f_B(theta), v) - dot(s_u, v) \
                    + theta*phi/dt - dot(u, grad(phi))*theta + dot(K/Pr*grad(theta), grad(phi)) - theta_n*phi/dt - s_theta*phi \
                    )*dx

            return F
    # Implement the Newton linearized form published in danaila2014newton
    elif linearize:

        w_w = TrialFunction(W)

        u_w, p_w, theta_w = split(w_w)

        w_k = Function(W)

        w_k.assign(w_n)

        u_k, p_k, theta_k = split(w_k)

        df_B_dtheta = Ra/(Pr*Re*Re)*g

        def linear_variational_form(dt):

            dt = Constant(dt)

            A = (\
                b(u_w,q) - gamma*p_w*q \
                + dot(u_w, v)/dt + c(u_w, u_k, v) + c(u_k, u_w, v) + a(mu, u_w, v) + b(v, p_w) + dot(theta_w*df_B_dtheta, v) \
                + theta_w*phi/dt - dot(u_k, grad(phi))*theta_w - dot(u_w, grad(phi))*theta_k + dot(K/Pr*grad(theta_w), grad(phi)) \
                )*dx

            L = (\
                b(u_k,q) - gamma*p_k*q + s_p*q \
                + dot(u_k - u_n, v)/dt + c(u_k, u_k, v) + a(mu, u_k, v) + b(v, p_k) + dot(f_B(theta_k), v) + dot(s_u, v) \
                + (theta_k - theta_n)*phi/dt - dot(u_k, grad(phi))*theta_k + dot(K/Pr*grad(theta_k), grad(phi)) + s_theta*phi \
                )*dx

            return A, L

    # Create progress bar
    progress = Progress('Time-stepping')

    set_log_level(PROGRESS)

    # Define method for writing values, and write initial values# Create VTK file for visualization output
    velocity_file = File(output_dir + '/velocity.pvd')

    pressure_file = File(output_dir + '/pressure.pvd')

    temperature_file = File(output_dir + '/temperature.pvd')

    def write_solution(_w, time):

        _velocity, _pressure, _temperature = _w.split()

        _velocity.rename("u", "velocity")

        _pressure.rename("p", "pressure")

        _temperature.rename("theta", "temperature")

        velocity_file << (_velocity, time)

        pressure_file << (_pressure, time)

        temperature_file << (_temperature, time)

    time = 0.

    w.assign(w_n)

    write_solution(w, time)

    # Solve each time step

    w_w = Function(W) # w_w was previously a TrialFunction, but must be a Function when calling solve()

    time_residual = Function(W)

    def solve_time_step(dt):

        if linearize:

            print '\nIterating Newton method'

            converged = False

            iteration_count = 0

            old_residual = 1e32

            w_k.assign(w_n)

            for k in range(max_newton_iterations):

                A, L = linear_variational_form(dt)

                solve(A == L, w_w, bcs=bc)

                w_k.assign(w_k - w_w)

                norm_residual = norm(w_w, 'L2')/norm(w_k, 'L2')

                print '\nL2 norm of relative residual, || w_w || / || w_k || = ' + str(norm_residual) + '\n'

                if norm_residual > old_residual:

                    diverging = True

                    if not adaptive_time:

                        assert(not diverging)

                    return diverging

                old_residual = norm_residual

                if norm_residual < newton_relative_tolerance:

                    converged = True

                    iteration_count = k + 1

                    print 'Converged after ' + str(k) + ' iterations'

                    break

            assert(converged)

            w.assign(w_k)

            diverging = False

            return diverging

        else:

            assert(not adaptive_time) # @todo How to get residual from solver.solve() to check if diverging?

            F = nonlinear_variational_form(dt)

            problem = NonlinearVariationalProblem(F, w, bc, derivative(F, w))

            solver = NonlinearVariationalSolver(problem)

            iteration_count, converged = solver.solve()

            assert(converged)

    EPSILON = 1.e-12

    while time < final_time - EPSILON:

        if adaptive_time:

            remaining_time = final_time - time

            if time_step_size > remaining_time:

                time_step_size = remaining_time

            diverging = True

            while diverging:

                diverging = solve_time_step(time_step_size)

                if diverging:

                    # @todo Expose parameters for maximum and minimum size.

                    time_step_size /= 2.

                    print 'Set time step size to dt = ' + str(time_step_size)

            time += time_step_size

            time_step_size *= 2.

            print 'Set time step size to dt = ' + str(time_step_size)

        else:

            solve_time_step(time_step_size)

            time += time_step_size

        print 'Reached time t = ' + str(time)

        # Save solution to files
        write_solution(w, time)

        if stop_when_steady:

            # Check for steady state
            time_residual.assign(w - w_n)

        # Update previous solution
        w_n.assign(w)

        # Show the time progress
        progress.update(time / final_time)

        if stop_when_steady:

            unsteadiness = norm(time_residual, 'L2')/norm(w_n, 'L2')

            print 'Unsteadiness (L2 norm of relative time residual), || w_{n+1} || / || w_n || = ' + str(unsteadiness)

            if (unsteadiness < steady_relative_tolerance):
                print 'Reached steady state at time t = ' + str(time)
                break

    if time >= final_time:

        print 'Reached final time, t = ' + str(final_time)

    if exact_solution_expression:

        w_e = Expression(exact_solution_expression, element=W_ele)

        error = errornorm(w_e, w_n, 'L2')

        print("Error = " + str(error))

def test():

    run()

    pass

if __name__=='__main__':

    test()<commit_msg>Updated default argument values to run natural convection example<commit_after>'''

    @brief  Solve the benchmark "differentially heated cavity" natural convection problem using finite elements.

    @detail

        Solve the natural convection test problem from

            @article
            {danaila2014newton,
              title={A Newton method with adaptive finite elements for solving phase-change problems with natural convection},
              author={Danaila, Ionut and Moglan, Raluca and Hecht, Fr{\'e}d{\'e}ric and Le Masson, St{\'e}phane},
              journal={Journal of Computational Physics},
              volume={274},
              pages={826--840},
              year={2014},
              publisher={Academic Press}
            }

        Match the notation in danaila2014newton as best as possible.

    @author Alexander G. Zimmerman <zimmerman@aices.rwth-aachen.de>

'''

from fenics import \
    UnitSquareMesh, FiniteElement, VectorElement, MixedElement, \
    FunctionSpace, VectorFunctionSpace, \
    Function, TrialFunction, TestFunctions, split, \
    DirichletBC, Constant, Expression, \
    dx, \
    dot, inner, grad, nabla_grad, sym, div, \
    errornorm, norm, \
    File, \
    Progress, set_log_level, PROGRESS, \
    project, interpolate, \
    solve, parameters, info, derivative, \
    LinearVariationalProblem, LinearVariationalSolver, NonlinearVariationalProblem, NonlinearVariationalSolver, \
    SubDomain, EdgeFunction, near, adapt

def arguments():
    """Returns tuple containing dictionary of calling function's
       named arguments and a list of calling function's unnamed
       positional arguments.
    """
    from inspect import getargvalues, stack
    posname, kwname, args = getargvalues(stack()[1][0])[-3:]
    posargs = args.pop(posname, [])
    args.update(args.pop(kwname, []))
    return args, posargs

def run(
    output_dir = "output/natural_convection", \
    Ra = 1.e6, \
    Pr = 0.71, \
    K = 1., \
    g = (0., -1.), \
    mu = 1., \
    mesh = UnitSquareMesh(20, 20, "crossed"), \
    s_u = ("0.", "0."), \
    s_p = "0.", \
    s_theta = "0.", \
    initial_values_expression = ( \
        "0.", \
        "0.", \
        "0.", \
        "0.5*near(x[0],  0.) -0.5*near(x[0],  1.)"), \
    bc_expressions = [ \
        [0, ("0.", "0."), 3, "near(x[0],  0.) | near(x[0],  1.) | near(x[1], 0.) | near(x[1],  1.)","topological"], \
        [2, "0.", 2, "near(x[0],  0.)", "topological"], \
        [2, "0.", 2, "near(x[0],  1.)", "topological"]], \
    final_time = 1., \
    time_step_size = 1.e-3, \
    adaptive_time = True, \
    gamma = 1.e-7, \
    pressure_degree = 1, \
    temperature_degree = 1, \
    linearize = True, \
    newton_relative_tolerance = 1.e-9, \
    max_newton_iterations = 10, \
    stop_when_steady = True, \
    steady_relative_tolerance = 1.e-8, \
    debug_b_factor = 1., \
    debug_c_factor = 1., \
    exact_solution_expression = [] \
    ):

    #
    print("Running nsb_pcm with the following arguments:")

    print(arguments())

    # @todo Try 3D
    dim = 2

    '''
    Per Equation 8 from danaila2014newton, we implement an equation scaled with v_ref = nu_liquid/H => t_ref = nu_liquid/H^2 => Re = 1.
    '''
    Re = 1.

    # Compute derived parameters
    velocity_degree = pressure_degree + 1

    # Define function spaces for the system
    VxV = VectorFunctionSpace(mesh, 'P', velocity_degree)

    Q = FunctionSpace(mesh, 'P', pressure_degree) # @todo mixing up test function space

    V = FunctionSpace(mesh, 'P', temperature_degree)

    '''
    MixedFunctionSpace used to be available but is now deprecated.
    The way that fenics separates function spaces and elements is confusing.
    To create the mixed space, I'm using the approach from https://fenicsproject.org/qa/11983/mixedfunctionspace-in-2016-2-0
    '''
    VxV_ele = VectorElement('P', mesh.ufl_cell(), velocity_degree)

    '''
    @todo How can we use the space $Q = \left{q \in L^2(\Omega) | \int{q = 0}\right}$ ?

    All Navier-Stokes FEniCS examples I've found simply use P2P1. danaila2014newton says that
    they are using the "classical Hilbert spaces" for velocity and pressure, but then they write
    down the space Q with less restrictions than H^1_0.

    '''
    Q_ele = FiniteElement('P', mesh.ufl_cell(), pressure_degree)

    V_ele = FiniteElement('P', mesh.ufl_cell(), temperature_degree)

    W_ele = MixedElement([VxV_ele, Q_ele, V_ele])

    W = FunctionSpace(mesh, W_ele)

    # Define function and test functions
    w = Function(W)

    v, q, phi = TestFunctions(W)

    # Split solution function to access variables separately
    u, p, theta = split(w)

    # Set source term expressions
    s_u = Expression(s_u, element=VxV_ele, mu=mu, gamma=gamma)

    s_p = Expression(s_p, element=Q_ele, mu=mu, gamma=gamma)

    s_theta = Expression(s_theta, element=V_ele)

    # Specify the initial values
    w_n = interpolate(Expression(initial_values_expression, element=W_ele), W)

    u_n, p_n, theta_n = split(w_n)

    # Define expressions needed for variational form
    Ra = Constant(Ra)

    Pr = Constant(Pr)

    Re = Constant(Re)

    K = Constant(K)

    mu = Constant(mu)

    g = Constant(g)

    gamma = Constant(gamma)

    # Define variational form
    def f_B(_theta):

        return _theta*Ra/(Pr*Re*Re)*g

    def a(_mu, _u, _v):

        def D(_u):

            return sym(grad(_u))

        return 2.*_mu*inner(D(_u), D(_v))

    def b(_u, _q):

        return -div(_u)*_q*debug_b_factor

    def c(_w, _z, _v):

        return dot(dot(_w, nabla_grad(_z)), _v)*debug_c_factor # @todo Is this use of nabla_grad correct?

    # Specify boundary conditions
    bc = []

    for subspace, expression, degree, coordinates, method in bc_expressions:

        bc.append(DirichletBC(W.sub(subspace), Expression(expression, degree=degree), coordinates, method=method))

    # Implement the nonlinear variational form
    if not linearize:

        def nonlinear_variational_form(dt):

            dt = Constant(dt)

            F = (\
                    b(u, q) - gamma*p*q - s_p*q \
                    + dot(u, v)/dt + c(u, u, v) + a(mu, u, v) + b(v, p) - dot(u_n, v)/dt + dot(f_B(theta), v) - dot(s_u, v) \
                    + theta*phi/dt - dot(u, grad(phi))*theta + dot(K/Pr*grad(theta), grad(phi)) - theta_n*phi/dt - s_theta*phi \
                    )*dx

            return F
    # Implement the Newton linearized form published in danaila2014newton
    elif linearize:

        w_w = TrialFunction(W)

        u_w, p_w, theta_w = split(w_w)

        w_k = Function(W)

        w_k.assign(w_n)

        u_k, p_k, theta_k = split(w_k)

        df_B_dtheta = Ra/(Pr*Re*Re)*g

        def linear_variational_form(dt):

            dt = Constant(dt)

            A = (\
                b(u_w,q) - gamma*p_w*q \
                + dot(u_w, v)/dt + c(u_w, u_k, v) + c(u_k, u_w, v) + a(mu, u_w, v) + b(v, p_w) + dot(theta_w*df_B_dtheta, v) \
                + theta_w*phi/dt - dot(u_k, grad(phi))*theta_w - dot(u_w, grad(phi))*theta_k + dot(K/Pr*grad(theta_w), grad(phi)) \
                )*dx

            L = (\
                b(u_k,q) - gamma*p_k*q + s_p*q \
                + dot(u_k - u_n, v)/dt + c(u_k, u_k, v) + a(mu, u_k, v) + b(v, p_k) + dot(f_B(theta_k), v) + dot(s_u, v) \
                + (theta_k - theta_n)*phi/dt - dot(u_k, grad(phi))*theta_k + dot(K/Pr*grad(theta_k), grad(phi)) + s_theta*phi \
                )*dx

            return A, L

    # Create progress bar
    progress = Progress('Time-stepping')

    set_log_level(PROGRESS)

    # Define method for writing values, and write initial values# Create VTK file for visualization output
    velocity_file = File(output_dir + '/velocity.pvd')

    pressure_file = File(output_dir + '/pressure.pvd')

    temperature_file = File(output_dir + '/temperature.pvd')

    def write_solution(_w, time):

        _velocity, _pressure, _temperature = _w.split()

        _velocity.rename("u", "velocity")

        _pressure.rename("p", "pressure")

        _temperature.rename("theta", "temperature")

        velocity_file << (_velocity, time)

        pressure_file << (_pressure, time)

        temperature_file << (_temperature, time)

    time = 0.

    w.assign(w_n)

    write_solution(w, time)

    # Solve each time step

    w_w = Function(W) # w_w was previously a TrialFunction, but must be a Function when calling solve()

    time_residual = Function(W)

    def solve_time_step(dt):

        if linearize:

            print '\nIterating Newton method'

            converged = False

            iteration_count = 0

            old_residual = 1e32

            w_k.assign(w_n)

            for k in range(max_newton_iterations):

                A, L = linear_variational_form(dt)

                solve(A == L, w_w, bcs=bc)

                w_k.assign(w_k - w_w)

                norm_residual = norm(w_w, 'L2')/norm(w_k, 'L2')

                print '\nL2 norm of relative residual, || w_w || / || w_k || = ' + str(norm_residual) + '\n'

                if norm_residual > old_residual:

                    diverging = True

                    if not adaptive_time:

                        assert(not diverging)

                    return diverging

                old_residual = norm_residual

                if norm_residual < newton_relative_tolerance:

                    converged = True

                    iteration_count = k + 1

                    print 'Converged after ' + str(k) + ' iterations'

                    break

            assert(converged)

            w.assign(w_k)

            diverging = False

            return diverging

        else:

            assert(not adaptive_time) # @todo How to get residual from solver.solve() to check if diverging?

            F = nonlinear_variational_form(dt)

            problem = NonlinearVariationalProblem(F, w, bc, derivative(F, w))

            solver = NonlinearVariationalSolver(problem)

            iteration_count, converged = solver.solve()

            assert(converged)

    EPSILON = 1.e-12

    while time < final_time - EPSILON:

        if adaptive_time:

            remaining_time = final_time - time

            if time_step_size > remaining_time:

                time_step_size = remaining_time

            diverging = True

            while diverging:

                diverging = solve_time_step(time_step_size)

                if diverging:

                    # @todo Expose parameters for maximum and minimum size.

                    time_step_size /= 2.

                    print 'Set time step size to dt = ' + str(time_step_size)

            time += time_step_size

            time_step_size *= 2.

            print 'Set time step size to dt = ' + str(time_step_size)

        else:

            solve_time_step(time_step_size)

            time += time_step_size

        print 'Reached time t = ' + str(time)

        # Save solution to files
        write_solution(w, time)

        if stop_when_steady:

            # Check for steady state
            time_residual.assign(w - w_n)

        # Update previous solution
        w_n.assign(w)

        # Show the time progress
        progress.update(time / final_time)

        if stop_when_steady:

            unsteadiness = norm(time_residual, 'L2')/norm(w_n, 'L2')

            print 'Unsteadiness (L2 norm of relative time residual), || w_{n+1} || / || w_n || = ' + str(unsteadiness)

            if (unsteadiness < steady_relative_tolerance):
                print 'Reached steady state at time t = ' + str(time)
                break

    if time >= final_time:

        print 'Reached final time, t = ' + str(final_time)

    if exact_solution_expression:

        w_e = Expression(exact_solution_expression, element=W_ele)

        error = errornorm(w_e, w_n, 'L2')

        print("Error = " + str(error))

def test():

    run()

    pass

if __name__=='__main__':

    test()<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import os, sys
sys.path.append('../scarplet')

import boto
from celery import *

import dem, scarplet
import WindowedTemplate as wt
from WindowedTemplate import Scarp as Template

import time
import datetime

import numpy as np
import numexpr

from timeit import default_timer as timer

app = Celery('scarplet-testing', broker='sqs://')
app.config_from_object('celeryconfig')

data = dem.DEMGrid('../tests/data/carrizo.tif')
#data = load_data_from

@app.task(ignore_result=False)
@app.task
def add(x, y):
    return x + y

@app.task(ignore_result=True)
def match_template(d, age, alpha):
    best_results = load_results_from_s3()
    amp, snr = scarplet.calculate_amplitude(data, wt.Scarp, d, age, alpha)
    this_results = [amp, age, alpha, snr]
    best_results = scarplet.compare_fits(best_results, this_results)
    save_results_to_s3(best_results)

@app.task(ignore_result=False)
def match_chunk(min_age, max_age, min_ang=0, max_ang=180, age_step = 0.1):
    d = 100
    ang_step = 1
    nages = (max_age - min_age)/age_step + 1
    nangles = (max_ang - min_ang)/ang_step

    ages = 10**np.linspace(min_age, max_age, num=nages)[:-1]
    orientations = np.linspace(-np.pi/2, np.pi/2, num=nangles)

    s = data._griddata.shape
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)
    ny, nx = s
    de = data._georef_info.dx

    # Pre-plain PyFFTW FFT calls (30 s speedup over whole loop)
    pyfftw.interfaces.cache.enable()
    pyfftw.interfaces.cache.set_keepalive_time(10)

    t = Template(d, 10, 0, nx, ny, de)
    template = t.template_numexpr()
    curv = data._calculate_directional_laplacian_numexpr(0)
    this_amp, this_snr = scarplet.match_template_numexpr(curv, template)

    for this_alpha in orientations:
        for this_age in ages:

            t = Template(d, this_age, this_alpha, nx, ny, de)
            template = t.template_numexpr()

            curv = data._calculate_directional_laplacian_numexpr(this_alpha)

            this_amp, this_snr = scarplet.match_template_numexpr(curv, template)
            mask = t.get_window_limits()
            this_amp[mask] = 0
            this_snr[mask] = 0

            best_amp = numexpr.evaluate("(best_snr > this_snr)*best_amp + (best_snr < this_snr)*this_amp")
            best_age = numexpr.evaluate("(best_snr > this_snr)*best_age + (best_snr < this_snr)*this_age")
            best_alpha = numexpr.evaluate("(best_snr > this_snr)*best_alpha + (best_snr < this_snr)*this_alpha")
            best_snr = numexpr.evaluate("(best_snr > this_snr)*best_snr + (best_snr < this_snr)*this_snr")

    save_results_to_s3([best_amp, best_age, best_alpha, best_snr])

def load_data_from_s3(filename, bucket_name='scarp-tmp'):
    connection = boto.connect_s3()
    bucket = connection.get_bucket(bucket_name, validate=False)
    key = bucket.new_key(filename)
    key.get_contents_to_filename(filename)
    return np.load(filename)

def save_data_to_s3(results, filename=None, bucket_name='scarp-tmp'):
    connection = boto.connect_s3()
    bucket = connection.get_bucket(bucket_name, validate=False)
    d = datetime.datetime.now()
    if not filename:
        filename = 'tmp_' + d.isoformat() + '.npy'
    key = bucket.new_key(filename)
    np.save(filename, results)
    key.set_contents_from_filename(filename)

def save_results_to_s3(results):
    connection = boto.connect_s3()
    bucket = connection.get_bucket('scarp-tmp', validate=False)
    d = datetime.datetime.now()
    filename = 'tmp_' + d.isoformat() + '.npy'
    key = bucket.new_key(filename)
    np.save(filename, results)
    key.set_contents_from_filename(filename)

def initialize_results():
    s = get_grid_size()
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)
    return best_amp, best_age, best_alpha, best_snr

@app.task(forget_results=False)
def compare_fits_from_s3():
    connection = boto.connect_s3()
    bucket = connection.get_bucket('scarp-tmp', validate=False)
    best_results = initialize_results()
    for key in bucket.list():
        key.get_contents_to_filename('tmp.npy')
        this_results = np.load('tmp.npy')
        best_results = scarplet.compare_fits(best_results, this_results)
        key.delete()
    return best_results

def pairs(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return itertools.izip(a, b)

def pairwise(iterable):
    a = iter(iterable)
    return itertools.izip(a, a)

def get_grid_size():
    return data._griddata.shape
<commit_msg>Save best results, don't return them<commit_after>from __future__ import absolute_import, unicode_literals

import os, sys
sys.path.append('../scarplet')

import boto
from celery import *

import dem, scarplet
import WindowedTemplate as wt
from WindowedTemplate import Scarp as Template

import time
import datetime

import pyfftw
import numpy as np
import numexpr

from timeit import default_timer as timer

app = Celery('scarplet-testing', broker='sqs://')
app.config_from_object('celeryconfig')

data = dem.DEMGrid('../tests/data/carrizo.tif')
#data = load_data_from

@app.task(ignore_result=False)
@app.task
def add(x, y):
    return x + y

@app.task(ignore_result=True)
def match_template(d, age, alpha):
    best_results = load_results_from_s3()
    amp, snr = scarplet.calculate_amplitude(data, wt.Scarp, d, age, alpha)
    this_results = [amp, age, alpha, snr]
    best_results = scarplet.compare_fits(best_results, this_results)
    save_results_to_s3(best_results)

@app.task(ignore_result=False)
def match_chunk(min_age, max_age, min_ang=0, max_ang=180, age_step = 0.1):
    d = 100
    ang_step = 1
    nages = (max_age - min_age)/age_step + 1
    nangles = (max_ang - min_ang)/ang_step

    ages = 10**np.linspace(min_age, max_age, num=nages)[:-1]
    orientations = np.linspace(-np.pi/2, np.pi/2, num=nangles)

    s = data._griddata.shape
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)
    ny, nx = s
    de = data._georef_info.dx

    # Pre-plain PyFFTW FFT calls (30 s speedup over whole loop)
    pyfftw.interfaces.cache.enable()
    pyfftw.interfaces.cache.set_keepalive_time(10)

    t = Template(d, 10, 0, nx, ny, de)
    template = t.template_numexpr()
    curv = data._calculate_directional_laplacian_numexpr(0)
    this_amp, this_snr = scarplet.match_template_numexpr(curv, template)

    for this_alpha in orientations:
        for this_age in ages:

            t = Template(d, this_age, this_alpha, nx, ny, de)
            template = t.template_numexpr()

            curv = data._calculate_directional_laplacian_numexpr(this_alpha)

            this_amp, this_snr = scarplet.match_template_numexpr(curv, template)
            mask = t.get_window_limits()
            this_amp[mask] = 0
            this_snr[mask] = 0

            best_amp = numexpr.evaluate("(best_snr > this_snr)*best_amp + (best_snr < this_snr)*this_amp")
            best_age = numexpr.evaluate("(best_snr > this_snr)*best_age + (best_snr < this_snr)*this_age")
            best_alpha = numexpr.evaluate("(best_snr > this_snr)*best_alpha + (best_snr < this_snr)*this_alpha")
            best_snr = numexpr.evaluate("(best_snr > this_snr)*best_snr + (best_snr < this_snr)*this_snr")

    save_results_to_s3([best_amp, best_age, best_alpha, best_snr])

def load_data_from_s3(filename, bucket_name='scarp-tmp'):
    connection = boto.connect_s3()
    bucket = connection.get_bucket(bucket_name, validate=False)
    key = bucket.new_key(filename)
    key.get_contents_to_filename(filename)
    return np.load(filename)

def save_data_to_s3(results, filename=None, bucket_name='scarp-tmp'):
    connection = boto.connect_s3()
    bucket = connection.get_bucket(bucket_name, validate=False)
    d = datetime.datetime.now()
    if not filename:
        filename = 'tmp_' + d.isoformat() + '.npy'
    key = bucket.new_key(filename)
    np.save(filename, results)
    key.set_contents_from_filename(filename)

def save_results_to_s3(results):
    connection = boto.connect_s3()
    bucket = connection.get_bucket('scarp-tmp', validate=False)
    d = datetime.datetime.now()
    filename = 'tmp_' + d.isoformat() + '.npy'
    key = bucket.new_key(filename)
    np.save(filename, results)
    key.set_contents_from_filename(filename)

def initialize_results():
    s = get_grid_size()
    best_snr = np.zeros(s)
    best_amp = np.zeros(s)
    best_age = np.zeros(s)
    best_alpha = np.zeros(s)
    return best_amp, best_age, best_alpha, best_snr

@app.task(forget_results=False)
def compare_fits_from_s3():
    connection = boto.connect_s3()
    bucket = connection.get_bucket('scarp-tmp', validate=False)
    best_results = initialize_results()
    for key in bucket.list():
        key.get_contents_to_filename('tmp.npy')
        this_results = np.load('tmp.npy')
        best_results = scarplet.compare_fits(best_results, this_results)
        key.delete()
    save_data_to_s3(best_results, filename='best_carrizo.npy', bucket='scarp-results')

def pairs(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return itertools.izip(a, b)

def pairwise(iterable):
    a = iter(iterable)
    return itertools.izip(a, a)

def get_grid_size():
    return data._griddata.shape
<|endoftext|>


<commit_before><commit_msg>examples: Update docstrings<commit_after><|endoftext|>


<commit_before>from collections import UserDict
from functools import partial
from fontTools.misc import sstruct
from fontTools.misc.textTools import safeEval
from . import DefaultTable
import array
import itertools
import logging
import struct
import sys
import fontTools.ttLib.tables.TupleVariation as tv

log = logging.getLogger(__name__)
TupleVariation = tv.TupleVariation

# https://www.microsoft.com/typography/otspec/gvar.htm
# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm
#
# Apple's documentation of 'gvar':
# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gvar.html
#
# FreeType2 source code for parsing 'gvar':
# http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c

GVAR_HEADER_FORMAT = """
	> # big endian
	version:			H
	reserved:			H
	axisCount:			H
	sharedTupleCount:		H
	offsetToSharedTuples:		I
	glyphCount:			H
	flags:				H
	offsetToGlyphVariationData:	I
"""

GVAR_HEADER_SIZE = sstruct.calcsize(GVAR_HEADER_FORMAT)

class _LazyDict(UserDict):
	def __getitem__(self, k):
		v = self.data[k]
		if callable(v):
			v = v()
			self.data[k] = v
		return v

class table__g_v_a_r(DefaultTable.DefaultTable):
	dependencies = ["fvar", "glyf"]

	def __init__(self, tag=None):
		DefaultTable.DefaultTable.__init__(self, tag)
		self.version, self.reserved = 1, 0
		self.variations = {}

	def compile(self, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		sharedTuples =  tv.compileSharedTuples(
			axisTags, itertools.chain(*self.variations.values()))
		sharedTupleIndices = {coord:i for i, coord in enumerate(sharedTuples)}
		sharedTupleSize = sum([len(c) for c in sharedTuples])
		compiledGlyphs = self.compileGlyphs_(
			ttFont, axisTags, sharedTupleIndices)
		offset = 0
		offsets = []
		for glyph in compiledGlyphs:
			offsets.append(offset)
			offset += len(glyph)
		offsets.append(offset)
		compiledOffsets, tableFormat = self.compileOffsets_(offsets)

		header = {}
		header["version"] = self.version
		header["reserved"] = self.reserved
		header["axisCount"] = len(axisTags)
		header["sharedTupleCount"] = len(sharedTuples)
		header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets)
		header["glyphCount"] = len(compiledGlyphs)
		header["flags"] = tableFormat
		header["offsetToGlyphVariationData"] = header["offsetToSharedTuples"] + sharedTupleSize
		compiledHeader = sstruct.pack(GVAR_HEADER_FORMAT, header)

		result = [compiledHeader, compiledOffsets]
		result.extend(sharedTuples)
		result.extend(compiledGlyphs)
		return b''.join(result)

	def compileGlyphs_(self, ttFont, axisTags, sharedCoordIndices):
		result = []
		glyf = ttFont['glyf']
		for glyphName in ttFont.getGlyphOrder():
			glyph = glyf[glyphName]
			pointCount = self.getNumPoints_(glyph)
			variations = self.variations.get(glyphName, [])
			result.append(compileGlyph_(variations, pointCount,
			                            axisTags, sharedCoordIndices))
		return result

	def decompile(self, data, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		glyphs = ttFont.getGlyphOrder()
		sstruct.unpack(GVAR_HEADER_FORMAT, data[0:GVAR_HEADER_SIZE], self)
		assert len(glyphs) == self.glyphCount
		assert len(axisTags) == self.axisCount
		offsets = self.decompileOffsets_(data[GVAR_HEADER_SIZE:], tableFormat=(self.flags & 1), glyphCount=self.glyphCount)
		sharedCoords = tv.decompileSharedTuples(
			axisTags, self.sharedTupleCount, data, self.offsetToSharedTuples)
		variations = {}
		offsetToData = self.offsetToGlyphVariationData
		glyf = ttFont['glyf']

		def decompileVarGlyph(glyphName, gid):
			glyph = glyf[glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			gvarData = data[offsetToData + offsets[gid] : offsetToData + offsets[gid + 1]]
			return decompileGlyph_(numPointsInGlyph, sharedCoords, axisTags, gvarData)

		for gid in range(self.glyphCount):
			glyphName = glyphs[gid]
			variations[glyphName] = partial(decompileVarGlyph, glyphName, gid)
		self.variations = _LazyDict()
		self.variations.data = variations

	@staticmethod
	def decompileOffsets_(data, tableFormat, glyphCount):
		if tableFormat == 0:
			# Short format: array of UInt16
			offsets = array.array("H")
			offsetsSize = (glyphCount + 1) * 2
		else:
			# Long format: array of UInt32
			offsets = array.array("I")
			offsetsSize = (glyphCount + 1) * 4
		offsets.frombytes(data[0 : offsetsSize])
		if sys.byteorder != "big": offsets.byteswap()

		# In the short format, offsets need to be multiplied by 2.
		# This is not documented in Apple's TrueType specification,
		# but can be inferred from the FreeType implementation, and
		# we could verify it with two sample GX fonts.
		if tableFormat == 0:
			offsets = [off * 2 for off in offsets]

		return offsets

	@staticmethod
	def compileOffsets_(offsets):
		"""Packs a list of offsets into a 'gvar' offset table.

		Returns a pair (bytestring, tableFormat). Bytestring is the
		packed offset table. Format indicates whether the table
		uses short (tableFormat=0) or long (tableFormat=1) integers.
		The returned tableFormat should get packed into the flags field
		of the 'gvar' header.
		"""
		assert len(offsets) >= 2
		for i in range(1, len(offsets)):
			assert offsets[i - 1] <= offsets[i]
		if max(offsets) <= 0xffff * 2:
			packed = array.array("H", [n >> 1 for n in offsets])
			tableFormat = 0
		else:
			packed = array.array("I", offsets)
			tableFormat = 1
		if sys.byteorder != "big": packed.byteswap()
		return (packed.tobytes(), tableFormat)

	def toXML(self, writer, ttFont):
		writer.simpletag("version", value=self.version)
		writer.newline()
		writer.simpletag("reserved", value=self.reserved)
		writer.newline()
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		for glyphName in ttFont.getGlyphNames():
			variations = self.variations.get(glyphName)
			if not variations:
				continue
			writer.begintag("glyphVariations", glyph=glyphName)
			writer.newline()
			for gvar in variations:
				gvar.toXML(writer, axisTags)
			writer.endtag("glyphVariations")
			writer.newline()

	def fromXML(self, name, attrs, content, ttFont):
		if name == "version":
			self.version = safeEval(attrs["value"])
		elif name == "reserved":
			self.reserved = safeEval(attrs["value"])
		elif name == "glyphVariations":
			if not hasattr(self, "variations"):
				self.variations = {}
			glyphName = attrs["glyph"]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			glyphVariations = []
			for element in content:
				if isinstance(element, tuple):
					name, attrs, content = element
					if name == "tuple":
						gvar = TupleVariation({}, [None] * numPointsInGlyph)
						glyphVariations.append(gvar)
						for tupleElement in content:
							if isinstance(tupleElement, tuple):
								tupleName, tupleAttrs, tupleContent = tupleElement
								gvar.fromXML(tupleName, tupleAttrs, tupleContent)
			self.variations[glyphName] = glyphVariations

	@staticmethod
	def getNumPoints_(glyph):
		NUM_PHANTOM_POINTS = 4
		if glyph.isComposite():
			return len(glyph.components) + NUM_PHANTOM_POINTS
		else:
			# Empty glyphs (eg. space, nonmarkingreturn) have no "coordinates" attribute.
			return len(getattr(glyph, "coordinates", [])) + NUM_PHANTOM_POINTS

def compileGlyph_(variations, pointCount, axisTags, sharedCoordIndices):
	tupleVariationCount, tuples, data = tv.compileTupleVariationStore(
		variations, pointCount, axisTags, sharedCoordIndices)
	if tupleVariationCount == 0:
		return b""
	result = [
		struct.pack(">HH", tupleVariationCount, 4 + len(tuples)),
		tuples,
		data
	]
	if (len(tuples) + len(data)) % 2 != 0:
		result.append(b"\0")  # padding
	return b''.join(result)

def decompileGlyph_(pointCount, sharedTuples, axisTags, data):
	if len(data) < 4:
		return []
	tupleVariationCount, offsetToData = struct.unpack(">HH", data[:4])
	dataPos = offsetToData
	return tv.decompileTupleVariationStore(
		"gvar", axisTags,
		tupleVariationCount, pointCount,
		sharedTuples, data, 4, offsetToData
	)
<commit_msg>add custom __init__ in favor of setting .data externally<commit_after>from collections import UserDict
from functools import partial
from fontTools.misc import sstruct
from fontTools.misc.textTools import safeEval
from . import DefaultTable
import array
import itertools
import logging
import struct
import sys
import fontTools.ttLib.tables.TupleVariation as tv

log = logging.getLogger(__name__)
TupleVariation = tv.TupleVariation

# https://www.microsoft.com/typography/otspec/gvar.htm
# https://www.microsoft.com/typography/otspec/otvarcommonformats.htm
#
# Apple's documentation of 'gvar':
# https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6gvar.html
#
# FreeType2 source code for parsing 'gvar':
# http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c

GVAR_HEADER_FORMAT = """
	> # big endian
	version:			H
	reserved:			H
	axisCount:			H
	sharedTupleCount:		H
	offsetToSharedTuples:		I
	glyphCount:			H
	flags:				H
	offsetToGlyphVariationData:	I
"""

GVAR_HEADER_SIZE = sstruct.calcsize(GVAR_HEADER_FORMAT)

class _LazyDict(UserDict):
	def __init__(self, data):
		super().__init__()
		self.data = data

	def __getitem__(self, k):
		v = self.data[k]
		if callable(v):
			v = v()
			self.data[k] = v
		return v

class table__g_v_a_r(DefaultTable.DefaultTable):
	dependencies = ["fvar", "glyf"]

	def __init__(self, tag=None):
		DefaultTable.DefaultTable.__init__(self, tag)
		self.version, self.reserved = 1, 0
		self.variations = {}

	def compile(self, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		sharedTuples =  tv.compileSharedTuples(
			axisTags, itertools.chain(*self.variations.values()))
		sharedTupleIndices = {coord:i for i, coord in enumerate(sharedTuples)}
		sharedTupleSize = sum([len(c) for c in sharedTuples])
		compiledGlyphs = self.compileGlyphs_(
			ttFont, axisTags, sharedTupleIndices)
		offset = 0
		offsets = []
		for glyph in compiledGlyphs:
			offsets.append(offset)
			offset += len(glyph)
		offsets.append(offset)
		compiledOffsets, tableFormat = self.compileOffsets_(offsets)

		header = {}
		header["version"] = self.version
		header["reserved"] = self.reserved
		header["axisCount"] = len(axisTags)
		header["sharedTupleCount"] = len(sharedTuples)
		header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets)
		header["glyphCount"] = len(compiledGlyphs)
		header["flags"] = tableFormat
		header["offsetToGlyphVariationData"] = header["offsetToSharedTuples"] + sharedTupleSize
		compiledHeader = sstruct.pack(GVAR_HEADER_FORMAT, header)

		result = [compiledHeader, compiledOffsets]
		result.extend(sharedTuples)
		result.extend(compiledGlyphs)
		return b''.join(result)

	def compileGlyphs_(self, ttFont, axisTags, sharedCoordIndices):
		result = []
		glyf = ttFont['glyf']
		for glyphName in ttFont.getGlyphOrder():
			glyph = glyf[glyphName]
			pointCount = self.getNumPoints_(glyph)
			variations = self.variations.get(glyphName, [])
			result.append(compileGlyph_(variations, pointCount,
			                            axisTags, sharedCoordIndices))
		return result

	def decompile(self, data, ttFont):
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		glyphs = ttFont.getGlyphOrder()
		sstruct.unpack(GVAR_HEADER_FORMAT, data[0:GVAR_HEADER_SIZE], self)
		assert len(glyphs) == self.glyphCount
		assert len(axisTags) == self.axisCount
		offsets = self.decompileOffsets_(data[GVAR_HEADER_SIZE:], tableFormat=(self.flags & 1), glyphCount=self.glyphCount)
		sharedCoords = tv.decompileSharedTuples(
			axisTags, self.sharedTupleCount, data, self.offsetToSharedTuples)
		variations = {}
		offsetToData = self.offsetToGlyphVariationData
		glyf = ttFont['glyf']

		def decompileVarGlyph(glyphName, gid):
			glyph = glyf[glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			gvarData = data[offsetToData + offsets[gid] : offsetToData + offsets[gid + 1]]
			return decompileGlyph_(numPointsInGlyph, sharedCoords, axisTags, gvarData)

		for gid in range(self.glyphCount):
			glyphName = glyphs[gid]
			variations[glyphName] = partial(decompileVarGlyph, glyphName, gid)
		self.variations = _LazyDict(variations)

	@staticmethod
	def decompileOffsets_(data, tableFormat, glyphCount):
		if tableFormat == 0:
			# Short format: array of UInt16
			offsets = array.array("H")
			offsetsSize = (glyphCount + 1) * 2
		else:
			# Long format: array of UInt32
			offsets = array.array("I")
			offsetsSize = (glyphCount + 1) * 4
		offsets.frombytes(data[0 : offsetsSize])
		if sys.byteorder != "big": offsets.byteswap()

		# In the short format, offsets need to be multiplied by 2.
		# This is not documented in Apple's TrueType specification,
		# but can be inferred from the FreeType implementation, and
		# we could verify it with two sample GX fonts.
		if tableFormat == 0:
			offsets = [off * 2 for off in offsets]

		return offsets

	@staticmethod
	def compileOffsets_(offsets):
		"""Packs a list of offsets into a 'gvar' offset table.

		Returns a pair (bytestring, tableFormat). Bytestring is the
		packed offset table. Format indicates whether the table
		uses short (tableFormat=0) or long (tableFormat=1) integers.
		The returned tableFormat should get packed into the flags field
		of the 'gvar' header.
		"""
		assert len(offsets) >= 2
		for i in range(1, len(offsets)):
			assert offsets[i - 1] <= offsets[i]
		if max(offsets) <= 0xffff * 2:
			packed = array.array("H", [n >> 1 for n in offsets])
			tableFormat = 0
		else:
			packed = array.array("I", offsets)
			tableFormat = 1
		if sys.byteorder != "big": packed.byteswap()
		return (packed.tobytes(), tableFormat)

	def toXML(self, writer, ttFont):
		writer.simpletag("version", value=self.version)
		writer.newline()
		writer.simpletag("reserved", value=self.reserved)
		writer.newline()
		axisTags = [axis.axisTag for axis in ttFont["fvar"].axes]
		for glyphName in ttFont.getGlyphNames():
			variations = self.variations.get(glyphName)
			if not variations:
				continue
			writer.begintag("glyphVariations", glyph=glyphName)
			writer.newline()
			for gvar in variations:
				gvar.toXML(writer, axisTags)
			writer.endtag("glyphVariations")
			writer.newline()

	def fromXML(self, name, attrs, content, ttFont):
		if name == "version":
			self.version = safeEval(attrs["value"])
		elif name == "reserved":
			self.reserved = safeEval(attrs["value"])
		elif name == "glyphVariations":
			if not hasattr(self, "variations"):
				self.variations = {}
			glyphName = attrs["glyph"]
			glyph = ttFont["glyf"][glyphName]
			numPointsInGlyph = self.getNumPoints_(glyph)
			glyphVariations = []
			for element in content:
				if isinstance(element, tuple):
					name, attrs, content = element
					if name == "tuple":
						gvar = TupleVariation({}, [None] * numPointsInGlyph)
						glyphVariations.append(gvar)
						for tupleElement in content:
							if isinstance(tupleElement, tuple):
								tupleName, tupleAttrs, tupleContent = tupleElement
								gvar.fromXML(tupleName, tupleAttrs, tupleContent)
			self.variations[glyphName] = glyphVariations

	@staticmethod
	def getNumPoints_(glyph):
		NUM_PHANTOM_POINTS = 4
		if glyph.isComposite():
			return len(glyph.components) + NUM_PHANTOM_POINTS
		else:
			# Empty glyphs (eg. space, nonmarkingreturn) have no "coordinates" attribute.
			return len(getattr(glyph, "coordinates", [])) + NUM_PHANTOM_POINTS

def compileGlyph_(variations, pointCount, axisTags, sharedCoordIndices):
	tupleVariationCount, tuples, data = tv.compileTupleVariationStore(
		variations, pointCount, axisTags, sharedCoordIndices)
	if tupleVariationCount == 0:
		return b""
	result = [
		struct.pack(">HH", tupleVariationCount, 4 + len(tuples)),
		tuples,
		data
	]
	if (len(tuples) + len(data)) % 2 != 0:
		result.append(b"\0")  # padding
	return b''.join(result)

def decompileGlyph_(pointCount, sharedTuples, axisTags, data):
	if len(data) < 4:
		return []
	tupleVariationCount, offsetToData = struct.unpack(">HH", data[:4])
	dataPos = offsetToData
	return tv.decompileTupleVariationStore(
		"gvar", axisTags,
		tupleVariationCount, pointCount,
		sharedTuples, data, 4, offsetToData
	)
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Elasticsearch manipulation script for deleting and creating Elasticsearch
indices.
"""

import argparse
import datetime
import os

from configman import configuration, Namespace

from socorro.external.es.base import generate_list_of_indexes
from socorro.external.es.connection_context import ConnectionContext
from socorro.scripts import WrappedTextHelpFormatter

DESCRIPTION = 'Create and delete Elasticsearch indices'

EPILOG = 'Requires Elasticsearch configuration to be set in environment.'

def get_conn():
    ns = Namespace()
    ns.add_option('elasticsearch_class', default=ConnectionContext)

    config = configuration(
        definition_source=ns,
        values_source_list=[{
            'elasticsearch_urls': os.environ.get('ELASTICSEARCH_URLS', 'http://localhost:9200')
        }]
    )

    return ConnectionContext(config)

def main(argv=None):
    parser = argparse.ArgumentParser(
        formatter_class=WrappedTextHelpFormatter,
        description=DESCRIPTION.strip(),
        epilog=EPILOG.strip()
    )
    subparsers = parser.add_subparsers(dest='cmd')
    subparsers.required = True
    create_parser = subparsers.add_parser('create', help='create indices')
    create_parser.add_argument(
        '--future', type=int, default=2,
        help='Number of weeks in the future to create.'
    )
    create_parser.add_argument(
        '--past', type=int, default=2,
        help='Number of weeks in the future to create.'
    )

    args = parser.parse_args()
    conn = get_conn()

    if args.cmd == 'create':
        # Create recent indices
        index_name_template = conn.get_index_template()

        # Figure out dates
        today = datetime.date.today()
        from_date = today - datetime.timedelta(weeks=args.past)
        to_date = today + datetime.timedelta(weeks=args.future)

        # Create indiices
        index_names = generate_list_of_indexes(from_date, to_date, index_name_template)
        for index_name in index_names:
            was_created = conn.create_index(index_name)
            if was_created:
                print('Index %s was created.' % index_name)
            else:
                print('Index %s already existed.' % index_name)
<commit_msg>bug 1546433: add "es list" and fix config<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Elasticsearch manipulation script for deleting and creating Elasticsearch
indices.
"""

import argparse
import datetime

from configman import ConfigurationManager

from socorro.external.es.base import generate_list_of_indexes
from socorro.external.es.connection_context import ConnectionContext
from socorro.scripts import WrappedTextHelpFormatter

DESCRIPTION = 'Create and delete Elasticsearch indices'

EPILOG = 'Requires Elasticsearch configuration to be set in environment.'

def get_conn():
    cm = ConfigurationManager(ConnectionContext.get_required_config())
    config = cm.get_config()
    return ConnectionContext(config)

def cmd_create(weeks_past, weeks_future):
    """Create recent indices."""
    conn = get_conn()

    # Create recent indices
    index_name_template = conn.get_index_template()

    # Figure out dates
    today = datetime.date.today()
    from_date = today - datetime.timedelta(weeks=weeks_past)
    to_date = today + datetime.timedelta(weeks=weeks_future)

    # Create indiices
    index_names = generate_list_of_indexes(from_date, to_date, index_name_template)
    for index_name in index_names:
        was_created = conn.create_index(index_name)
        if was_created:
            print('Index %s was created.' % index_name)
        else:
            print('Index %s already existed.' % index_name)

def cmd_list():
    """List indices."""
    conn = get_conn()
    indices_client = conn.indices_client()
    status = indices_client.status()
    indices = status['indices'].keys()
    if indices:
        print('Indices:')
        for index in indices:
            print('   %s' % index)
    else:
        print('No indices.')

def main(argv=None):
    parser = argparse.ArgumentParser(
        formatter_class=WrappedTextHelpFormatter,
        description=DESCRIPTION.strip(),
        epilog=EPILOG.strip()
    )
    subparsers = parser.add_subparsers(dest='cmd')
    subparsers.required = True

    create_parser = subparsers.add_parser('create', help='create indices')
    create_parser.add_argument(
        '--future', type=int, default=2,
        help='Number of weeks in the future to create.'
    )
    create_parser.add_argument(
        '--past', type=int, default=2,
        help='Number of weeks in the future to create.'
    )

    subparsers.add_parser('list', help='list indices')

    args = parser.parse_args()

    if args.cmd == 'create':
        return cmd_create(args.past, args.future)

    if args.cmd == 'list':
        return cmd_list()
<|endoftext|>


<commit_before>"""celery.views"""
from django.http import HttpResponse, Http404
from celery.task import tasks, is_done, apply_async
from celery.result import AsyncResult
from anyjson import serialize as JSON_dump

def apply(request, task_name, *args):
    """View applying a task.

    Example:
        http://e.com/celery/apply/task_name/arg1/arg2//?kwarg1=a&kwarg2=b

    **NOTE** Use with caution, preferably not make this publicly accessible
    without ensuring your code is safe!

    """
    kwargs = request.method == "POST" and \
            request.POST.copy() or request.GET.copy()
    kwargs = [(key.encode("utf-8"), value)
                    for key, value in kwargs.items()]
    if task_name not in tasks:
        raise Http404("apply: no such task")

    task = tasks[task_name]
    result = apply_async(task, args=args, kwargs=kwargs)
    return JSON_dump({"ok": "true", "task_id": result.task_id})

def is_task_done(request, task_id):
    """Returns task execute status in JSON format."""
    response_data = {"task": {"id": task_id, "executed": is_done(task_id)}}
    return HttpResponse(JSON_dump(response_data), mimetype="application/json")

def task_status(request, task_id):
    """Returns task status and result in JSON format."""
    async_result = AsyncResult(task_id)
    status = async_result.status
    if status == "FAILURE":
        response_data = {
            "id": task_id,
            "status": status,
            "result": async_result.result.args[0],
        }
    else:
        response_data = {
            "id": task_id,
            "status": status,
            "result": async_result.result,
        }
    return HttpResponse(JSON_dump({"task": response_data}),
            mimetype="application/json")
<commit_msg>kwargs should be a dict<commit_after>"""celery.views"""
from django.http import HttpResponse, Http404
from celery.task import tasks, is_done, apply_async
from celery.result import AsyncResult
from anyjson import serialize as JSON_dump

def apply(request, task_name, *args):
    """View applying a task.

    Example:
        http://e.com/celery/apply/task_name/arg1/arg2//?kwarg1=a&kwarg2=b

    **NOTE** Use with caution, preferably not make this publicly accessible
    without ensuring your code is safe!

    """
    kwargs = request.method == "POST" and \
            request.POST.copy() or request.GET.copy()
    kwargs = dict((key.encode("utf-8"), value)
                    for key, value in kwargs.items())
    if task_name not in tasks:
        raise Http404("apply: no such task")

    task = tasks[task_name]
    result = apply_async(task, args=args, kwargs=kwargs)
    return JSON_dump({"ok": "true", "task_id": result.task_id})

def is_task_done(request, task_id):
    """Returns task execute status in JSON format."""
    response_data = {"task": {"id": task_id, "executed": is_done(task_id)}}
    return HttpResponse(JSON_dump(response_data), mimetype="application/json")

def task_status(request, task_id):
    """Returns task status and result in JSON format."""
    async_result = AsyncResult(task_id)
    status = async_result.status
    if status == "FAILURE":
        response_data = {
            "id": task_id,
            "status": status,
            "result": async_result.result.args[0],
        }
    else:
        response_data = {
            "id": task_id,
            "status": status,
            "result": async_result.result,
        }
    return HttpResponse(JSON_dump({"task": response_data}),
            mimetype="application/json")
<|endoftext|>