<commit_before>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyspark.sql.column import Column
from pyspark.sql.functions import udf

from smv.smvapp import SmvApp
from smv.utils import smv_copy_array, is_string

def nGram2(c1, c2):
    """2-gram UDF with formula (number of overlaped gramCnt)/max(c1.gramCnt, c2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 2-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.nGram2(c1._jc, c2._jc))

def nGram3(c1, c2):
    """3-gram UDF with formula (number of overlaped gramCnt)/max(s1.gramCnt, s2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 3-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.nGram3(c1._jc, c2._jc))

def diceSorensen(c1, c2):
    """2-gram UDF with formula (2 * number of overlaped gramCnt)/(s1.gramCnt + s2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 2-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.diceSorensen(c1._jc, c2._jc))

def normlevenshtein(c1, c2):
    """Levenshtein edit distance metric UDF

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): distances
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.normlevenshtein(c1._jc, c2._jc))

def jaroWinkler(c1, c2):
    """Jaro-Winkler edit distance metric UDF

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): distances
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.jaroWinkler(c1._jc, c2._jc))

def smvFirst(c, nonNull = False):
    """Variation of Spark "first" which also returns null values

        Since Spark "first" will return the first non-null value, we have to
        create our version smvFirst which to retune the real first value, even
        if it's null. Alternatively can return the first non-null value.

        Args:
            c (Column: column to extract first value from
            nonNull (bool): If false, return first value even if null. If true, return first non-null value. Defaults to false.

        Returns:
            (object): first value
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.smvFirst(c._jc, nonNull))

def smvCreateLookUp(m, default, outputType):
    """Return a Python UDF which will perform a dictionary lookup on a column

        Args:
            m (dictionary): a Python dictionary to be applied
            default (any): default value if dictionary lookup failed
            outputType (DataType): output value's data type

        Returns:
            (udf): an udf which can apply to a column and apply the lookup
    """
    return udf(lambda k: m.get(k, default), outputType)

def smvArrayCat(sep, col):
    """For an array typed column, concat the elements to a string with the given separater.

       Args:
            sep: a Python string to separate the fields
            col: a Column with ArrayType

       Return:
            (col): a Column in StringType with array elements concatenated
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.smvArrayCat(sep, col._jc))

def smvCollectSet(col, datatype):
    """An aggregate function, which will collect all the values of the given column and create a set as an array typed column.
       Since Spark 1.6, a spark function collect_set was introduced, so as migrate to Spark 1.6 and later, this smvCollectSet
       will be depricated.

       Args:
            col (Column): column to be aggregated on
            datatype (DataType): datatype of the input column
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvCollectSet(col._jc, datatype.json()))

def smvStrCat(head, *others):
    """Concatenate multiple columns to a single string. Similar to `concat` and `concat_ws` functions in Spark but behaves differently
       when some columns are nulls.
       The Spark version will return null if any of the inputs is null.
       smvStrCat will return null if all of the inputs are nulls, otherwise it will coalesce null cols to blank.

       This function can take 2 forms:
       - smvStrCat(sep, col1, col2, ...)
       - smvStrCat(col1, col2, ...)

       Args:
           sep (String): separater for the concats
           col. (Column): columns to be concatenated

       Return:
           (col): a StringType column
    """
    if is_string(head):
        sep = head
        cols = list(others)
    elif isinstance(head, Column):
        sep = ""
        cols = [head] + list(others)
    else:
        raise RuntimeError("first parameter must be either a String or a Column")
    app = SmvApp.getInstance()
    return Column(app._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvStrCat(sep, smv_copy_array(app.sc, *cols)))

def smvHashKey(head, *others):
    """Create MD5 on concatenated columns.
    Return "Prefix" + MD5 Hex string(size 32 string) as the unique key

    MD5's collisions rate on real data records could be ignored based on the following discussion.

    https://marc-stevens.nl/research/md5-1block-collision/
    The shortest messages have the same MD5 are 512-bit (64-byte) messages as below

    4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa200a8284bf36e8e4b55b35f427593d849676da0d1555d8360fb5f07fea2
    and the (different by two bits)
    4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa202a8284bf36e8e4b55b35f427593d849676da0d1d55d8360fb5f07fea2
    both have MD5 hash
    008ee33a9d58b51cfeb425b0959121c9

    There are other those pairs, but all carefully constructed.
    Theoretically the random collisions will happen on data size approaching 2^64 (since MD5 has
    128-bit), which is much larger than the number of records we deal with (a billion is about 2^30)
    There for using MD5 to hash primary key columns is good enough for creating an unique key

    This function can take 2 forms:
    - smvHashKey(prefix, col1, col2, ...)
    - smvHashKey(col1, col2, ...)

    Args:
     prefix (String): return string's prefix
     col. (Column): columns to be part of hash

    Return:
     (col): a StringType column as Prefix + MD5 Hex string
    """

    if is_string(head):
        pre = head
        cols = list(others)
    elif isinstance(head):
        pre = ""
        cols = [head] + list(others)
    else:
        raise RuntimeError("first parameter must be either a String or a Column")
    app = SmvApp.getInstance()
    return Column(app._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvHashKey(pre, smv_copy_array(app.sc, *cols)))
<commit_msg> #915 Fixed bug in smvHashKey<commit_after>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyspark.sql.column import Column
from pyspark.sql.functions import udf

from smv.smvapp import SmvApp
from smv.utils import smv_copy_array, is_string

def nGram2(c1, c2):
    """2-gram UDF with formula (number of overlaped gramCnt)/max(c1.gramCnt, c2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 2-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.nGram2(c1._jc, c2._jc))

def nGram3(c1, c2):
    """3-gram UDF with formula (number of overlaped gramCnt)/max(s1.gramCnt, s2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 3-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.nGram3(c1._jc, c2._jc))

def diceSorensen(c1, c2):
    """2-gram UDF with formula (2 * number of overlaped gramCnt)/(s1.gramCnt + s2.gramCnt)

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): 2-gram
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.diceSorensen(c1._jc, c2._jc))

def normlevenshtein(c1, c2):
    """Levenshtein edit distance metric UDF

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): distances
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.normlevenshtein(c1._jc, c2._jc))

def jaroWinkler(c1, c2):
    """Jaro-Winkler edit distance metric UDF

        Args:
            c1 (Column): first column
            c2 (Column): second column

        Returns:
            (Column): distances
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.jaroWinkler(c1._jc, c2._jc))

def smvFirst(c, nonNull = False):
    """Variation of Spark "first" which also returns null values

        Since Spark "first" will return the first non-null value, we have to
        create our version smvFirst which to retune the real first value, even
        if it's null. Alternatively can return the first non-null value.

        Args:
            c (Column: column to extract first value from
            nonNull (bool): If false, return first value even if null. If true, return first non-null value. Defaults to false.

        Returns:
            (object): first value
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.smvFirst(c._jc, nonNull))

def smvCreateLookUp(m, default, outputType):
    """Return a Python UDF which will perform a dictionary lookup on a column

        Args:
            m (dictionary): a Python dictionary to be applied
            default (any): default value if dictionary lookup failed
            outputType (DataType): output value's data type

        Returns:
            (udf): an udf which can apply to a column and apply the lookup
    """
    return udf(lambda k: m.get(k, default), outputType)

def smvArrayCat(sep, col):
    """For an array typed column, concat the elements to a string with the given separater.

       Args:
            sep: a Python string to separate the fields
            col: a Column with ArrayType

       Return:
            (col): a Column in StringType with array elements concatenated
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.smvfuncs.smvArrayCat(sep, col._jc))

def smvCollectSet(col, datatype):
    """An aggregate function, which will collect all the values of the given column and create a set as an array typed column.
       Since Spark 1.6, a spark function collect_set was introduced, so as migrate to Spark 1.6 and later, this smvCollectSet
       will be depricated.

       Args:
            col (Column): column to be aggregated on
            datatype (DataType): datatype of the input column
    """
    return Column(SmvApp.getInstance()._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvCollectSet(col._jc, datatype.json()))

def smvStrCat(head, *others):
    """Concatenate multiple columns to a single string. Similar to `concat` and `concat_ws` functions in Spark but behaves differently
       when some columns are nulls.
       The Spark version will return null if any of the inputs is null.
       smvStrCat will return null if all of the inputs are nulls, otherwise it will coalesce null cols to blank.

       This function can take 2 forms:
       - smvStrCat(sep, col1, col2, ...)
       - smvStrCat(col1, col2, ...)

       Args:
           sep (String): separater for the concats
           col. (Column): columns to be concatenated

       Return:
           (col): a StringType column
    """
    if is_string(head):
        sep = head
        cols = list(others)
    elif isinstance(head, Column):
        sep = ""
        cols = [head] + list(others)
    else:
        raise RuntimeError("first parameter must be either a String or a Column")
    app = SmvApp.getInstance()
    return Column(app._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvStrCat(sep, smv_copy_array(app.sc, *cols)))

def smvHashKey(head, *others):
    """Create MD5 on concatenated columns.
    Return "Prefix" + MD5 Hex string(size 32 string) as the unique key

    MD5's collisions rate on real data records could be ignored based on the following discussion.

    https://marc-stevens.nl/research/md5-1block-collision/
    The shortest messages have the same MD5 are 512-bit (64-byte) messages as below

    4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa200a8284bf36e8e4b55b35f427593d849676da0d1555d8360fb5f07fea2
    and the (different by two bits)
    4dc968ff0ee35c209572d4777b721587d36fa7b21bdc56b74a3dc0783e7b9518afbfa202a8284bf36e8e4b55b35f427593d849676da0d1d55d8360fb5f07fea2
    both have MD5 hash
    008ee33a9d58b51cfeb425b0959121c9

    There are other those pairs, but all carefully constructed.
    Theoretically the random collisions will happen on data size approaching 2^64 (since MD5 has
    128-bit), which is much larger than the number of records we deal with (a billion is about 2^30)
    There for using MD5 to hash primary key columns is good enough for creating an unique key

    This function can take 2 forms:
    - smvHashKey(prefix, col1, col2, ...)
    - smvHashKey(col1, col2, ...)

    Args:
     prefix (String): return string's prefix
     col. (Column): columns to be part of hash

    Return:
     (col): a StringType column as Prefix + MD5 Hex string
    """

    if is_string(head):
        pre = head
        cols = list(others)
    elif isinstance(head, Column):
        pre = ""
        cols = [head] + list(others)
    else:
        raise RuntimeError("first parameter must be either a String or a Column")
    app = SmvApp.getInstance()
    return Column(app._jvm.org.tresamigos.smv.python.SmvPythonHelper.smvHashKey(pre, smv_copy_array(app.sc, *cols)))
<|endoftext|>


<commit_before>'''
Given n non-negative integers representing the histogram's bar height where the width of each bar is 1, find the area of largest rectangle in the histogram.

For example,
Given height = [2,1,5,6,2,3],
return 10.
'''

#思路分析:
#[G4G Set 2 O(n)](http://www.geeksforgeeks.org/largest-rectangle-under-histogram/)
#[G4G Set 1 O(nlogn)](http://www.geeksforgeeks.org/largest-rectangular-area-in-a-histogram-set-1/)

class Solution:
    # @param height, a list of integer
    # @return an integer

    def largestRectangleArea(self, height):
        height.append(0) # Corner case: [1]
        n = len(height)
        stack = []
        maxarea = 0
        i = 0
        while i < n: # 注意这里用while，不用for，否则会跳过一个
            if len(stack) == 0 or height[i] > height[stack[-1]]:
                stack.append(i)
                i += 1
                # Note: we only need to i += 1 here! we do not need to add 1 in the else case
                # since after we calculate the area, still need to push that value into stack!
            else:
                tmp_height = height[stack.pop()]
                if len(stack) == 0:
                    width = i # 注意这里 width = i 不是stack.pop()
                else:
                    width = i - stack[-1] -1
                    # left_bound: stack[-1] + 1
                    # right_bound: i-1
                    # right_bound-left_bound+1: (i-1)-(stack[-1]+1) + 1 = i - stack[-1] -1
                tmparea = tmp_height * width
                maxarea = max(maxarea, tmparea)
        return maxarea
<commit_msg>Update Largest-Rectangle-in-Histogram.py<commit_after>'''
Given n non-negative integers representing the histogram's bar height where the width of each bar is 1, find the area of largest rectangle in the histogram.

For example,
Given height = [2,1,5,6,2,3],
return 10.
'''

#思路分析:
#[G4G Set 2 O(n)](http://www.geeksforgeeks.org/largest-rectangle-under-histogram/)
#[G4G Set 1 O(nlogn)](http://www.geeksforgeeks.org/largest-rectangular-area-in-a-histogram-set-1/)

'''
For every bar ‘x’, we calculate the area with ‘x’ as the smallest bar in the rectangle. If we calculate such area for every bar ‘x’ and find the maximum of all areas,
our task is done. How to calculate area with ‘x’ as smallest bar? We need to know index of the first smaller (smaller than ‘x’) bar on left of ‘x’ and index of first
smaller bar on right of ‘x’. Let us call these indexes as ‘left index’ and ‘right index’ respectively.
We traverse all bars from left to right, maintain a stack of bars. Every bar is pushed to stack once. A bar is popped from stack when a bar of smaller height is seen.
When a bar is popped, we calculate the area with the popped bar as smallest bar. How do we get left and right indexes of the popped bar – the current index tells us the
‘right index’ and index of previous item in stack is the ‘left index’.
'''

class Solution:
    # @param height, a list of integer
    # @return an integer

    def largestRectangleArea(self, height):
        height.append(0) # Corner case: [1]
        n = len(height)
        stack = []
        maxarea = 0
        i = 0
        while i < n: # 注意这里用while，不用for，否则会跳过一个
            if len(stack) == 0 or height[i] > height[stack[-1]]:  #注意这里不是比较height[i] > height[i-1]!
                stack.append(i)
                i += 1
                # Note: we only need to i += 1 here! we do not need to add 1 in the else case
                # since after we calculate the area, still need to push that value into stack!
            else:
                tmp_height = height[stack.pop()]
                if len(stack) == 0:
                    width = i # 注意这里 width = i 不是stack.pop()
                else:
                    width = i - stack[-1] -1
                    # left_bound: stack[-1] + 1
                    # right_bound: i-1
                    # right_bound-left_bound+1: (i-1)-(stack[-1]+1) + 1 = i - stack[-1] -1
                tmparea = tmp_height * width
                maxarea = max(maxarea, tmparea)
        return maxarea
<|endoftext|>


<commit_before>
import json
from django.http import HttpResponse
from .api import Twitter
from datetime import datetime
from .utils import get_client_ip
from .models import Tweet, TwitterConfiguration, Country
from django.contrib.gis.geoip2 import GeoIP2

def index(request):
    try:
        not_countries = ['uknown', 'localhost']
        ip = get_client_ip(request)
        g = GeoIP2()
        if ip == '127.0.0.1':
            country = 'localhost'
        else:
            country = g.city(ip)
    except Exception:
        country = 'uknown'
    try:
        conf = TwitterConfiguration.objects.first()
        twitter = Twitter()
        if Country.objects.count() > 0 and country not in not_countries and conf.geolocation:
            country_object = Country.objects.filter(
                country_code=country['country_code'].lower()).values('topics', 'languages')
            if len(country_object) > 0:
                topics = country_object[0]['topics'].split()
                languages = country_object[0]['languages'].split()
            else:
                topics = conf.topics.split()
                languages = conf.languages.split()
        else:
            topics = conf.topics.split()
            languages = conf.languages.split()
        twitter.stream(topics, languages)
        tweets = Tweet.objects.all()
        tweet = tweets[len(tweets) - 1]
        tweet_object = {
            'name': tweet.name,
            'user': '@{0}'.format(tweet.user),
            'text': tweet.text,
            'date': tweet.twitter_date_created.strftime('%d/%b/%Y                       %H:%M')
        }
        context = json.dumps(tweet_object, ensure_ascii=False)
        return HttpResponse(context, content_type="application/json;charset=utf-8")
    except Exception:
        tweet_object = {
            'name': '@Penelope',
            'user': 'Penelope',
            'text': 'I can\'t find you love.',
            'date': datetime.now().strftime('%d/%b/%Y \t \t \t \t \t \t \t %H:%M'),
            'error': True
        }
        context = json.dumps(tweet_object, ensure_ascii=False)
        return HttpResponse(context, content_type="application/json;charset=utf-8")
<commit_msg>update<commit_after>
import json
from django.http import HttpResponse
from .api import Twitter
from datetime import datetime
from .utils import get_client_ip
from .models import Tweet, TwitterConfiguration, Country
from django.contrib.gis.geoip2 import GeoIP2

def index(request):
    try:
        not_countries = ['uknown', 'localhost']
        ip = get_client_ip(request)
        g = GeoIP2()
        if ip == '127.0.0.1':
            country = 'localhost'
        else:
            country = g.city(ip)
    except Exception:
        country = 'uknown'
    try:
        conf = TwitterConfiguration.objects.first()
        twitter = Twitter()
        if Country.objects.count() > 0 and country not in not_countries and conf.geolocation:
            country_object = Country.objects.filter(
                country_code=country['country_code'].lower()).values('topics', 'languages')
            if len(country_object) > 0:
                topics = country_object[0]['topics'].split()
                languages = country_object[0]['languages'].split()
            else:
                topics = conf.topics.split()
                languages = conf.languages.split()
        else:
            topics = conf.topics.split()
            languages = conf.languages.split()
        twitter.stream(topics, languages)
        tweets = Tweet.objects.all()
        tweet = tweets[len(tweets) - 1]
        tweet_object = {
            'name': tweet.name,
            'user': '@{0}'.format(tweet.user),
            'text': tweet.text,
            'date': tweet.twitter_date_created.strftime('%d/%b/%Y              %H:%M')
        }
        context = json.dumps(tweet_object, ensure_ascii=False)
        return HttpResponse(context, content_type="application/json;charset=utf-8")
    except Exception:
        tweet_object = {
            'name': '@Penelope',
            'user': 'Penelope',
            'text': 'I can\'t find you love.',
            'date': datetime.now().strftime('%d/%b/%Y              %H:%M'),
            'error': True
        }
        context = json.dumps(tweet_object, ensure_ascii=False)
        return HttpResponse(context, content_type="application/json;charset=utf-8")
<|endoftext|>


<commit_before>#! /usr/bin/env python

# Copyright 2012 Jtmorgan

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import MySQLdb
import wikitools
import settings
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

report_title = settings.rootpage + '/Badge/Awarded'

report_template = u'''== Badges awarded ==
Please do not edit this table manually. It is updated daily by [[User:HostBot|HostBot]] and your edits will be overwritten. The page was last updated at {{subst:DATETIME}}
<onlyinclude>
%s badges have been given out so far.

{| class="wikitable sortable plainlinks"
|-
! Badge
! Number of recipients
|-
%s
|}
</onlyinclude>

=== Recently awarded badges ===
{| class="wikitable sortable plainlinks"
|-
! Username
! Badge
! Award date
|-
%s
|}
'''

wiki = wikitools.Wiki(settings.apiurl)
wiki.login(settings.username, settings.password)
conn = MySQLdb.connect(host = 'db67.pmtpa.wmnet', db = 'jmorgan', read_default_file = '~/.my.cnf', use_unicode=1, charset="utf8" )
cursor = conn.cursor()

#adds pages to the badge page list
def updatePageList():
	cursor.execute('INSERT IGNORE INTO th_badges (bpage_title, bpage_id) SELECT page_title, page_id FROM enwiki.page WHERE page_namespace = 4 AND page_title LIKE "Teahouse/Badge/%"')
	conn.commit()

#gets badge pages and titles
def getPages(cursor):
	list = []
	cursor.execute('SELECT bpage_id, bpage_title, b_id FROM th_badges WHERE b_id IS NOT NULL')
	rows = cursor.fetchall()
	for row in rows:
		page = str(row[0])
		page_title = row[1]
		page_title = MySQLdb.escape_string(page_title)
		some_id = str(row[2])
# 		print page_title
		list.append([page, page_title, some_id])
	return list

#finds new links to badges.
def findLinks(cursor, list):
	badges_inserted = 0
	for l in list:
		cursor.execute('''INSERT IGNORE INTO th_up_badges_awarded (b_id, ba_date, ba_page, bap_ns, bap_title) SELECT %s, DATE(NOW()), page_id, page_namespace, page_title FROM enwiki.page AS p, enwiki.pagelinks AS pl WHERE pl.pl_namespace = 4 AND pl.pl_title = "%s" AND p.page_namespace IN (2,3) AND p.page_id = pl.pl_from AND p.page_title NOT LIKE "%s"
	''' % (l[2], l[1], "%/%"))
		conn.commit()
		badges_inserted += cursor.rowcount
	return badges_inserted
# 		print cursor._executed

#gets recipient user data
def getUserData(cursor):
		cursor.execute('UPDATE th_up_badges_awarded as t, enwiki.user AS u SET t.user_id = u.user_id, t.user_name = u.user_name, t.user_registration = u.user_registration, t.not_badge = 0 WHERE REPLACE(t.bap_title,"_"," ") = u.user_name AND t.user_id IS NULL')
		conn.commit()

def updateBadgeTable(cursor):
	cursor.execute('UPDATE th_badges AS p, (SELECT b_id, COUNT(ba_id) as num FROM th_up_badges_awarded WHERE bap_ns = 3 AND not_badge = 0 GROUP BY b_id) AS tmp SET p.b_awarded = tmp.num WHERE p.b_id = tmp.b_id')

#print a summary table of badges to Teahouse/Badge/Awarded
def outputBadgeTable(cursor):
	output = [] #list that contains each row for the summary table
	#gets total badge counts for output
	cursor.execute('SELECT SUM(b_awarded) AS sum FROM th_badges WHERE b_id IS NOT NULL AND b_id != 9 AND b_awarded IS NOT NULL')
	row = cursor.fetchone()
	total = row[0] #total badges awarded. all types
	output.append(total)
	#gets individual badge counts for output
	cursor.execute('SELECT b_name, bpage_title, b_awarded FROM th_badges WHERE b_id IS NOT NULL AND b_id != 9 AND b_awarded IS NOT NULL ORDER BY b_awarded DESC')
	rows = cursor.fetchall()
	for row in rows:
		name = row[0]
		title = row[1]
		count = row[2]
		link = '[[Wikipedia:%s|%s]]' % (title, name)
		table_row = u'''| %s
| %s
|-''' % (link, count)
		output.append(table_row)

	output2 = [] #the recent by-user badge table
	cursor.execute('SELECT user_name, b_name, bpage_title, ba_date FROM th_badges as b, th_up_badges_awarded as ba WHERE b.b_id = ba.b_id AND ba.bap_ns = 3 AND ba.b_id != 9 ORDER BY ba_date DESC LIMIT 100')
	rows = cursor.fetchall()
	for row in rows:
		uname = row[0]
		bname = row[1]
		bpage = row[2]
		bdate = row[3]
		table_row = u'''|[[User_talk:%s|%s]]
| [[WP:%s|%s]]
| %s
|-''' % (uname, uname, bpage, bname, bdate)
		output2.append(table_row)

	#prints the report to WP:Teahouse/Badge/Awarded
	report = wikitools.Page(wiki, report_title)
	report_text = report_template % (output[0], '\n'.join(output[1:]), '\n'.join(output2))
	report_text = report_text.encode('utf-8')
# 	print report_text
	report.edit(report_text, section=1, summary="[[User:HostBot|HostBot]] is updating [[WP:Teahouse/Badge|Badge]] counts", bot=1)

##MAIN##
updatePageList()
badge_list = getPages(cursor)
badges_inserted = findLinks(cursor, badge_list)
# if badges_inserted > 0:
getUserData(cursor)
updateBadgeTable(cursor)
outputBadgeTable(cursor)

cursor.close()
conn.close()<commit_msg>update badge reporting script output<commit_after>#! /usr/bin/env python

# Copyright 2012 Jtmorgan

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import MySQLdb
import wikitools
import settings
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

report_title = settings.rootpage + '/Badge/Awarded'

report_template = u'''== Badges awarded ==
Please do not edit this table manually. It is updated daily by [[User:HostBot|HostBot]] and your edits will be overwritten. The page was last updated on {{subst:REVISIONMONTH}}/{{subst:REVISIONDAY}}/{{subst:REVISIONYEAR}} by {{subst:REVISIONUSER}}.
<onlyinclude>
%s badges have been given out so far.

{| class="wikitable sortable plainlinks"
|-
! Badge
! Number of recipients
|-
%s
|}
</onlyinclude>

=== Recently awarded badges ===
{| class="wikitable sortable plainlinks"
|-
! Username
! Badge
! Award date
|-
%s
|}
'''

wiki = wikitools.Wiki(settings.apiurl)
wiki.login(settings.username, settings.password)
conn = MySQLdb.connect(host = 'db67.pmtpa.wmnet', db = 'jmorgan', read_default_file = '~/.my.cnf', use_unicode=1, charset="utf8" )
cursor = conn.cursor()

#adds pages to the badge page list
def updatePageList():
	cursor.execute('INSERT IGNORE INTO th_badges (bpage_title, bpage_id) SELECT page_title, page_id FROM enwiki.page WHERE page_namespace = 4 AND page_title LIKE "Teahouse/Badge/%"')
	conn.commit()

#gets badge pages and titles
def getPages(cursor):
	list = []
	cursor.execute('SELECT bpage_id, bpage_title, b_id FROM th_badges WHERE b_id IS NOT NULL')
	rows = cursor.fetchall()
	for row in rows:
		page = str(row[0])
		page_title = row[1]
		page_title = MySQLdb.escape_string(page_title)
		some_id = str(row[2])
# 		print page_title
		list.append([page, page_title, some_id])
	return list

#finds new links to badges.
def findLinks(cursor, list):
	badges_inserted = 0
	for l in list:
		cursor.execute('''INSERT IGNORE INTO th_up_badges_awarded (b_id, ba_date, ba_page, bap_ns, bap_title) SELECT %s, DATE(NOW()), page_id, page_namespace, page_title FROM enwiki.page AS p, enwiki.pagelinks AS pl WHERE pl.pl_namespace = 4 AND pl.pl_title = "%s" AND p.page_namespace IN (2,3) AND p.page_id = pl.pl_from AND p.page_title NOT LIKE "%s"
	''' % (l[2], l[1], "%/%"))
		conn.commit()
		badges_inserted += cursor.rowcount
# 		print badges_inserted
	return badges_inserted
# 		print cursor._executed

#gets recipient user data
def getUserData(cursor):
		cursor.execute('UPDATE th_up_badges_awarded as t, enwiki.user AS u SET t.user_id = u.user_id, t.user_name = u.user_name, t.user_registration = u.user_registration, t.not_badge = 0 WHERE REPLACE(t.bap_title,"_"," ") = u.user_name AND t.user_id IS NULL')
		conn.commit()

def updateBadgeTable(cursor):
	cursor.execute('UPDATE th_badges AS p, (SELECT b_id, COUNT(ba_id) as num FROM th_up_badges_awarded WHERE bap_ns = 3 AND not_badge = 0 GROUP BY b_id) AS tmp SET p.b_awarded = tmp.num WHERE p.b_id = tmp.b_id')

#print a summary table of badges to Teahouse/Badge/Awarded
def outputBadgeTable(cursor):
	output = [] #list that contains each row for the summary table
	#gets total badge counts for output
	cursor.execute('SELECT SUM(b_awarded) AS sum FROM th_badges WHERE b_id IS NOT NULL AND b_id != 9 AND b_awarded IS NOT NULL')
	row = cursor.fetchone()
	total = row[0] #total badges awarded. all types
	output.append(total)
	#gets individual badge counts for output
	cursor.execute('SELECT b_name, bpage_title, b_awarded FROM th_badges WHERE b_id IS NOT NULL AND b_id != 9 AND b_awarded IS NOT NULL ORDER BY b_awarded DESC')
	rows = cursor.fetchall()
	for row in rows:
		name = row[0]
		title = row[1]
		count = row[2]
		link = '[[Wikipedia:%s|%s]]' % (title, name)
		table_row = u'''| %s
| %s
|-''' % (link, count)
		output.append(table_row)

	output2 = [] #the recent by-user badge table
	cursor.execute('SELECT user_name, b_name, bpage_title, ba_date FROM th_badges as b, th_up_badges_awarded as ba WHERE b.b_id = ba.b_id AND ba.bap_ns = 3 AND ba.b_id != 9 AND ba.user_id IS NOT NULL ORDER BY ba_date DESC LIMIT 100')
	rows = cursor.fetchall()
	for row in rows:
		uname = row[0]
		bname = row[1]
		bpage = row[2]
		bdate = row[3]
		table_row = u'''|[[User_talk:%s|%s]]
| [[WP:%s|%s]]
| %s
|-''' % (uname, uname, bpage, bname, bdate)
		output2.append(table_row)

	#prints the report to WP:Teahouse/Badge/Awarded
	report = wikitools.Page(wiki, report_title)
	report_text = report_template % (output[0], '\n'.join(output[1:]), '\n'.join(output2))
	report_text = report_text.encode('utf-8')
# 	print report_text
	report.edit(report_text, section=1, summary="[[User:HostBot|HostBot]] is updating [[WP:Teahouse/Badge|Badge]] counts", bot=1)

##MAIN##
updatePageList()
badge_list = getPages(cursor)
badges_inserted = findLinks(cursor, badge_list)
if badges_inserted > 0:
	getUserData(cursor)
	updateBadgeTable(cursor)
	outputBadgeTable(cursor)

cursor.close()
conn.close()<|endoftext|>


<commit_before>from itertools import groupby
from collections import defaultdict
from xml.etree.ElementTree import Element
from casexml.apps.phone.models import OTARestoreUser
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq import toggles

class LocationSet(object):
    """
    Very simple class for keeping track of a set of locations
    """

    def __init__(self, locations=None):
        self.by_id = {}
        self.root_locations = set()
        self.by_parent = defaultdict(set)
        if locations is not None:
            for loc in locations:
                self.add_location(loc)

    def add_location(self, location):
        if _valid_parent_type(location):
            self.by_id[location.location_id] = location
            parent = location.parent
            parent_id = parent.location_id if parent else None
            if parent_id is None:  # this is a root
                self.add_root(location)
            self.by_parent[parent_id].add(location)

    def add_root(self, location):
        self.root_locations.add(location)

    def __contains__(self, item):
        return item in self.by_id

def should_sync_locations(last_sync, location_db, restore_user):
    """
    Determine if any locations (already filtered to be relevant
    to this user) require syncing.
    """
    if (
        not last_sync or
        not last_sync.date or
        restore_user.get_fixture_last_modified() >= last_sync.date
    ):
        return True

    for location in location_db.by_id.values():
        if (
            not location.last_modified or
            location.last_modified >= last_sync.date or
            location.location_type.last_modified >= last_sync.date
        ):
            return True

    return False

class LocationFixtureProvider(object):

    def __init__(self, id, serializer):
        self.id = id
        self.serializer = serializer

    def __call__(self, restore_user, version, last_sync=None, app=None):
        """
        By default this will generate a fixture for the users
        location and it's "footprint", meaning the path
        to a root location through parent hierarchies.

        There is an admin feature flag that will make this generate
        a fixture with ALL locations for the domain.
        """
        assert isinstance(restore_user, OTARestoreUser)
        all_locations = restore_user.get_locations_to_sync()
        if not should_sync_locations(last_sync, all_locations, restore_user):
            return []

        return self.serializer.get_xml_nodes(self.id, restore_user, all_locations)

class HierarchicalLocationSerializer(object):

    def get_xml_nodes(self, fixture_id, restore_user, all_locations):
        if not restore_user.project.uses_locations:
            return []

        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        root_locations = all_locations.root_locations

        if root_locations:
            _append_children(root_node, all_locations, root_locations)
        return [root_node]

class FlatLocationSerializer(object):

    def get_xml_nodes(self, fixture_id, restore_user, all_locations):
        if not toggles.FLAT_LOCATION_FIXTURE.enabled(restore_user.domain):
            return []

        all_types = LocationType.objects.filter(domain=restore_user.domain).values_list(
            'code', flat=True
        )
        base_attrs = {'{}_id'.format(t): '' for t in all_types if t is not None}
        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        outer_node = Element('locations')
        root_node.append(outer_node)
        for location in sorted(all_locations.by_id.values(), key=lambda l: l.site_code):
            attrs = {
                'type': location.location_type.code,
                'id': location.location_id,
            }
            attrs.update(base_attrs)
            attrs['{}_id'.format(location.location_type.code)] = location.location_id
            tmp_location = location
            while tmp_location.parent:
                tmp_location = tmp_location.parent
                attrs['{}_id'.format(tmp_location.location_type.code)] = tmp_location.location_id

            location_node = Element('location', attrs)
            _fill_in_location_element(location_node, location)
            outer_node.append(location_node)

        return [root_node]

location_fixture_generator = LocationFixtureProvider(
    id='commtrack:locations', serializer=HierarchicalLocationSerializer()
)
flat_location_fixture_generator = LocationFixtureProvider(
    id='locations', serializer=FlatLocationSerializer()
)

def get_all_locations_to_sync(user):
    if toggles.SYNC_ALL_LOCATIONS.enabled(user.domain):
        return LocationSet(SQLLocation.active_objects.filter(domain=user.domain))
    else:
        all_locations = set()

        user_locations = set(user.sql_locations)
        # old flagged multi-locations, ToDo remove in next phase
        user_locations |= {location for location in _gather_multiple_locations(user)}
        for user_location in user_locations:
            location_type = user_location.location_type
            expand_from = location_type.expand_from or location_type
            expand_to = location_type.expand_to
            expand_from_locations = _get_expand_from_level(user.domain, user_location, expand_from)

            for expand_from_location in expand_from_locations:
                for child in _get_children(user.domain, expand_from_location, expand_to):
                    # Walk down the tree and get all the children we want to sync
                    all_locations.add(child)

                for ancestor in expand_from_location.get_ancestors():
                    # We sync all ancestors of the highest location
                    all_locations.add(ancestor)

        return LocationSet(all_locations)

def _gather_multiple_locations(user):
    """If the project has multiple locations enabled, returns all the extra
    locations the user is assigned to.
    """
    if user.project.supports_multiple_locations_per_user:
        location_ids = [loc.location_id for loc in user.locations]
        for location in SQLLocation.active_objects.filter(location_id__in=location_ids):
            yield location

def _get_expand_from_level(domain, user_location, expand_from):
    """From the users current location, returns the highest location they want to start expanding from
    """
    if user_location.location_type.expand_from_root:
        return SQLLocation.root_locations(domain=domain)
    else:
        ancestors = (
            user_location
            .get_ancestors(include_self=True)
            .filter(location_type=expand_from, is_archived=False)
        )
        return ancestors

def _get_children(domain, root, expand_to):
    """From the topmost location, get all the children we want to sync
    """
    expand_to_level = set(
        SQLLocation.active_objects.
        filter(domain__exact=domain, location_type=expand_to).
        values_list('level', flat=True)
    ) or None

    children = root.get_descendants(include_self=True).filter(is_archived=False)
    if expand_to_level is not None:
        assert len(expand_to_level) == 1
        children = children.filter(level__lte=expand_to_level.pop())

    return children

def _valid_parent_type(location):
    parent = location.parent
    parent_type = parent.location_type if parent else None
    return parent_type == location.location_type.parent_type

def _append_children(node, location_db, locations):
    for type, locs in _group_by_type(locations):
        locs = sorted(locs, key=lambda loc: loc.name)
        node.append(_types_to_fixture(location_db, type, locs))

def _group_by_type(locations):
    key = lambda loc: (loc.location_type.code, loc.location_type)
    for (code, type), locs in groupby(sorted(locations, key=key), key=key):
        yield type, list(locs)

def _types_to_fixture(location_db, type, locs):
    type_node = Element('%ss' % type.code)  # hacky pluralization
    for loc in locs:
        type_node.append(_location_to_fixture(location_db, loc, type))
    return type_node

def _get_metadata_node(location):
    node = Element('location_data')
    for key, value in location.metadata.items():
        element = Element(key)
        element.text = value
        node.append(element)
    return node

def _location_to_fixture(location_db, location, type):
    root = Element(type.code, {'id': location.location_id})
    _fill_in_location_element(root, location)
    _append_children(root, location_db, location_db.by_parent[location.location_id])
    return root

def _fill_in_location_element(xml_root, location):
    fixture_fields = [
        'name',
        'site_code',
        'external_id',
        'latitude',
        'longitude',
        'location_type',
        'supply_point_id',
    ]
    for field in fixture_fields:
        field_node = Element(field)
        val = getattr(location, field)
        field_node.text = unicode(val if val is not None else '')
        xml_root.append(field_node)

    xml_root.append(_get_metadata_node(location))
<commit_msg>Add included locations and their ancestors to fixture<commit_after>from itertools import groupby
from collections import defaultdict
from xml.etree.ElementTree import Element
from casexml.apps.phone.models import OTARestoreUser
from corehq.apps.locations.models import SQLLocation, LocationType
from corehq import toggles

class LocationSet(object):
    """
    Very simple class for keeping track of a set of locations
    """

    def __init__(self, locations=None):
        self.by_id = {}
        self.root_locations = set()
        self.by_parent = defaultdict(set)
        if locations is not None:
            for loc in locations:
                self.add_location(loc)

    def add_location(self, location):
        if _valid_parent_type(location):
            self.by_id[location.location_id] = location
            parent = location.parent
            parent_id = parent.location_id if parent else None
            if parent_id is None:  # this is a root
                self.add_root(location)
            self.by_parent[parent_id].add(location)

    def add_root(self, location):
        self.root_locations.add(location)

    def __contains__(self, item):
        return item in self.by_id

def should_sync_locations(last_sync, location_db, restore_user):
    """
    Determine if any locations (already filtered to be relevant
    to this user) require syncing.
    """
    if (
        not last_sync or
        not last_sync.date or
        restore_user.get_fixture_last_modified() >= last_sync.date
    ):
        return True

    for location in location_db.by_id.values():
        if (
            not location.last_modified or
            location.last_modified >= last_sync.date or
            location.location_type.last_modified >= last_sync.date
        ):
            return True

    return False

class LocationFixtureProvider(object):

    def __init__(self, id, serializer):
        self.id = id
        self.serializer = serializer

    def __call__(self, restore_user, version, last_sync=None, app=None):
        """
        By default this will generate a fixture for the users
        location and it's "footprint", meaning the path
        to a root location through parent hierarchies.

        There is an admin feature flag that will make this generate
        a fixture with ALL locations for the domain.
        """
        assert isinstance(restore_user, OTARestoreUser)
        all_locations = restore_user.get_locations_to_sync()
        if not should_sync_locations(last_sync, all_locations, restore_user):
            return []

        return self.serializer.get_xml_nodes(self.id, restore_user, all_locations)

class HierarchicalLocationSerializer(object):

    def get_xml_nodes(self, fixture_id, restore_user, all_locations):
        if not restore_user.project.uses_locations:
            return []

        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        root_locations = all_locations.root_locations

        if root_locations:
            _append_children(root_node, all_locations, root_locations)
        return [root_node]

class FlatLocationSerializer(object):

    def get_xml_nodes(self, fixture_id, restore_user, all_locations):
        if not toggles.FLAT_LOCATION_FIXTURE.enabled(restore_user.domain):
            return []

        all_types = LocationType.objects.filter(domain=restore_user.domain).values_list(
            'code', flat=True
        )
        base_attrs = {'{}_id'.format(t): '' for t in all_types if t is not None}
        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        outer_node = Element('locations')
        root_node.append(outer_node)
        for location in sorted(all_locations.by_id.values(), key=lambda l: l.site_code):
            attrs = {
                'type': location.location_type.code,
                'id': location.location_id,
            }
            attrs.update(base_attrs)
            attrs['{}_id'.format(location.location_type.code)] = location.location_id
            tmp_location = location
            while tmp_location.parent:
                tmp_location = tmp_location.parent
                attrs['{}_id'.format(tmp_location.location_type.code)] = tmp_location.location_id

            location_node = Element('location', attrs)
            _fill_in_location_element(location_node, location)
            outer_node.append(location_node)

        return [root_node]

location_fixture_generator = LocationFixtureProvider(
    id='commtrack:locations', serializer=HierarchicalLocationSerializer()
)
flat_location_fixture_generator = LocationFixtureProvider(
    id='locations', serializer=FlatLocationSerializer()
)

def get_all_locations_to_sync(user):
    if toggles.SYNC_ALL_LOCATIONS.enabled(user.domain):
        return LocationSet(SQLLocation.active_objects.filter(domain=user.domain))
    else:
        all_locations = set()

        user_locations = set(user.sql_locations)
        # old flagged multi-locations, ToDo remove in next phase
        user_locations |= {location for location in _gather_multiple_locations(user)}
        for user_location in user_locations:
            location_type = user_location.location_type
            expand_from = location_type.expand_from or location_type
            expand_to = location_type.expand_to
            expand_from_locations = _get_expand_from_level(user.domain, user_location, expand_from)

            for expand_from_location in expand_from_locations:
                for child in _get_children(user.domain, expand_from_location, expand_to):
                    # Walk down the tree and get all the children we want to sync
                    all_locations.add(child)

                for ancestor in expand_from_location.get_ancestors():
                    # We sync all ancestors of the highest location
                    all_locations.add(ancestor)

            all_locations |= _get_include_without_expanding_locations(user.domain, location_type)

        return LocationSet(all_locations)

def _gather_multiple_locations(user):
    """If the project has multiple locations enabled, returns all the extra
    locations the user is assigned to.
    """
    if user.project.supports_multiple_locations_per_user:
        location_ids = [loc.location_id for loc in user.locations]
        for location in SQLLocation.active_objects.filter(location_id__in=location_ids):
            yield location

def _get_expand_from_level(domain, user_location, expand_from):
    """From the users current location, returns the highest location they want to start expanding from
    """
    if user_location.location_type.expand_from_root:
        return SQLLocation.root_locations(domain=domain)
    else:
        ancestors = (
            user_location
            .get_ancestors(include_self=True)
            .filter(location_type=expand_from, is_archived=False)
        )
        return ancestors

def _get_children(domain, root, expand_to):
    """From the topmost location, get all the children we want to sync
    """
    expand_to_level = set(
        SQLLocation.active_objects.
        filter(domain__exact=domain, location_type=expand_to).
        values_list('level', flat=True)
    ) or None

    children = root.get_descendants(include_self=True).filter(is_archived=False)
    if expand_to_level is not None:
        assert len(expand_to_level) == 1
        children = children.filter(level__lte=expand_to_level.pop())

    return children

def _get_include_without_expanding_locations(domain, location_type):
    """returns all locations set for inclusion along with their ancestors
    """
    include_without_expanding = location_type.include_without_expanding
    include_root_without_expanding = location_type.include_root_without_expanding
    if include_without_expanding is not None:
        # add in other "forced" locations the user should have, along with their ancestors
        forced_locations = set(SQLLocation.active_objects.filter(
            domain__exact=domain,
            location_type=include_without_expanding
        ))
        forced_ancestors = {
            ancestor
            for location in forced_locations
            for ancestor in location.get_ancestors()
        }

        return forced_locations | forced_ancestors

    if include_root_without_expanding:
        # return only the root locations
        return set(SQLLocation.root_locations(domain=domain))

    return set()

def _valid_parent_type(location):
    parent = location.parent
    parent_type = parent.location_type if parent else None
    return parent_type == location.location_type.parent_type

def _append_children(node, location_db, locations):
    for type, locs in _group_by_type(locations):
        locs = sorted(locs, key=lambda loc: loc.name)
        node.append(_types_to_fixture(location_db, type, locs))

def _group_by_type(locations):
    key = lambda loc: (loc.location_type.code, loc.location_type)
    for (code, type), locs in groupby(sorted(locations, key=key), key=key):
        yield type, list(locs)

def _types_to_fixture(location_db, type, locs):
    type_node = Element('%ss' % type.code)  # hacky pluralization
    for loc in locs:
        type_node.append(_location_to_fixture(location_db, loc, type))
    return type_node

def _get_metadata_node(location):
    node = Element('location_data')
    for key, value in location.metadata.items():
        element = Element(key)
        element.text = value
        node.append(element)
    return node

def _location_to_fixture(location_db, location, type):
    root = Element(type.code, {'id': location.location_id})
    _fill_in_location_element(root, location)
    _append_children(root, location_db, location_db.by_parent[location.location_id])
    return root

def _fill_in_location_element(xml_root, location):
    fixture_fields = [
        'name',
        'site_code',
        'external_id',
        'latitude',
        'longitude',
        'location_type',
        'supply_point_id',
    ]
    for field in fixture_fields:
        field_node = Element(field)
        val = getattr(location, field)
        field_node.text = unicode(val if val is not None else '')
        xml_root.append(field_node)

    xml_root.append(_get_metadata_node(location))
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import
from __future__ import unicode_literals

from .ttLib import TTFont, newTable
from .ttLib.tables._c_m_a_p import cmap_classes
from .ttLib.tables._n_a_m_e import NameRecord, makeName
from .misc.timeTools import timestampNow
import struct

_headDefaults = dict(
    tableVersion = 1.0,
    fontRevision = 1.0,
    checkSumAdjustment = 0,
    magicNumber = 0x5F0F3CF5,
    flags = 0x0003,
    unitsPerEm = 1000,
    created = 0,
    modified = 0,
    xMin = 0,
    yMin = 0,
    xMax = 0,
    yMax = 0,
    macStyle = 0,
    lowestRecPPEM = 3,
    fontDirectionHint = 2,
    indexToLocFormat = 0,
    glyphDataFormat = 0,
)

_maxpDefaultsTTF = dict(
    tableVersion = 0x00010000,
    numGlyphs = 0,
    maxPoints = 0,
    maxContours = 0,
    maxCompositePoints = 0,
    maxCompositeContours = 0,
    maxZones = 2,
    maxTwilightPoints = 0,
    maxStorage = 0,
    maxFunctionDefs = 0,
    maxInstructionDefs = 0,
    maxStackElements = 0,
    maxSizeOfInstructions = 0,
    maxComponentElements = 0,
    maxComponentDepth = 0,
)
_maxpDefaultsOTF = dict(
    tableVersion = 0x00005000,
    numGlyphs = 0,
)

_postDefaults = dict(
    formatType = 3.0,
    italicAngle = 0,
    underlinePosition = 0,
    underlineThickness = 0,
    isFixedPitch = 0,
    minMemType42 = 0,
    maxMemType42 = 0,
    minMemType1 = 0,
    maxMemType1 = 0,
)

_hheaDefaults = dict(
    tableVersion = 0x00010000,
    ascent = 0,
    descent = 0,
    lineGap = 0,
    advanceWidthMax = 0,
    minLeftSideBearing = 0,
    minRightSideBearing = 0,
    xMaxExtent = 0,
    caretSlopeRise = 1,
    caretSlopeRun = 0,
    caretOffset = 0,
    reserved0 = 0,
    reserved1 = 0,
    reserved2 = 0,
    reserved3 = 0,
    metricDataFormat = 0,
    numberOfHMetrics = 0,
)

_vheaDefaults = dict(
    tableVersion = 0x00010000,
    ascent = 0,
    descent = 0,
    lineGap = 0,
    advanceHeightMax = 0,
    minTopSideBearing = 0,
    minBottomSideBearing = 0,
    yMaxExtent = 0,
    caretSlopeRise = 0,
    caretSlopeRun = 0,
    reserved0 = 0,
    reserved1 = 0,
    reserved2 = 0,
    reserved3 = 0,
    reserved4 = 0,
    metricDataFormat = 0,
    numberOfVMetrics = 0,
)

_nameIDs = dict(
             copyright = 0,
            familyName = 1,
             styleName = 2,
            identifier = 3,
              fullName = 4,
               version = 5,
                psName = 6,
             trademark = 7,
          manufacturer = 8,
     typographicFamily = 16,
  typographicSubfamily = 17,
# XXX this needs to be extended with legal things, etc.
)

_panoseDefaults = dict(
    bFamilyType = 0,
    bSerifStyle = 0,
    bWeight = 0,
    bProportion = 0,
    bContrast = 0,
    bStrokeVariation = 0,
    bArmStyle = 0,
    bLetterForm = 0,
    bMidline = 0,
    bXHeight = 0,
)

_OS2Defaults = dict(
    version = 3,
    xAvgCharWidth = 0,
    usWeightClass = 400,
    usWidthClass = 5,
    fsType = 0x0004,  # default: Preview & Print embedding
    ySubscriptXSize = 0,
    ySubscriptYSize = 0,
    ySubscriptXOffset = 0,
    ySubscriptYOffset = 0,
    ySuperscriptXSize = 0,
    ySuperscriptYSize = 0,
    ySuperscriptXOffset = 0,
    ySuperscriptYOffset = 0,
    yStrikeoutSize = 0,
    yStrikeoutPosition = 0,
    sFamilyClass = 0,
    panose = _panoseDefaults,
    ulUnicodeRange1 = 0,
    ulUnicodeRange2 = 0,
    ulUnicodeRange3 = 0,
    ulUnicodeRange4 = 0,
    achVendID = "????",
    fsSelection = 0,
    usFirstCharIndex = 0,
    usLastCharIndex = 0,
    sTypoAscender = 0,
    sTypoDescender = 0,
    sTypoLineGap = 0,
    usWinAscent = 0,
    usWinDescent = 0,
    ulCodePageRange1 = 0,
    ulCodePageRange2 = 0,
    sxHeight = 0,
    sCapHeight = 0,
    usDefaultChar = 0,  # .notdef
    usBreakChar = 32,   # space
    usMaxContext = 2,   # just kerning
    usLowerOpticalPointSize = 0,
    usUpperOpticalPointSize = 0,
)

class FontBuilder(object):

    def __init__(self, unitsPerEm=None, font=None, isTTF=True):
        if font is None:
            self.font = TTFont(recalcTimestamp=False)
            self.isTTF = isTTF
            now = timestampNow()
            assert unitsPerEm is not None
            self.setupHead(unitsPerEm=unitsPerEm, created=now, modified=now)
            self.setupMaxp()
        else:
            assert unitsPerEm is None
            self.font = font
            self.isTTF = "glyf" in font

    def save(self, path):
        self.font.save(path)

    def _initTableWithValues(self, tableTag, defaults, values):
        table = self.font[tableTag] = newTable(tableTag)
        for k, v in defaults.items():
            setattr(table, k, v)
        for k, v in values.items():
            setattr(table, k, v)
        return table

    def _updateTableWithValues(self, tableTag, values):
        table = self.font[tableTag]
        for k, v in values.items():
            setattr(table, k, v)

    def setupHead(self, **values):
        self._initTableWithValues("head", _headDefaults, values)

    def updateHead(self, **values):
        self._updateTableWithValues("head", values)

    def setupGlyphOrder(self, glyphOrder):
        self.font.setGlyphOrder(glyphOrder)

    def setupCharacterMap(self, cmapping, allowFallback=False):
        subTables = []
        highestUnicode = max(cmapping)
        if highestUnicode > 0xffff:
            cmapping_3_1 = dict((k, v) for k, v in cmapping.items() if k < 0x10000)
            subTable_3_10 = buildCmapSubTable(cmapping, 12, 3, 10)
            subTables.append(subTable_3_10)
        else:
            cmapping_3_1 = cmapping
        format = 4
        subTable_3_1 = buildCmapSubTable(cmapping_3_1, format, 3, 1)
        try:
            subTable_3_1.compile(self.font)
        except struct.error:
            # format 4 overflowed, fall back to format 12
            if not allowFallback:
                raise ValueError("cmap format 4 subtable overflowed; sort glyph order by unicode to fix.")
            format = 12
            subTable_3_1 = buildCmapSubTable(cmapping_3_1, format, 3, 1)
        subTables.append(subTable_3_1)
        subTable_0_3 = buildCmapSubTable(cmapping_3_1, format, 0, 3)
        subTables.append(subTable_0_3)

        self.font["cmap"] = newTable("cmap")
        self.font["cmap"].tableVersion = 0
        self.font["cmap"].tables = subTables

    def setupNameTable(self, nameStrings):
        nameTable = self.font["name"] = newTable("name")
        nameTable.names = []

        for nameName, nameValue in nameStrings.items():
            if isinstance(nameName, int):
                nameID = nameName
            else:
                nameID = _nameIDs[nameName]
            nameTable.addMultilingualName(dict(en=nameValue), ttFont=self.font, nameID=nameID)

    def setupOS2(self, **values):
        if "xAvgCharWidth" not in values:
            gs = self.font.getGlyphSet()
            widths = [gs[glyphName].width for glyphName in gs.keys() if gs[glyphName].width > 0]
            values["xAvgCharWidth"] = int(round(sum(widths) / float(len(widths))))
        self._initTableWithValues("OS/2", _OS2Defaults, values)
        if not ("ulUnicodeRange1" in values or "ulUnicodeRange2" in values or
                "ulUnicodeRange3" in values or "ulUnicodeRange3" in values):
            assert "cmap" in self.font
            self.font["OS/2"].recalcUnicodeRanges(self.font)

    def setupCFF(self, psName, fontInfo, charStringsDict, privateDict):
        assert not self.isTTF
        from .cffLib import CFFFontSet, TopDictIndex, TopDict, CharStrings, \
                GlobalSubrsIndex, PrivateDict
        self.font.sfntVersion = "OTTO"
        fontSet = CFFFontSet()
        fontSet.major = 1
        fontSet.minor = 0
        fontSet.fontNames = [psName]
        fontSet.topDictIndex = TopDictIndex()

        globalSubrs = GlobalSubrsIndex()
        fontSet.GlobalSubrs = globalSubrs
        private = PrivateDict()
        for key, value in privateDict.items():
            setattr(private, key, value)
        fdSelect = None
        fdArray = None

        topDict = TopDict()
        topDict.charset = self.font.getGlyphOrder()
        topDict.Private = private
        for key, value in fontInfo.items():
            setattr(topDict, key, value)

        charStrings = CharStrings(None, topDict.charset, globalSubrs, private, fdSelect, fdArray)
        for glypnName, charString in charStringsDict.items():
            charString.private = private
            charString.globalSubrs = globalSubrs
            charStrings[glypnName] = charString
        topDict.CharStrings = charStrings

        fontSet.topDictIndex.append(topDict)

        self.font["CFF "] = newTable("CFF ")
        self.font["CFF "].cff = fontSet

    def setupGlyf(self, glyphs):
        assert self.isTTF
        self.font["loca"] = newTable("loca")
        self.font["glyf"] = newTable("glyf")
        self.font["glyf"].glyphs = glyphs
        if hasattr(self.font, "glyphOrder"):
            self.font["glyf"].glyphOrder = self.font.glyphOrder

    def setupFvar(self, axes, instances):
        addFvar(self.font, axes, instances)

    def setupGvar(self, variations):
        gvar = self.font["gvar"] = newTable('gvar')
        gvar.version = 1
        gvar.reserved = 0
        gvar.variations = variations

    def calcGlyphBounds(self):
        glyphTable = self.font["glyf"]
        for glyph in glyphTable.glyphs.values():
            glyph.recalcBounds(glyphTable)

    def setupMetrics(self, tableTag, metrics):
        assert tableTag in ("hmtx", "vmtx")
        mtxTable = self.font[tableTag] = newTable(tableTag)
        roundedMetrics = {}
        for gn in metrics:
            w, lsb = metrics[gn]
            roundedMetrics[gn] = int(round(w)), int(round(lsb))
        mtxTable.metrics = roundedMetrics

    def setupHorizontalHeader(self, **values):
        self._initTableWithValues("hhea", _hheaDefaults, values)

    def setupVerticalHeader(self, **values):
        self._initTableWithValues("vhea", _vheaDefaults, values)

    def setupVerticalOrigins(self, verticalOrigins, defaultVerticalOrigin=None):
        if defaultVerticalOrigin is None:
            # find the most frequent vorg value
            bag = {}
            for gn in verticalOrigins:
                vorg = verticalOrigins[gn]
                if vorg not in bag:
                    bag[vorg] = 1
                else:
                    bag[vorg] += 1
            defaultVerticalOrigin = sorted(bag, key=lambda vorg: bag[vorg], reverse=True)[0]
        self._initTableWithValues("VORG", {}, dict(VOriginRecords={}, defaultVertOriginY=defaultVerticalOrigin))
        vorgTable = self.font["VORG"]
        vorgTable.majorVersion = 1
        vorgTable.minorVersion = 0
        for gn in verticalOrigins:
            vorgTable[gn] = verticalOrigins[gn]

    def setupPost(self, keepGlyphNames=True, **values):
        postTable = self._initTableWithValues("post", _postDefaults, values)
        if self.isTTF and keepGlyphNames:
            postTable.formatType = 2.0
            postTable.extraNames = []
            postTable.mapping = {}
        else:
            postTable.formatType = 3.0

    def setupMaxp(self):
        if self.isTTF:
            defaults = _maxpDefaultsTTF
        else:
            defaults = _maxpDefaultsOTF
        self._initTableWithValues("maxp", defaults, {})

    def setupDSIG(self):
        from .ttLib.tables.D_S_I_G_ import SignatureRecord

        sig = SignatureRecord()
        sig.ulLength = 20
        sig.cbSignature = 12
        sig.usReserved2 = 0
        sig.usReserved1 = 0
        sig.pkcs7 = '\xd3M4\xd3M5\xd3M4\xd3M4'
        sig.ulFormat = 1
        sig.ulOffset = 20

        values = dict(
            ulVersion = 1,
            usFlag = 1,
            usNumSigs = 1,
            signatureRecords = [sig],
        )
        self._initTableWithValues("DSIG", {}, values)

    def addOpenTypeFeatures(self, features, filename=None, tables=None):
        from .feaLib.builder import addOpenTypeFeaturesFromString
        addOpenTypeFeaturesFromString(self.font, features, filename=filename, tables=tables)

def buildCmapSubTable(cmapping, format, platformID, platEncID):
    subTable = cmap_classes[format](format)
    subTable.cmap = cmapping
    subTable.platformID = platformID
    subTable.platEncID = platEncID
    subTable.language = 0
    return subTable

def addFvar(font, axes, instances):
    from .misc.py23 import Tag, tounicode
    from .ttLib.tables._f_v_a_r import Axis, NamedInstance

    assert axes

    fvar = newTable('fvar')
    nameTable = font['name']

    for tag, minValue, defaultValue, maxValue, name in axes:
        axis = Axis()
        axis.axisTag = Tag(tag)
        axis.minValue, axis.defaultValue, axis.maxValue = minValue, defaultValue, maxValue
        axis.axisNameID = nameTable.addName(tounicode(name))
        fvar.axes.append(axis)

    for instance in instances:
        coordinates = instance['location']
        name = tounicode(instance['stylename'])
        psname = instance.get('postscriptfontname')

        inst = NamedInstance()
        inst.subfamilyNameID = nameTable.addName(name)
        if psname is not None:
            psname = tounicode(psname)
            inst.postscriptNameID = nameTable.addName(psname)
        inst.coordinates = coordinates
        fvar.instances.append(inst)

    font['fvar'] = fvar
<commit_msg>allow localized name table entries<commit_after>from __future__ import print_function, division, absolute_import
from __future__ import unicode_literals

from .misc.py23 import *
from .ttLib import TTFont, newTable
from .ttLib.tables._c_m_a_p import cmap_classes
from .ttLib.tables._n_a_m_e import NameRecord, makeName
from .misc.timeTools import timestampNow
import struct

_headDefaults = dict(
    tableVersion = 1.0,
    fontRevision = 1.0,
    checkSumAdjustment = 0,
    magicNumber = 0x5F0F3CF5,
    flags = 0x0003,
    unitsPerEm = 1000,
    created = 0,
    modified = 0,
    xMin = 0,
    yMin = 0,
    xMax = 0,
    yMax = 0,
    macStyle = 0,
    lowestRecPPEM = 3,
    fontDirectionHint = 2,
    indexToLocFormat = 0,
    glyphDataFormat = 0,
)

_maxpDefaultsTTF = dict(
    tableVersion = 0x00010000,
    numGlyphs = 0,
    maxPoints = 0,
    maxContours = 0,
    maxCompositePoints = 0,
    maxCompositeContours = 0,
    maxZones = 2,
    maxTwilightPoints = 0,
    maxStorage = 0,
    maxFunctionDefs = 0,
    maxInstructionDefs = 0,
    maxStackElements = 0,
    maxSizeOfInstructions = 0,
    maxComponentElements = 0,
    maxComponentDepth = 0,
)
_maxpDefaultsOTF = dict(
    tableVersion = 0x00005000,
    numGlyphs = 0,
)

_postDefaults = dict(
    formatType = 3.0,
    italicAngle = 0,
    underlinePosition = 0,
    underlineThickness = 0,
    isFixedPitch = 0,
    minMemType42 = 0,
    maxMemType42 = 0,
    minMemType1 = 0,
    maxMemType1 = 0,
)

_hheaDefaults = dict(
    tableVersion = 0x00010000,
    ascent = 0,
    descent = 0,
    lineGap = 0,
    advanceWidthMax = 0,
    minLeftSideBearing = 0,
    minRightSideBearing = 0,
    xMaxExtent = 0,
    caretSlopeRise = 1,
    caretSlopeRun = 0,
    caretOffset = 0,
    reserved0 = 0,
    reserved1 = 0,
    reserved2 = 0,
    reserved3 = 0,
    metricDataFormat = 0,
    numberOfHMetrics = 0,
)

_vheaDefaults = dict(
    tableVersion = 0x00010000,
    ascent = 0,
    descent = 0,
    lineGap = 0,
    advanceHeightMax = 0,
    minTopSideBearing = 0,
    minBottomSideBearing = 0,
    yMaxExtent = 0,
    caretSlopeRise = 0,
    caretSlopeRun = 0,
    reserved0 = 0,
    reserved1 = 0,
    reserved2 = 0,
    reserved3 = 0,
    reserved4 = 0,
    metricDataFormat = 0,
    numberOfVMetrics = 0,
)

_nameIDs = dict(
             copyright = 0,
            familyName = 1,
             styleName = 2,
            identifier = 3,
              fullName = 4,
               version = 5,
                psName = 6,
             trademark = 7,
          manufacturer = 8,
     typographicFamily = 16,
  typographicSubfamily = 17,
# XXX this needs to be extended with legal things, etc.
)

_panoseDefaults = dict(
    bFamilyType = 0,
    bSerifStyle = 0,
    bWeight = 0,
    bProportion = 0,
    bContrast = 0,
    bStrokeVariation = 0,
    bArmStyle = 0,
    bLetterForm = 0,
    bMidline = 0,
    bXHeight = 0,
)

_OS2Defaults = dict(
    version = 3,
    xAvgCharWidth = 0,
    usWeightClass = 400,
    usWidthClass = 5,
    fsType = 0x0004,  # default: Preview & Print embedding
    ySubscriptXSize = 0,
    ySubscriptYSize = 0,
    ySubscriptXOffset = 0,
    ySubscriptYOffset = 0,
    ySuperscriptXSize = 0,
    ySuperscriptYSize = 0,
    ySuperscriptXOffset = 0,
    ySuperscriptYOffset = 0,
    yStrikeoutSize = 0,
    yStrikeoutPosition = 0,
    sFamilyClass = 0,
    panose = _panoseDefaults,
    ulUnicodeRange1 = 0,
    ulUnicodeRange2 = 0,
    ulUnicodeRange3 = 0,
    ulUnicodeRange4 = 0,
    achVendID = "????",
    fsSelection = 0,
    usFirstCharIndex = 0,
    usLastCharIndex = 0,
    sTypoAscender = 0,
    sTypoDescender = 0,
    sTypoLineGap = 0,
    usWinAscent = 0,
    usWinDescent = 0,
    ulCodePageRange1 = 0,
    ulCodePageRange2 = 0,
    sxHeight = 0,
    sCapHeight = 0,
    usDefaultChar = 0,  # .notdef
    usBreakChar = 32,   # space
    usMaxContext = 2,   # just kerning
    usLowerOpticalPointSize = 0,
    usUpperOpticalPointSize = 0,
)

class FontBuilder(object):

    def __init__(self, unitsPerEm=None, font=None, isTTF=True):
        if font is None:
            self.font = TTFont(recalcTimestamp=False)
            self.isTTF = isTTF
            now = timestampNow()
            assert unitsPerEm is not None
            self.setupHead(unitsPerEm=unitsPerEm, created=now, modified=now)
            self.setupMaxp()
        else:
            assert unitsPerEm is None
            self.font = font
            self.isTTF = "glyf" in font

    def save(self, path):
        self.font.save(path)

    def _initTableWithValues(self, tableTag, defaults, values):
        table = self.font[tableTag] = newTable(tableTag)
        for k, v in defaults.items():
            setattr(table, k, v)
        for k, v in values.items():
            setattr(table, k, v)
        return table

    def _updateTableWithValues(self, tableTag, values):
        table = self.font[tableTag]
        for k, v in values.items():
            setattr(table, k, v)

    def setupHead(self, **values):
        self._initTableWithValues("head", _headDefaults, values)

    def updateHead(self, **values):
        self._updateTableWithValues("head", values)

    def setupGlyphOrder(self, glyphOrder):
        self.font.setGlyphOrder(glyphOrder)

    def setupCharacterMap(self, cmapping, allowFallback=False):
        subTables = []
        highestUnicode = max(cmapping)
        if highestUnicode > 0xffff:
            cmapping_3_1 = dict((k, v) for k, v in cmapping.items() if k < 0x10000)
            subTable_3_10 = buildCmapSubTable(cmapping, 12, 3, 10)
            subTables.append(subTable_3_10)
        else:
            cmapping_3_1 = cmapping
        format = 4
        subTable_3_1 = buildCmapSubTable(cmapping_3_1, format, 3, 1)
        try:
            subTable_3_1.compile(self.font)
        except struct.error:
            # format 4 overflowed, fall back to format 12
            if not allowFallback:
                raise ValueError("cmap format 4 subtable overflowed; sort glyph order by unicode to fix.")
            format = 12
            subTable_3_1 = buildCmapSubTable(cmapping_3_1, format, 3, 1)
        subTables.append(subTable_3_1)
        subTable_0_3 = buildCmapSubTable(cmapping_3_1, format, 0, 3)
        subTables.append(subTable_0_3)

        self.font["cmap"] = newTable("cmap")
        self.font["cmap"].tableVersion = 0
        self.font["cmap"].tables = subTables

    def setupNameTable(self, nameStrings):
        nameTable = self.font["name"] = newTable("name")
        nameTable.names = []

        for nameName, nameValue in nameStrings.items():
            if isinstance(nameName, int):
                nameID = nameName
            else:
                nameID = _nameIDs[nameName]
            if not isinstance(nameValue, basestring):
                nameValue, language = nameValue
            else:
                language = "en"
            nameTable.addMultilingualName({language: nameValue}, ttFont=self.font, nameID=nameID)

    def setupOS2(self, **values):
        if "xAvgCharWidth" not in values:
            gs = self.font.getGlyphSet()
            widths = [gs[glyphName].width for glyphName in gs.keys() if gs[glyphName].width > 0]
            values["xAvgCharWidth"] = int(round(sum(widths) / float(len(widths))))
        self._initTableWithValues("OS/2", _OS2Defaults, values)
        if not ("ulUnicodeRange1" in values or "ulUnicodeRange2" in values or
                "ulUnicodeRange3" in values or "ulUnicodeRange3" in values):
            assert "cmap" in self.font
            self.font["OS/2"].recalcUnicodeRanges(self.font)

    def setupCFF(self, psName, fontInfo, charStringsDict, privateDict):
        assert not self.isTTF
        from .cffLib import CFFFontSet, TopDictIndex, TopDict, CharStrings, \
                GlobalSubrsIndex, PrivateDict
        self.font.sfntVersion = "OTTO"
        fontSet = CFFFontSet()
        fontSet.major = 1
        fontSet.minor = 0
        fontSet.fontNames = [psName]
        fontSet.topDictIndex = TopDictIndex()

        globalSubrs = GlobalSubrsIndex()
        fontSet.GlobalSubrs = globalSubrs
        private = PrivateDict()
        for key, value in privateDict.items():
            setattr(private, key, value)
        fdSelect = None
        fdArray = None

        topDict = TopDict()
        topDict.charset = self.font.getGlyphOrder()
        topDict.Private = private
        for key, value in fontInfo.items():
            setattr(topDict, key, value)

        charStrings = CharStrings(None, topDict.charset, globalSubrs, private, fdSelect, fdArray)
        for glypnName, charString in charStringsDict.items():
            charString.private = private
            charString.globalSubrs = globalSubrs
            charStrings[glypnName] = charString
        topDict.CharStrings = charStrings

        fontSet.topDictIndex.append(topDict)

        self.font["CFF "] = newTable("CFF ")
        self.font["CFF "].cff = fontSet

    def setupGlyf(self, glyphs):
        assert self.isTTF
        self.font["loca"] = newTable("loca")
        self.font["glyf"] = newTable("glyf")
        self.font["glyf"].glyphs = glyphs
        if hasattr(self.font, "glyphOrder"):
            self.font["glyf"].glyphOrder = self.font.glyphOrder

    def setupFvar(self, axes, instances):
        addFvar(self.font, axes, instances)

    def setupGvar(self, variations):
        gvar = self.font["gvar"] = newTable('gvar')
        gvar.version = 1
        gvar.reserved = 0
        gvar.variations = variations

    def calcGlyphBounds(self):
        glyphTable = self.font["glyf"]
        for glyph in glyphTable.glyphs.values():
            glyph.recalcBounds(glyphTable)

    def setupMetrics(self, tableTag, metrics):
        assert tableTag in ("hmtx", "vmtx")
        mtxTable = self.font[tableTag] = newTable(tableTag)
        roundedMetrics = {}
        for gn in metrics:
            w, lsb = metrics[gn]
            roundedMetrics[gn] = int(round(w)), int(round(lsb))
        mtxTable.metrics = roundedMetrics

    def setupHorizontalHeader(self, **values):
        self._initTableWithValues("hhea", _hheaDefaults, values)

    def setupVerticalHeader(self, **values):
        self._initTableWithValues("vhea", _vheaDefaults, values)

    def setupVerticalOrigins(self, verticalOrigins, defaultVerticalOrigin=None):
        if defaultVerticalOrigin is None:
            # find the most frequent vorg value
            bag = {}
            for gn in verticalOrigins:
                vorg = verticalOrigins[gn]
                if vorg not in bag:
                    bag[vorg] = 1
                else:
                    bag[vorg] += 1
            defaultVerticalOrigin = sorted(bag, key=lambda vorg: bag[vorg], reverse=True)[0]
        self._initTableWithValues("VORG", {}, dict(VOriginRecords={}, defaultVertOriginY=defaultVerticalOrigin))
        vorgTable = self.font["VORG"]
        vorgTable.majorVersion = 1
        vorgTable.minorVersion = 0
        for gn in verticalOrigins:
            vorgTable[gn] = verticalOrigins[gn]

    def setupPost(self, keepGlyphNames=True, **values):
        postTable = self._initTableWithValues("post", _postDefaults, values)
        if self.isTTF and keepGlyphNames:
            postTable.formatType = 2.0
            postTable.extraNames = []
            postTable.mapping = {}
        else:
            postTable.formatType = 3.0

    def setupMaxp(self):
        if self.isTTF:
            defaults = _maxpDefaultsTTF
        else:
            defaults = _maxpDefaultsOTF
        self._initTableWithValues("maxp", defaults, {})

    def setupDSIG(self):
        from .ttLib.tables.D_S_I_G_ import SignatureRecord

        sig = SignatureRecord()
        sig.ulLength = 20
        sig.cbSignature = 12
        sig.usReserved2 = 0
        sig.usReserved1 = 0
        sig.pkcs7 = '\xd3M4\xd3M5\xd3M4\xd3M4'
        sig.ulFormat = 1
        sig.ulOffset = 20

        values = dict(
            ulVersion = 1,
            usFlag = 1,
            usNumSigs = 1,
            signatureRecords = [sig],
        )
        self._initTableWithValues("DSIG", {}, values)

    def addOpenTypeFeatures(self, features, filename=None, tables=None):
        from .feaLib.builder import addOpenTypeFeaturesFromString
        addOpenTypeFeaturesFromString(self.font, features, filename=filename, tables=tables)

def buildCmapSubTable(cmapping, format, platformID, platEncID):
    subTable = cmap_classes[format](format)
    subTable.cmap = cmapping
    subTable.platformID = platformID
    subTable.platEncID = platEncID
    subTable.language = 0
    return subTable

def addFvar(font, axes, instances):
    from .misc.py23 import Tag, tounicode
    from .ttLib.tables._f_v_a_r import Axis, NamedInstance

    assert axes

    fvar = newTable('fvar')
    nameTable = font['name']

    for tag, minValue, defaultValue, maxValue, name in axes:
        axis = Axis()
        axis.axisTag = Tag(tag)
        axis.minValue, axis.defaultValue, axis.maxValue = minValue, defaultValue, maxValue
        axis.axisNameID = nameTable.addName(tounicode(name))
        fvar.axes.append(axis)

    for instance in instances:
        coordinates = instance['location']
        name = tounicode(instance['stylename'])
        psname = instance.get('postscriptfontname')

        inst = NamedInstance()
        inst.subfamilyNameID = nameTable.addName(name)
        if psname is not None:
            psname = tounicode(psname)
            inst.postscriptNameID = nameTable.addName(psname)
        inst.coordinates = coordinates
        fvar.instances.append(inst)

    font['fvar'] = fvar
<|endoftext|>


<commit_before># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.website.website_generator import WebsiteGenerator
from frappe.website.render import clear_cache
from frappe.templates.pages.blog import get_context

class BlogCategory(WebsiteGenerator):
	def autoname(self):
		# to override autoname of WebsiteGenerator
		self.name = self.category_name

	def on_update(self):
		WebsiteGenerator.on_update(self)
		clear_cache()

	def validate(self):
		self.parent_website_route = "blog"
		super(BlogCategory, self).validate()
<commit_msg>[hotfix] blog category<commit_after># Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
from frappe.website.website_generator import WebsiteGenerator
from frappe.website.render import clear_cache

class BlogCategory(WebsiteGenerator):
	def autoname(self):
		# to override autoname of WebsiteGenerator
		self.name = self.category_name

	def on_update(self):
		WebsiteGenerator.on_update(self)
		clear_cache()

	def validate(self):
		self.parent_website_route = "blog"
		super(BlogCategory, self).validate()
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Worker status api handler for reporting SDK harness debug info."""

from __future__ import absolute_import
from __future__ import division

import queue
import sys
import threading
import traceback
from collections import defaultdict

import grpc

from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.portability.api import beam_fn_api_pb2_grpc
from apache_beam.runners.worker.channel_factory import GRPCChannelFactory
from apache_beam.runners.worker.worker_id_interceptor import WorkerIdInterceptor

def thread_dump():
  """Get a thread dump for the current SDK worker harness. """
  # deduplicate threads with same stack trace
  stack_traces = defaultdict(list)
  frames = sys._current_frames()  # pylint: disable=protected-access

  for t in threading.enumerate():
    stack_trace = ''.join(traceback.format_stack(frames[t.ident]))
    thread_ident_name = (t.ident, t.name)
    stack_traces[stack_trace].append(thread_ident_name)

  all_traces = ['=' * 10 + 'THREAD DUMP' + '=' * 10]
  for stack, identity in stack_traces.items():
    ident, name = identity[0]
    trace = '--- Thread #%s name: %s %s---\n' % (
        ident,
        name,
        'and other %d threads' %
        (len(identity) - 1) if len(identity) > 1 else '')
    if len(identity) > 1:
      trace += 'threads: %s\n' % identity
    trace += stack
    all_traces.append(trace)
  all_traces.append('=' * 30)
  return '\n'.join(all_traces)

def _active_processing_bundles_state(bundle_process_cache):
  """Gather information about the currently in-processing active bundles.

  The result only keeps the longest lasting 10 bundles to avoid excessive
  spamming.
  """
  active_bundles = ['=' * 10 + 'ACTIVE PROCESSING BUNDLES' + '=' * 10]
  if not bundle_process_cache.active_bundle_processors:
    active_bundles.append("No active processing bundles.")
  else:
    cache = []
    for instruction in list(
        bundle_process_cache.active_bundle_processors.keys()):
      processor = bundle_process_cache.lookup(instruction)
      if processor:
        info = processor.state_sampler.get_info()
        cache.append((
            instruction,
            processor.process_bundle_descriptor.id,
            info.tracked_thread,
            info.time_since_transition))
    # reverse sort active bundle by time since last transition, keep top 10.
    cache.sort(key=lambda x: x[-1], reverse=True)
    for s in cache[:10]:
      state = '--- instruction %s ---\n' % s[0]
      state += 'ProcessBundleDescriptorId: %s\n' % s[1]
      state += "tracked thread: %s\n" % s[2]
      state += "time since transition: %.2f seconds\n" % (s[3] / 1e9)
      active_bundles.append(state)

  active_bundles.append('=' * 30)
  return '\n'.join(active_bundles)

DONE = object()

class FnApiWorkerStatusHandler(object):
  """FnApiWorkerStatusHandler handles worker status request from Runner. """
  def __init__(self, status_address, bundle_process_cache=None):
    """Initialize FnApiWorkerStatusHandler.

    Args:
      status_address: The URL Runner uses to host the WorkerStatus server.
      bundle_process_cache: The BundleProcessor cache dict from sdk worker.
    """
    self._alive = True
    self._bundle_process_cache = bundle_process_cache
    ch = GRPCChannelFactory.insecure_channel(status_address)
    grpc.channel_ready_future(ch).result(timeout=60)
    self._status_channel = grpc.intercept_channel(ch, WorkerIdInterceptor())
    self._status_stub = beam_fn_api_pb2_grpc.BeamFnWorkerStatusStub(
        self._status_channel)
    self._responses = queue.Queue()
    self._server = threading.Thread(
        target=lambda: self._serve(), name='fn_api_status_handler')
    self._server.daemon = True
    self._server.start()

  def _get_responses(self):
    while True:
      response = self._responses.get()
      if response is DONE:
        self._alive = False
        return
      yield response

  def _serve(self):
    while self._alive:
      for request in self._status_stub.WorkerStatus(self._get_responses()):
        try:
          self._responses.put(
              beam_fn_api_pb2.WorkerStatusResponse(
                  id=request.id, status_info=self.generate_status_response()))
        except Exception:
          traceback_string = traceback.format_exc()
          self._responses.put(
              beam_fn_api_pb2.WorkerStatusResponse(
                  id=request.id,
                  error="Exception encountered while generating "
                  "status page: %s" % traceback_string))

  def generate_status_response(self):
    all_status_sections = [
        _active_processing_bundles_state(self._bundle_process_cache)
    ] if self._bundle_process_cache else []
    all_status_sections.append(thread_dump())
    return '\n'.join(all_status_sections)

  def close(self):
    self._responses.put(DONE, timeout=5)
<commit_msg>Fix race exception in python worker status thread dump<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Worker status api handler for reporting SDK harness debug info."""

from __future__ import absolute_import
from __future__ import division

import queue
import sys
import threading
import traceback
from collections import defaultdict

import grpc

from apache_beam.portability.api import beam_fn_api_pb2
from apache_beam.portability.api import beam_fn_api_pb2_grpc
from apache_beam.runners.worker.channel_factory import GRPCChannelFactory
from apache_beam.runners.worker.worker_id_interceptor import WorkerIdInterceptor

def thread_dump():
  """Get a thread dump for the current SDK worker harness. """
  # deduplicate threads with same stack trace
  stack_traces = defaultdict(list)
  frames = sys._current_frames()  # pylint: disable=protected-access

  for t in threading.enumerate():
    try:
      stack_trace = ''.join(traceback.format_stack(frames[t.ident]))
    except KeyError:
      # the thread may have been destroyed already while enumerating, in such
      # case, skip to next thread.
      continue
    thread_ident_name = (t.ident, t.name)
    stack_traces[stack_trace].append(thread_ident_name)

  all_traces = ['=' * 10 + 'THREAD DUMP' + '=' * 10]
  for stack, identity in stack_traces.items():
    ident, name = identity[0]
    trace = '--- Thread #%s name: %s %s---\n' % (
        ident,
        name,
        'and other %d threads' %
        (len(identity) - 1) if len(identity) > 1 else '')
    if len(identity) > 1:
      trace += 'threads: %s\n' % identity
    trace += stack
    all_traces.append(trace)
  all_traces.append('=' * 30)
  return '\n'.join(all_traces)

def _active_processing_bundles_state(bundle_process_cache):
  """Gather information about the currently in-processing active bundles.

  The result only keeps the longest lasting 10 bundles to avoid excessive
  spamming.
  """
  active_bundles = ['=' * 10 + 'ACTIVE PROCESSING BUNDLES' + '=' * 10]
  if not bundle_process_cache.active_bundle_processors:
    active_bundles.append("No active processing bundles.")
  else:
    cache = []
    for instruction in list(
        bundle_process_cache.active_bundle_processors.keys()):
      processor = bundle_process_cache.lookup(instruction)
      if processor:
        info = processor.state_sampler.get_info()
        cache.append((
            instruction,
            processor.process_bundle_descriptor.id,
            info.tracked_thread,
            info.time_since_transition))
    # reverse sort active bundle by time since last transition, keep top 10.
    cache.sort(key=lambda x: x[-1], reverse=True)
    for s in cache[:10]:
      state = '--- instruction %s ---\n' % s[0]
      state += 'ProcessBundleDescriptorId: %s\n' % s[1]
      state += "tracked thread: %s\n" % s[2]
      state += "time since transition: %.2f seconds\n" % (s[3] / 1e9)
      active_bundles.append(state)

  active_bundles.append('=' * 30)
  return '\n'.join(active_bundles)

DONE = object()

class FnApiWorkerStatusHandler(object):
  """FnApiWorkerStatusHandler handles worker status request from Runner. """
  def __init__(self, status_address, bundle_process_cache=None):
    """Initialize FnApiWorkerStatusHandler.

    Args:
      status_address: The URL Runner uses to host the WorkerStatus server.
      bundle_process_cache: The BundleProcessor cache dict from sdk worker.
    """
    self._alive = True
    self._bundle_process_cache = bundle_process_cache
    ch = GRPCChannelFactory.insecure_channel(status_address)
    grpc.channel_ready_future(ch).result(timeout=60)
    self._status_channel = grpc.intercept_channel(ch, WorkerIdInterceptor())
    self._status_stub = beam_fn_api_pb2_grpc.BeamFnWorkerStatusStub(
        self._status_channel)
    self._responses = queue.Queue()
    self._server = threading.Thread(
        target=lambda: self._serve(), name='fn_api_status_handler')
    self._server.daemon = True
    self._server.start()

  def _get_responses(self):
    while True:
      response = self._responses.get()
      if response is DONE:
        self._alive = False
        return
      yield response

  def _serve(self):
    while self._alive:
      for request in self._status_stub.WorkerStatus(self._get_responses()):
        try:
          self._responses.put(
              beam_fn_api_pb2.WorkerStatusResponse(
                  id=request.id, status_info=self.generate_status_response()))
        except Exception:
          traceback_string = traceback.format_exc()
          self._responses.put(
              beam_fn_api_pb2.WorkerStatusResponse(
                  id=request.id,
                  error="Exception encountered while generating "
                  "status page: %s" % traceback_string))

  def generate_status_response(self):
    all_status_sections = [
        _active_processing_bundles_state(self._bundle_process_cache)
    ] if self._bundle_process_cache else []
    all_status_sections.append(thread_dump())
    return '\n'.join(all_status_sections)

  def close(self):
    self._responses.put(DONE, timeout=5)
<|endoftext|>