<commit_before>#!/usr/bin/env python

"""
Alarm script
============

Author:  Laszlo Szathmary, 2011 (jabba.laci@gmail.com)
Website: https://ubuntuincident.wordpress.com/2011/04/17/alarm-script/
GitHub:  https://github.com/jabbalaci/Bash-Utils

A simple alarm script that plays a list of MP3s at a given time.
Very useful if you leave your computer switched on during the night.

Usage:
------

./alarm.py -p
    Play music. First do this to adjust volume! If the volume is low,
    you won't hear it in the morning.

./alarm.py -t 7h15
    Set alarm time. The format is HhM, where H is the hour (24-hour system),
    M is the minute, 'h' is the separator.

./alarm.py
    Set alarm with the default time. In my case it's 6h55.
"""

import os
import sys
import glob

from optparse import OptionParser
from datetime import datetime
from time import sleep

MUSIC_DIR = '/home/jabba/bin/alarm/at_a_given_time/assets/rock'
MEDIA_PLAYER = '/usr/bin/mplayer'
DEFAULT_TIME = '6h55'

def play_music():
    for f in sorted(glob.glob(os.path.join(MUSIC_DIR, '*.mp3'))):
        val = os.system("{0} \"{1}\"".format(MEDIA_PLAYER, f))
        if val == 2:    # interrupted with CTRL-C
            sys.exit(val)

def set_alarm(hour, minute):
    # autoflush
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    sep = "=" * 19
    print sep
    print "| Alarm at {0:2}h{1:02}. |".format(hour, minute)
    print sep

    alarm_time = hour * 100 + minute

    while True:
        now = datetime.now()
        time = datetime.time(now)
        current_time = time.hour * 100 + time.minute
        if (current_time >= alarm_time) and (current_time - alarm_time <= 100):
            play_music()
            sys.exit(0)
        else:
            sys.stdout.write('.')
            try:
                sleep(10)
            except KeyboardInterrupt:
                print
                break   # break out of 'while True'

def check_alarm(alarm_time):
    msg = "{0} error: there is a problem with the alarm time.".format(sys.argv[0])
    try:
        alarm_time = alarm_time.lower()
        if 'h' not in alarm_time:
            alarm_time += 'h'
        hour, minute = alarm_time.split('h')
        if not minute:
            minute = '0'
        hour = int(hour)
        minute = int(minute)
        if not ( (0 <= hour <= 23) and (0 <= minute <= 59) ):
            print >>sys.stderr, msg
            sys.exit(1)
    except ValueError:
        print >>sys.stderr, msg
        sys.exit(1)

    return hour, minute

def main(default=DEFAULT_TIME):
    parser = OptionParser(usage='%prog [options]')

    #[options]
    parser.add_option('-t',
                      '--time',
                      action='store',
                      default=default,
                      type='string',
                      dest='alarm_time',
                      help='Alarm time, ex.: 6h55.')

    parser.add_option('-p',
                      '--play',
                      action='store_true',
                      default=False,
                      dest='is_play',
                      help = 'Play music. Useful for adjusting the volume.')

    options, arguments = parser.parse_args()
    #print options
    #print arguments
    if options.is_play:
        play_music()    # play and
        sys.exit(0)     # quit
    # else
    if options.alarm_time:
        hour, minute = check_alarm(options.alarm_time)
        set_alarm(hour, minute)

if __name__ == "__main__":
    main()
<commit_msg>alarm.py can collect songs recursively<commit_after>#!/usr/bin/env python

"""
Alarm script
============

Author:  Laszlo Szathmary, 2011 (jabba.laci@gmail.com)
Website: https://ubuntuincident.wordpress.com/2011/04/17/alarm-script/
GitHub:  https://github.com/jabbalaci/Bash-Utils

A simple alarm script that plays a list of MP3s at a given time.
Very useful if you leave your computer switched on during the night.

Usage:
------

./alarm.py -p
    Play music. First do this to adjust volume! If the volume is low,
    you won't hear it in the morning.

./alarm.py -t 7h15
    Set alarm time. The format is HhM, where H is the hour (24-hour system),
    M is the minute, 'h' is the separator.

./alarm.py
    Set alarm with the default time. In my case it's 6h55.
"""

import os
import sys
import glob

from optparse import OptionParser
from datetime import datetime
from time import sleep
from random import shuffle

MUSIC_DIR = '/media/XMAS/mp3.alarm'
TRAVERSE_RECURSIVELY = True
MPLAYER = '/usr/bin/mplayer'
MPLAYER_OPTIONS = '-endpos 00:00:60'    # play first 60 seconds; disabled when -p is used
DEFAULT_TIME = '6h55'

class CollectMp3:
    """Collect music files recursively in a given directory."""
    def __init__(self, music_dir):
        self.music_dir = music_dir
        self.songs = []

    def traverse(self, directory):
        """Traverse directory recursively. Symlinks are skipped."""
        content = [os.path.join(directory, x) for x in os.listdir(directory)]
        dirs = sorted([x for x in content if os.path.isdir(x)])
        files = sorted([x for x in content if os.path.isfile(x)])

        for f in files:
            if os.path.islink(f):
                continue
            ext = os.path.splitext(f)[1]
            if ext in ('.mp3', '.flac', '.ogg', '.flv'):
                self.songs.append(f)

        if TRAVERSE_RECURSIVELY:
            for d in dirs:
                if os.path.islink(d):
                    continue
                self.traverse(d)

    def collect(self):
        """Collect songs, shuffle order, and print a little statistics."""
        self.traverse(self.music_dir)
        if self.get_number_of_songs() == 0:
            print "Error: there are no songs available."
            sys.exit(-1)
        # else
        shuffle(self.songs)
        header = "number of songs: {0}".format(self.get_number_of_songs())
        sep = '#' * (len(header) + 2 + 2)
        print sep
        print '# ' + header + ' #'
        print sep
        print

    def get_number_of_songs(self):
        return len(self.songs)

    def get_songs(self):
        return self.songs

collector = CollectMp3(MUSIC_DIR)

#############################################################################

def play_music():
    songs = collector.get_songs()
    for f in songs:
        val = os.system("{mplayer} {options} \"{song}\"".format(mplayer=MPLAYER, options=MPLAYER_OPTIONS, song=f))
        if val == 2:    # interrupted with CTRL-C
            sys.exit(val)

def set_alarm(hour, minute):
    # autoflush
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    sep = "=" * 19
    print sep
    print "| Alarm at {0:2}h{1:02}. |".format(hour, minute)
    print sep

    alarm_time = hour * 100 + minute

    while True:
        now = datetime.now()
        time = datetime.time(now)
        current_time = time.hour * 100 + time.minute
        if (current_time >= alarm_time) and (current_time - alarm_time <= 100):
            play_music()
            sys.exit(0)
        else:
            sys.stdout.write('.')
            try:
                sleep(10)
            except KeyboardInterrupt:
                print
                break   # break out of 'while True'

def check_alarm(alarm_time):
    msg = "{0} error: there is a problem with the alarm time.".format(sys.argv[0])
    try:
        alarm_time = alarm_time.lower()
        if 'h' not in alarm_time:
            alarm_time += 'h'
        hour, minute = alarm_time.split('h')
        if not minute:
            minute = '0'
        hour = int(hour)
        minute = int(minute)
        if not ( (0 <= hour <= 23) and (0 <= minute <= 59) ):
            print >>sys.stderr, msg
            sys.exit(1)
    except ValueError:
        print >>sys.stderr, msg
        sys.exit(1)

    return hour, minute

def main(default=DEFAULT_TIME):
    parser = OptionParser(usage='%prog [options]')

    #[options]
    parser.add_option('-t',
                      '--time',
                      action='store',
                      default=default,
                      type='string',
                      dest='alarm_time',
                      help='Alarm time, ex.: 6h55.')

    parser.add_option('-p',
                      '--play',
                      action='store_true',
                      default=False,
                      dest='is_play',
                      help = 'Play music. Useful for adjusting the volume.')

    options, arguments = parser.parse_args()

    if options.is_play:
        global MPLAYER_OPTIONS
        MPLAYER_OPTIONS = ''
        print '# MPLAYER_OPTIONS is disabled'

    collector.collect()

    if options.is_play:
        play_music()    # play and
        sys.exit(0)     # quit
    # else
    if options.alarm_time:
        hour, minute = check_alarm(options.alarm_time)
        set_alarm(hour, minute)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from typing import Callable
from time import sleep
from math import ceil
from abc import ABC, abstractmethod
from threading import Lock, Timer, Thread

class RateLimiter(ABC):
    @property
    @abstractmethod
    def permits_issued(self) -> int:
        pass

    @abstractmethod
    def reset_permits_issued(self) -> None:
        pass

    @abstractmethod
    def __enter__(self) -> "RateLimiter":
        pass

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        pass

    def limit(self, method: Callable) -> Callable:
        def limited(*args, **kwargs):
            with self:
                return method(*args, **kwargs)
        return limited

class FixedWindowRateLimiter(RateLimiter):
    def __init__(self, window_seconds: int, window_permits: int) -> None:
        self._window_seconds = window_seconds
        self._window_permits = window_permits

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

        self._permitter = Lock()
        self._permits = window_permits
        self._permits_lock = Lock()

        self._currently_processing = 0
        self._currently_processing_lock = Lock()

        self._resetter = None
        self._resetter_lock = Lock()

    def __enter__(self) -> "FixedWindowRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._permits_lock:
            self._permits -= 1
            if self._permits > 0:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        # Increment current count
        with self._currently_processing_lock:
            self._currently_processing += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # Decrement current count
        with self._currently_processing_lock:
            self._currently_processing -= 1

        # The fixed window starts after processing the first permit ends. The presence of a resetter indicates whether we're in a fixed window already, or need to start one.
        with self._resetter_lock:
            if not self._resetter:
                self._resetter = Timer(self._window_seconds, self._reset)
                self._resetter.daemon = True
                self._resetter.start()

    def _reset(self):
        with self._resetter_lock:
            self._resetter = None

        with self._permits_lock:
            with self._currently_processing_lock:
                self._permits = self._window_permits - self._currently_processing
                try:
                    self._permitter.release()
                except RuntimeError:
                    # Wasn't waiting on any acquire
                    pass

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0

class TokenBucketRateLimiter(RateLimiter):
    def __init__(self, epoch_seconds: int, epoch_permits: int, max_burst: int, token_update_frequency: float) -> None:
        if max_burst < 1 or max_burst > epoch_permits:
            raise ValueError("Max burst must be >= 1 and <= epoch permits!")

        if token_update_frequency <= 0 or token_update_frequency > epoch_seconds:
            raise ValueError("Token update frequency must be > 0 and <= epoch seconds!")

        self._epoch_seconds = epoch_seconds
        self._epoch_permits = epoch_permits

        self._permitter = Lock()
        self._token_limit = max_burst
        self._tokens = max_burst
        self._tokens_lock = Lock()

        self._token_update = token_update_frequency

        self._token_provider = None
        self._token_provider_lock = Lock()

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

    def __enter__(self) -> "TokenBucketRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._tokens_lock:
            self._tokens -= 1
            if self._tokens > 0:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # We don't want a thread sitting around dumping tokens into a filled bucket during downtimes. We construct one when needed and it cleans itself up if the bucket is full for an entire epoch.
        with self._token_provider_lock:
            if not self._token_provider:
                self._token_provider = Thread(target=self._provide_tokens)
                self._token_provider.daemon = True
                self._token_provider.start()

    def _provide_tokens(self):
        tokens_per_segment = int(self._epoch_permits // (self._epoch_seconds / self._token_update))
        segments_full = 0

        # If we go an entire epoch with a full bucket we'll stop this provider
        segments_per_epoch = int(ceil(self._epoch_seconds / self._token_update))
        while segments_full < segments_per_epoch:
            sleep(self._token_update)

            with self._tokens_lock:
                if self._tokens == self._token_limit:
                    segments_full += 1
                elif self._tokens == 0:
                    self._tokens = min(tokens_per_segment, self._token_limit)
                    segments_full = 0
                    try:
                        self._permitter.release()
                    except RuntimeError:
                        # Wasn't waiting on any acquire
                        pass
                else:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0

        # Clean up
        with self._token_provider_lock:
            self._token_provider = None

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0
<commit_msg>allow partial tokens in token bucket limiter<commit_after>from typing import Callable
from time import sleep
from math import ceil
from abc import ABC, abstractmethod
from threading import Lock, Timer, Thread

class RateLimiter(ABC):
    @property
    @abstractmethod
    def permits_issued(self) -> int:
        pass

    @abstractmethod
    def reset_permits_issued(self) -> None:
        pass

    @abstractmethod
    def __enter__(self) -> "RateLimiter":
        pass

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        pass

    def limit(self, method: Callable) -> Callable:
        def limited(*args, **kwargs):
            with self:
                return method(*args, **kwargs)
        return limited

class FixedWindowRateLimiter(RateLimiter):
    def __init__(self, window_seconds: int, window_permits: int) -> None:
        self._window_seconds = window_seconds
        self._window_permits = window_permits

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

        self._permitter = Lock()
        self._permits = window_permits
        self._permits_lock = Lock()

        self._currently_processing = 0
        self._currently_processing_lock = Lock()

        self._resetter = None
        self._resetter_lock = Lock()

    def __enter__(self) -> "FixedWindowRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._permits_lock:
            self._permits -= 1
            if self._permits > 0:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        # Increment current count
        with self._currently_processing_lock:
            self._currently_processing += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # Decrement current count
        with self._currently_processing_lock:
            self._currently_processing -= 1

        # The fixed window starts after processing the first permit ends. The presence of a resetter indicates whether we're in a fixed window already, or need to start one.
        with self._resetter_lock:
            if not self._resetter:
                self._resetter = Timer(self._window_seconds, self._reset)
                self._resetter.daemon = True
                self._resetter.start()

    def _reset(self):
        with self._resetter_lock:
            self._resetter = None

        with self._permits_lock:
            with self._currently_processing_lock:
                self._permits = self._window_permits - self._currently_processing
                try:
                    self._permitter.release()
                except RuntimeError:
                    # Wasn't waiting on any acquire
                    pass

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0

class TokenBucketRateLimiter(RateLimiter):
    def __init__(self, epoch_seconds: int, epoch_permits: int, max_burst: int, token_update_frequency: float) -> None:
        if max_burst < 1 or max_burst > epoch_permits:
            raise ValueError("Max burst must be >= 1 and <= epoch permits!")

        if token_update_frequency <= 0 or token_update_frequency > epoch_seconds:
            raise ValueError("Token update frequency must be > 0 and <= epoch seconds!")

        self._epoch_seconds = epoch_seconds
        self._epoch_permits = epoch_permits

        self._permitter = Lock()
        self._token_limit = max_burst
        self._tokens = max_burst
        self._tokens_lock = Lock()

        self._token_update = token_update_frequency

        self._token_provider = None
        self._token_provider_lock = Lock()

        self._total_permits_issued = 0
        self._total_permits_issued_lock = Lock()

    def __enter__(self) -> "TokenBucketRateLimiter":
        # Grab the permit lock and decrement remaining permits. If this leaves it at 0, don't release the permit lock. It will be released by the resetter.
        self._permitter.acquire()
        with self._tokens_lock:
            self._tokens -= 1
            if self._tokens >= 1:
                self._permitter.release()

        # Increment total count
        with self._total_permits_issued_lock:
            self._total_permits_issued += 1

        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        # We don't want a thread sitting around dumping tokens into a filled bucket during downtimes. We construct one when needed and it cleans itself up if the bucket is full for an entire epoch.
        with self._token_provider_lock:
            if not self._token_provider:
                self._token_provider = Thread(target=self._provide_tokens)
                self._token_provider.daemon = True
                self._token_provider.start()

    def _provide_tokens(self):
        tokens_per_segment = self._epoch_permits / (self._epoch_seconds / self._token_update)
        segments_full = 0

        # If we go an entire epoch with a full bucket we'll stop this provider
        segments_per_epoch = int(ceil(self._epoch_seconds / self._token_update))
        while segments_full < segments_per_epoch:
            sleep(self._token_update)

            with self._tokens_lock:
                if self._tokens == self._token_limit:
                    segments_full += 1
                elif self._tokens == 0:
                    self._tokens = min(tokens_per_segment, self._token_limit)
                    segments_full = 0
                    try:
                        self._permitter.release()
                    except RuntimeError:
                        # Wasn't waiting on any acquire
                        pass
                else:
                    self._tokens = min(self._tokens + tokens_per_segment, self._token_limit)
                    segments_full = 0

        # Clean up
        with self._token_provider_lock:
            self._token_provider = None

    @property
    def permits_issued(self) -> int:
        with self._total_permits_issued_lock:
            return self._total_permits_issued

    def reset_permits_issued(self) -> None:
        with self._total_permits_issued_lock:
            self._total_permits_issued = 0
<|endoftext|>


<commit_before>import operator
import typing as tp
import abc
import functools

from metafunctions.core._decorators import binary_operation
from metafunctions.core._decorators import inject_call_state
from metafunctions.core._call_state import CallState
from metafunctions.operators import concat

class MetaFunction(metaclass=abc.ABCMeta):

    # Metafunctions will pass call state to any function with this attribute set to true
    _receives_call_state = True
    _function_join_str = ''

    @abc.abstractmethod
    def __init__(self, *args, **kwargs):
        '''A MetaFunction is a function that contains other functions. When executed, it calls the
        functions it contains.
        '''
        self._functions = []

    @abc.abstractmethod
    def __call__(self, *args, call_state=None, **kwargs):
        '''Call the functions contained in this MetaFunction'''

    def __str__(self):
        return f'({f" {self._function_join_str} ".join(str(f) for f in self.functions)})'

    @property
    def functions(self):
        return self._functions

    @staticmethod
    def make_meta(function):
        '''Wrap the given function in a metafunction, unless it's already a metafunction'''
        if not isinstance(function, MetaFunction):
            return SimpleFunction(function)
        return function

    @staticmethod
    def defer_value(value):
        '''Wrap the given value in a DeferredValue object, (which returns its value when called).'''
        return DeferredValue(value)

    @staticmethod
    def new_call_state():
        return CallState()

    ### Operator overloads ###
    @binary_operation
    def __or__(self, other):
        return FunctionChain.combine(self, other)

    @binary_operation
    def __ror__(self, other):
        return FunctionChain.combine(other, self)

    @binary_operation
    def __and__(self, other):
        return FunctionMerge.combine(concat, self, other)

    @binary_operation
    def __rand__(self, other):
        return FunctionMerge.combine(concat, other, self)

    @binary_operation
    def __add__(self, other):
        return FunctionMerge(operator.add, (self, other))

    @binary_operation
    def __radd__(self, other):
        return FunctionMerge(operator.add, (other, self))

    @binary_operation
    def __sub__(self, other):
        return FunctionMerge(operator.sub, (self, other))

    @binary_operation
    def __rsub__(self, other):
        return FunctionMerge(operator.sub, (other, self))

    @binary_operation
    def __mul__(self, other):
        return FunctionMerge(operator.mul, (self, other))

    @binary_operation
    def __rmul__(self, other):
        return FunctionMerge(operator.mul, (other, self))

    @binary_operation
    def __truediv__(self, other):
        return FunctionMerge(operator.truediv, (self, other))

    @binary_operation
    def __rtruediv__(self, other):
        return FunctionMerge(operator.truediv, (other, self))

    @binary_operation
    def __matmul__(self, other):
        asdf

    @binary_operation
    def __rmatmul__(self, other):
        asdf

class FunctionChain(MetaFunction):
    _function_join_str = '|'
    def __init__(self, functions:tuple):
        '''A FunctionChain is a metafunction that calls its functions in sequence, passing the
        results of the first function subsequent functions.
        '''
        super().__init__()
        self._functions = functions

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(result, **kwargs)
        return result

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions})'

    @classmethod
    def combine(cls, *funcs):
        '''Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain.
        '''
        new_funcs = []
        for f in funcs:
            if type(f) is cls:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(tuple(new_funcs))

class FunctionMerge(MetaFunction):
    _character_to_operator = {
        '+': operator.add,
        '-': operator.sub,
        '*': operator.mul,
        '/': operator.truediv,
        '&': concat,
    }
    _operator_to_character = {v: k for k, v in _character_to_operator.items()}

    def __init__(self, merge_func:tp.Callable, functions:tuple, function_join_str=''):
        '''A FunctionMerge merges its functions by executing all of them and passing their results to `merge_func`

        Args:
            function_join_str: If you're using a `merge_func` that is not one of the standard operator
            functions, use this argument to provide a custom character to use in string formatting. If not provided, we default to using str(merge_func).
        '''
        super().__init__()
        self._merge_func = merge_func
        self._functions = functions
        self._function_join_str = function_join_str or self._operator_to_character.get(
                merge_func, str(merge_func))

    @inject_call_state
    def __call__(self, *args, **kwargs):
        results = (f(*args, **kwargs) for f in self.functions)
        return self._merge_func(*results)

    def __repr__(self):
        return f'{self.__class__.__name__}({self._merge_func}, {self.functions})'

    @classmethod
    def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None):
        '''Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine
        them into a single FunctionMerge.

        NOTE: combine does not check to make sure the merge_func can accept the new number of
        arguments.
        '''
        new_funcs = []
        for f in funcs:
            if isinstance(f, cls) and f._merge_func == merge_func:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str)

class SimpleFunction(MetaFunction):
    def __init__(self, function, print_location_in_traceback=True):
        '''A MetaFunction-aware wrapper around a single function
        The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction.
        '''
        # An interesting side effect of wraps: it causes simplefunctions to collapse into each
        # other. Because calling wraps on a function copies all that function's attributes to the
        # new function, we copy _function, etc from the wrapped function. Essentially
        # absorbing it. I'm not sure if that's good or bad.
        functools.wraps(function)(self)

        super().__init__()
        self._function = function
        self.add_location_to_traceback = print_location_in_traceback

    @inject_call_state
    def __call__(self, *args, call_state, **kwargs):
        call_state._called_functions.append(self)
        if getattr(self._function, '_receives_call_state', False):
            kwargs['call_state'] = call_state
        try:
            return self._function(*args, **kwargs)
        except Exception as e:
            self._handle_exception(call_state, e)

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions[0]!r})'

    def __str__(self):
        try:
            return self.__name__
        except AttributeError:
            # We're usually wrapping a function, but it's possible we're wrapping another
            # metafunction
            return str(self.functions[0])

    @property
    def functions(self):
        return (self._function, )

    def _handle_exception(self, call_state, e):
        if self.add_location_to_traceback:
            from metafunctions.util import highlight_current_function
            detailed_message = str(e)
            if call_state:
                detailed_message = f"{str(e)} \n\nOccured in the following function: {highlight_current_function(call_state)}"
            raise type(e)(detailed_message).with_traceback(e.__traceback__)
        raise

class DeferredValue(SimpleFunction):
    def __init__(self, value):
        '''A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x.
        '''
        self._value = value
        self.__name__ = repr(value)

    def __call__(self, *args, **kwargs):
        return self._value

    def __repr__(self):
        return f'{self.__class__.__name__}({self._value!r})'

    @property
    def functions(self):
        return (self, )
<commit_msg>Initial BroadcastChain implementation<commit_after>import operator
import typing as tp
import abc
import functools

from metafunctions.core._decorators import binary_operation
from metafunctions.core._decorators import inject_call_state
from metafunctions.core._call_state import CallState
from metafunctions.operators import concat

class MetaFunction(metaclass=abc.ABCMeta):

    # Metafunctions will pass call state to any function with this attribute set to true
    _receives_call_state = True
    _function_join_str = ''

    @abc.abstractmethod
    def __init__(self, *args, **kwargs):
        '''A MetaFunction is a function that contains other functions. When executed, it calls the
        functions it contains.
        '''
        self._functions = []

    @abc.abstractmethod
    def __call__(self, *args, call_state=None, **kwargs):
        '''Call the functions contained in this MetaFunction'''

    def __str__(self):
        return f'({f" {self._function_join_str} ".join(str(f) for f in self.functions)})'

    @property
    def functions(self):
        return self._functions

    @staticmethod
    def make_meta(function):
        '''Wrap the given function in a metafunction, unless it's already a metafunction'''
        if not isinstance(function, MetaFunction):
            return SimpleFunction(function)
        return function

    @staticmethod
    def defer_value(value):
        '''Wrap the given value in a DeferredValue object, (which returns its value when called).'''
        return DeferredValue(value)

    @staticmethod
    def new_call_state():
        return CallState()

    ### Operator overloads ###
    @binary_operation
    def __or__(self, other):
        return FunctionChain.combine(self, other)

    @binary_operation
    def __ror__(self, other):
        return FunctionChain.combine(other, self)

    @binary_operation
    def __and__(self, other):
        return FunctionMerge.combine(concat, self, other)

    @binary_operation
    def __rand__(self, other):
        return FunctionMerge.combine(concat, other, self)

    @binary_operation
    def __add__(self, other):
        return FunctionMerge(operator.add, (self, other))

    @binary_operation
    def __radd__(self, other):
        return FunctionMerge(operator.add, (other, self))

    @binary_operation
    def __sub__(self, other):
        return FunctionMerge(operator.sub, (self, other))

    @binary_operation
    def __rsub__(self, other):
        return FunctionMerge(operator.sub, (other, self))

    @binary_operation
    def __mul__(self, other):
        return FunctionMerge(operator.mul, (self, other))

    @binary_operation
    def __rmul__(self, other):
        return FunctionMerge(operator.mul, (other, self))

    @binary_operation
    def __truediv__(self, other):
        return FunctionMerge(operator.truediv, (self, other))

    @binary_operation
    def __rtruediv__(self, other):
        return FunctionMerge(operator.truediv, (other, self))

    @binary_operation
    def __matmul__(self, other):
        return BroadcastChain.combine(self, other)

    @binary_operation
    def __rmatmul__(self, other):
        return BroadcastChain.combine(other, self)

class FunctionChain(MetaFunction):
    _function_join_str = '|'
    def __init__(self, functions:tuple):
        '''A FunctionChain is a metafunction that calls its functions in sequence, passing the
        results of the first function subsequent functions.
        '''
        super().__init__()
        self._functions = functions

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(result, **kwargs)
        return result

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions})'

    @classmethod
    def combine(cls, *funcs):
        '''Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain.
        '''
        new_funcs = []
        for f in funcs:
            if type(f) is cls:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(tuple(new_funcs))

class BroadcastChain(FunctionChain):
    _function_join_str = '@'

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(*result, **kwargs)
        return result

class FunctionMerge(MetaFunction):
    _character_to_operator = {
        '+': operator.add,
        '-': operator.sub,
        '*': operator.mul,
        '/': operator.truediv,
        '&': concat,
    }
    _operator_to_character = {v: k for k, v in _character_to_operator.items()}

    def __init__(self, merge_func:tp.Callable, functions:tuple, function_join_str=''):
        '''A FunctionMerge merges its functions by executing all of them and passing their results to `merge_func`

        Args:
            function_join_str: If you're using a `merge_func` that is not one of the standard operator
            functions, use this argument to provide a custom character to use in string formatting. If not provided, we default to using str(merge_func).
        '''
        super().__init__()
        self._merge_func = merge_func
        self._functions = functions
        self._function_join_str = function_join_str or self._operator_to_character.get(
                merge_func, str(merge_func))

    @inject_call_state
    def __call__(self, *args, **kwargs):
        results = (f(*args, **kwargs) for f in self.functions)
        return self._merge_func(*results)

    def __repr__(self):
        return f'{self.__class__.__name__}({self._merge_func}, {self.functions})'

    @classmethod
    def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None):
        '''Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine
        them into a single FunctionMerge.

        NOTE: combine does not check to make sure the merge_func can accept the new number of
        arguments.
        '''
        new_funcs = []
        for f in funcs:
            if isinstance(f, cls) and f._merge_func == merge_func:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str)

class SimpleFunction(MetaFunction):
    def __init__(self, function, print_location_in_traceback=True):
        '''A MetaFunction-aware wrapper around a single function
        The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction.
        '''
        # An interesting side effect of wraps: it causes simplefunctions to collapse into each
        # other. Because calling wraps on a function copies all that function's attributes to the
        # new function, we copy _function, etc from the wrapped function. Essentially
        # absorbing it. I'm not sure if that's good or bad.
        functools.wraps(function)(self)

        super().__init__()
        self._function = function
        self.add_location_to_traceback = print_location_in_traceback

    @inject_call_state
    def __call__(self, *args, call_state, **kwargs):
        call_state._called_functions.append(self)
        if getattr(self._function, '_receives_call_state', False):
            kwargs['call_state'] = call_state
        try:
            return self._function(*args, **kwargs)
        except Exception as e:
            self._handle_exception(call_state, e)

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions[0]!r})'

    def __str__(self):
        try:
            return self.__name__
        except AttributeError:
            # We're usually wrapping a function, but it's possible we're wrapping another
            # metafunction
            return str(self.functions[0])

    @property
    def functions(self):
        return (self._function, )

    def _handle_exception(self, call_state, e):
        if self.add_location_to_traceback:
            from metafunctions.util import highlight_current_function
            detailed_message = str(e)
            if call_state:
                detailed_message = f"{str(e)} \n\nOccured in the following function: {highlight_current_function(call_state)}"
            raise type(e)(detailed_message).with_traceback(e.__traceback__)
        raise

class DeferredValue(SimpleFunction):
    def __init__(self, value):
        '''A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x.
        '''
        self._value = value
        self.__name__ = repr(value)

    def __call__(self, *args, **kwargs):
        return self._value

    def __repr__(self):
        return f'{self.__class__.__name__}({self._value!r})'

    @property
    def functions(self):
        return (self, )
<|endoftext|>


<commit_before>#!/usr/bin/python

"""
Usage:
	update_translations.py -p <project-key>
"""

import urllib, sys, getopt, os, json, shutil
from zipfile import ZipFile

PROJECT_ID = "bemyeyes"
ENDPOINT = "https://api.crowdin.com/api/project/{projectID}/download/{package}.zip?key={projectKey}"
LANG_STATUS_ENDPOINT = "https://api.crowdin.com/api/project/{projectID}/status?key={projectKey}&jsonp"
FILENAME = "languages.zip"
EXTRACTION_DIR = "extracted_languages"
RENAME_RULES = {'zh-CN':'zh-Hant', 'zh-TW' : 'zh-Hans', 'ur-PK' : 'ur', 'es-ES' : 'es', 'sv-SE' : 'sv', 'pt-PT' : 'pt'}

def usage():
	print(__doc__)

def fetchLanguages(projectKey):
	try:
		languagesURL = ENDPOINT.format(projectID=PROJECT_ID, package="all", projectKey=projectKey)
		(filename, headers) = urllib.urlretrieve(languagesURL, FILENAME)
	except BaseException as e:
		return (None, e)
	else:
		return (filename, None)

def unzipFile(filename):
	try:
		file = ZipFile(filename)
		file.extractall(EXTRACTION_DIR)
	except zipfile.BadZipFile as e:
		print("Error while unzipping: {0}".format(e.strerror))
		sys.exist(2)
	except zipfile.LargeZipFile as e:
		print("Error while unzipping: {0}".format(e.strerror))
		sys.exist(2)
	return EXTRACTION_DIR

def renameAndRemoveDirectories(parentDir, renameRules, fullyTranslated):
	extractedDir = os.path.join(os.getcwd(), parentDir)
	dirsList = [name for name in os.listdir(extractedDir)
	            if os.path.isdir(os.path.join(extractedDir, name))]
	try:
		for dir in dirsList:
			if dir in fullyTranslated:
				dirName = dir + ".lproj"
				if dir in renameRules:
					dirName = renameRules[dir] + ".lproj"
				print("{0} -> {1}".format(dir, dirName))
				os.rename(os.path.join(extractedDir, dir), os.path.join(extractedDir, dirName))
			else:
				pass
				shutil.rmtree(os.path.join(extractedDir, dir))
	except OSError as error:
		print(error.strerror)

def fetchLanguageStatus(projectKey):
	fullyTranslated = []
	try:
		statusURL = LANG_STATUS_ENDPOINT.format(projectID=PROJECT_ID, projectKey=projectKey)
		f = urllib.urlopen(statusURL)
		r = f.read()[1:-2]
		response = json.loads(r)
		f.close()
	except BaseException as e:
		print(e)
	else:
		for status in response:
			progressString = status["translated_progress"]
			progress = int(progressString)
			if isAcceptable(progress):
				fullyTranslated.append(status["code"])
	return fullyTranslated

def isAcceptable(progress):
	if progress == 100:
		return True
	else:
		return False

if __name__ == "__main__":
	try:
		opts, args = getopt.getopt(sys.argv[1:], "p:")
		if len(opts) != 1:
			usage()
			sys.exit(2)
	except getopt.GetoptError:
		usage()
		sys.exit(2)
	else:
		projectKey = ''
		for opt, arg in opts:
			if opt == "-p":
				projectKey = arg
			else:
				usage()
				sys.exit(2)

		fullyTranslated = fetchLanguageStatus(projectKey)
		(filename, exception) = fetchLanguages(projectKey)

		if filename:
			dirName = unzipFile(filename)
			renameAndRemoveDirectories(dirName, RENAME_RULES, fullyTranslated)
		else:
			print(exception.strerror)
	<commit_msg>A new version of the CrowdIn script that puts all the AppStore.txt files in a separate folder.<commit_after>#!/usr/bin/python

"""
Usage:
	update_translations.py -p <project-key>
"""

import urllib, sys, getopt, os, json, shutil
from zipfile import ZipFile

PROJECT_ID = "bemyeyes"
ENDPOINT = "https://api.crowdin.com/api/project/{projectID}/download/{package}.zip?key={projectKey}"
LANG_STATUS_ENDPOINT = "https://api.crowdin.com/api/project/{projectID}/status?key={projectKey}&jsonp"
FILENAME = "languages.zip"
APP_STORE_DESCRIPTION = "AppStore.txt"
APP_STORE_DESCRIPTIONS_DIR = "app_store_descriptions"
EXTRACTION_DIR = "extracted_languages"
RENAME_RULES = {'zh-CN':'zh-Hant', 'zh-TW' : 'zh-Hans', 'ur-PK' : 'ur', 'es-ES' : 'es', 'sv-SE' : 'sv', 'pt-PT' : 'pt'}

def usage():
	print(__doc__)

def fetchLanguages(projectKey):
	"""
	This function fetches all translations in a zipped file
	and returns the name of the downloaded file or an error.
	"""
	try:
		languagesURL = ENDPOINT.format(projectID=PROJECT_ID, package="all", projectKey=projectKey)
		(filename, headers) = urllib.urlretrieve(languagesURL, FILENAME)
	except BaseException as e:
		return (None, e)
	else:
		return (filename, None)

def unzipFile(filename):
	"""
	Unzips the file containing translations.
	Returns name of the directory where files were extracted.
	"""
	try:
		file = ZipFile(filename)
		file.extractall(EXTRACTION_DIR)
	except zipfile.BadZipFile as e:
		print("Error while unzipping: {0}".format(e.strerror))
		sys.exist(2)
	except zipfile.LargeZipFile as e:
		print("Error while unzipping: {0}".format(e.strerror))
		sys.exist(2)
	return EXTRACTION_DIR

def renameAndRemoveDirectories(parentDir, renameRules, fullyTranslated):
	"""
	Renames folders by following the RENAME_RULES and adds the .lproj suffix.
	Removes folders for languages that have not been fully translated.
	"""
	extractedDir = os.path.join(os.getcwd(), parentDir)
	dirsList = [name for name in os.listdir(extractedDir)
	            if os.path.isdir(os.path.join(extractedDir, name))]
	try:
		for dir in dirsList:
			if dir in fullyTranslated:
				dirName = dir + ".lproj"
				if dir in renameRules:
					dirName = renameRules[dir] + ".lproj"
				print("{0} -> {1}".format(dir, dirName))
				os.rename(os.path.join(extractedDir, dir), os.path.join(extractedDir, dirName))
			else:
				shutil.rmtree(os.path.join(extractedDir, dir))
	except OSError as error:
		print(error.strerror)

def moveAppStoreDescriptions(dirName):
	"""
	Moves all AppStore.txt files into a separate directory.
	"""
	descriptionsDir = ''
	try:
		descriptionsDir = os.path.join(os.getcwd(), APP_STORE_DESCRIPTIONS_DIR)
		os.mkdir(descriptionsDir)

		extractedDir = os.path.join(os.getcwd(), dirName)
		dirsList = [name for name in os.listdir(extractedDir)
				            if os.path.isdir(os.path.join(extractedDir, name))]
		for dir in dirsList:
			fromPath = os.path.join(extractedDir, dir, APP_STORE_DESCRIPTION)
			toPath = os.path.join(descriptionsDir, dir, APP_STORE_DESCRIPTION)
			newDir = os.path.join(descriptionsDir, dir)
			os.mkdir(newDir)
			shutil.move(fromPath, toPath)

	except OSError as e:
		print(e.strerror)
		sys.exit(2)

def fetchLanguageStatus(projectKey):
	"""
	Fetches info about languages to determine which ones
	have been fully translated.
	"""
	fullyTranslated = []
	try:
		statusURL = LANG_STATUS_ENDPOINT.format(projectID=PROJECT_ID, projectKey=projectKey)
		f = urllib.urlopen(statusURL)
		r = f.read()[1:-2]
		response = json.loads(r)
		f.close()
	except BaseException as e:
		print(e)
	else:
		for status in response:
			progressString = status["translated_progress"]
			progress = int(progressString)
			if isAcceptable(progress):
				fullyTranslated.append(status["code"])
	return fullyTranslated

def isAcceptable(progress):
	"""
	Function evaluating whether a translation is acceptable based
	on its progress.
	"""
	if progress == 100:
		return True
	else:
		return False

if __name__ == "__main__":
	try:
		opts, args = getopt.getopt(sys.argv[1:], "p:")
		if len(opts) != 1:
			usage()
			sys.exit(2)
	except getopt.GetoptError:
		usage()
		sys.exit(2)
	else:
		projectKey = ''
		for opt, arg in opts:
			if opt == "-p":
				projectKey = arg
			else:
				usage()
				sys.exit(2)

		fullyTranslated = fetchLanguageStatus(projectKey)
		(filename, exception) = fetchLanguages(projectKey)

		if filename:
			dirName = unzipFile(filename)
			renameAndRemoveDirectories(dirName, RENAME_RULES, fullyTranslated)
			moveAppStoreDescriptions(dirName)
		else:
			print(exception.strerror)
	<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7

import os
import hashlib
import gzip
import time
import datetime
import mimetypes

from boto.s3.connection import S3Connection, OrdinaryCallingFormat
from boto.s3.key import Key
from main.config import (AWS_KEY, AWS_SECRET_KEY, AWS_BUCKET, AWS_DIRECTORY,
    HTML_EXPIRES, STATIC_EXPIRES, ABSOLUTE_PATH)

PUSH_FROM = ABSOLUTE_PATH + 'build'

def get_files(directory):
    """Gets full path of all files within directory, including subdirectories
    Returns a list of paths"""

    file_paths = []
    for root, dirs, files in os.walk(directory):
        for f in files:
            filepath = os.path.join(root, f)
            file_paths.append(filepath)

    return file_paths

def expires_header(duration):
    """Takes a number of seconds and turns it into an expires header"""

    return time.strftime("%a, %d-%b-%Y %T GMT",
        time.gmtime(time.time() + duration))

def gzip_file(filename):
    """Does what you think it does"""

    f_in = open(filename, 'rb')
    with gzip.open(filename + '.gz', 'w+') as f:
        f.writelines(f_in)
    f_in.close()
    f = filename + '.gz'
    return f

def set_metadata():
    """Take a list of files to be uploaded to s3 and gzip CSS, JS, and HTML,
    setting metadata for all files"""

    gzip_extensions = ['.html', '.js', '.css']

    upload_list = get_files(PUSH_FROM)
    conn = S3Connection(AWS_KEY, AWS_SECRET_KEY,calling_format=OrdinaryCallingFormat())
    mybucket = conn.get_bucket(AWS_BUCKET)

    static_expires = expires_header(STATIC_EXPIRES)
    html_expires = expires_header(HTML_EXPIRES)

    # define all necessary attributes of each file for s3
    for filename in upload_list:
        k = Key(mybucket)
        ext = os.path.splitext(filename)[1]
        if not ext:
            ext = '.html'
        web_path = filename.split('build/')[1]

        # Set expires for HTML, remove extension if not index
        if ext == '.html':
            if 'index' not in web_path:
                k.key = AWS_DIRECTORY + os.path.splitext(web_path)[0]
            else:
                k.key = AWS_DIRECTORY + web_path
            k.set_metadata('Expires', html_expires)
        else:
            k.key = AWS_DIRECTORY + web_path  # strip leading 0
            k.set_metadata('Expires', static_expires)

        if ext in gzip_extensions:
            f = gzip_file(filename)
            k.set_metadata('Content-Encoding', 'gzip')
        else:
            f = filename

        k.set_metadata('Content-Type', 'text/xml')
        etag_hash = hashlib.sha1(f + str(time.time())).hexdigest()
        k.set_metadata('ETag', etag_hash)
        k.set_contents_from_filename(f)
        k.make_public()

    print '\nPage successfully updated'
    print "On " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
<commit_msg>Update upload_s3.py to set appropriate Content-Type headers<commit_after>#!/usr/local/bin/python2.7

import os
import hashlib
import gzip
import time
import datetime
import mimetypes

from boto.s3.connection import S3Connection, OrdinaryCallingFormat
from boto.s3.key import Key
from main.config import (AWS_KEY, AWS_SECRET_KEY, AWS_BUCKET, AWS_DIRECTORY,
    HTML_EXPIRES, STATIC_EXPIRES, ABSOLUTE_PATH)

PUSH_FROM = ABSOLUTE_PATH + 'build'

def get_files(directory):
    """Gets full path of all files within directory, including subdirectories
    Returns a list of paths"""

    file_paths = []
    for root, dirs, files in os.walk(directory):
        for f in files:
            filepath = os.path.join(root, f)
            file_paths.append(filepath)

    return file_paths

def expires_header(duration):
    """Takes a number of seconds and turns it into an expires header"""

    return time.strftime("%a, %d-%b-%Y %T GMT",
        time.gmtime(time.time() + duration))

def gzip_file(filename):
    """Does what you think it does"""

    f_in = open(filename, 'rb')
    with gzip.open(filename + '.gz', 'w+') as f:
        f.writelines(f_in)
    f_in.close()
    f = filename + '.gz'
    return f

def set_metadata():
    """Take a list of files to be uploaded to s3 and gzip CSS, JS, and HTML,
    setting metadata for all files"""

    gzip_extensions = ['.html', '.js', '.css']

    upload_list = get_files(PUSH_FROM)
    conn = S3Connection(AWS_KEY, AWS_SECRET_KEY,calling_format=OrdinaryCallingFormat())
    mybucket = conn.get_bucket(AWS_BUCKET)

    static_expires = expires_header(STATIC_EXPIRES)
    html_expires = expires_header(HTML_EXPIRES)

    # define all necessary attributes of each file for s3
    for filename in upload_list:
        k = Key(mybucket)
        ext = os.path.splitext(filename)[1]
        if not ext:
            ext = '.html'
    	if filename.split('/')[-1]=='index.html':
    		content_type = 'text/xml'
        else:
        	content_type = mimetypes.types_map[ext]
        web_path = filename.split('build/')[1]

        # Set expires for HTML, remove extension if not index
        if ext == '.html':
            if 'index' not in web_path:
                k.key = AWS_DIRECTORY + os.path.splitext(web_path)[0]
            else:
                k.key = AWS_DIRECTORY + web_path
            k.set_metadata('Expires', html_expires)
        else:
            k.key = AWS_DIRECTORY + web_path  # strip leading 0
            k.set_metadata('Expires', static_expires)

        if ext in gzip_extensions:
            f = gzip_file(filename)
            k.set_metadata('Content-Encoding', 'gzip')
        else:
            f = filename

        k.set_metadata('Content-Type', content_type)
        etag_hash = hashlib.sha1(f + str(time.time())).hexdigest()
        k.set_metadata('ETag', etag_hash)
        k.set_contents_from_filename(f)
        k.make_public()

    print '\nPage successfully updated'
    print "On " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from data_validation import consts

class ConfigManager(object):

    _config: dict = None
    source_client = None
    target_client = None

    def __init__(self, config, source_client, target_client, verbose=False):
        """ Initialize a ValidationBuilder client which supplies the
            source and target queries tto run.

        Args:
            config (Dict): The Validation config supplied
            source_client (IbisClient): The Ibis client for the source DB
            target_client (IbisClient): The Ibis client for the target DB
            verbose (Bool): If verbose, the Data Validation client will print queries run
        """
        self._config = config

        self.source_client = source_client
        self.target_client = target_client

        self.verbose = verbose

    @property
    def config(self):
        """Return config object."""
        return self._config

    @property
    def validation_type(self):
        """Return string validation type (Column|GroupedColumn)."""
        return self._config[consts.CONFIG_TYPE]

    @property
    def aggregates(self):
        """Return Aggregates from Config """
        return self._config.get(consts.CONFIG_AGGREGATES, [])

    def append_aggregates(self, aggregate_configs):
        """Append aggregate configs to existing config."""
        self._config[consts.CONFIG_AGGREGATES] = (
            self.aggregates + aggregate_configs
        )

    @property
    def query_groups(self):
        """ Return Query Groups from Config """
        return self._config.get(consts.CONFIG_GROUPED_COLUMNS, [])

    def append_query_groups(self, grouped_column_configs):
        """Append aggregate configs to existing config."""
        self._config[consts.CONFIG_GROUPED_COLUMNS] = (
            self.query_groups + grouped_column_configs
        )

    @property
    def source_schema(self):
        """Return string value of source schema."""
        return self._config[consts.CONFIG_SCHEMA_NAME]

    @property
    def source_table(self):
        """Return string value of source table."""
        return self._config[consts.CONFIG_TABLE_NAME]

    @property
    def target_schema(self):
        """Return string value of target schema."""
        return self._config.get(
            consts.CONFIG_TARGET_SCHEMA_NAME,
            self._config[consts.CONFIG_SCHEMA_NAME]
        )

    @property
    def target_table(self):
        """Return string value of target table."""
        return self._config.get(
            consts.CONFIG_TARGET_TABLE_NAME,
            self._config[consts.CONFIG_TABLE_NAME]
        )

    @property
    def query_limit(self):
        """Return int limit for query executions."""
        return self._config.get(consts.CONFIG_LIMIT)

    def get_source_ibis_table(self):
        """Return IbisTable from source."""
        if not hasattr(self, "_source_ibis_table"):
            self._source_ibis_table = self.source_client.table(
                self.source_table, database=self.source_schema
            )

        return self._source_ibis_table

    def get_target_ibis_table(self):
        """Return IbisTable from target."""
        if not hasattr(self, "_target_ibis_table"):
            self._target_ibis_table = self.target_client.table(
                self.target_table, database=self.target_schema
            )

        return self._target_ibis_table

    @staticmethod
    def build_config_manager(
        config_type,
        source_conn,
        target_conn,
        source_client,
        target_client,
        table_obj,
        verbose=False,
    ):
        """Return a ConfigManager instance with available config."""
        config = {
            consts.CONFIG_TYPE: config_type,
            consts.CONFIG_SOURCE_CONN: source_conn,
            consts.CONFIG_TARGET_CONN: target_conn,
            consts.CONFIG_SCHEMA_NAME: table_obj[consts.CONFIG_SCHEMA_NAME],
            consts.CONFIG_TABLE_NAME: table_obj[consts.CONFIG_TABLE_NAME],
            consts.CONFIG_TARGET_SCHEMA_NAME: table_obj.get(
                consts.CONFIG_TARGET_SCHEMA_NAME
            )
            or table_obj[consts.CONFIG_SCHEMA_NAME],
            consts.CONFIG_TARGET_TABLE_NAME: table_obj.get(
                consts.CONFIG_TARGET_TABLE_NAME
            )
            or table_obj[consts.CONFIG_TABLE_NAME],
        }

        return ConfigManager(config, source_client, target_client, verbose=verbose)

    def build_config_grouped_columns(self, grouped_columns):
        """Return list of grouped column config objects."""
        grouped_column_configs = []
        source_table = self.get_source_ibis_table()
        for column in grouped_columns:
            if column not in source_table.columns:
                raise ValueError(
                    f"GroupedColumn DNE: {source_table.op().name}.{column}"
                )
            column_config = {
                consts.CONFIG_SOURCE_COLUMN: column,
                consts.CONFIG_TARGET_COLUMN: column,
                consts.CONFIG_FIELD_ALIAS: column,
                consts.CONFIG_CAST: None,
            }
            grouped_column_configs.append(column_config)

        return grouped_column_configs

    def build_config_aggregates(self, agg_type, arg_value, supported_types):
        """Return list of aggregate objects of given agg_type."""
        aggregate_configs = []
        source_table = self.get_source_ibis_table()
        target_table = self.get_target_ibis_table()
        whitelist_columns = arg_value or source_table.columns
        for column in source_table.columns:
            if column not in whitelist_columns:
                continue
            elif column not in target_table.columns:
                logging.info(
                    f"Skipping Agg {agg_type}: {source_table.op().name}.{column}"
                )
                continue
            elif (
                supported_types
                and str(source_table[column].type()) not in supported_types
            ):
                continue

            aggregate_config = {
                consts.CONFIG_SOURCE_COLUMN: column,
                consts.CONFIG_TARGET_COLUMN: column,
                consts.CONFIG_FIELD_ALIAS: f"{agg_type}__{column}",
                consts.CONFIG_TYPE: agg_type,
            }
            aggregate_configs.append(aggregate_config)

        return aggregate_configs
<commit_msg>rename to allow list<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from data_validation import consts

class ConfigManager(object):

    _config: dict = None
    source_client = None
    target_client = None

    def __init__(self, config, source_client, target_client, verbose=False):
        """ Initialize a ValidationBuilder client which supplies the
            source and target queries tto run.

        Args:
            config (Dict): The Validation config supplied
            source_client (IbisClient): The Ibis client for the source DB
            target_client (IbisClient): The Ibis client for the target DB
            verbose (Bool): If verbose, the Data Validation client will print queries run
        """
        self._config = config

        self.source_client = source_client
        self.target_client = target_client

        self.verbose = verbose

    @property
    def config(self):
        """Return config object."""
        return self._config

    @property
    def validation_type(self):
        """Return string validation type (Column|GroupedColumn)."""
        return self._config[consts.CONFIG_TYPE]

    @property
    def aggregates(self):
        """Return Aggregates from Config """
        return self._config.get(consts.CONFIG_AGGREGATES, [])

    def append_aggregates(self, aggregate_configs):
        """Append aggregate configs to existing config."""
        self._config[consts.CONFIG_AGGREGATES] = (
            self.aggregates + aggregate_configs
        )

    @property
    def query_groups(self):
        """ Return Query Groups from Config """
        return self._config.get(consts.CONFIG_GROUPED_COLUMNS, [])

    def append_query_groups(self, grouped_column_configs):
        """Append aggregate configs to existing config."""
        self._config[consts.CONFIG_GROUPED_COLUMNS] = (
            self.query_groups + grouped_column_configs
        )

    @property
    def source_schema(self):
        """Return string value of source schema."""
        return self._config[consts.CONFIG_SCHEMA_NAME]

    @property
    def source_table(self):
        """Return string value of source table."""
        return self._config[consts.CONFIG_TABLE_NAME]

    @property
    def target_schema(self):
        """Return string value of target schema."""
        return self._config.get(
            consts.CONFIG_TARGET_SCHEMA_NAME,
            self._config[consts.CONFIG_SCHEMA_NAME]
        )

    @property
    def target_table(self):
        """Return string value of target table."""
        return self._config.get(
            consts.CONFIG_TARGET_TABLE_NAME,
            self._config[consts.CONFIG_TABLE_NAME]
        )

    @property
    def query_limit(self):
        """Return int limit for query executions."""
        return self._config.get(consts.CONFIG_LIMIT)

    def get_source_ibis_table(self):
        """Return IbisTable from source."""
        if not hasattr(self, "_source_ibis_table"):
            self._source_ibis_table = self.source_client.table(
                self.source_table, database=self.source_schema
            )

        return self._source_ibis_table

    def get_target_ibis_table(self):
        """Return IbisTable from target."""
        if not hasattr(self, "_target_ibis_table"):
            self._target_ibis_table = self.target_client.table(
                self.target_table, database=self.target_schema
            )

        return self._target_ibis_table

    @staticmethod
    def build_config_manager(
        config_type,
        source_conn,
        target_conn,
        source_client,
        target_client,
        table_obj,
        verbose=False,
    ):
        """Return a ConfigManager instance with available config."""
        config = {
            consts.CONFIG_TYPE: config_type,
            consts.CONFIG_SOURCE_CONN: source_conn,
            consts.CONFIG_TARGET_CONN: target_conn,
            consts.CONFIG_SCHEMA_NAME: table_obj[consts.CONFIG_SCHEMA_NAME],
            consts.CONFIG_TABLE_NAME: table_obj[consts.CONFIG_TABLE_NAME],
            consts.CONFIG_TARGET_SCHEMA_NAME: table_obj.get(
                consts.CONFIG_TARGET_SCHEMA_NAME
            )
            or table_obj[consts.CONFIG_SCHEMA_NAME],
            consts.CONFIG_TARGET_TABLE_NAME: table_obj.get(
                consts.CONFIG_TARGET_TABLE_NAME
            )
            or table_obj[consts.CONFIG_TABLE_NAME],
        }

        return ConfigManager(config, source_client, target_client, verbose=verbose)

    def build_config_grouped_columns(self, grouped_columns):
        """Return list of grouped column config objects."""
        grouped_column_configs = []
        source_table = self.get_source_ibis_table()
        for column in grouped_columns:
            if column not in source_table.columns:
                raise ValueError(
                    f"GroupedColumn DNE: {source_table.op().name}.{column}"
                )
            column_config = {
                consts.CONFIG_SOURCE_COLUMN: column,
                consts.CONFIG_TARGET_COLUMN: column,
                consts.CONFIG_FIELD_ALIAS: column,
                consts.CONFIG_CAST: None,
            }
            grouped_column_configs.append(column_config)

        return grouped_column_configs

    def build_config_aggregates(self, agg_type, arg_value, supported_types):
        """Return list of aggregate objects of given agg_type."""
        aggregate_configs = []
        source_table = self.get_source_ibis_table()
        target_table = self.get_target_ibis_table()
        allowlist_columns = arg_value or source_table.columns
        for column in source_table.columns:
            if column not in allowlist_columns:
                continue
            elif column not in target_table.columns:
                logging.info(
                    f"Skipping Agg {agg_type}: {source_table.op().name}.{column}"
                )
                continue
            elif (
                supported_types
                and str(source_table[column].type()) not in supported_types
            ):
                continue

            aggregate_config = {
                consts.CONFIG_SOURCE_COLUMN: column,
                consts.CONFIG_TARGET_COLUMN: column,
                consts.CONFIG_FIELD_ALIAS: f"{agg_type}__{column}",
                consts.CONFIG_TYPE: agg_type,
            }
            aggregate_configs.append(aggregate_config)

        return aggregate_configs
<|endoftext|>


<commit_before>import json
import flask
import sys
import traceback
from flask import Flask, request, make_response, session, g, redirect, url_for, \
     abort, render_template, flash
from dataactcore.utils.responseException import ResponseException

class JsonResponse :
    """ Used to create an http response object containing JSON """
    debugMode = True
    printDebug = False # Can cause errors when printing trace on ec2 if set to True
    logDebug = True

    @staticmethod
    def create(code,dictionaryData):
        """
        Creates a JSON response object
        if debugMode is enabled errors are added
        """
        jsondata  =  flask.Response()
        jsondata.headers["Content-Type"] = "application/json"
        jsondata.status_code = code
        jsondata.set_data(json.dumps(dictionaryData))
        return jsondata

    @staticmethod
    def error(exception, errorCode, extraDict = {}):
        """ Create an http response object for specified error

        Args:
            exception: Exception to be represented by response object
            errorCode: Status code to be used in response
            extraDict: Extra fields and values to be included in response

        Returns:
            Http response object containing specified error
        """
        responseDict = {}
        for key in extraDict:
            responseDict[key] = extraDict[key]
        if(JsonResponse.debugMode):
            responseDict["message"] = str(exception)
            responseDict["errorType"] = str(type(exception))
            if(type(exception)==type(ResponseException("")) and exception.wrappedException != None):
                responseDict["wrappedType"] = str(type(exception.wrappedException))
                responseDict["wrappedMessage"] = str(exception.wrappedException)
            exc_type, exc_obj, exc_tb = sys.exc_info()
            trace = traceback.extract_tb(exc_tb, 10)
            responseDict["trace"] = trace
            if(JsonResponse.printDebug):
                print(str(type(exception)))
                print(str(exception))
                print(str(trace))
            if(JsonResponse.logDebug):
                open("responseErrorLog","a").write(str(type(exception)))
                open("responseErrorLog","a").write(str(exception))
                open("responseErrorLog","a").write(str(trace))
            del exc_tb
            return JsonResponse.create(errorCode, responseDict)
        else:
            responseDict["message"] = "An error has occurred"
            return JsonResponse.create(errorCode, responseDict)
<commit_msg>Made error log more readable<commit_after>import json
import flask
import sys
import traceback
from flask import Flask, request, make_response, session, g, redirect, url_for, \
     abort, render_template, flash
from dataactcore.utils.responseException import ResponseException

class JsonResponse :
    """ Used to create an http response object containing JSON """
    debugMode = True
    printDebug = False # Can cause errors when printing trace on ec2 if set to True
    logDebug = True

    @staticmethod
    def create(code,dictionaryData):
        """
        Creates a JSON response object
        if debugMode is enabled errors are added
        """
        jsondata  =  flask.Response()
        jsondata.headers["Content-Type"] = "application/json"
        jsondata.status_code = code
        jsondata.set_data(json.dumps(dictionaryData))
        return jsondata

    @staticmethod
    def error(exception, errorCode, extraDict = {}):
        """ Create an http response object for specified error

        Args:
            exception: Exception to be represented by response object
            errorCode: Status code to be used in response
            extraDict: Extra fields and values to be included in response

        Returns:
            Http response object containing specified error
        """
        responseDict = {}
        for key in extraDict:
            responseDict[key] = extraDict[key]
        if(JsonResponse.debugMode):
            responseDict["message"] = str(exception)
            responseDict["errorType"] = str(type(exception))
            if(type(exception)==type(ResponseException("")) and exception.wrappedException != None):
                responseDict["wrappedType"] = str(type(exception.wrappedException))
                responseDict["wrappedMessage"] = str(exception.wrappedException)
            exc_type, exc_obj, exc_tb = sys.exc_info()
            trace = traceback.extract_tb(exc_tb, 10)
            responseDict["trace"] = trace
            if(JsonResponse.printDebug):
                print(str(type(exception)))
                print(str(exception))
                print(str(trace))
            if(JsonResponse.logDebug):
                open("responseErrorLog","a").write(str(type(exception)) + ": ")
                open("responseErrorLog","a").write(str(exception) + "\n")
                open("responseErrorLog","a").write(str(trace) + "\n")
            del exc_tb
            return JsonResponse.create(errorCode, responseDict)
        else:
            responseDict["message"] = "An error has occurred"
            return JsonResponse.create(errorCode, responseDict)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Extract i19 messages from HTML, write gettext pot file and include cache
"""

import sys
from HTMLParser import HTMLParser
import time
from cPickle import dump
import re

# from pygettext
POT_HEADER = '''\
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR ORGANIZATION
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\\n"
"POT-Creation-Date: %s\\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"
"Language-Team: LANGUAGE <LL@li.org>\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=utf-8\\n"
"Content-Transfer-Encoding: ENCODING\\n"
"Generated-By: i19extract.py 0.1\\n"

'''

POT_SINGULAR = '''
#. %s
#. Default: %s
#: %s
msgid "%s"
msgstr ""
'''

POT_PLURAL = '''
#. %s
#. Default: %s
#: %s
msgid "%s"
msgid_plural "%s"
msgstr ""
'''

# List of void HTML elements (ie without end tag)
VOID_TAGS = (
    # HTML4
    # http://www.w3.org/TR/html4/index/elements.html
    'area', 'base', 'basefont', 'br', 'col', 'frame', 'hr', 'img',
    'input', 'isindex', 'link', 'meta', 'param',
    # New in HTML5
    # http://www.w3.org/TR/html-markup/syntax.html#void-element
    'command', 'embed', 'keygen', 'source', 'track', 'wbr',
)

def fmttag(tag, attr, exclude=()):
    """
    Unparse HTML element ``tag`` with attributes ``attr`` into a string
    Skip attributes in ``exclude``
    """
    return "<%s>" % (" ".join([tag] +
        ['%s="%s"' % (k, v) for k, v in attr if not k in exclude]),)

def sanitize(i19id):
    """
    Filter i18n IDs to me alphanumeric characters and - or _ only
    """
    e19id = re.sub(r'\\.', '', i19id)
    return "".join(c for c in e19id if c.isalnum() or c in "-_")

class i19Parser(HTMLParser):
    """
    Parse HTML and extract i19, i19-attr, and i19-name strings
    """
    def __init__(self, filename):
        HTMLParser.__init__(self)
        # stack of i18n IDs
        self._i19 = []
        # tag nesting level counter
        self._nest = 0
        # id, nesting level, and raw html of current include directive
        self._include = ['', 0, '']
        # collect {i18n ID: translation strings}
        self.strs, self.includes = dict(), dict()

        self._fn = filename
        with file(filename) as inf:
            self.feed(inf.read())

    def handle_starttag(self, tag, attrs):
        if not tag in VOID_TAGS:
            self._nest += 1
        attrdict = dict(attrs)

        # handle i19-name attribute for nested translations
        if 'i19-name' in attrdict:
            i19id = "${" + attrdict['i19-name'] + "}"
            self._i19[-1][1] += i19id
            self._include = [i19id, self._nest, '']

        # record raw html for i19 string if closest parent is not i19-name
        if self._i19 and self._i19[-1][2] >= self._include[1]:
            self._i19[-1][1] += fmttag(tag, attrs)

        # record raw html for i19-name directive
        if self._include[1]:
            self._include[2] += fmttag(tag, attrs, ('i19-name',))

        # handle i19 tag/attribute for translating the inner HTML
        if tag == 'i19':
            self._i19.append(['', '', self._nest])
        elif 'i19' in attrdict:
            self._i19.append([attrdict.get('i19') or '', '', self._nest])

        # handle i19-attr attribute for translating attributes
        trans_attr = 'i19-attr' in attrdict and \
                attrdict.get('i19-attr').split(',') or []
        for attribute in trans_attr:
            spec = attribute.strip().split(' ')
            value = attrdict[spec[0]]
            if len(spec) == 1: # no translation id given
                spec.append(value)
            self.strs[spec[1]] = ("%s:%d" % (self._fn, self.lineno,), value,
                    '<%s %s>' % (tag, spec[0]))

    def handle_endtag(self, tag):
        if not tag in VOID_TAGS:
            self._nest -= 1

        if not self._i19:
            return

        # store raw html
        if self._include[1]:
            self._include[2] += "</" + tag + ">"

        # i19 directive not closed yet
        if self._i19[-1][2] <= self._nest:
            self._i19[-1][1] += "</" + tag + ">"
            return

        i19id, data, _ = self._i19.pop()

        if self._i19 and self._include[0]:
            # documentation for nested i19 tags
            doc = "Referenced in %r as %s" % \
                    (self._i19[-1][0], self._include[0],)
        else:
            doc = ''

        # end of include directive
        if self._include[1] >= self._nest:
            inid, _, raw = self._include
            self.includes[inid] = raw
            self._include = ['', 0, '']

        data = data.strip().replace('\n', '\\n')
        self.strs[(i19id or sanitize(data))] = \
                ("%s:%d" % (self._fn, self.lineno,), data, doc)

    def handle_data(self, data):
        if not self._i19:
            return
        self._i19[-1][1] += data
        if self._include[1]:
            self._include[2] += data

def main():
    assert len(sys.argv) > 3, \
            "Usage: i19extract POT_FILE CACHE_FILE [SOURCES..]"

    strs, inc = dict(), dict()
    for src_file in sys.argv[3:]:
        parser = i19Parser(src_file)
        strs.update(parser.strs.items())
        inc.update(parser.includes.items())

    with file(sys.argv[1], 'w') as pot:
        pot.write(POT_HEADER % (time.strftime('%Y-%m-%d %H:%M+0000'),))
        for i19id, dt in strs.items():
            if i19id.endswith(')'):
                pot.write(POT_PLURAL % (dt[2], dt[1], dt[0], i19id, i19id,))
            else:
                pot.write(POT_SINGULAR % (dt[2], dt[1], dt[0], i19id,))

    with file(sys.argv[2], 'wb') as i19n:
        dump((inc, strs,), i19n)

if __name__ == '__main__':
    main()
<commit_msg>i19extract: allow data- prefix on attributes.<commit_after>#!/usr/bin/env python

"""
Extract i19 messages from HTML, write gettext pot file and include cache
"""

import sys
from HTMLParser import HTMLParser
import time
from cPickle import dump
import re

# from pygettext
POT_HEADER = '''\
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR ORGANIZATION
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\\n"
"POT-Creation-Date: %s\\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"
"Language-Team: LANGUAGE <LL@li.org>\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=utf-8\\n"
"Content-Transfer-Encoding: ENCODING\\n"
"Generated-By: i19extract.py 0.1\\n"

'''

POT_SINGULAR = '''
#. %s
#. Default: %s
#: %s
msgid "%s"
msgstr ""
'''

POT_PLURAL = '''
#. %s
#. Default: %s
#: %s
msgid "%s"
msgid_plural "%s"
msgstr ""
'''

# List of void HTML elements (ie without end tag)
VOID_TAGS = (
    # HTML4
    # http://www.w3.org/TR/html4/index/elements.html
    'area', 'base', 'basefont', 'br', 'col', 'frame', 'hr', 'img',
    'input', 'isindex', 'link', 'meta', 'param',
    # New in HTML5
    # http://www.w3.org/TR/html-markup/syntax.html#void-element
    'command', 'embed', 'keygen', 'source', 'track', 'wbr',
)

def fmttag(tag, attr, exclude=()):
    """
    Unparse HTML element ``tag`` with attributes ``attr`` into a string
    Skip attributes in ``exclude``
    """
    return "<%s>" % (" ".join([tag] +
        ['%s="%s"' % (k, v) for k, v in attr if not k in exclude]),)

def sanitize(i19id):
    """
    Filter i18n IDs to me alphanumeric characters and - or _ only
    """
    e19id = re.sub(r'\\.', '', i19id)
    return "".join(c for c in e19id if c.isalnum() or c in "-_")

class i19Parser(HTMLParser):
    """
    Parse HTML and extract i19, i19-attr, and i19-name strings
    """
    def __init__(self, filename):
        HTMLParser.__init__(self)
        # stack of i18n IDs
        self._i19 = []
        # tag nesting level counter
        self._nest = 0
        # id, nesting level, and raw html of current include directive
        self._include = ['', 0, '']
        # collect {i18n ID: translation strings}
        self.strs, self.includes = dict(), dict()

        self._fn = filename
        with file(filename) as inf:
            self.feed(inf.read())

    def handle_starttag(self, tag, attrs):
        if not tag in VOID_TAGS:
            self._nest += 1
        attrdict = dict(attrs)

        # handle i19-name attribute for nested translations
        if 'i19-name' in attrdict:
            i19id = "${" + attrdict['i19-name'] + "}"
            self._i19[-1][1] += i19id
            self._include = [i19id, self._nest, '']

        # record raw html for i19 string if closest parent is not i19-name
        if self._i19 and self._i19[-1][2] >= self._include[1]:
            self._i19[-1][1] += fmttag(tag, attrs)

        # record raw html for i19-name directive
        if self._include[1]:
            self._include[2] += fmttag(tag, attrs, ('i19-name',))

        # handle i19 tag/attribute for translating the inner HTML
        if tag == 'i19':
            self._i19.append(['', '', self._nest])
        elif 'i19' in attrdict:
            self._i19.append([attrdict.get('i19') or '', '', self._nest])
        elif 'data-i19' in attrdict:
            self._i19.append([attrdict.get('data-i19') or '', '', self._nest])

        # handle i19-attr attribute for translating attributes
        trans_attr = []
        if 'i19-attr' in attrdict:
            trans_attr = attrdict.get('i19-attr').split(',')
        elif 'data-i19attr' in attrdict:
            trans_attr = attrdict.get('data-i19attr').split(',')

        for attribute in trans_attr:
            spec = attribute.strip().split(' ')
            value = attrdict[spec[0]]
            if len(spec) == 1: # no translation id given
                spec.append(value)
            self.strs[spec[1]] = ("%s:%d" % (self._fn, self.lineno,), value,
                    '<%s %s>' % (tag, spec[0]))

    def handle_endtag(self, tag):
        if not tag in VOID_TAGS:
            self._nest -= 1

        if not self._i19:
            return

        # store raw html
        if self._include[1]:
            self._include[2] += "</" + tag + ">"

        # i19 directive not closed yet
        if self._i19[-1][2] <= self._nest:
            self._i19[-1][1] += "</" + tag + ">"
            return

        i19id, data, _ = self._i19.pop()

        if self._i19 and self._include[0]:
            # documentation for nested i19 tags
            doc = "Referenced in %r as %s" % \
                    (self._i19[-1][0], self._include[0],)
        else:
            doc = ''

        # end of include directive
        if self._include[1] >= self._nest:
            inid, _, raw = self._include
            self.includes[inid] = raw
            self._include = ['', 0, '']

        data = data.strip().replace('\n', '\\n')
        self.strs[(i19id or sanitize(data))] = \
                ("%s:%d" % (self._fn, self.lineno,), data, doc)

    def handle_data(self, data):
        if not self._i19:
            return
        self._i19[-1][1] += data
        if self._include[1]:
            self._include[2] += data

def main():
    assert len(sys.argv) > 3, \
            "Usage: i19extract POT_FILE CACHE_FILE [SOURCES..]"

    strs, inc = dict(), dict()
    for src_file in sys.argv[3:]:
        parser = i19Parser(src_file)
        strs.update(parser.strs.items())
        inc.update(parser.includes.items())

    with file(sys.argv[1], 'w') as pot:
        pot.write(POT_HEADER % (time.strftime('%Y-%m-%d %H:%M+0000'),))
        for i19id, dt in strs.items():
            if i19id.endswith(')'):
                pot.write(POT_PLURAL % (dt[2], dt[1], dt[0], i19id, i19id,))
            else:
                pot.write(POT_SINGULAR % (dt[2], dt[1], dt[0], i19id,))

    with file(sys.argv[2], 'wb') as i19n:
        dump((inc, strs,), i19n)

if __name__ == '__main__':
    main()
<|endoftext|>