<commit_before>"""A collection of tools to handle genomic tests, in particular Tajima's D.

Part of the biostructmap package.
"""
from __future__ import absolute_import, division, print_function

from io import StringIO
import warnings
from math import log
from Bio import AlignIO
from Bio.Data import IUPACData
from numpy import mean
import dendropy

from .seqtools import _sliding_window_var_sites

def shannon_entropy(alignment, table='Standard',
                    protein_letters=IUPACData.protein_letters,
                    normalized=False, gap='-'):
    '''
    Calculate mean Shannon entropy for all residues in a genomic alignment.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.
        table: A codon lookup table used by the Bio.Seq.translate() method.
            See BioPython docs for possible options.
        protein_letters (str, optional): String of all protein letters being
            used to define the amino acid alphabet. Defaults to standard 20
            amino acids. If another alphabet is used (if your sequence contains
            non-standard amino acid), then the maximum Shannon entropy values
            will change accordingly.
        normalized (bool): Normalize such the entropy is in the range [0, 1]

    Returns:
        float: Shannon entropy value.
    '''
    if isinstance(alignment, str):
        alignment = AlignIO.read(StringIO(alignment), format='fasta')
    if not alignment or len(alignment[0]) == 0:
        return None
    translated_positions = list(zip(*[str(x.seq.translate(table=table, gap=gap)) for
                                      x in alignment]))
    not_in_alphabet = set([res for x in translated_positions
                           for res in x]).difference(protein_letters)
    if not_in_alphabet:
        warnings.warn("Multiple sequence alignment contains residues that aren't "\
                      "in the provided alphabet. Entropy values will not be "\
                      "accurate - consider supplying an extended amino acid "\
                      "alphabet to the `protein_letters` keyword argument. "\
                      "Offending residue(s) are: {res}".format(res=str(not_in_alphabet)))
    entropy_values = [_calculate_shannon_entropy(x, protein_letters, normalized)
                      for x in translated_positions]
    entropy = mean(entropy_values)
    return entropy

def _calculate_shannon_entropy(seq, protein_letters=IUPACData.protein_letters,
                               normalized=False):
    '''
    Calculate Shannon entropy from a set of residues from a single position.

    Args:
        seq (str): Set of amino acid residues, assuming standard extended alphabet.
        protein_letters (str, optional): String of all protein letters being
            used to define the amino acid alphabet. Defaults to standard 20
            amino acids. If another alphabet is used (if your sequence contains
            non-standard amino acid), then the maximum Shannon entropy values
            will change accordingly.
        normalized (bool): Normalize such the entropy is in the range [0, 1]
    Returns:
        entropy (float): Shannon entropy.
    '''
    # Normalization involves dividing entropy values by the maximum entropy,
    # which is mathematically the same as changing the base of the logarithm
    # to be the size of the protein alphabet.
    if normalized:
        base = len(protein_letters)
    else:
        base = 2
    letters = protein_letters
    entropy = -sum((0 if seq.count(letter) == 0 else
                    seq.count(letter) / len(seq) *
                    log(seq.count(letter)/len(seq), base) for letter in letters))
    return entropy

def tajimas_d(alignment, window=None, step=3):
    """
    Uses DendroPy package to calculate Tajimas D.

    Notes:
        Several optimisations are used to speed up the calculation, including
        memoisation of previous window result, which is used if no new
        polymorphic sites are added/subtracted.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.
        window (int, optional): The size of the sliding window over which
            Tajima's D is calculated. Default is None, in which case a
            single Tajima's D value is calculated for the multiple sequence
            alignment.
        step (int, optional): Step size for sliding window calculation.
            Default step size of 3 (ie. one codon).
    Returns:
        float/dict: If window parameter is None, returns a single value for
            Tajima's D. Otherwise a dict mapping genome window midpoint to
            calculated Tajima's D values is returned.
    """
    if window:
        if isinstance(alignment, str):
            alignment = AlignIO.read(StringIO(alignment), 'fasta')
        results = {}
        prev_win = None
        prev_d = None
        slide = _sliding_window_var_sites(alignment, window, step=step)
        for i, win in enumerate(slide):
            centre = i*step + 1 + (window-1)/2
            if win == prev_win:
                results[centre] = prev_d
            else:
                current_d = _tajimas_d(win)
                results[centre] = current_d
                prev_d = current_d
                prev_win = win
        return results
    else:
        return _tajimas_d(alignment)

def _tajimas_d(alignment):
    """
    Uses DendroPy to calculate tajimas D.

    If Tajima's D is undefined (ie. Dendropy Tajima's D method raises a
    ZeroDivisionError), then this method returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Tajima's D value. Returns None if Tajima's D is undefined.
    """
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    if not alignment or len(alignment[0]) == 0:
        return None
    try:
        seq = dendropy.DnaCharacterMatrix.get(data=data,
                                              schema='fasta')
        taj_d = dendropy.calculate.popgenstat.tajimas_d(seq)
    except ZeroDivisionError:
        taj_d = None
    return taj_d

def nucleotide_diversity(alignment):
    """
    Use DendroPy to calculate nucleotide diversity.

    If nucleotide diversity is undefined, returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Nucleotide diversity value. Returns None if nucleotide
            diversity is undefined.
    """
    if is_empty_alignment(alignment):
        return None
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    seq = dendropy.DnaCharacterMatrix.get(data=data, schema='fasta')
    diversity = dendropy.calculate.popgenstat.nucleotide_diversity(seq)
    return diversity

def is_empty_alignment(alignment):
    """Returns True if alignment is empty.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        bool: True if alignment is empty, False otherwise.
    """
    if isinstance(alignment, str) and len(alignment.split('\n')[1]) == 0:
        return True
    elif not alignment or len(alignment[0]) == 0:
        return True
    else:
        return False

def wattersons_theta(alignment):
    """
    Use DendroPy to calculate Watterson's Theta.

    If Watterson's Theta is undefined, returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Watterson's Theta value. Returns None if Watterson's Theta is
            undefined.
    """
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    if not alignment or len(alignment[0]) == 0:
        return None
    seq = dendropy.DnaCharacterMatrix.get(data=data, schema='fasta')
    theta = dendropy.calculate.popgenstat.wattersons_theta(seq)
    return theta
<commit_msg>Fix empty alignments causing issues when in raw fasta format (again).<commit_after>"""A collection of tools to handle genomic tests, in particular Tajima's D.

Part of the biostructmap package.
"""
from __future__ import absolute_import, division, print_function

from io import StringIO
import warnings
from math import log
from Bio import AlignIO
from Bio.Data import IUPACData
from numpy import mean
import dendropy

from .seqtools import _sliding_window_var_sites

def shannon_entropy(alignment, table='Standard',
                    protein_letters=IUPACData.protein_letters,
                    normalized=False, gap='-'):
    '''
    Calculate mean Shannon entropy for all residues in a genomic alignment.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.
        table: A codon lookup table used by the Bio.Seq.translate() method.
            See BioPython docs for possible options.
        protein_letters (str, optional): String of all protein letters being
            used to define the amino acid alphabet. Defaults to standard 20
            amino acids. If another alphabet is used (if your sequence contains
            non-standard amino acid), then the maximum Shannon entropy values
            will change accordingly.
        normalized (bool): Normalize such the entropy is in the range [0, 1]

    Returns:
        float: Shannon entropy value.
    '''
    if isinstance(alignment, str):
        alignment = AlignIO.read(StringIO(alignment), format='fasta')
    if is_empty_alignment(alignment):
        return None
    translated_positions = list(zip(*[str(x.seq.translate(table=table, gap=gap)) for
                                      x in alignment]))
    not_in_alphabet = set([res for x in translated_positions
                           for res in x]).difference(protein_letters)
    if not_in_alphabet:
        warnings.warn("Multiple sequence alignment contains residues that aren't "\
                      "in the provided alphabet. Entropy values will not be "\
                      "accurate - consider supplying an extended amino acid "\
                      "alphabet to the `protein_letters` keyword argument. "\
                      "Offending residue(s) are: {res}".format(res=str(not_in_alphabet)))
    entropy_values = [_calculate_shannon_entropy(x, protein_letters, normalized)
                      for x in translated_positions]
    entropy = mean(entropy_values)
    return entropy

def _calculate_shannon_entropy(seq, protein_letters=IUPACData.protein_letters,
                               normalized=False):
    '''
    Calculate Shannon entropy from a set of residues from a single position.

    Args:
        seq (str): Set of amino acid residues, assuming standard extended alphabet.
        protein_letters (str, optional): String of all protein letters being
            used to define the amino acid alphabet. Defaults to standard 20
            amino acids. If another alphabet is used (if your sequence contains
            non-standard amino acid), then the maximum Shannon entropy values
            will change accordingly.
        normalized (bool): Normalize such the entropy is in the range [0, 1]
    Returns:
        entropy (float): Shannon entropy.
    '''
    # Normalization involves dividing entropy values by the maximum entropy,
    # which is mathematically the same as changing the base of the logarithm
    # to be the size of the protein alphabet.
    if normalized:
        base = len(protein_letters)
    else:
        base = 2
    letters = protein_letters
    entropy = -sum((0 if seq.count(letter) == 0 else
                    seq.count(letter) / len(seq) *
                    log(seq.count(letter)/len(seq), base) for letter in letters))
    return entropy

def tajimas_d(alignment, window=None, step=3):
    """
    Uses DendroPy package to calculate Tajimas D.

    Notes:
        Several optimisations are used to speed up the calculation, including
        memoisation of previous window result, which is used if no new
        polymorphic sites are added/subtracted.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.
        window (int, optional): The size of the sliding window over which
            Tajima's D is calculated. Default is None, in which case a
            single Tajima's D value is calculated for the multiple sequence
            alignment.
        step (int, optional): Step size for sliding window calculation.
            Default step size of 3 (ie. one codon).
    Returns:
        float/dict: If window parameter is None, returns a single value for
            Tajima's D. Otherwise a dict mapping genome window midpoint to
            calculated Tajima's D values is returned.
    """
    if window:
        if isinstance(alignment, str):
            alignment = AlignIO.read(StringIO(alignment), 'fasta')
        results = {}
        prev_win = None
        prev_d = None
        slide = _sliding_window_var_sites(alignment, window, step=step)
        for i, win in enumerate(slide):
            centre = i*step + 1 + (window-1)/2
            if win == prev_win:
                results[centre] = prev_d
            else:
                current_d = _tajimas_d(win)
                results[centre] = current_d
                prev_d = current_d
                prev_win = win
        return results
    else:
        return _tajimas_d(alignment)

def _tajimas_d(alignment):
    """
    Uses DendroPy to calculate tajimas D.

    If Tajima's D is undefined (ie. Dendropy Tajima's D method raises a
    ZeroDivisionError), then this method returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Tajima's D value. Returns None if Tajima's D is undefined.
    """
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    if is_empty_alignment(alignment):
        return None
    try:
        seq = dendropy.DnaCharacterMatrix.get(data=data,
                                              schema='fasta')
        taj_d = dendropy.calculate.popgenstat.tajimas_d(seq)
    except ZeroDivisionError:
        taj_d = None
    return taj_d

def nucleotide_diversity(alignment):
    """
    Use DendroPy to calculate nucleotide diversity.

    If nucleotide diversity is undefined, returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Nucleotide diversity value. Returns None if nucleotide
            diversity is undefined.
    """
    if is_empty_alignment(alignment):
        return None
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    seq = dendropy.DnaCharacterMatrix.get(data=data, schema='fasta')
    diversity = dendropy.calculate.popgenstat.nucleotide_diversity(seq)
    return diversity

def is_empty_alignment(alignment):
    """Returns True if alignment is empty.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        bool: True if alignment is empty, False otherwise.
    """
    if isinstance(alignment, str) and len(alignment.split('\n')[1]) == 0:
        return True
    elif not alignment or len(alignment[0]) == 0:
        return True
    else:
        return False

def wattersons_theta(alignment):
    """
    Use DendroPy to calculate Watterson's Theta.

    If Watterson's Theta is undefined, returns None.

    Args:
        alignment (str/Bio.Align.MultipleSequenceAlignment): A multiple sequence
            alignment string in FASTA format or a multiple sequence alignment
            object, either as a Bio.Align.MultipleSequenceAlignment or a
            biostructmap.SequenceAlignment object.

    Returns:
        float: Watterson's Theta value. Returns None if Watterson's Theta is
            undefined.
    """
    if not isinstance(alignment, str):
        data = alignment.format('fasta')
    else:
        data = alignment
    if is_empty_alignment(alignment):
        return None
    seq = dendropy.DnaCharacterMatrix.get(data=data, schema='fasta')
    theta = dendropy.calculate.popgenstat.wattersons_theta(seq)
    return theta
<|endoftext|>


<commit_before>import logging
from logging.handlers import BufferingHandler
import threading

import six

from nose2.events import Plugin
from nose2.util import ln

log = logging.getLogger(__name__)

class LogCapture(Plugin):
    """TODO: document"""

    configSection = 'logging'
    commandLineSwitch = (None, 'log-capture', 'Enable log capture')
    logformat = '%(name)s: %(levelname)s: %(message)s'
    logdatefmt = None
    clear = False
    filters = ['-nose']

    def __init__(self):
        self.logformat = self.config.as_str('format', self.logformat)
        self.logdatefmt = self.config.as_str('date-format', self.logdatefmt)
        self.filters = self.config.as_list('filter', self.filters)
        self.clear = self.config.as_bool('clear-handlers', self.clear)
        # FIXME loglevel

    def setupLoghandler(self):
        # setup our handler with root logger
        root_logger = logging.getLogger()
        if self.clear:
            if hasattr(root_logger, "handlers"):
                for handler in root_logger.handlers:
                    root_logger.removeHandler(handler)
            for logger in logging.Logger.manager.loggerDict.values():
                if hasattr(logger, "handlers"):
                    for handler in logger.handlers:
                        logger.removeHandler(handler)
        # make sure there isn't one already
        # you can't simply use "if self.handler not in root_logger.handlers"
        # since at least in unit tests this doesn't work --
        # LogCapture() is instantiated for each test case while root_logger
        # is module global
        # so we always add new MyMemoryHandler instance
        for handler in root_logger.handlers[:]:
            if isinstance(handler, MyMemoryHandler):
                root_logger.handlers.remove(handler)
        root_logger.addHandler(self.handler)
        # to make sure everything gets captured FIXME use level config
        root_logger.setLevel(logging.NOTSET)

    def addCapturedLogs(self, event):
        format = self.handler.format
        records = [format(r) for r in self.handler.buffer]
        event.metadata['logs'] = records
        # FIXME when there's a better way, do that instead
        records = [ln('>> begin captured logging <<')] + records + [
            ln('>> end captured logging <<')]
        records = six.u('\n').join(records)
        event.traceback = "%s\n%s" % (
            event.traceback, records.encode('utf-8'))

    def startTestRun(self, event):
        self.handler = MyMemoryHandler(1000, self.logformat, self.logdatefmt,
                                       self.filters)
        self.setupLoghandler()

    def startTest(self, event):
        self.setupLoghandler()

    def stopTest(self, event):
        if event.error or event.failed:
            self.addCapturedLogs(event)
        self.handler.truncate()

class FilterSet(object):
    def __init__(self, filter_components):
        self.inclusive, self.exclusive = self._partition(filter_components)

    @staticmethod
    def _partition(components):
        inclusive, exclusive = [], []
        for component in components:
            if component.startswith('-'):
                exclusive.append(component[1:])
            else:
                inclusive.append(component)
        return inclusive, exclusive

    def allow(self, record):
        """returns whether this record should be printed"""
        if not self:
            # nothing to filter
            return True
        return self._allow(record) and not self._deny(record)

    @staticmethod
    def _any_match(matchers, record):
        """return the bool of whether `record` starts with
        any item in `matchers`"""
        def record_matches_key(key):
            return record == key or record.startswith(key + '.')
        return any(map(record_matches_key, matchers))

    def _allow(self, record):
        if not self.inclusive:
            return True
        return self._any_match(self.inclusive, record)

    def _deny(self, record):
        if not self.exclusive:
            return False
        return self._any_match(self.exclusive, record)

class MyMemoryHandler(BufferingHandler):
    def __init__(self, capacity, logformat, logdatefmt, filters):
        BufferingHandler.__init__(self, capacity)
        fmt = logging.Formatter(logformat, logdatefmt)
        self.setFormatter(fmt)
        self.filterset = FilterSet(filters)

    def flush(self):
        pass # do nothing

    def truncate(self):
        self.buffer = []

    def filter(self, record):
        return self.filterset.allow(record.name)

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['lock']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.lock = threading.RLock()
<commit_msg>Reimplement log capture plugin<commit_after>import logging
from logging.handlers import BufferingHandler
import threading

from nose2.events import Plugin
from nose2.util import ln

log = logging.getLogger(__name__)

class LogCapture(Plugin):
    """TODO: document"""

    configSection = 'log-capture'
    commandLineSwitch = (None, 'log-capture', 'Enable log capture')
    logformat = '%(name)s: %(levelname)s: %(message)s'
    logdatefmt = None
    clear = False
    filters = ['-nose']

    def __init__(self):
        self.logformat = self.config.as_str('format', self.logformat)
        self.logdatefmt = self.config.as_str('date-format', self.logdatefmt)
        self.filters = self.config.as_list('filter', self.filters)
        self.clear = self.config.as_bool('clear-handlers', self.clear)
        # FIXME support symbolic level names
        self.loglevel = self.config.as_int('log-level', logging.NOTSET)
        self.handler = MyMemoryHandler(1000, self.logformat, self.logdatefmt,
                                       self.filters)

    def startTestRun(self, event):
        self._setupLoghandler()

    def startTest(self, event):
        self._setupLoghandler()

    def testOutcome(self, event):
        self._addCapturedLogs(event)

    def stopTest(self, event):
        self.handler.truncate()

    def outcomeDetail(self, event):
        logs = event.outcomeEvent.metadata.get('logs', None)
        if logs:
            event.extraDetail.append(ln('>> begin captured logging <<'))
            event.extraDetail.extend(logs)
            event.extraDetail.append(ln('>> end captured logging <<'))

    def _setupLoghandler(self):
        # setup our handler with root logger
        root_logger = logging.getLogger()
        if self.clear:
            if hasattr(root_logger, "handlers"):
                for handler in root_logger.handlers:
                    root_logger.removeHandler(handler)
            for logger in logging.Logger.manager.loggerDict.values():
                if hasattr(logger, "handlers"):
                    for handler in logger.handlers:
                        logger.removeHandler(handler)
        # make sure there isn't one already
        # you can't simply use "if self.handler not in root_logger.handlers"
        # since at least in unit tests this doesn't work --
        # LogCapture() is instantiated for each test case while root_logger
        # is module global
        # so we always add new MyMemoryHandler instance
        for handler in root_logger.handlers[:]:
            if isinstance(handler, MyMemoryHandler):
                root_logger.handlers.remove(handler)
        root_logger.addHandler(self.handler)
        root_logger.setLevel(self.loglevel)

    def _addCapturedLogs(self, event):
        format = self.handler.format
        records = [format(r) for r in self.handler.buffer]
        event.metadata['logs'] = records

class FilterSet(object):
    def __init__(self, filter_components):
        self.inclusive, self.exclusive = self._partition(filter_components)

    @staticmethod
    def _partition(components):
        inclusive, exclusive = [], []
        for component in components:
            if component.startswith('-'):
                exclusive.append(component[1:])
            else:
                inclusive.append(component)
        return inclusive, exclusive

    def allow(self, record):
        """returns whether this record should be printed"""
        if not self:
            # nothing to filter
            return True
        return self._allow(record) and not self._deny(record)

    @staticmethod
    def _any_match(matchers, record):
        """return the bool of whether `record` starts with
        any item in `matchers`"""
        def record_matches_key(key):
            return record == key or record.startswith(key + '.')
        return any(map(record_matches_key, matchers))

    def _allow(self, record):
        if not self.inclusive:
            return True
        return self._any_match(self.inclusive, record)

    def _deny(self, record):
        if not self.exclusive:
            return False
        return self._any_match(self.exclusive, record)

class MyMemoryHandler(BufferingHandler):
    def __init__(self, capacity, logformat, logdatefmt, filters):
        BufferingHandler.__init__(self, capacity)
        fmt = logging.Formatter(logformat, logdatefmt)
        self.setFormatter(fmt)
        self.filterset = FilterSet(filters)

    def flush(self):
        pass # do nothing

    def truncate(self):
        self.buffer = []

    def filter(self, record):
        return self.filterset.allow(record.name)

    def __getstate__(self):
        state = self.__dict__.copy()
        del state['lock']
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        self.lock = threading.RLock()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division, absolute_import, print_function

import re
import click
import sys
import time
import subprocess
import os
import colors
import string
import locale

class Format(object):
    def __init__(self, fmt, fmts, colors=True):
        self.fmt = fmt
        self.val_fmts = fmts
        self.colors = colors

    def __repr__(self):
        return self.fmt

    def colorize(self, val, fmt, use_colors=None, **kwargs):
        if use_colors is None: use_colors = self.colors
        s = fmt.format(val, **kwargs)
        if use_colors:
            if val > 0:
                return colors.green(s)
            elif val < 0:
                return colors.red(s)
        return s

    def format(self, *values, **kwargs):
        formatted_vals = [self.colorize(val, fmt, **kwargs) for fmt, val in zip(self.val_fmts, values)]
        return self.fmt.format(*formatted_vals)

    def format_str_with_spaces(self):
        last_seen = None
        fmt = []
        for c in self.fmt:
            if c == u'{':
                if last_seen == u'{': # escaped {
                    fmt.append(u' ')
                    last_seen = None
                    continue
            elif c == u'}':
                if last_seen == u'}': # escaped }
                    fmt.append(u' ')
                    last_seen = None
                    continue
                elif last_seen == u'{': # real format str
                    fmt.append(u'{}')
                    last_seen = None
                    continue
            elif c in string.whitespace:
                fmt.append(c)
            else:
                fmt.append(u' ')
            last_seen = c
        return u''.join(fmt)

    def format_wsp(self, *values, **kwargs):
        formatted_vals = [self.colorize(val, fmt, **kwargs) for fmt, val in zip(self.val_fmts, values)]
        return self.format_str_with_spaces().format(*formatted_vals)

class ValueFormat(object):
    def __init__(self, prefix, align, plus, width, fmt):
        self.prefix = prefix
        self.align = align
        self.plus = plus
        self.width = width
        self.fmt = fmt

    def format_str(self, prefix=None, align=None, plus=None, width=None, fmt=None):
        if prefix is None: prefix = self.prefix
        if align is None: align = self.align
        if plus is None: plus = self.plus
        if width is None: width = self.width
        if fmt is None: fmt = self.fmt
        return u'%s{:%s%s%s%s}' % (prefix, align, plus, width, fmt)

    def format(self, *values, **kwargs):
        return self.format_str(**kwargs).format(*values)

formats = {}

def parse_num(match, plus, flex):
    spaces, n = match.groups()

    prefix = u''
    width = len(n)
    align = u'<'

    if spaces:
        prefix = u' '
        width += len(spaces) - 1
        align = u''
    elif match.start(1) == 0:
        align = u''

    if u'.' not in n:
        if align == u'' and flex and width < 2:
            width = 2
        if len(n) > 1 and n.startswith(u'0'):
            width = u'0{}'.format(width)
            align = u''

        return int(n), ValueFormat(prefix, align, plus, width, u'')

    whole, frac = n.split(u'.', 1)
    frac_len = len(frac)
    if align == u'' and flex and width < frac_len + 3:
        width = frac_len + 3
    if len(whole) > 1 and whole.startswith(u'0'):
        width = u'0{}'.format(width)
        align = u''
    return float(n), ValueFormat(prefix, align, plus, width, u'.%df' % len(frac))

def num(n):
    if u'.' not in n:
        return int(n)
    return float(n)

def parse(line, flex=True):
    global formats
    values = []
    val_formats = []
    def value(v):
        val, fmt = parse_num(v, u'+', flex)
        values.append(val)
        val_formats.append(fmt)
        return u'{}'

    raw_fmt = line.replace(u'{', u'{{').replace(u'}', u'}}')
    raw_fmt = re.sub(r'(\s*)([0-9]+(?:\.[0-9]+)?)', value, raw_fmt)
    fmt = Format(raw_fmt, val_formats)

    rx_line = re.sub(r'([()\[\]^$\\|])', r'\\\1', line)
    rx_str = re.sub(r'(\s*[0-9]+(?:\.[0-9]+)?)', r'(\s*[0-9]+(?:\.[0-9]+)?)', rx_line)
    rx = re.compile(rx_str)

    formats[rx] = (fmt, values)
    return fmt.format(*values, plus=u'', use_colors=False)

def process(line, flex, absolute):
    global formats
    for rx, (fmt, old_values) in formats.items():
        m = rx.match(line)
        if m:
            values = [num(v) for v in m.groups()]
            deltas = [n-o for n, o in zip(values, old_values)]
            if not absolute:
                formats[rx] = (fmt, values)
            return fmt, deltas, values

    return parse(line, flex), None, None

def stdin_feed(sep_interval):
    while True:
        ts = time.time()
        line = sys.stdin.readline()
        if not line:
            break
        yield line, (time.time() - ts) > sep_interval

def command_feed(cmd, interval):
    _, encoding = locale.getdefaultlocale()
    if len(cmd) == 1:
        shell = os.getenv(u'SHELL', u'/bin/sh')
        cmd = (shell, u'-c') + cmd
    while True:
        output = subprocess.check_output(cmd)
        first = True
        for line in output.splitlines():
            yield line.decode(encoding) + u'\n', first
            first = False
        time.sleep(interval)

@click.command()
@click.option(u'-t/-T', u'--timestamps/--no-timestamps', help=u'Show timestamps on all output lines')
@click.option(u'-i', u'--interval', metavar=u'SECONDS', type=click.INT,
    help=u'Interval between command runs', default=1)
@click.option(u'-f/-F', u'--flex/--no-flex', help=u'Tweak column widths for better output (default is on)', default=True)
@click.option(u'-s/-S', u'--separators/--no-separators', help=u'Show separators between chunks of output (default is on)', default=True)
@click.option(u'-c', u'--color', type=click.Choice([u'never', u'auto', u'always']), help=u'Color output', default=u'auto')
@click.option(u'-o/-O', u'--orig/--no-orig', help=u'Show original output interleaved with deltas')
@click.option(u'-z/-Z', u'--skip-zeros/--with-zeros', help=u'Skip all-zero deltas')
@click.option(u'-a/-A', u'--absolute/--relative', help=u'Show deltas from original value, not last')
@click.argument(u'cmd', nargs=-1, required=False)
def cli(timestamps, cmd, interval, flex, separators, color, orig, skip_zeros, absolute):
    if cmd:
        feed = command_feed(cmd, interval)
    else:
        feed = stdin_feed(interval * 0.8)

    if color == u'never':
        color = False
    elif color == u'always':
        color = True
    else:
        color = os.isatty(sys.stdout.fileno())

    def p(line, with_sep=False):
        if with_sep:
            if timestamps:
                sys.stdout.write(u'{}\n'.format(time.asctime()))
            else:
                sys.stdout.write(u'--- {}\n'.format(time.asctime()))
        if timestamps:
            sys.stdout.write(u'{}: '.format(time.asctime()))
        sys.stdout.write(line)
        sys.stdout.flush()

    try:
        print_sep = True
        for line, want_sep in feed:
            fmt, deltas, values = process(line, flex, absolute)
            if values is None:
                p(fmt)
                continue
            all_zeros = all(d == 0 for d in deltas)
            print_sep = print_sep or want_sep
            with_sep = separators and print_sep and len(formats) > 1
            if orig:
                print_sep = False
                if len(values):
                    p(fmt.format(*values, plus=u'', use_colors=False), with_sep=with_sep)
                    if not skip_zeros or not all_zeros:
                        p(fmt.format_wsp(*deltas, use_colors=color))
                else:
                    p(fmt.format(*values), with_sep=with_sep)
            else:
                if not skip_zeros or not all_zeros:
                    print_sep = False
                    p(fmt.format(*deltas, use_colors=color), with_sep=with_sep)

    except KeyboardInterrupt:
        pass
    except IOError:
        return

if __name__ == u'__main__':
    cli()
<commit_msg>Unicode-safe I/O<commit_after>#!/usr/bin/env python

from __future__ import division, absolute_import, print_function

import re
import click
import sys
import time
import subprocess
import os
import colors
import string
import locale

if sys.version_info[0] == 2:
    import codecs
    _, encoding = locale.getdefaultlocale()
    sys.stdout = codecs.getwriter(encoding)(sys.stdout)

class Format(object):
    def __init__(self, fmt, fmts, colors=True):
        self.fmt = fmt
        self.val_fmts = fmts
        self.colors = colors

    def __repr__(self):
        return self.fmt

    def colorize(self, val, fmt, use_colors=None, **kwargs):
        if use_colors is None: use_colors = self.colors
        s = fmt.format(val, **kwargs)
        if use_colors:
            if val > 0:
                return colors.green(s)
            elif val < 0:
                return colors.red(s)
        return s

    def format(self, *values, **kwargs):
        formatted_vals = [self.colorize(val, fmt, **kwargs) for fmt, val in zip(self.val_fmts, values)]
        return self.fmt.format(*formatted_vals)

    def format_str_with_spaces(self):
        last_seen = None
        fmt = []
        for c in self.fmt:
            if c == u'{':
                if last_seen == u'{': # escaped {
                    fmt.append(u' ')
                    last_seen = None
                    continue
            elif c == u'}':
                if last_seen == u'}': # escaped }
                    fmt.append(u' ')
                    last_seen = None
                    continue
                elif last_seen == u'{': # real format str
                    fmt.append(u'{}')
                    last_seen = None
                    continue
            elif c in string.whitespace:
                fmt.append(c)
            else:
                fmt.append(u' ')
            last_seen = c
        return u''.join(fmt)

    def format_wsp(self, *values, **kwargs):
        formatted_vals = [self.colorize(val, fmt, **kwargs) for fmt, val in zip(self.val_fmts, values)]
        return self.format_str_with_spaces().format(*formatted_vals)

class ValueFormat(object):
    def __init__(self, prefix, align, plus, width, fmt):
        self.prefix = prefix
        self.align = align
        self.plus = plus
        self.width = width
        self.fmt = fmt

    def format_str(self, prefix=None, align=None, plus=None, width=None, fmt=None):
        if prefix is None: prefix = self.prefix
        if align is None: align = self.align
        if plus is None: plus = self.plus
        if width is None: width = self.width
        if fmt is None: fmt = self.fmt
        return u'%s{:%s%s%s%s}' % (prefix, align, plus, width, fmt)

    def format(self, *values, **kwargs):
        return self.format_str(**kwargs).format(*values)

formats = {}

def parse_num(match, plus, flex):
    spaces, n = match.groups()

    prefix = u''
    width = len(n)
    align = u'<'

    if spaces:
        prefix = u' '
        width += len(spaces) - 1
        align = u''
    elif match.start(1) == 0:
        align = u''

    if u'.' not in n:
        if align == u'' and flex and width < 2:
            width = 2
        if len(n) > 1 and n.startswith(u'0'):
            width = u'0{}'.format(width)
            align = u''

        return int(n), ValueFormat(prefix, align, plus, width, u'')

    whole, frac = n.split(u'.', 1)
    frac_len = len(frac)
    if align == u'' and flex and width < frac_len + 3:
        width = frac_len + 3
    if len(whole) > 1 and whole.startswith(u'0'):
        width = u'0{}'.format(width)
        align = u''
    return float(n), ValueFormat(prefix, align, plus, width, u'.%df' % len(frac))

def num(n):
    if u'.' not in n:
        return int(n)
    return float(n)

def parse(line, flex=True):
    global formats
    values = []
    val_formats = []
    def value(v):
        val, fmt = parse_num(v, u'+', flex)
        values.append(val)
        val_formats.append(fmt)
        return u'{}'

    raw_fmt = line.replace(u'{', u'{{').replace(u'}', u'}}')
    raw_fmt = re.sub(r'(\s*)([0-9]+(?:\.[0-9]+)?)', value, raw_fmt)
    fmt = Format(raw_fmt, val_formats)

    rx_line = re.sub(r'([()\[\]^$\\|])', r'\\\1', line)
    rx_str = re.sub(r'(\s*[0-9]+(?:\.[0-9]+)?)', r'(\s*[0-9]+(?:\.[0-9]+)?)', rx_line)
    rx = re.compile(rx_str)

    formats[rx] = (fmt, values)
    return fmt.format(*values, plus=u'', use_colors=False)

def process(line, flex, absolute):
    global formats
    for rx, (fmt, old_values) in formats.items():
        m = rx.match(line)
        if m:
            values = [num(v) for v in m.groups()]
            deltas = [n-o for n, o in zip(values, old_values)]
            if not absolute:
                formats[rx] = (fmt, values)
            return fmt, deltas, values

    return parse(line, flex), None, None

def stdin_feed(sep_interval):
    while True:
        ts = time.time()
        line = sys.stdin.readline()
        if sys.version_info[0] == 2:
            line = line.decode(encoding)
        if not line:
            break
        yield line, (time.time() - ts) > sep_interval

def command_feed(cmd, interval):
    _, encoding = locale.getdefaultlocale()
    if len(cmd) == 1:
        shell = os.getenv(u'SHELL', u'/bin/sh')
        cmd = (shell, u'-c') + cmd
    while True:
        output = subprocess.check_output(cmd)
        first = True
        for line in output.splitlines():
            yield line.decode(encoding) + u'\n', first
            first = False
        time.sleep(interval)

@click.command()
@click.option(u'-t/-T', u'--timestamps/--no-timestamps', help=u'Show timestamps on all output lines')
@click.option(u'-i', u'--interval', metavar=u'SECONDS', type=click.INT,
    help=u'Interval between command runs', default=1)
@click.option(u'-f/-F', u'--flex/--no-flex', help=u'Tweak column widths for better output (default is on)', default=True)
@click.option(u'-s/-S', u'--separators/--no-separators', help=u'Show separators between chunks of output (default is on)', default=True)
@click.option(u'-c', u'--color', type=click.Choice([u'never', u'auto', u'always']), help=u'Color output', default=u'auto')
@click.option(u'-o/-O', u'--orig/--no-orig', help=u'Show original output interleaved with deltas')
@click.option(u'-z/-Z', u'--skip-zeros/--with-zeros', help=u'Skip all-zero deltas')
@click.option(u'-a/-A', u'--absolute/--relative', help=u'Show deltas from original value, not last')
@click.argument(u'cmd', nargs=-1, required=False)
def cli(timestamps, cmd, interval, flex, separators, color, orig, skip_zeros, absolute):
    if cmd:
        feed = command_feed(cmd, interval)
    else:
        feed = stdin_feed(interval * 0.8)

    if color == u'never':
        color = False
    elif color == u'always':
        color = True
    else:
        color = os.isatty(sys.stdout.fileno())

    def p(line, with_sep=False):
        if with_sep:
            if timestamps:
                sys.stdout.write(u'{}\n'.format(time.asctime()))
            else:
                sys.stdout.write(u'--- {}\n'.format(time.asctime()))
        if timestamps:
            sys.stdout.write(u'{}: '.format(time.asctime()))
        sys.stdout.write(line)
        sys.stdout.flush()

    try:
        print_sep = True
        for line, want_sep in feed:
            fmt, deltas, values = process(line, flex, absolute)
            if values is None:
                p(fmt)
                continue
            all_zeros = all(d == 0 for d in deltas)
            print_sep = print_sep or want_sep
            with_sep = separators and print_sep and len(formats) > 1
            if orig:
                print_sep = False
                if len(values):
                    p(fmt.format(*values, plus=u'', use_colors=False), with_sep=with_sep)
                    if not skip_zeros or not all_zeros:
                        p(fmt.format_wsp(*deltas, use_colors=color))
                else:
                    p(fmt.format(*values), with_sep=with_sep)
            else:
                if not skip_zeros or not all_zeros:
                    print_sep = False
                    p(fmt.format(*deltas, use_colors=color), with_sep=with_sep)

    except KeyboardInterrupt:
        pass
    except IOError:
        return

if __name__ == u'__main__':
    cli()
<|endoftext|>


<commit_before>
memory = {}
with open('input') as fin:
    m = fin.read
    memory_n, queries_n, table_r_addr = [int(x) for x in next(fin).split()]
    for i in range(memory_n):
        paddr, value = [int(x) for x in next(fin).split()]
        memory[paddr] = value

#memory_n = m # phys addr = value(8byte)
qr = [
42,
0,
4096,
131313,
        ]

def mem(phys_addr):
    return memory.get(phys_addr, 0)

#table_r_addr = mem(root_table)
#print('root table @ ', hex(table_r_addr), table_r_addr)

def get_addr_from_record(r):
    is_present = r & 1
    if not is_present:
        print('fault')
        raise Exception()
    addr = r & 0xFFFFF000
    return addr
    # 31-11
    # * 2^12

def get_addr(laddr):
    # Remember where addr and where value, where logic addr
    # parse logic addr
    #Use bits 39-47 (9 bits) as an index into P4
    #laddr  & 0x01FF
    # test 0x?
    # 2^9 = 512 records
    offset = laddr & 0x1FF
    p1_idx = (laddr >> 9) & 0x1FF
    p2_idx = (laddr >> 18) & 0x1FF
    p3_idx = (laddr >> 27) & 0x1FF
    p4_idx = (laddr >> 36) #& 0x1FF
    # get record from root table at p4 idx:

    rec = mem(table_r_addr + (p4_idx * 8))
    t3addr = get_addr_from_record(rec)

    rec = mem(t3addr + (p3_idx * 8))
    t2addr = get_addr_from_record(rec)

    rec = mem(t2addr + (p2_idx * 8))
    t1addr = get_addr_from_record(rec)
    # print('LOGIC: ', laddr)
    # print("p4 idx", p4_idx)
    # print("p3 idx", p3_idx)
    # print("p2 idx", p2_idx)
    # print("p1 idx", p1_idx)
    # print('rec ', rec)
    # print('table 3 phys addr', t3addr)
    # print('rec ', hex(rec))
    # print('table 2 phys addr', t2addr)
    # print('rec ', hex(rec))
    # print('table 1 phys addr', t1addr)

    rec = mem(t1addr + (p1_idx * 8))
    # print('rec ', hex(rec), rec)
    phys = get_addr_from_record(rec)
    phys += offset
    #print('PHYS ', hex(phys), phys)
    print(phys)

# https://github.com/0xAX/linux-insides/blob/master/Theory/Paging.md

for q in qr:
    try:
        get_addr(q)
    except:
        pass
<commit_msg>continue paging task<commit_after>input_file = 'input'
input_file = '/home/ksi/Downloads/dataset_44327_15.txt'
memory = {}
qr = list()
with open(input_file) as fin:
    m = fin.read
    memory_n, queries_n, table_r_addr = [int(x) for x in next(fin).split()]
    for i in range(memory_n):
        paddr, value = [int(x) for x in next(fin).split()]
        memory[paddr] = value
    for i in range(queries_n):
        qr.append(int(fin.readline()))

#memory_n = m # phys addr = value(8byte)
def mem(phys_addr):
    return memory.get(phys_addr, 0)

#table_r_addr = mem(root_table)
#print('root table @ ', hex(table_r_addr), table_r_addr)

def get_addr_from_record(r):
    is_present = r & 1
    if not is_present:
        print('fault')
        raise Exception()
    addr = r & 0xFFFFF000
    return addr
    # 31-11
    # * 2^12

def get_addr(laddr):
    # Remember where addr and where value, where logic addr
    # parse logic addr
    #Use bits 39-47 (9 bits) as an index into P4
    #laddr  & 0x01FF
    # test 0x?
    # 2^9 = 512 records
    offset = laddr & 0x1FF
    p1_idx = (laddr >> 9) & 0x1FF
    p2_idx = (laddr >> 18) & 0x1FF
    p3_idx = (laddr >> 27) & 0x1FF
    p4_idx = (laddr >> 36) #& 0x1FF
    # get record from root table at p4 idx:

    rec = mem(table_r_addr + (p4_idx * 8))
    t3addr = get_addr_from_record(rec)

    rec = mem(t3addr + (p3_idx * 8))
    t2addr = get_addr_from_record(rec)

    rec = mem(t2addr + (p2_idx * 8))
    t1addr = get_addr_from_record(rec)
    # print('LOGIC: ', laddr)
    # print("p4 idx", p4_idx)
    # print("p3 idx", p3_idx)
    # print("p2 idx", p2_idx)
    # print("p1 idx", p1_idx)
    # print('rec ', rec)
    # print('table 3 phys addr', t3addr)
    # print('rec ', hex(rec))
    # print('table 2 phys addr', t2addr)
    # print('rec ', hex(rec))
    # print('table 1 phys addr', t1addr)

    rec = mem(t1addr + (p1_idx * 8))
    # print('rec ', hex(rec), rec)
    phys = get_addr_from_record(rec)
    phys += offset
    #print('PHYS ', hex(phys), phys)
    print(phys)

# https://github.com/0xAX/linux-insides/blob/master/Theory/Paging.md

for q in qr:
    try:
        get_addr(q)
    except:
        pass
<|endoftext|>


<commit_before>import json
import logging
import os
import re
import sys

import requests
import utils

logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
logging.StreamHandler(sys.stdout)

"""
    These utility functions allow us to maintain the noto source manifest file and
    download source files from Google's Github repos.
"""

# local paths
FONTS_SOURCE = os.path.abspath(os.path.join(os.path.dirname(__file__), "noto_source"))

FONT_MANIFEST_PATH = os.path.join(FONTS_SOURCE, "manifest.json")
with open(FONT_MANIFEST_PATH, "r") as mf:
    MANIFEST = json.load(mf)

KEY_REF = "ref"
KEY_FONTS = "fonts"
KEY_REG_URL = "reg_url"
KEY_BOLD_URL = "bold_url"

FONT_MANIFEST = MANIFEST[KEY_FONTS]

TTF_PATH = os.path.join(FONTS_SOURCE, "sources")

# Github API object keys
GH_PATH = "path"

# filename and directory patterns
OLD_STYLE_PATH = "^hinted/NotoSans\w*-\w*.ttf$"
PHASE_3_PATH = "^phaseIII_only/hinted/ttf/NotoSans\w*/NotoSans\w*-\w+\.ttf$"

DIRECTORY_PATTERN = "^NotoSans\w*$"
REG_PATTERN = "^(NotoSans\w*)-Regular\.ttf$"
BOLD_PATTERN = "^(NotoSans\w*)-Bold\.ttf$"

DOWNLOAD_URL = "https://raw.githubusercontent.com/googlei18n/noto-fonts/{ref}/{path}"

def _request(path):
    url = "https://api.github.com/repos/googlei18n/noto-fonts/" + path
    token = os.environ.get("GITHUB_TOKEN")
    headers = {"Authorization": "token {}".format(token)} if token else {}
    r = requests.get(url, headers=headers)
    if r.status_code == 403:
        logging.error("You've hit the Github API rate limit.")
        if not token:
            logging.info(
                "You can set a GITHUB_TOKEN environment variable with a github API token.\n"
                + "Generate a github token at https://github.com/settings/tokens and give it read-only permission."
            )
        sys.exit(1)
    else:
        r.raise_for_status()
    return r.json()

def _is_base_font(name):
    """
    Used to filter out some special variants that we don't need
    """
    MODIFIERS = ["Display", "Mono", "Slanted"]
    for m in MODIFIERS:
        if name.endswith(m):
            return False
    return True

def _manifest_object(reg_url, bold_url):
    """
    Primary font record in the manifest
    """
    return {KEY_REG_URL: reg_url, KEY_BOLD_URL: bold_url}

def _p3_download_url(base_font_name, is_bold, ref):
    path = "phaseIII_only/hinted/ttf/{base}/{base}-{weight}.ttf".format(
        base=base_font_name, weight=("Bold" if is_bold else "Regular")
    )
    return DOWNLOAD_URL.format(ref=ref, path=path)

def _old_download_url(base_font_name, is_bold, ref):
    path = "hinted/{}-{}.ttf".format(base_font_name, "Bold" if is_bold else "Regular")
    return DOWNLOAD_URL.format(ref=ref, path=path)

def _font_info(recursive_tree, ref, path_pattern, download_url_generator):
    """
    Grab info on all relevant older fonts. Returns a dict of objects generated by

        _manifest_object(reg_url, bold_url)

    which contain a download URL for the bold and regular font weights. Keys of the
    object are font base names, such as NotoSans or NotoSansArabic
    """
    file_names = [
        os.path.basename(item["path"])
        for item in recursive_tree["tree"]
        if re.match(path_pattern, item["path"])
    ]

    # Accumulate dicts of github info objects for regular and bold fonts
    reg_items = set()
    bold_items = set()
    for file_name in file_names:
        reg = re.match(pattern=REG_PATTERN, string=file_name)
        bold = re.match(pattern=BOLD_PATTERN, string=file_name)
        if reg:
            font_name = reg.group(1)
            if not _is_base_font(font_name):
                continue
            reg_items.add(font_name)
        elif bold:
            font_name = bold.group(1)
            if not _is_base_font(font_name):
                continue
            bold_items.add(font_name)

    # these are the fonts that have both regular and bold variants
    bold_and_reg = reg_items & bold_items

    # generate manifest objects, referring to the 'UI' variants when possible
    output = {}
    for font_name in bold_and_reg:
        # skip UI variants because these will always have a normal variant too
        if font_name.endswith("UI"):
            continue
        # for non-UI items, reference the UI variant when it exists
        if font_name + "UI" in bold_and_reg:
            base_font_name = font_name + "UI"
        else:
            base_font_name = font_name

        output[font_name] = _manifest_object(
            download_url_generator(base_font_name, False, ref),
            download_url_generator(base_font_name, True, ref),
        )

    return output

def update_manifest(ref=None):
    """
    Given a git reference in the Noto repo, such as a git commit hash or tag, extract
    information about the fonts available for use and save that information to the
    manifest file.

    The Noto repo currently contains both an older style and the newer "Phase 3"
    fonts. Phase 3 fonts have more consistent internal metrics which makes them amenable
    to being merged together, which we make use of. The older fonts are still usable,
    but cannot be merged together.

    Noto also contains both standard and "UI" variants of many fonts. When a font has a
    UI variant, it means that some of the glyphs in the standard variant are very tall
    and might overflow a typical line of text; the UI variant has the glypsh redrawn
    to fit.

    When searching for fonts to include, we take all language fonts that have both a
    regular and a bold variant, with preference given to Phase 3 and UI variants.
    """

    # grab the head of master
    if not ref:
        logging.info("Using head of master")
        ref = _request("git/refs/heads/master")["object"]["sha"]

    logging.info("Generating new manifest for reference '{}'".format(ref))

    git_tree = _request("git/trees/{}?recursive=1".format(ref))

    # backups
    font_info = _font_info(git_tree, ref, OLD_STYLE_PATH, _old_download_url)
    # prefer phase 3, replacing old-styles when possible
    font_info.update(_font_info(git_tree, ref, PHASE_3_PATH, _p3_download_url))

    new_manifest = {KEY_REF: ref, KEY_FONTS: font_info}
    utils.json_dump_formatted(new_manifest, FONT_MANIFEST_PATH)

def fetch_fonts():
    """
    download ttf files from the manifest
    """

    if not os.path.exists(TTF_PATH):
        os.makedirs(TTF_PATH)

    # out with the old
    for file_name in os.listdir(TTF_PATH):
        os.unlink(os.path.join(TTF_PATH, file_name))

    # in with the new
    for font_name in FONT_MANIFEST:
        font_info = FONT_MANIFEST[font_name]

        # regular
        output_path = get_path(font_name, False)
        logging.info("Writing {}".format(output_path))
        r = requests.get(font_info[KEY_REG_URL])
        r.raise_for_status()
        with open(output_path, "wb") as f:
            f.write(r.content)

        # bold
        output_path = get_path(font_name, True)
        logging.info("Writing {}".format(output_path))
        r = requests.get(font_info[KEY_BOLD_URL])
        r.raise_for_status()
        with open(output_path, "wb") as f:
            f.write(r.content)

@utils.memoize
def get_path(font_name, is_bold=False):
    info = FONT_MANIFEST[font_name]
    path = info[KEY_BOLD_URL] if is_bold else info[KEY_REG_URL]
    return os.path.join(TTF_PATH, os.path.basename(path))
<commit_msg>fix for manifest, ref #6796 (issue 3)<commit_after>import json
import logging
import os
import re
import sys

import requests
import utils

logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
logging.StreamHandler(sys.stdout)

"""
    These utility functions allow us to maintain the noto source manifest file and
    download source files from Google's Github repos.
"""

# local paths
FONTS_SOURCE = os.path.abspath(os.path.join(os.path.dirname(__file__), "noto_source"))

FONT_MANIFEST_NAME = "manifest.json"
FONT_MANIFEST_PATH = os.path.join(FONTS_SOURCE, FONT_MANIFEST_NAME)
with open(FONT_MANIFEST_PATH, "r") as mf:
    MANIFEST = json.load(mf)

KEY_REF = "ref"
KEY_FONTS = "fonts"
KEY_REG_URL = "reg_url"
KEY_BOLD_URL = "bold_url"

FONT_MANIFEST = MANIFEST[KEY_FONTS]

TTF_PATH = os.path.join(FONTS_SOURCE, "sources")

# Github API object keys
GH_PATH = "path"

# filename and directory patterns
OLD_STYLE_PATH = "^hinted/NotoSans\w*-\w*.ttf$"
PHASE_3_PATH = "^phaseIII_only/hinted/ttf/NotoSans\w*/NotoSans\w*-\w+\.ttf$"

DIRECTORY_PATTERN = "^NotoSans\w*$"
REG_PATTERN = "^(NotoSans\w*)-Regular\.ttf$"
BOLD_PATTERN = "^(NotoSans\w*)-Bold\.ttf$"

DOWNLOAD_URL = "https://raw.githubusercontent.com/googlei18n/noto-fonts/{ref}/{path}"

def _request(path):
    url = "https://api.github.com/repos/googlei18n/noto-fonts/" + path
    token = os.environ.get("GITHUB_TOKEN")
    headers = {"Authorization": "token {}".format(token)} if token else {}
    r = requests.get(url, headers=headers)
    if r.status_code == 403:
        logging.error("You've hit the Github API rate limit.")
        if not token:
            logging.info(
                "You can set a GITHUB_TOKEN environment variable with a github API token.\n"
                + "Generate a github token at https://github.com/settings/tokens and give it read-only permission."
            )
        sys.exit(1)
    else:
        r.raise_for_status()
    return r.json()

def _is_base_font(name):
    """
    Used to filter out some special variants that we don't need
    """
    MODIFIERS = ["Display", "Mono", "Slanted"]
    for m in MODIFIERS:
        if name.endswith(m):
            return False
    return True

def _manifest_object(reg_url, bold_url):
    """
    Primary font record in the manifest
    """
    return {KEY_REG_URL: reg_url, KEY_BOLD_URL: bold_url}

def _p3_download_url(base_font_name, is_bold, ref):
    path = "phaseIII_only/hinted/ttf/{base}/{base}-{weight}.ttf".format(
        base=base_font_name, weight=("Bold" if is_bold else "Regular")
    )
    return DOWNLOAD_URL.format(ref=ref, path=path)

def _old_download_url(base_font_name, is_bold, ref):
    path = "hinted/{}-{}.ttf".format(base_font_name, "Bold" if is_bold else "Regular")
    return DOWNLOAD_URL.format(ref=ref, path=path)

def _font_info(recursive_tree, ref, path_pattern, download_url_generator):
    """
    Grab info on all relevant older fonts. Returns a dict of objects generated by

        _manifest_object(reg_url, bold_url)

    which contain a download URL for the bold and regular font weights. Keys of the
    object are font base names, such as NotoSans or NotoSansArabic
    """
    file_names = [
        os.path.basename(item["path"])
        for item in recursive_tree["tree"]
        if re.match(path_pattern, item["path"])
    ]

    # Accumulate dicts of github info objects for regular and bold fonts
    reg_items = set()
    bold_items = set()
    for file_name in file_names:
        reg = re.match(pattern=REG_PATTERN, string=file_name)
        bold = re.match(pattern=BOLD_PATTERN, string=file_name)
        if reg:
            font_name = reg.group(1)
            if not _is_base_font(font_name):
                continue
            reg_items.add(font_name)
        elif bold:
            font_name = bold.group(1)
            if not _is_base_font(font_name):
                continue
            bold_items.add(font_name)

    # these are the fonts that have both regular and bold variants
    bold_and_reg = reg_items & bold_items

    # generate manifest objects, referring to the 'UI' variants when possible
    output = {}
    for font_name in bold_and_reg:
        # skip UI variants because these will always have a normal variant too
        if font_name.endswith("UI"):
            continue
        # for non-UI items, reference the UI variant when it exists
        if font_name + "UI" in bold_and_reg:
            base_font_name = font_name + "UI"
        else:
            base_font_name = font_name

        output[font_name] = _manifest_object(
            download_url_generator(base_font_name, False, ref),
            download_url_generator(base_font_name, True, ref),
        )

    return output

def update_manifest(ref=None):
    """
    Given a git reference in the Noto repo, such as a git commit hash or tag, extract
    information about the fonts available for use and save that information to the
    manifest file.

    The Noto repo currently contains both an older style and the newer "Phase 3"
    fonts. Phase 3 fonts have more consistent internal metrics which makes them amenable
    to being merged together, which we make use of. The older fonts are still usable,
    but cannot be merged together.

    Noto also contains both standard and "UI" variants of many fonts. When a font has a
    UI variant, it means that some of the glyphs in the standard variant are very tall
    and might overflow a typical line of text; the UI variant has the glypsh redrawn
    to fit.

    When searching for fonts to include, we take all language fonts that have both a
    regular and a bold variant, with preference given to Phase 3 and UI variants.
    """

    # grab the head of master
    if not ref:
        logging.info("Using head of master")
        ref = _request("git/refs/heads/master")["object"]["sha"]

    logging.info("Generating new manifest for reference '{}'".format(ref))

    git_tree = _request("git/trees/{}?recursive=1".format(ref))

    # backups
    font_info = _font_info(git_tree, ref, OLD_STYLE_PATH, _old_download_url)
    # prefer phase 3, replacing old-styles when possible
    font_info.update(_font_info(git_tree, ref, PHASE_3_PATH, _p3_download_url))

    new_manifest = {KEY_REF: ref, KEY_FONTS: font_info}
    utils.json_dump_formatted(new_manifest, FONTS_SOURCE, FONT_MANIFEST_NAME)

def fetch_fonts():
    """
    download ttf files from the manifest
    """

    if not os.path.exists(TTF_PATH):
        os.makedirs(TTF_PATH)

    # out with the old
    for file_name in os.listdir(TTF_PATH):
        os.unlink(os.path.join(TTF_PATH, file_name))

    # in with the new
    for font_name in FONT_MANIFEST:
        font_info = FONT_MANIFEST[font_name]

        # regular
        output_path = get_path(font_name, False)
        logging.info("Writing {}".format(output_path))
        r = requests.get(font_info[KEY_REG_URL])
        r.raise_for_status()
        with open(output_path, "wb") as f:
            f.write(r.content)

        # bold
        output_path = get_path(font_name, True)
        logging.info("Writing {}".format(output_path))
        r = requests.get(font_info[KEY_BOLD_URL])
        r.raise_for_status()
        with open(output_path, "wb") as f:
            f.write(r.content)

@utils.memoize
def get_path(font_name, is_bold=False):
    info = FONT_MANIFEST[font_name]
    path = info[KEY_BOLD_URL] if is_bold else info[KEY_REG_URL]
    return os.path.join(TTF_PATH, os.path.basename(path))
<|endoftext|>


<commit_before>"""Notifier that sends email messages through SMTP"""

import logging
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from notifiers.base_notifier import Notifier

_logger = logging.getLogger(__name__)

class EmailNotifier(Notifier):
    """Notifier class to work with email"""

    def __init__(self, config):
        """Override init to make SMTP-settings check"""
        self.fromaddr = config['from_email']
        # smtp user may be different from email
        self.fromuser = config.get('from_user', self.fromaddr)
        self.frompwd = config['from_pwd']
        self.host = config['from_smtp_host']
        self.port = config['from_smtp_port']
        self.toaddr = config['to_email']
        super(EmailNotifier, self).__init__(config)

    def check_requirements(self):
        """Logs in to SMTP server to check credentials and settings"""
        server = smtplib.SMTP(self.host, self.port)
        server.ehlo()
        server.starttls()
        server.ehlo()
        try:
            server.login(self.fromuser, self.frompwd)
        except Exception as ex:
            _logger.error("Cannot connect to your SMTP account. "
                          "Correct your config and try again. Error details:")
            _logger.error(ex)
            raise
        _logger.info("SMTP server check passed")

    def notify(self, title, text, url=False):
        """Send email notification using SMTP"""
        msg = MIMEMultipart()
        msg['From'] = self.fromaddr
        msg['To'] = self.toaddr
        msg['Subject'] = title
        body = text + '\nURL: ' + url
        msg.attach(MIMEText(body, 'plain'))
        server = smtplib.SMTP(self.host, self.port)
        server.ehlo()
        server.starttls()
        server.ehlo()
        server.login(self.fromuser, self.frompwd)
        text = msg.as_string()
        server.sendmail(self.fromaddr, self.toaddr, text)
<commit_msg>Do not try to login if from_user and from_pwd are empty string.<commit_after>"""Notifier that sends email messages through SMTP"""

import logging
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from notifiers.base_notifier import Notifier

_logger = logging.getLogger(__name__)

class EmailNotifier(Notifier):
    """Notifier class to work with email"""

    def __init__(self, config):
        """Override init to make SMTP-settings check"""
        self.fromaddr = config['from_email']
        # smtp user may be different from email
        self.fromuser = config.get('from_user', self.fromaddr)
        self.frompwd = config['from_pwd']
        self.host = config['from_smtp_host']
        self.port = config['from_smtp_port']
        self.toaddr = config['to_email']
        self.login_required = config['from_user'] and config['from_pwd']
        super(EmailNotifier, self).__init__(config)

    def check_requirements(self):
        """Logs in to SMTP server to check credentials and settings"""
        server = smtplib.SMTP(self.host, self.port)
        server.ehlo()
        server.starttls()
        server.ehlo()
        try:
            if self.login_required:
                server.login(self.fromuser, self.frompwd)
        except Exception as ex:
            _logger.error("Cannot connect to your SMTP account. "
                          "Correct your config and try again. Error details:")
            _logger.error(ex)
            raise
        _logger.info("SMTP server check passed")

    def notify(self, title, text, url=False):
        """Send email notification using SMTP"""
        msg = MIMEMultipart()
        msg['From'] = self.fromaddr
        msg['To'] = self.toaddr
        msg['Subject'] = title
        body = text + '\nURL: ' + url
        msg.attach(MIMEText(body, 'plain'))
        server = smtplib.SMTP(self.host, self.port)
        server.ehlo()
        server.starttls()
        server.ehlo()
        if self.login_required:
          server.login(self.fromuser, self.frompwd)
        text = msg.as_string()
        server.sendmail(self.fromaddr, self.toaddr, text)
<|endoftext|>


<commit_before>import ast
import logging
import sys
import time

from django.core.mail import send_mass_mail
from django.core.management.base import BaseCommand
from django.template import loader

from cl import settings
from cl.lib.argparse_types import csv_list
from cl.lib.utils import chunks
from cl.stats import tally_stat
from cl.users.models import UserProfile

logger = logging.getLogger(__name__)
CHUNK_SIZE = 50

class Command(BaseCommand):
    help = ('Sends a PR email to the people listed in recipients field and/or '
            'that are subscribed to the CL newsletter')

    def add_arguments(self, parser):
        parser.add_argument(
            '--recipients',
            type=csv_list,
            required=True,
            help='A list of recipients you wish to send the email to as comma '
                 'separated values.'
        ),
        parser.add_argument(
            '--subject',
            required=True,
            help='The subject of the email you wish to send.',
        ),
        parser.add_argument(
            '--sender',
            default=settings.DEFAULT_FROM_EMAIL,
            help='The sender for the email, default is from your settings.py '
                 'file.'
        ),
        parser.add_argument(
            '--template',
            required=True,
            help='What is the path, relative to the template directory, where '
                 'your email template lives?'
        ),
        parser.add_argument(
            '--context',
            default='{}',
            help='A dict containing the context for the email, if desired.'
        ),
        parser.add_argument(
            '--simulate',
            action='store_true',
            default=False,
            help="Don\'t send any emails, just pretend.",
        )

    def send_emails(self, recipients):
        """Send the emails using the templates and contexts requested."""
        email_subject = self.options['subject']
        email_sender = self.options['sender']
        txt_template = loader.get_template(self.options['template'])
        context = ast.literal_eval(self.options['context'])
        email_txt = txt_template.render(context)

        emails_sent_count = 0
        messages = []
        for recipient in recipients:
            messages.append((
                email_subject,
                email_txt,
                email_sender,
                [recipient],
            ))
            emails_sent_count += 1

        if not self.options['simulate']:
            send_mass_mail(messages, fail_silently=False)
            tally_stat('pr.emails.sent', inc=emails_sent_count)
            logger.info("Sent %s pr emails." % emails_sent_count)
        else:
            sys.stdout.write('Simulation mode. Imagine that we just sent %s '
                             'emails!\n' % emails_sent_count)

    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        self.options = options

        recipients = set()

        if options.get('subscribers'):
            for up in UserProfile.objects.filter(wants_newsletter=True):
                recipients.add(up.user.email)
        if options.get('recipients'):
            recipients.update(options['recipients'])
        if 'courtlistener' not in self.options['sender']:
            press_on = raw_input(
                "'CourtListener' isn't in your sender email address. In the "
                "past this has resulted in a lot of emails being marked as "
                "spam. Continue [y/n]? "
            )
            if press_on.lower() != 'y':
                exit(1)
        if not options['subject']:
            sys.stderr.write('No subject provided. Aborting.\n')
            exit(1)
        if not options['template']:
            sys.stderr.write('No template provided. Aborting.\n')
            exit(1)
        if recipients:
            if options['simulate']:
                sys.stdout.write("**********************************\n")
                sys.stdout.write("* SIMULATE MODE - NO EMAILS SENT *\n")
                sys.stdout.write("**********************************\n")
            if not options['simulate']:
                for chunk in chunks(recipients, CHUNK_SIZE):
                    self.send_emails(chunk)
                    logger.info("Waiting 300s before sending another chunk of "
                                "%s emails." % CHUNK_SIZE)
                    time.sleep(300)
            else:
                self.send_emails(recipients)
        else:
            sys.stderr.write("No recipients defined. Aborting.\n")
            exit(1)
<commit_msg>Fixes chunking.<commit_after>import ast
import logging
import sys
import time

from django.core.mail import send_mass_mail
from django.core.management.base import BaseCommand
from django.template import loader

from cl import settings
from cl.lib.argparse_types import csv_list
from cl.lib.utils import chunks
from cl.stats import tally_stat
from cl.users.models import UserProfile

logger = logging.getLogger(__name__)
CHUNK_SIZE = 50

class Command(BaseCommand):
    help = ('Sends a PR email to the people listed in recipients field and/or '
            'that are subscribed to the CL newsletter')

    def add_arguments(self, parser):
        parser.add_argument(
            '--recipients',
            type=csv_list,
            required=True,
            help='A list of recipients you wish to send the email to as comma '
                 'separated values.'
        ),
        parser.add_argument(
            '--subject',
            required=True,
            help='The subject of the email you wish to send.',
        ),
        parser.add_argument(
            '--sender',
            default=settings.DEFAULT_FROM_EMAIL,
            help='The sender for the email, default is from your settings.py '
                 'file.'
        ),
        parser.add_argument(
            '--template',
            required=True,
            help='What is the path, relative to the template directory, where '
                 'your email template lives?'
        ),
        parser.add_argument(
            '--context',
            default='{}',
            help='A dict containing the context for the email, if desired.'
        ),
        parser.add_argument(
            '--simulate',
            action='store_true',
            default=False,
            help="Don\'t send any emails, just pretend.",
        )

    def send_emails(self, recipients):
        """Send the emails using the templates and contexts requested."""
        email_subject = self.options['subject']
        email_sender = self.options['sender']
        txt_template = loader.get_template(self.options['template'])
        context = ast.literal_eval(self.options['context'])
        email_txt = txt_template.render(context)

        emails_sent_count = 0
        messages = []
        for recipient in recipients:
            messages.append((
                email_subject,
                email_txt,
                email_sender,
                [recipient],
            ))
            emails_sent_count += 1

        if not self.options['simulate']:
            send_mass_mail(messages, fail_silently=False)
            tally_stat('pr.emails.sent', inc=emails_sent_count)
            logger.info("Sent %s pr emails." % emails_sent_count)
        else:
            sys.stdout.write('Simulation mode. Imagine that we just sent %s '
                             'emails!\n' % emails_sent_count)

    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        self.options = options

        recipients = set()

        if options.get('subscribers'):
            for up in UserProfile.objects.filter(wants_newsletter=True):
                recipients.add(up.user.email)
        if options.get('recipients'):
            recipients.update(options['recipients'])
        if 'courtlistener' not in self.options['sender']:
            press_on = raw_input(
                "'CourtListener' isn't in your sender email address. In the "
                "past this has resulted in a lot of emails being marked as "
                "spam. Continue [y/n]? "
            )
            if press_on.lower() != 'y':
                exit(1)
        if not options['subject']:
            sys.stderr.write('No subject provided. Aborting.\n')
            exit(1)
        if not options['template']:
            sys.stderr.write('No template provided. Aborting.\n')
            exit(1)
        if recipients:
            if options['simulate']:
                sys.stdout.write("**********************************\n")
                sys.stdout.write("* SIMULATE MODE - NO EMAILS SENT *\n")
                sys.stdout.write("**********************************\n")
            if not options['simulate']:
                for chunk in chunks(list(recipients), CHUNK_SIZE):
                    self.send_emails(chunk)
                    logger.info("Waiting 300s before sending another chunk of "
                                "%s emails." % CHUNK_SIZE)
                    time.sleep(300)
            else:
                self.send_emails(recipients)
        else:
            sys.stderr.write("No recipients defined. Aborting.\n")
            exit(1)
<|endoftext|>


<commit_before>database = 'gpadmin'
host = 'localhost'
port = '5431'
username = 'gpadmin'
password = ''
<commit_msg>Updated params.py<commit_after>database = #
host = #
port = #
username = #
password = #
<|endoftext|>


<commit_before>__version__ = '0.0.1a'<commit_msg>version bump to 0.0.2a1<commit_after>__version__ = '0.0.2a1'
<|endoftext|>