<commit_before>""" Gaussian process emulator """

import logging
import pickle

import numpy as np
from sklearn.decomposition import PCA
from sklearn.externals import joblib
from sklearn.gaussian_process import GaussianProcessRegressor as GPR
from sklearn.gaussian_process import kernels
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

from . import cachedir, lazydict, model
from .design import Design

class Emulator:
    """
    Multidimensional Gaussian process emulator using principal component
    analysis.

    """
    # observables to emulate
    # list of 2-tuples: (obs, [list of subobs])
    observables = [
        ('dNch_deta', [None]),
        ('dET_deta', [None]),
        ('dN_dy', ['pion', 'kaon', 'proton']),
        ('mean_pT', ['pion', 'kaon', 'proton']),
        ('vnk', [(2, 2), (3, 2), (4, 2)]),
    ]

    def __init__(self, system, npc=10, nrestarts=0):
        logging.info(
            'training emulator for system %s (%d PC, %d restarts)',
            system, npc, nrestarts
        )

        arrays = []
        self._slices = {}

        index = 0
        for obs, subobslist in self.observables:
            self._slices[obs] = {}
            for subobs in subobslist:
                Y = model.data[system][obs][subobs]['Y']
                arrays.append(Y)
                n = Y.shape[1]
                self._slices[obs][subobs] = slice(index, index + n)
                index += n

        # StandardScaler with_mean is unnecessary since PCA removes the mean
        self.scaler = StandardScaler(with_mean=False, copy=False)
        self.pca = PCA(npc, copy=False, whiten=True, svd_solver='full')
        self.pipeline = make_pipeline(self.scaler, self.pca)

        Z = self.pipeline.fit_transform(np.concatenate(arrays, axis=1))

        design = Design(system)
        ptp = design.max - design.min
        kernel = (
            1. * kernels.RBF(
                length_scale=ptp,
                length_scale_bounds=np.outer(ptp, (.1, 10))
            ) +
            kernels.WhiteKernel(
                noise_level=.01,
                noise_level_bounds=(1e-8, 10)
            )
        )

        self.gps = [
            GPR(
                kernel=kernel, alpha=0,
                n_restarts_optimizer=nrestarts,
                copy_X_train=False
            ).fit(design, z)
            for z in Z.T
        ]

    @classmethod
    def from_cache(cls, system, retrain=False, **kwargs):
        """
        Load from the cache if available, otherwise create and cache a new
        instance.

        """
        cachefile = cachedir / 'emulator' / '{}.pkl'.format(system)

        # cache the __dict__ rather than the Emulator instance itself
        # this way the __name__ doesn't matter, e.g. a pickled
        # __main__.Emulator can be unpickled as a src.emulator.Emulator
        if not retrain and cachefile.exists():
            logging.debug('loading emulator for system %s from cache', system)
            emu = cls.__new__(cls)
            emu.__dict__ = joblib.load(cachefile)
            return emu

        emu = cls(system, **kwargs)

        logging.info('writing cache file %s', cachefile)
        cachefile.parent.mkdir(exist_ok=True)
        joblib.dump(emu.__dict__, cachefile, protocol=pickle.HIGHEST_PROTOCOL)

        return emu

    def _results_dict(self, Y):
        """
        Unpack an array of predictions into a nested dict of observables.

        """
        return {
            obs: {
                subobs: Y[..., s]
                for subobs, s in slices.items()
            } for obs, slices in self._slices.items()
        }

    def predict(self, X):
        """
        Predict mean model output at X.

        Returns a nested dict of observable arrays, each with shape
        (n_samples_X, n_cent_bins).

        """
        return self._results_dict(
            self.pipeline.inverse_transform(
                np.concatenate([
                    gp.predict(X)[:, np.newaxis] for gp in self.gps
                ], axis=1)
            )
        )

    def var(self):
        """
        Compute the predictive variance.

        Returns a nested dict of the variance for each observable.

        The variance is computed by transforming the GP noise terms into
        physical space and taking the diagonal components; this is a good
        estimate except near the edges of the design space.

        """
        # TODO this is a rudimentary implementation, to be improved
        return self._results_dict(
            np.einsum(
                'ij,ij,i,i,j->j',
                self.pca.components_, self.pca.components_,
                self.pca.explained_variance_,
                [gp.kernel_.k2.noise_level for gp in self.gps],
                self.scaler.var_
            )
        )

    def sample_y(self, X, n_samples=1, random_state=None):
        """
        Sample model output at X.

        Returns a nested dict of observable arrays, each with shape
        (n_samples_X, n_samples, n_cent_bins).

        """
        return self._results_dict(
            self.pipeline.inverse_transform(
                np.concatenate([
                    gp.sample_y(
                        X, n_samples=n_samples, random_state=random_state
                    ).reshape(-1, 1)
                    for gp in self.gps
                ], axis=1)
            ).reshape(X.shape[0], n_samples, -1)
        )

emulators = lazydict(Emulator.from_cache)

if __name__ == '__main__':
    import argparse
    from . import systems

    def arg_to_system(arg):
        if arg not in systems:
            raise argparse.ArgumentTypeError(arg)
        return arg

    parser = argparse.ArgumentParser(
        description='train emulators for each collision system',
        argument_default=argparse.SUPPRESS
    )

    parser.add_argument(
        '--npc', type=int,
        help='number of principal components'
    )
    parser.add_argument(
        '--nrestarts', type=int,
        help='number of optimizer restarts'
    )

    parser.add_argument(
        '--retrain', action='store_true',
        help='retrain even if emulator is cached'
    )
    parser.add_argument(
        'systems', nargs='*', type=arg_to_system,
        default=systems, metavar='SYSTEM',
        help='system(s) to train'
    )

    args = parser.parse_args()
    kwargs = vars(args)

    for s in kwargs.pop('systems'):
        emu = Emulator.from_cache(s, **kwargs)

        print(s)
        print('{} PCs explain {:.5f} of variance'.format(
            emu.pca.n_components_,
            emu.pca.explained_variance_ratio_.sum()
        ))

        for n, (evr, gp) in enumerate(zip(
                emu.pca.explained_variance_ratio_, emu.gps
        )):
            print(
                'GP {}: {:.5f} of variance, kernel: {}'
                .format(n, evr, gp.kernel_)
            )
<commit_msg>Implement inverse transform of emulator predictive covariance.<commit_after>""" Gaussian process emulator """

import logging
import pickle

import numpy as np
from sklearn.decomposition import PCA
from sklearn.externals import joblib
from sklearn.gaussian_process import GaussianProcessRegressor as GPR
from sklearn.gaussian_process import kernels
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

from . import cachedir, lazydict, model
from .design import Design

class _Covariance:
    """
    Proxy object to extract observable sub-blocks from a covariance array.
    Returned by Emulator.predict().

    """
    def __init__(self, array, slices):
        self.array = array
        self._slices = slices

    def __getitem__(self, key):
        (obs1, subobs1), (obs2, subobs2) = key
        return self.array[
            ...,
            self._slices[obs1][subobs1],
            self._slices[obs2][subobs2]
        ]

class Emulator:
    """
    Multidimensional Gaussian process emulator using principal component
    analysis.

    """
    # observables to emulate
    # list of 2-tuples: (obs, [list of subobs])
    observables = [
        ('dNch_deta', [None]),
        ('dET_deta', [None]),
        ('dN_dy', ['pion', 'kaon', 'proton']),
        ('mean_pT', ['pion', 'kaon', 'proton']),
        ('vnk', [(2, 2), (3, 2), (4, 2)]),
    ]

    def __init__(self, system, npc=10, nrestarts=0):
        logging.info(
            'training emulator for system %s (%d PC, %d restarts)',
            system, npc, nrestarts
        )

        arrays = []
        self._slices = {}

        index = 0
        for obs, subobslist in self.observables:
            self._slices[obs] = {}
            for subobs in subobslist:
                Y = model.data[system][obs][subobs]['Y']
                arrays.append(Y)
                n = Y.shape[1]
                self._slices[obs][subobs] = slice(index, index + n)
                index += n

        # StandardScaler with_mean is unnecessary since PCA removes the mean
        self.scaler = StandardScaler(with_mean=False, copy=False)
        self.pca = PCA(npc, copy=False, whiten=True, svd_solver='full')
        # XXX Although a pipeline is used here for convenience, the scaler and
        # pca objects are accessed directly when inverse transforming
        # predictive covariances.  If the pipeline is modified, self.predict()
        # must be updated accordingly.
        self.pipeline = make_pipeline(self.scaler, self.pca)

        Z = self.pipeline.fit_transform(np.concatenate(arrays, axis=1))

        design = Design(system)
        ptp = design.max - design.min
        kernel = (
            1. * kernels.RBF(
                length_scale=ptp,
                length_scale_bounds=np.outer(ptp, (.1, 10))
            ) +
            kernels.WhiteKernel(
                noise_level=.01,
                noise_level_bounds=(1e-8, 10)
            )
        )

        self.gps = [
            GPR(
                kernel=kernel, alpha=0,
                n_restarts_optimizer=nrestarts,
                copy_X_train=False
            ).fit(design, z)
            for z in Z.T
        ]

    @classmethod
    def from_cache(cls, system, retrain=False, **kwargs):
        """
        Load from the cache if available, otherwise create and cache a new
        instance.

        """
        cachefile = cachedir / 'emulator' / '{}.pkl'.format(system)

        # cache the __dict__ rather than the Emulator instance itself
        # this way the __name__ doesn't matter, e.g. a pickled
        # __main__.Emulator can be unpickled as a src.emulator.Emulator
        if not retrain and cachefile.exists():
            logging.debug('loading emulator for system %s from cache', system)
            emu = cls.__new__(cls)
            emu.__dict__ = joblib.load(cachefile)
            return emu

        emu = cls(system, **kwargs)

        logging.info('writing cache file %s', cachefile)
        cachefile.parent.mkdir(exist_ok=True)
        joblib.dump(emu.__dict__, cachefile, protocol=pickle.HIGHEST_PROTOCOL)

        return emu

    def _results_dict(self, Y):
        """
        Unpack an array of predictions into a nested dict of observables.

        """
        return {
            obs: {
                subobs: Y[..., s]
                for subobs, s in slices.items()
            } for obs, slices in self._slices.items()
        }

    def predict(self, X, return_cov=False):
        """
        Predict model output at X.

        X must be a 2D array-like with shape (nsamples, ndim).  It is passed
        directly to sklearn GaussianProcessRegressor.predict().

        If return_cov is true, return a tuple (mean, cov), otherwise only
        return the mean.

        The mean is returned as a nested dict of observable arrays, each with
        shape (nsamples, n_cent_bins).

        The covariance is returned as a proxy object which extracts observable
        sub-blocks using a dict-like interface:

        >>> mean, cov = emulator.predict(X, return_cov=True)

        >>> mean['dN_dy']['pion']
        <mean prediction of pion dN/dy>

        >>> cov[('dN_dy', 'pion'), ('dN_dy', 'pion')]
        <covariance matrix of pion dN/dy>

        >>> cov[('dN_dy', 'pion'), ('mean_pT', 'kaon')]
        <covariance matrix between pion dN/dy and kaon mean pT>

        The shape of the extracted covariance blocks are
        (nsamples, n_cent_bins_1, n_cent_bins_2).

        NB: the covariance is only computed between observables and centrality
        bins, not between sample points.

        """
        gp_mean = [gp.predict(X, return_cov=return_cov) for gp in self.gps]

        if return_cov:
            gp_mean, gp_cov = zip(*gp_mean)

        # inverse transform the predictive mean
        mean = self._results_dict(
            self.pipeline.inverse_transform(
                np.concatenate([
                    m[:, np.newaxis] for m in gp_mean
                ], axis=1)
            )
        )

        if return_cov:
            # Now transform the predictive variance in PC space to physical
            # space.  Assuming the PCs are uncorrelated, the transformation is
            #
            #   cov_ij = sum_k A_ik var_k A_jk
            #
            # where A is the PC transformation matrix and var_k is the
            # variance of the kth PC.
            # https://en.wikipedia.org/wiki/Propagation_of_uncertainty

            # In the einsum calls below, explicitly disable optimization (which
            # attempts to find the fastest computational path) because the path
            # is already optimized "by hand".

            # Create the linear transformation matrix.
            # shape: (npc, nobs)
            A = np.einsum(
                'ij,i,j->ij',
                self.pca.components_,
                np.sqrt(self.pca.explained_variance_),
                self.scaler.scale_,
                optimize=False
            )

            # At this point, the transformation can be accomplished by
            #
            #   einsum('ki,ka,kj->aij', A, gp_var, A)
            #
            # however it's faster to first expand the ij indices and then sum
            # over k using a dot product.

            # Perform the first part of the transformation.
            # shape: (npc, nobs, nobs)
            A = np.einsum('ki,kj->kij', A, A, optimize=False)

            # Build array of the GP predictive variances at each sample point.
            # shape: (nsamples, npc)
            gp_var = np.concatenate([
                c.diagonal()[:, np.newaxis] for c in gp_cov
            ], axis=1)

            # Compute the covariance at each sample point.
            # (Reshaping A to a 2D array allows np.dot to use BLAS.)
            npc, nobs = self.pca.components_.shape
            cov = np.dot(gp_var, A.reshape(npc, -1)).reshape(-1, nobs, nobs)

            return mean, _Covariance(cov, self._slices)
        else:
            return mean

    def var(self):
        """
        Compute the predictive variance.

        Returns a nested dict of the variance for each observable.

        The variance is computed by transforming the GP noise terms into
        physical space and taking the diagonal components; this is a good
        estimate except near the edges of the design space.

        """
        # TODO this is a rudimentary implementation, to be improved
        return self._results_dict(
            np.einsum(
                'ij,ij,i,i,j->j',
                self.pca.components_, self.pca.components_,
                self.pca.explained_variance_,
                [gp.kernel_.k2.noise_level for gp in self.gps],
                self.scaler.var_
            )
        )

    def sample_y(self, X, n_samples=1, random_state=None):
        """
        Sample model output at X.

        Returns a nested dict of observable arrays, each with shape
        (n_samples_X, n_samples, n_cent_bins).

        """
        return self._results_dict(
            self.pipeline.inverse_transform(
                np.concatenate([
                    gp.sample_y(
                        X, n_samples=n_samples, random_state=random_state
                    ).reshape(-1, 1)
                    for gp in self.gps
                ], axis=1)
            ).reshape(X.shape[0], n_samples, -1)
        )

emulators = lazydict(Emulator.from_cache)

if __name__ == '__main__':
    import argparse
    from . import systems

    def arg_to_system(arg):
        if arg not in systems:
            raise argparse.ArgumentTypeError(arg)
        return arg

    parser = argparse.ArgumentParser(
        description='train emulators for each collision system',
        argument_default=argparse.SUPPRESS
    )

    parser.add_argument(
        '--npc', type=int,
        help='number of principal components'
    )
    parser.add_argument(
        '--nrestarts', type=int,
        help='number of optimizer restarts'
    )

    parser.add_argument(
        '--retrain', action='store_true',
        help='retrain even if emulator is cached'
    )
    parser.add_argument(
        'systems', nargs='*', type=arg_to_system,
        default=systems, metavar='SYSTEM',
        help='system(s) to train'
    )

    args = parser.parse_args()
    kwargs = vars(args)

    for s in kwargs.pop('systems'):
        emu = Emulator.from_cache(s, **kwargs)

        print(s)
        print('{} PCs explain {:.5f} of variance'.format(
            emu.pca.n_components_,
            emu.pca.explained_variance_ratio_.sum()
        ))

        for n, (evr, gp) in enumerate(zip(
                emu.pca.explained_variance_ratio_, emu.gps
        )):
            print(
                'GP {}: {:.5f} of variance, kernel: {}'
                .format(n, evr, gp.kernel_)
            )
<|endoftext|>


<commit_before>"""
Perform prediction results evaluation
"""
from itertools import zip_longest
import json
import argparse

def evaluate(text_file, correct_file, submission_file):
    with open(text_file) as f:
        text = json.load(f)
    with open(correct_file) as f:
        correct = json.load(f)
    with open(submission_file) as f:
        submission = json.load(f)
    data = []
    for sent, cor, sub in zip_longest(text, correct, submission):
        for w, c, s in zip_longest(sent, cor, sub):
            if w.lower() in ['a', 'an', 'the']:
                if s is None or s[0] == w.lower():
                    s = ['', float('-inf')]
                data.append((-s[1], s[0] == c, c is not None))
    data.sort()
    fp2 = 0
    fp = 0
    tp = 0
    all_mistakes = sum(x[2] for x in data)
    score = 0
    acc = 0
    for _, c, r in data:
        fp2 += not c
        fp += not r
        tp += c
        acc = max(acc, 1 - (0. + fp + all_mistakes - tp) / len(data))
        if fp2 * 1. / len(data) <= 0.02:
            score = tp * 1. / all_mistakes
    print("# of mistakes: %d" % (all_mistakes))
    print( 'target score = %.2f %%' % (score * 100))
    print( 'accuracy (just for info) = %.2f %%' % (acc * 100))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='The prediction results evaluator')
    parser.add_argument('--results_file', required = True,
                        help='the path to the file with predictions resulst')
    parser.add_argument('--test_sentences_file', required = True,
                        help="the text's corpora file for test data")
    parser.add_argument('--test_corrections_file', required = True,
                        help='the path to the file with ethalon corrections')
    args = parser.parse_args()

    evaluate(text_file = args.test_sentences_file,
             correct_file = args.test_corrections_file,
             submission_file = args.results_file)
<commit_msg>Removed obsolete output<commit_after>"""
Perform prediction results evaluation
"""
from itertools import zip_longest
import json
import argparse

def evaluate(text_file, correct_file, submission_file):
    with open(text_file) as f:
        text = json.load(f)
    with open(correct_file) as f:
        correct = json.load(f)
    with open(submission_file) as f:
        submission = json.load(f)
    data = []
    for sent, cor, sub in zip_longest(text, correct, submission):
        for w, c, s in zip_longest(sent, cor, sub):
            if w.lower() in ['a', 'an', 'the']:
                if s is None or s[0] == w.lower():
                    s = ['', float('-inf')]
                data.append((-s[1], s[0] == c, c is not None))
    data.sort()
    fp2 = 0
    fp = 0
    tp = 0
    all_mistakes = sum(x[2] for x in data)
    score = 0
    acc = 0
    for _, c, r in data:
        fp2 += not c
        fp += not r
        tp += c
        acc = max(acc, 1 - (0. + fp + all_mistakes - tp) / len(data))
        if fp2 * 1. / len(data) <= 0.02:
            score = tp * 1. / all_mistakes

    print( 'target score = %.2f %%' % (score * 100))
    print( 'accuracy (just for info) = %.2f %%' % (acc * 100))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='The prediction results evaluator')
    parser.add_argument('--results_file', required = True,
                        help='the path to the file with predictions resulst')
    parser.add_argument('--test_sentences_file', required = True,
                        help="the text's corpora file for test data")
    parser.add_argument('--test_corrections_file', required = True,
                        help='the path to the file with ethalon corrections')
    args = parser.parse_args()

    evaluate(text_file = args.test_sentences_file,
             correct_file = args.test_corrections_file,
             submission_file = args.results_file)
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" Converts an .exr (OpenEXR format) file's depth channgel to a .png.
http://www.blender.org/forum/viewtopic.php?p=98212&sid=4cef7d0c329b3c3fe366e66832482fc0

Peter Battaglia
2013.04.08
"""
# Standard
from argparse import Action, ArgumentError, ArgumentParser
import glob
from math import log10
from operator import add
import os
import sys
# External
import Imath
import OpenEXR
import numpy as np
from scipy.misc import imsave
#
from pdb import set_trace as BP

def load_exr(filename):
    """ Loads .exr file."""
    exrimg = OpenEXR.InputFile(filename)
    return exrimg

def get_exr_dims(exrimg):
    dw = exrimg.header()['dataWindow']
    width = dw.max.x - dw.min.x + 1
    height = dw.max.y - dw.min.y + 1
    return width, height

def get_channels(exrimg, outchans="RGBA"):
    """ Get the separate channels.
    Possible values are: R, G, B, A, Z"""
    def fromstr(s, width, height):
        mat = np.fromstring(s, dtype=np.float16)
        mat = mat.reshape(height, width)
        return mat

    w, h = get_exr_dims(exrimg)
    pt = Imath.PixelType(Imath.PixelType.HALF)
    channels = [fromstr(s, w, h) for s in exrimg.channels(outchans, pt)]
    return channels

def convert(inname, outname, outchans, normchans, nanfill):
    """ Convert one file."""
    # Load the .exr file.
    exrimg = load_exr(inname)
    # Get the channels.
    channels = get_channels(exrimg, outchans=outchans)
    # Compose image data into what is necessary for output.
    outimg = np.dstack(channels)
    # Fix nans and infs.
    badidx = ~np.isfinite(outimg)
    if np.any(badidx):
        outimg[badidx] = nanfill
    # Normalize (optionally).
    if normchans:
        # Normalize A and Z channels independently from RGB channels.
        nchans = [set(normchans).intersection(c) for c in ("RGB", "A", "Z")]
        for c in nchans:
            idx = np.array([oc in c for oc in outchans])
            if idx.any():
                outimg[:, :, idx] /= outimg[:, :, idx].max()
        # Reset any nanfill values that were changed by normalizing.
        # what was specified.
        outimg[badidx] = nanfill
    # Squeeze out length-1 dimensions.
    outimg = np.squeeze(outimg)
    # Write output file.
    imsave(outname, outimg)

if __name__ == "__main__":
    # Parse input arguments.

    ## Cmd line interface.
    # Create parser object.
    parser = ArgumentParser(description="Output options.")
    group = parser.add_mutually_exclusive_group()
    # Argument: input file name.
    parser.add_argument("innames", nargs="+", help="Input filename(s).")
    # Argument: output file name.
    parser.add_argument("-o", dest="outname", default="",
                        help=("Output file name. If extension is "
                              "present, it overrides the default."))
    # Argument: suffix to append to output file names.
    parser.add_argument("--suffix", default="",
                        help="Suffix to append to output file names.")
    # Argument: output file format.
    parser.add_argument("-f", dest="outfmt", default=None,
                        help="Output file format. Overrides -o extension.")
    # Argument: output file channels.
    group.add_argument("-c", dest="outchans",
                        default="RGBA", help="Output channels.")
    # Argument: normalize output image.
    parser.add_argument("-n", dest="normchans", default="", const="RGBAZ",
                        nargs="?", help="Normalize output image's channels.")
    # Argument: Z-channel for output file.
    class ZAction(Action):
        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, "outchans", "Z")
            normchans = getattr(namespace, "normchans")
            if "Z" not in normchans:
                setattr(namespace, "normchans", normchans + "Z")
            delattr(namespace, "Z")
    group.add_argument("-Z", nargs=0, action=ZAction,
                        help="Output normalized Z channel. Same as -c Z -n.")
    # Argument: value to fill nans with.
    parser.add_argument("--nanfill", default=0,
                        help="Value to replace nans with.")
    # Argument: allow output files to overwrite existing ones.
    parser.add_argument("--force", action="store_true",
                        help="Allow output files to overwrite existing ones.")
    # Create parser and parse args.
    parsed = parser.parse_args()
    innames = reduce(add, [sorted(glob.glob(fn)) for fn in parsed.innames])
    outname = parsed.outname
    suffix = parsed.suffix
    outfmt = parsed.outfmt
    outchans = parsed.outchans
    normchans = parsed.normchans
    nanfill = parsed.nanfill
    force = parsed.force
    # Set parameters according to inputs.
    N = len(innames)
    if N == 0:
        raise IOError("Cannot find input files that match: %s" %
                      ", ".join(parsed.innames))
    # Output format.
    if outfmt is None:
        # If no outfmt was input, try getting it from the outname.
        outfmt = os.path.splitext(outname)[1].lstrip(".")
        if not outfmt:
            # If outname also doesn't specify outfmt, default to .png.
            outfmt = "png"
    # Output file's base name without extension.
    outname = os.path.splitext(outname)[0]
    # Output file names.
    nametemplate1 = "%%s%s.%s" % (suffix, outfmt)
    nametemplateN = "%%s_%%0%dd%s.%s" % (int(log10(N)) + 1, suffix, outfmt)
    outnames = []
    for i, inname in enumerate(innames):
        if not outname:
            # If outname is empty, use the inname.
            on = os.path.splitext(inname)[0]
        else:
            # If outname was supplied, use it and append count if
            # there is more than one inname.
            if N == 1:
                on = nametemplate1 % outname
            else:
                on = nametemplateN % (outname, i)
        # Raise error if: we try to overwrite the input file; we try
        # to overwrite an existing file and haven't input 'force'.
        if inname == on or not force and os.path.isfile(on):
            raise IOError("Attempted to overwrite: %s" % on)
        # Store output file name.
        outnames.append(on)
    # Run the converter over all files.
    for inname, outname in zip(innames, outnames):
        # Do the conversion.
        convert(inname, outname, outchans, normchans, nanfill)
<commit_msg>Comments<commit_after>#!/usr/bin/env python
""" Converts an .exr (OpenEXR format) file's depth channgel to a .png.
http://www.blender.org/forum/viewtopic.php?p=98212&sid=4cef7d0c329b3c3fe366e66832482fc0

Peter Battaglia
2013.04.08
"""
# Standard
from argparse import Action, ArgumentError, ArgumentParser
import glob
from math import log10
from operator import add
import os
import sys
# External
import Imath
import OpenEXR
import numpy as np
from scipy.misc import imsave
#
from pdb import set_trace as BP

def load_exr(filename):
    """ Loads .exr file."""
    exrimg = OpenEXR.InputFile(filename)
    return exrimg

def get_exr_dims(exrimg):
    dw = exrimg.header()['dataWindow']
    width = dw.max.x - dw.min.x + 1
    height = dw.max.y - dw.min.y + 1
    return width, height

def get_channels(exrimg, outchans="RGBA"):
    """ Get the separate channels.
    Possible values are: R, G, B, A, Z"""
    def fromstr(s, width, height):
        mat = np.fromstring(s, dtype=np.float16)
        mat = mat.reshape(height, width)
        return mat

    w, h = get_exr_dims(exrimg)
    pt = Imath.PixelType(Imath.PixelType.HALF)
    channels = [fromstr(s, w, h) for s in exrimg.channels(outchans, pt)]
    return channels

def convert(inname, outname, outchans, normchans, nanfill):
    """ Convert one file."""
    # Load the .exr file.
    exrimg = load_exr(inname)
    # Get the channels.
    channels = get_channels(exrimg, outchans=outchans)
    # Compose image data into what is necessary for output.
    outimg = np.dstack(channels)
    # Fix nans and infs.
    badidx = ~np.isfinite(outimg)
    if np.any(badidx):
        outimg[badidx] = nanfill
    # Normalize (optionally).
    if normchans:
        # Normalize A and Z channels independently from RGB channels.
        nchans = [set(normchans).intersection(c) for c in ("RGB", "A", "Z")]
        for c in nchans:
            idx = np.array([oc in c for oc in outchans])
            if idx.any():
                outimg[:, :, idx] /= outimg[:, :, idx].max()
        # Reset any nanfill values that were changed by normalizing.
        # what was specified.
        outimg[badidx] = nanfill
    # Squeeze out length-1 dimensions.
    outimg = np.squeeze(outimg)
    # Write output file.
    imsave(outname, outimg)

if __name__ == "__main__":
    # Parse input arguments.

    ## Cmd line interface.
    # Create parser object.
    parser = ArgumentParser(description="Output options.")
    group = parser.add_mutually_exclusive_group()
    # Argument: input file name.
    parser.add_argument("innames", nargs="+", help="Input filename(s).")
    # Argument: output file name.
    parser.add_argument("-o", dest="outname", default="",
                        help=("Output file name. If extension is "
                              "present, it overrides the default."))
    # Argument: suffix to append to output file names.
    parser.add_argument("--suffix", default="",
                        help="Suffix to append to output file names.")
    # Argument: output file format.
    parser.add_argument("-f", dest="outfmt", default=None,
                        help="Output file format. Overrides -o extension.")
    # Argument: output file channels.
    group.add_argument("-c", dest="outchans",
                        default="RGBA", help="Output channels.")
    # Argument: normalize output image.
    parser.add_argument("-n", dest="normchans", default="", const="RGBAZ",
                        nargs="?", help="Normalize output image's channels.")
    class ZAction(Action):
        """ Custom action for "-Z" argument. Adds Z as outchan and
        normchan."""
        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, "outchans", "Z")
            normchans = getattr(namespace, "normchans")
            if "Z" not in normchans:
                setattr(namespace, "normchans", normchans + "Z")
            delattr(namespace, "Z")
    # Argument: Z-channel for output file.
    group.add_argument("-Z", nargs=0, action=ZAction,
                        help="Output normalized Z channel. Same as -c Z -n.")
    # Argument: value to fill nans with.
    parser.add_argument("--nanfill", default=0,
                        help="Value to replace nans with.")
    # Argument: allow output files to overwrite existing ones.
    parser.add_argument("--force", action="store_true",
                        help="Allow output files to overwrite existing ones.")
    # Create parser and parse args.
    parsed = parser.parse_args()
    innames = reduce(add, [sorted(glob.glob(fn)) for fn in parsed.innames])
    outname = parsed.outname
    suffix = parsed.suffix
    outfmt = parsed.outfmt
    outchans = parsed.outchans
    normchans = parsed.normchans
    nanfill = parsed.nanfill
    force = parsed.force
    # Set parameters according to inputs.
    N = len(innames)
    if N == 0:
        raise IOError("Cannot find input files that match: %s" %
                      ", ".join(parsed.innames))
    # Output format.
    if outfmt is None:
        # If no outfmt was input, try getting it from the outname.
        outfmt = os.path.splitext(outname)[1].lstrip(".")
        if not outfmt:
            # If outname also doesn't specify outfmt, default to .png.
            outfmt = "png"
    # Output file's base name without extension.
    outname = os.path.splitext(outname)[0]
    # Output file names.
    nametemplate1 = "%%s%s.%s" % (suffix, outfmt)
    nametemplateN = "%%s_%%0%dd%s.%s" % (int(log10(N)) + 1, suffix, outfmt)
    outnames = []
    for i, inname in enumerate(innames):
        if not outname:
            # If outname is empty, use the inname.
            on = os.path.splitext(inname)[0]
        else:
            # If outname was supplied, use it and append count if
            # there is more than one inname.
            if N == 1:
                on = nametemplate1 % outname
            else:
                on = nametemplateN % (outname, i)
        # Raise error if: we try to overwrite the input file; we try
        # to overwrite an existing file and haven't input 'force'.
        if inname == on or not force and os.path.isfile(on):
            raise IOError("Attempted to overwrite: %s" % on)
        # Store output file name.
        outnames.append(on)
    # Run the converter over all files.
    for inname, outname in zip(innames, outnames):
        # Do the conversion.
        convert(inname, outname, outchans, normchans, nanfill)
<|endoftext|>


<commit_before>import os

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatter import Formatter

## custom formatter

PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))

# function to run formatter and capture output

class NullFormatter(Formatter):
    def format(self, tokensource, outfile):
        for ttype, value in tokensource:
            print ttype, value
            outfile.write(value)

def parse(codeInput):
    return highlight(codeInput, PythonLexer(), NullFormatter())

if __name__ == '__main__':
    fin = open(os.path.join(
        PROJECT_PATH,
        '..',
        'test',
        'fixtures',
        'target.py'))
    code = fin.read()
    # code = 'print "Hello World"'
    print parse(code)

    # run it on `/test/fixtures/target.py`

    # For good fun, try:
    # src $ python findTodo.py
<commit_msg>fingers crossed<commit_after>import os

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.lexers import guess_lexer, guess_lexer_for_filename
from pygments.formatter import Formatter

## custom formatter

PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))

# function to run formatter and capture output

class NullFormatter(Formatter):
    def format(self, tokensource, outfile):
        for ttype, value in tokensource:
            print ttype, value
            outfile.write(value)

def parse(codeInput):
    return highlight(codeInput, PythonLexer(), NullFormatter())

def walk(dir):
    for dirname, dirnames, filenames in os.walk(dir):
        # print path to all subdirectories first.
        for subdirname in dirnames:
            print os.path.join(dirname, subdirname)

        # print path to all filenames.
        for filename in filenames:
            print os.path.join(dirname, filename)
            fin = open(os.path.join(dirname, filename))
            code = fin.read()
            print parse(code)

        # Advanced usage:
        # editing the 'dirnames' list will stop os.walk() from recursing into there.
        if '.git' in dirnames:
            # don't go into any .git directories.
            dirnames.remove('.git')

if __name__ == '__main__':
    testDir = os.path.join(
        PROJECT_PATH,
        '..',
        'test',
        'fixtures')
    walk(testDir)

    # for fin in os.listdir(testDir):
    #     pass

    fin = open(os.path.join(
        PROJECT_PATH,
        '..',
        'test',
        'fixtures',
        'target.py'))
    code = fin.read()
    # code = 'print "Hello World"'
    print parse(code)

    # run it on `/test/fixtures/target.py`

    # For good fun, try:
    # src $ python findTodo.py

<|endoftext|>


<commit_before><commit_msg>change all URL from lotes to producao<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python2.4
#
# Copyright 2008 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generate an HTML chart with the emoji4unicode.xml and related data."""

__author__ = "Darick Tong"
__author__ = "Markus Scherer"

import cgi
import codecs
import sys
import emoji4unicode
import utf

# Flags from command-line options.
_only_in_proposal = False
_no_unified = False
_no_fallbacks = False
_no_codes = False
_no_symbol_numbers = False
_show_font_chars = False

_HEADER = """
<title>Table for Working Draft Proposal for Encoding Emoji Symbols</title>

<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>

Table for Working Draft Proposal for Encoding Emoji Symbols

The images in this file point to images on other sites.
  The images are only for comparison and may change.

If the proposal font is available, then the Symbol column shows the proposal
  font glyphs for symbols that are proposed for addition to the Unicode
  Standard. If the proposal font is not available, then unrelated glyphs from
  other installed fonts will be visible in the Symbol column.

See the chart legend
  for an explanation of the data presentation in this chart.

Each symbol row has an anchor to allow direct linking by appending
  #e-4B0 (for example) to this page's URL in the
  address bar.

 Symbol ID
 Symbol
 Name & Annotations
 DoCoMo
 KDDI
 SoftBank
 Google

"""

_FOOTER = """"""

def _WriteEmoji4UnicodeHTML(writer):
  number_symbols_in_chart = 0
  number_symbols_unified = 0
  number_symbols_new = 0
  writer.write(_HEADER)
  for category in emoji4unicode.GetCategories():
    category_string = category.name
    if not category.in_proposal:
      if _only_in_proposal: continue  # Skip this category.
      category_string += (" (This section is for comparison only -- "
                          "not part of the Emoji proposal.)")
    _WriteSingleCelledRow(writer, "category", category_string)
    for subcategory in category.GetSubcategories():
      if not subcategory.in_proposal and _only_in_proposal:
        continue  # Skip this subcategory.
      _WriteSingleCelledRow(writer,
                            "subcategory",
                            "%s (%s)" % (subcategory.name, category.name))
      for symbol in subcategory.GetSymbols():
        if symbol.in_proposal:
          row_style = ""
        elif _only_in_proposal:
          continue  # Skip this symbol.
        else:
          row_style = " class=not_in_proposal"
        if symbol.GetUnicode():
          if _no_unified: continue  # Skip this symbol.
          number_symbols_unified += 1
        elif symbol.in_proposal:
          number_symbols_new += 1
        number_symbols_in_chart += 1
        e_id = "e-" + symbol.id
        writer.write("
%s	" %
                     (e_id, row_style, e_id, e_id))
        writer.write("%s	" % _RepresentationHTML(symbol))
        writer.write("%s	" % _NameAnnotationHTML(symbol))
        for carrier in emoji4unicode.carriers:
          code = symbol.GetCarrierUnicode(carrier)
          if code:
            if code.startswith(">"):
              if _no_fallbacks:
                writer.write("-	")
                continue
              template = "%s	"
              code = code[1:]
            else:
              template = "%s	"
            cell = template % _CarrierSymbolHTML(
                carrier,
                emoji4unicode.all_carrier_data[carrier],
                code)
          elif _no_fallbacks:
            cell = "-	"
          else:
            text_fallback = symbol.GetTextFallback()
            if not text_fallback: text_fallback = u"\u3013"  # geta mark
            cell = "%s	" % text_fallback
          writer.write(cell)
        writer.write("
\n")
  writer.write("
\n")
  writer.write("
Number of symbols in this chart: %d
\n" %
               number_symbols_in_chart)
  if _no_unified:
    writer.write("
Number of symbols unified with existing "
                 "Unicode characters: None shown in this chart.
\n")
  else:
    writer.write("
Number of symbols unified with existing "
                 "Unicode characters: %d
\n" %
                 number_symbols_unified)
  writer.write("
Number of proposed new symbols: %d
\n" %
               number_symbols_new)
  writer.write(_FOOTER)

def _RepresentationHTML(e4u_symbol):
  """Return HTML with the symbol representation."""
  uni = e4u_symbol.GetUnicode()
  if uni:
    if e4u_symbol.IsUnifiedWithUpcomingCharacter():
      # Print only code points, not also characters,
      # because no one will have a font for these.
      return (u"U5.2
"
               "U+" + uni.replace("+", " U+"))
    else:
      return _UnicodeHTML(uni)
  img = e4u_symbol.ImageHTML()
  font_uni = e4u_symbol.GetFontUnicode()
  font_img = u"" % font_uni
  if e4u_symbol.in_proposal:
    glyph_id = e4u_symbol.GetGlyphRefID()
    if _show_font_chars:
      font_str = utf.UTF.CodePointString(int(font_uni, 16))
      repr = (u"%sglyph%d=%s" %
              (font_str, glyph_id, font_img))
      if img: repr += u"\u2248" + img
    else:
      # repr = u"glyph%d" % glyph_id
      repr = font_img
    proposed_uni = e4u_symbol.GetProposedUnicode()
    if proposed_uni:
      return (repr + u"
U+" +
              proposed_uni.replace("+", " U+"))
    else:
      return repr + u"
U+xxxxx"
  if img: return img
  text_repr = e4u_symbol.GetTextRepresentation()
  if text_repr: return text_repr
  return "?repr?"

def _UnicodeHTML(uni):
  """Turn '0041+005A' into pretty HTML.

  Args:
    uni: Input string like '0041' or '0041+005A'.

  Returns:
    Corresponding pretty HTML.
  """
  # Turn '0041+005A' into 'Az U+0041 U+005A'.
  chars = unicode()
  code_points = unicode()
  for code in uni.split("+"):
    chars += utf.UTF.CodePointString(int(code, 16))
    code_points += " U+" + code
  return (u"" + chars + u"" + u"
" +
          code_points[1:])

def _NameAnnotationHTML(e4u_symbol):
  """Return HTML with the symbol name, annotations, etc."""
  lines = [e4u_symbol.GetName()]
  arib = e4u_symbol.GetARIB()
  if arib: lines.append(u"= ARIB-%s" % arib)
  if e4u_symbol.IsUnifiedWithUpcomingCharacter():
    lines.append(u"Temporary Note: "
                  "Unified with an upcoming Unicode 5.2/AMD6 character; "
                  "code point and name are preliminary.")
  for line in e4u_symbol.GetAnnotations(): lines.append(cgi.escape(line))
  desc = e4u_symbol.GetDescription()
  if desc: lines.append(u"Temporary Notes: " +
                        cgi.escape(desc) + u"")
  design = e4u_symbol.GetDesign()
  if design: lines.append(u"Design Note: " +
                          cgi.escape(design) + u"")
  return "
".join(lines)

def _CarrierSymbolHTML(carrier, one_carrier_data, code_string):
  codes = code_string.split("+")
  img_string = ""
  number_string = ""
  old_number_string = ""
  new_number_string = ""
  english_string = ""
  japanese_string = ""
  uni_string = ""
  shift_jis_string = ""
  jis_string = ""
  for code in codes:
    symbol = one_carrier_data.SymbolFromUnicode(code)
    img_html = emoji4unicode.CarrierImageHTML(carrier, symbol)
    if img_html: img_string += "+%s" % img_html
    if not _no_symbol_numbers:
      if symbol.number:
        if carrier == "docomo" and symbol.number >= 300:
          # DoCoMo shows symbol numbers 1..176 for "Basic Pictograms" and
          # symbol numbers "Exp.1".."Exp.76" in the English publication of
          # "Expansion Pictograms".
          # Our data uses an offset of 300 for the integer numbers of
          # Expansion Pictograms.
          number_string += "+#Exp." + str(symbol.number - 300)
        else:
          number_string += "+#" + str(symbol.number)
      if symbol.old_number:
        old_number_string += "+#old" + str(symbol.old_number)
      if symbol.new_number:
        new_number_string += "+#new" + str(symbol.new_number)
    name_en = symbol.GetEnglishName()
    if name_en: english_string += "+'%s'" % name_en
    name_ja = symbol.GetJapaneseName()
    if name_ja: japanese_string += "+" + name_ja
    if not _no_codes:
      uni_string += "+U+" + code
      if symbol.shift_jis: shift_jis_string += "+SJIS-" + symbol.shift_jis
      if symbol.jis: jis_string += "+JIS-" + symbol.jis
  result_pieces = []
  if len(codes) == 1:
    # Reduce the cell height by putting multiple data pieces on each line.
    groups = [[img_string,
               number_string, old_number_string, new_number_string],
              [english_string, japanese_string],
              [uni_string],
              [shift_jis_string, jis_string]]
    for group in groups:
      group_pieces = []
      for line in group:
        if line: group_pieces.append(line[1:])  # Remove leading separator.
      if group_pieces: result_pieces.append(" ".join(group_pieces))
  else:
    # For code sequences, use a line per type of data.
    for line in (img_string,
                 number_string, old_number_string, new_number_string,
                 english_string, japanese_string,
                 uni_string, shift_jis_string, jis_string):
      if line: result_pieces.append(line[1:])  # Remove leading separator.
  if not result_pieces:
    # Show *something* despite _no_codes.
    return u"-"
  return "
".join(result_pieces)

def _WriteSingleCelledRow(writer, style, contents):
  """Writes a single cell that spans an entire row."""
  writer.write("
%s
\n" % (style, contents))

def main():
  global _only_in_proposal, _no_unified, _no_fallbacks
  global _no_codes, _no_symbol_numbers, _show_font_chars
  for i in range(1, len(sys.argv)):
    if sys.argv[i] == "--only_in_proposal": _only_in_proposal = True
    if sys.argv[i] == "--no_codes": _no_codes = True
    if sys.argv[i] == "--design":
      _only_in_proposal = True
      _no_unified = True
      _no_fallbacks = True
      _no_codes = True
      _no_symbol_numbers = True
      _show_font_chars = True
  emoji4unicode.Load()
  _WriteEmoji4UnicodeHTML(codecs.getwriter("UTF-8")(sys.stdout))

if __name__ == "__main__":
  main()
<commit_msg>issue 70: remove glyph IDs from design chart<commit_after>#!/usr/bin/python2.4
#
# Copyright 2008 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generate an HTML chart with the emoji4unicode.xml and related data."""

__author__ = "Darick Tong"
__author__ = "Markus Scherer"

import cgi
import codecs
import sys
import emoji4unicode
import utf

# Flags from command-line options.
_only_in_proposal = False
_no_unified = False
_no_fallbacks = False
_no_codes = False
_no_symbol_numbers = False
_show_font_chars = False

_HEADER = """
<title>Table for Working Draft Proposal for Encoding Emoji Symbols</title>

<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>

Table for Working Draft Proposal for Encoding Emoji Symbols

The images in this file point to images on other sites.
  The images are only for comparison and may change.

If the proposal font is available, then the Symbol column shows the proposal
  font glyphs for symbols that are proposed for addition to the Unicode
  Standard. If the proposal font is not available, then unrelated glyphs from
  other installed fonts will be visible in the Symbol column.

See the chart legend
  for an explanation of the data presentation in this chart.

Each symbol row has an anchor to allow direct linking by appending
  #e-4B0 (for example) to this page's URL in the
  address bar.

 Symbol ID
 Symbol
 Name & Annotations
 DoCoMo
 KDDI
 SoftBank
 Google

"""

_FOOTER = """"""

def _WriteEmoji4UnicodeHTML(writer):
  number_symbols_in_chart = 0
  number_symbols_unified = 0
  number_symbols_new = 0
  writer.write(_HEADER)
  for category in emoji4unicode.GetCategories():
    category_string = category.name
    if not category.in_proposal:
      if _only_in_proposal: continue  # Skip this category.
      category_string += (" (This section is for comparison only -- "
                          "not part of the Emoji proposal.)")
    _WriteSingleCelledRow(writer, "category", category_string)
    for subcategory in category.GetSubcategories():
      if not subcategory.in_proposal and _only_in_proposal:
        continue  # Skip this subcategory.
      _WriteSingleCelledRow(writer,
                            "subcategory",
                            "%s (%s)" % (subcategory.name, category.name))
      for symbol in subcategory.GetSymbols():
        if symbol.in_proposal:
          row_style = ""
        elif _only_in_proposal:
          continue  # Skip this symbol.
        else:
          row_style = " class=not_in_proposal"
        if symbol.GetUnicode():
          if _no_unified: continue  # Skip this symbol.
          number_symbols_unified += 1
        elif symbol.in_proposal:
          number_symbols_new += 1
        number_symbols_in_chart += 1
        e_id = "e-" + symbol.id
        writer.write("
%s	" %
                     (e_id, row_style, e_id, e_id))
        writer.write("%s	" % _RepresentationHTML(symbol))
        writer.write("%s	" % _NameAnnotationHTML(symbol))
        for carrier in emoji4unicode.carriers:
          code = symbol.GetCarrierUnicode(carrier)
          if code:
            if code.startswith(">"):
              if _no_fallbacks:
                writer.write("-	")
                continue
              template = "%s	"
              code = code[1:]
            else:
              template = "%s	"
            cell = template % _CarrierSymbolHTML(
                carrier,
                emoji4unicode.all_carrier_data[carrier],
                code)
          elif _no_fallbacks:
            cell = "-	"
          else:
            text_fallback = symbol.GetTextFallback()
            if not text_fallback: text_fallback = u"\u3013"  # geta mark
            cell = "%s	" % text_fallback
          writer.write(cell)
        writer.write("
\n")
  writer.write("
\n")
  writer.write("
Number of symbols in this chart: %d
\n" %
               number_symbols_in_chart)
  if _no_unified:
    writer.write("
Number of symbols unified with existing "
                 "Unicode characters: None shown in this chart.
\n")
  else:
    writer.write("
Number of symbols unified with existing "
                 "Unicode characters: %d
\n" %
                 number_symbols_unified)
  writer.write("
Number of proposed new symbols: %d
\n" %
               number_symbols_new)
  writer.write(_FOOTER)

def _RepresentationHTML(e4u_symbol):
  """Return HTML with the symbol representation."""
  uni = e4u_symbol.GetUnicode()
  if uni:
    if e4u_symbol.IsUnifiedWithUpcomingCharacter():
      # Print only code points, not also characters,
      # because no one will have a font for these.
      return (u"U5.2
"
               "U+" + uni.replace("+", " U+"))
    else:
      return _UnicodeHTML(uni)
  img = e4u_symbol.ImageHTML()
  font_uni = e4u_symbol.GetFontUnicode()
  font_img = u"" % font_uni
  if e4u_symbol.in_proposal:
    if _show_font_chars:
      font_str = utf.UTF.CodePointString(int(font_uni, 16))
      repr = u"%s=%s" % (font_str, font_img)
      if img: repr += u"\u2248" + img
    else:
      repr = font_img
    proposed_uni = e4u_symbol.GetProposedUnicode()
    if proposed_uni:
      return (repr + u"
U+" +
              proposed_uni.replace("+", " U+"))
    else:
      return repr + u"
U+xxxxx"
  if img: return img
  text_repr = e4u_symbol.GetTextRepresentation()
  if text_repr: return text_repr
  return "?repr?"

def _UnicodeHTML(uni):
  """Turn '0041+005A' into pretty HTML.

  Args:
    uni: Input string like '0041' or '0041+005A'.

  Returns:
    Corresponding pretty HTML.
  """
  # Turn '0041+005A' into 'Az U+0041 U+005A'.
  chars = unicode()
  code_points = unicode()
  for code in uni.split("+"):
    chars += utf.UTF.CodePointString(int(code, 16))
    code_points += " U+" + code
  return (u"" + chars + u"" + u"
" +
          code_points[1:])

def _NameAnnotationHTML(e4u_symbol):
  """Return HTML with the symbol name, annotations, etc."""
  lines = [e4u_symbol.GetName()]
  arib = e4u_symbol.GetARIB()
  if arib: lines.append(u"= ARIB-%s" % arib)
  if e4u_symbol.IsUnifiedWithUpcomingCharacter():
    lines.append(u"Temporary Note: "
                  "Unified with an upcoming Unicode 5.2/AMD6 character; "
                  "code point and name are preliminary.")
  for line in e4u_symbol.GetAnnotations(): lines.append(cgi.escape(line))
  desc = e4u_symbol.GetDescription()
  if desc: lines.append(u"Temporary Notes: " +
                        cgi.escape(desc) + u"")
  design = e4u_symbol.GetDesign()
  if design: lines.append(u"Design Note: " +
                          cgi.escape(design) + u"")
  return "
".join(lines)

def _CarrierSymbolHTML(carrier, one_carrier_data, code_string):
  codes = code_string.split("+")
  img_string = ""
  number_string = ""
  old_number_string = ""
  new_number_string = ""
  english_string = ""
  japanese_string = ""
  uni_string = ""
  shift_jis_string = ""
  jis_string = ""
  for code in codes:
    symbol = one_carrier_data.SymbolFromUnicode(code)
    img_html = emoji4unicode.CarrierImageHTML(carrier, symbol)
    if img_html: img_string += "+%s" % img_html
    if not _no_symbol_numbers:
      if symbol.number:
        if carrier == "docomo" and symbol.number >= 300:
          # DoCoMo shows symbol numbers 1..176 for "Basic Pictograms" and
          # symbol numbers "Exp.1".."Exp.76" in the English publication of
          # "Expansion Pictograms".
          # Our data uses an offset of 300 for the integer numbers of
          # Expansion Pictograms.
          number_string += "+#Exp." + str(symbol.number - 300)
        else:
          number_string += "+#" + str(symbol.number)
      if symbol.old_number:
        old_number_string += "+#old" + str(symbol.old_number)
      if symbol.new_number:
        new_number_string += "+#new" + str(symbol.new_number)
    name_en = symbol.GetEnglishName()
    if name_en: english_string += "+'%s'" % name_en
    name_ja = symbol.GetJapaneseName()
    if name_ja: japanese_string += "+" + name_ja
    if not _no_codes:
      uni_string += "+U+" + code
      if symbol.shift_jis: shift_jis_string += "+SJIS-" + symbol.shift_jis
      if symbol.jis: jis_string += "+JIS-" + symbol.jis
  result_pieces = []
  if len(codes) == 1:
    # Reduce the cell height by putting multiple data pieces on each line.
    groups = [[img_string,
               number_string, old_number_string, new_number_string],
              [english_string, japanese_string],
              [uni_string],
              [shift_jis_string, jis_string]]
    for group in groups:
      group_pieces = []
      for line in group:
        if line: group_pieces.append(line[1:])  # Remove leading separator.
      if group_pieces: result_pieces.append(" ".join(group_pieces))
  else:
    # For code sequences, use a line per type of data.
    for line in (img_string,
                 number_string, old_number_string, new_number_string,
                 english_string, japanese_string,
                 uni_string, shift_jis_string, jis_string):
      if line: result_pieces.append(line[1:])  # Remove leading separator.
  if not result_pieces:
    # Show *something* despite _no_codes.
    return u"-"
  return "
".join(result_pieces)

def _WriteSingleCelledRow(writer, style, contents):
  """Writes a single cell that spans an entire row."""
  writer.write("
%s
\n" % (style, contents))

def main():
  global _only_in_proposal, _no_unified, _no_fallbacks
  global _no_codes, _no_symbol_numbers, _show_font_chars
  for i in range(1, len(sys.argv)):
    if sys.argv[i] == "--only_in_proposal": _only_in_proposal = True
    if sys.argv[i] == "--no_codes": _no_codes = True
    if sys.argv[i] == "--design":
      _only_in_proposal = True
      _no_unified = True
      _no_fallbacks = True
      _no_codes = True
      _no_symbol_numbers = True
      _show_font_chars = True
  emoji4unicode.Load()
  _WriteEmoji4UnicodeHTML(codecs.getwriter("UTF-8")(sys.stdout))

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>import logging
import os
import sys
import click
from functools import partial

from .config import load_config
from .models import Snapshot, Table, Base
from .operations import (
    copy_database,
    create_database,
    database_exists,
    remove_database,
    rename_database,
    terminate_database_connections,
    list_of_databases,
)
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import ProgrammingError
from psutil import pid_exists

__version__ = '0.4.1'
logger = logging.getLogger(__name__)

class Operations(object):
    def __init__(self, raw_connection, config):
        self.terminate_database_connections = partial(
            terminate_database_connections, raw_connection
        )
        self.create_database = partial(create_database, raw_connection)
        self.copy_database = partial(copy_database, raw_connection)
        self.database_exists = partial(database_exists, raw_connection)
        self.rename_database = partial(rename_database, raw_connection)
        self.remove_database = partial(remove_database, raw_connection)
        self.list_of_databases = partial(list_of_databases, raw_connection)

class Stellar(object):
    def __init__(self):
        logger.debug('Initialized Stellar()')
        self.load_config()
        self.init_database()

    def load_config(self):
        self.config = load_config()
        logging.basicConfig(level=self.config['logging'])

    def init_database(self):
        self.raw_db = create_engine(self.config['url'], echo=False)
        self.raw_conn = self.raw_db.connect()
        self.operations = Operations(self.raw_conn, self.config)

        try:
            self.raw_conn.connection.set_isolation_level(0)
        except AttributeError:
            logger.info('Could not set isolation level to 0')

        self.db = create_engine(self.config['stellar_url'], echo=False)
        self.db.session = sessionmaker(bind=self.db)()
        self.raw_db.session = sessionmaker(bind=self.raw_db)()
        tables_missing = self.create_stellar_database()

        self.create_stellar_tables()

        # logger.getLogger('sqlalchemy.engine').setLevel(logger.WARN)

    def create_stellar_database(self):
        if not self.operations.database_exists('stellar_data'):
            self.operations.create_database('stellar_data')
            return True
        else:
            return False

    def create_stellar_tables(self):
        Base.metadata.create_all(self.db)
        self.db.session.commit()

    def get_snapshot(self, snapshot_name):
        return self.db.session.query(Snapshot).filter(
            Snapshot.snapshot_name == snapshot_name,
            Snapshot.project_name == self.config['project_name']
        ).first()

    def get_snapshots(self):
        return self.db.session.query(Snapshot).filter(
            Snapshot.project_name == self.config['project_name']
        ).order_by(
            Snapshot.created_at.desc()
        ).all()

    def get_latest_snapshot(self):
        return self.db.session.query(Snapshot).filter(
            Snapshot.project_name == self.config['project_name']
        ).order_by(Snapshot.created_at.desc()).first()

    def create_snapshot(self, snapshot_name, before_copy=None):
        snapshot = Snapshot(
            snapshot_name=snapshot_name,
            project_name=self.config['project_name']
        )
        self.db.session.add(snapshot)
        self.db.session.flush()

        for table_name in self.config['tracked_databases']:
            if before_copy:
                before_copy(table_name)
            table = Table(
                table_name=table_name,
                snapshot=snapshot
            )
            logger.debug('Copying %s to %s' % (
                table_name,
                table.get_table_name('master')
            ))
            self.operations.copy_database(
                table_name,
                table.get_table_name('master')
            )
            self.db.session.add(table)
        self.db.session.commit()

        self.start_background_slave_copy(snapshot)

    def remove_snapshot(self, snapshot):
        for table in snapshot.tables:
            try:
                self.operations.remove_database(
                    table.get_table_name('master')
                )
            except ProgrammingError:
                pass
            try:
                self.operations.remove_database(
                    table.get_table_name('slave')
                )
            except ProgrammingError:
                pass
            self.db.session.delete(table)
        self.db.session.delete(snapshot)
        self.db.session.commit()

    def rename_snapshot(self, snapshot, new_name):
        snapshot.snapshot_name = new_name
        self.db.session.commit()

    def restore(self, snapshot):
        for table in snapshot.tables:
            click.echo("Restoring database %s" % table.table_name)
            if not self.operations.database_exists(
                table.get_table_name('slave')
            ):
                click.echo(
                    "Database %s does not exist."
                    % table.get_table_name('slave')
                )
                sys.exit(1)
            try:
                self.operations.remove_database(table.table_name)
            except ProgrammingError:
                logger.warn('Database %s does not exist.' % table.table_name)
            self.operations.rename_database(
                table.get_table_name('slave'),
                table.table_name
            )
        snapshot.worker_pid = 1
        self.db.session.commit()

        self.start_background_slave_copy(snapshot)

    def start_background_slave_copy(self, snapshot):
        logger.debug('Starting background slave copy')
        snapshot_id = snapshot.id

        self.raw_conn.close()
        self.raw_db.session.close()
        self.db.session.close()

        pid = os.fork() if hasattr(os, 'fork') else None
        if pid:
            return

        self.init_database()
        self.operations = Operations(self.raw_conn, self.config)

        snapshot = self.db.session.query(Snapshot).get(snapshot_id)
        snapshot.worker_pid = os.getpid()
        self.db.session.commit()
        self.inline_slave_copy(snapshot)
        sys.exit()

    def inline_slave_copy(self, snapshot):
        for table in snapshot.tables:
            self.operations.copy_database(
                table.get_table_name('master'),
                table.get_table_name('slave')
            )
        snapshot.worker_pid = None
        self.db.session.commit()

    def is_copy_process_running(self, snapshot):
        return pid_exists(snapshot.worker_pid)

    def is_old_database(self):
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                for postfix in ('master', 'slave'):
                    old_name = table.get_table_name(postfix=postfix, old=True)
                    if self.operations.database_exists(old_name):
                        return True
        return False

    def update_database_names_to_new_version(self, after_rename=None):
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                for postfix in ('master', 'slave'):
                    old_name = table.get_table_name(postfix=postfix, old=True)
                    new_name = table.get_table_name(postfix=postfix, old=False)
                    if self.operations.database_exists(old_name):
                        self.operations.rename_database(old_name, new_name)
                        if after_rename:
                            after_rename(old_name, new_name)

    def delete_orphan_snapshots(self, after_delete=None):
        stellar_databases = set()
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                stellar_databases.add(table.get_table_name('master'))
                stellar_databases.add(table.get_table_name('slave'))

        databases = set(self.operations.list_of_databases())

        for database in filter(
            lambda database: (
                database.startswith('stellar_') and
                database != 'stellar_data'
            ),
            (databases-stellar_databases)
        ):
            self.operations.remove_database(database)
            if after_delete:
                after_delete(database)

    @property
    def default_snapshot_name(self):
        n = 1
        while self.db.session.query(Snapshot).filter(
            Snapshot.snapshot_name == 'snap%d' % n,
            Snapshot.project_name == self.config['project_name']
        ).count():
            n += 1
        return 'snap%d' % n
<commit_msg>Bump version<commit_after>import logging
import os
import sys
import click
from functools import partial

from .config import load_config
from .models import Snapshot, Table, Base
from .operations import (
    copy_database,
    create_database,
    database_exists,
    remove_database,
    rename_database,
    terminate_database_connections,
    list_of_databases,
)
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import ProgrammingError
from psutil import pid_exists

__version__ = '0.4.2'
logger = logging.getLogger(__name__)

class Operations(object):
    def __init__(self, raw_connection, config):
        self.terminate_database_connections = partial(
            terminate_database_connections, raw_connection
        )
        self.create_database = partial(create_database, raw_connection)
        self.copy_database = partial(copy_database, raw_connection)
        self.database_exists = partial(database_exists, raw_connection)
        self.rename_database = partial(rename_database, raw_connection)
        self.remove_database = partial(remove_database, raw_connection)
        self.list_of_databases = partial(list_of_databases, raw_connection)

class Stellar(object):
    def __init__(self):
        logger.debug('Initialized Stellar()')
        self.load_config()
        self.init_database()

    def load_config(self):
        self.config = load_config()
        logging.basicConfig(level=self.config['logging'])

    def init_database(self):
        self.raw_db = create_engine(self.config['url'], echo=False)
        self.raw_conn = self.raw_db.connect()
        self.operations = Operations(self.raw_conn, self.config)

        try:
            self.raw_conn.connection.set_isolation_level(0)
        except AttributeError:
            logger.info('Could not set isolation level to 0')

        self.db = create_engine(self.config['stellar_url'], echo=False)
        self.db.session = sessionmaker(bind=self.db)()
        self.raw_db.session = sessionmaker(bind=self.raw_db)()
        tables_missing = self.create_stellar_database()

        self.create_stellar_tables()

        # logger.getLogger('sqlalchemy.engine').setLevel(logger.WARN)

    def create_stellar_database(self):
        if not self.operations.database_exists('stellar_data'):
            self.operations.create_database('stellar_data')
            return True
        else:
            return False

    def create_stellar_tables(self):
        Base.metadata.create_all(self.db)
        self.db.session.commit()

    def get_snapshot(self, snapshot_name):
        return self.db.session.query(Snapshot).filter(
            Snapshot.snapshot_name == snapshot_name,
            Snapshot.project_name == self.config['project_name']
        ).first()

    def get_snapshots(self):
        return self.db.session.query(Snapshot).filter(
            Snapshot.project_name == self.config['project_name']
        ).order_by(
            Snapshot.created_at.desc()
        ).all()

    def get_latest_snapshot(self):
        return self.db.session.query(Snapshot).filter(
            Snapshot.project_name == self.config['project_name']
        ).order_by(Snapshot.created_at.desc()).first()

    def create_snapshot(self, snapshot_name, before_copy=None):
        snapshot = Snapshot(
            snapshot_name=snapshot_name,
            project_name=self.config['project_name']
        )
        self.db.session.add(snapshot)
        self.db.session.flush()

        for table_name in self.config['tracked_databases']:
            if before_copy:
                before_copy(table_name)
            table = Table(
                table_name=table_name,
                snapshot=snapshot
            )
            logger.debug('Copying %s to %s' % (
                table_name,
                table.get_table_name('master')
            ))
            self.operations.copy_database(
                table_name,
                table.get_table_name('master')
            )
            self.db.session.add(table)
        self.db.session.commit()

        self.start_background_slave_copy(snapshot)

    def remove_snapshot(self, snapshot):
        for table in snapshot.tables:
            try:
                self.operations.remove_database(
                    table.get_table_name('master')
                )
            except ProgrammingError:
                pass
            try:
                self.operations.remove_database(
                    table.get_table_name('slave')
                )
            except ProgrammingError:
                pass
            self.db.session.delete(table)
        self.db.session.delete(snapshot)
        self.db.session.commit()

    def rename_snapshot(self, snapshot, new_name):
        snapshot.snapshot_name = new_name
        self.db.session.commit()

    def restore(self, snapshot):
        for table in snapshot.tables:
            click.echo("Restoring database %s" % table.table_name)
            if not self.operations.database_exists(
                table.get_table_name('slave')
            ):
                click.echo(
                    "Database %s does not exist."
                    % table.get_table_name('slave')
                )
                sys.exit(1)
            try:
                self.operations.remove_database(table.table_name)
            except ProgrammingError:
                logger.warn('Database %s does not exist.' % table.table_name)
            self.operations.rename_database(
                table.get_table_name('slave'),
                table.table_name
            )
        snapshot.worker_pid = 1
        self.db.session.commit()

        self.start_background_slave_copy(snapshot)

    def start_background_slave_copy(self, snapshot):
        logger.debug('Starting background slave copy')
        snapshot_id = snapshot.id

        self.raw_conn.close()
        self.raw_db.session.close()
        self.db.session.close()

        pid = os.fork() if hasattr(os, 'fork') else None
        if pid:
            return

        self.init_database()
        self.operations = Operations(self.raw_conn, self.config)

        snapshot = self.db.session.query(Snapshot).get(snapshot_id)
        snapshot.worker_pid = os.getpid()
        self.db.session.commit()
        self.inline_slave_copy(snapshot)
        sys.exit()

    def inline_slave_copy(self, snapshot):
        for table in snapshot.tables:
            self.operations.copy_database(
                table.get_table_name('master'),
                table.get_table_name('slave')
            )
        snapshot.worker_pid = None
        self.db.session.commit()

    def is_copy_process_running(self, snapshot):
        return pid_exists(snapshot.worker_pid)

    def is_old_database(self):
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                for postfix in ('master', 'slave'):
                    old_name = table.get_table_name(postfix=postfix, old=True)
                    if self.operations.database_exists(old_name):
                        return True
        return False

    def update_database_names_to_new_version(self, after_rename=None):
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                for postfix in ('master', 'slave'):
                    old_name = table.get_table_name(postfix=postfix, old=True)
                    new_name = table.get_table_name(postfix=postfix, old=False)
                    if self.operations.database_exists(old_name):
                        self.operations.rename_database(old_name, new_name)
                        if after_rename:
                            after_rename(old_name, new_name)

    def delete_orphan_snapshots(self, after_delete=None):
        stellar_databases = set()
        for snapshot in self.db.session.query(Snapshot):
            for table in snapshot.tables:
                stellar_databases.add(table.get_table_name('master'))
                stellar_databases.add(table.get_table_name('slave'))

        databases = set(self.operations.list_of_databases())

        for database in filter(
            lambda database: (
                database.startswith('stellar_') and
                database != 'stellar_data'
            ),
            (databases-stellar_databases)
        ):
            self.operations.remove_database(database)
            if after_delete:
                after_delete(database)

    @property
    def default_snapshot_name(self):
        n = 1
        while self.db.session.query(Snapshot).filter(
            Snapshot.snapshot_name == 'snap%d' % n,
            Snapshot.project_name == self.config['project_name']
        ).count():
            n += 1
        return 'snap%d' % n
<|endoftext|>