<commit_before># Copyright (c) 2016, the GPyOpt Authors
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import numpy as np
import GPy

from .base import BOModel

class GPModel(BOModel):
    """
    General class for handling a Gaussian Process in GPyOpt.

    :param kernel: GPy kernel to use in the GP model.
    :param noise_var: value of the noise variance if known.
    :param exact_feval: whether noiseless evaluations are available. IMPORTANT to make the optimization work well in noiseless scenarios (default, False).
    :param optimizer: optimizer of the model. Check GPy for details.
    :param max_iters: maximum number of iterations used to optimize the parameters of the model.
    :param optimize_restarts: number of restarts in the optimization.
    :param sparse: whether to use a sparse GP (default, False). This is useful when many observations are available.
    :param num_inducing: number of inducing points if a sparse GP is used.
    :param verbose: print out the model messages (default, False).
    :param ARD: whether ARD is used in the kernel (default, False).

    .. Note:: This model does Maximum likelihood estimation of the hyper-parameters.

    """

    analytical_gradient_prediction = True  # --- Needed in all models to check is the gradients of acquisitions are computable.

    def __init__(self, kernel=None, noise_var=None, exact_feval=False, optimizer='bfgs', max_iters=1000, optimize_restarts=5, sparse = False, num_inducing = 10,  verbose=True, ARD=False):
        self.kernel = kernel
        self.noise_var = noise_var
        self.exact_feval = exact_feval
        self.optimize_restarts = optimize_restarts
        self.optimizer = optimizer
        self.max_iters = max_iters
        self.verbose = verbose
        self.sparse = sparse
        self.num_inducing = num_inducing
        self.model = None
        self.ARD = ARD

    @staticmethod
    def fromConfig(config):
        return GPModel(**config)

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """

        # --- define kernel
        self.input_dim = X.shape[1]
        if self.kernel is None:
            kern = GPy.kern.Matern52(self.input_dim, variance=1., ARD=self.ARD) #+ GPy.kern.Bias(self.input_dim)
        else:
            kern = self.kernel
            self.kernel = None

        # --- define model
        noise_var = Y.var()*0.01 if self.noise_var is None else self.noise_var

        if not self.sparse:
            self.model = GPy.models.GPRegression(X, Y, kernel=kern, noise_var=noise_var)
        else:
            self.model = GPy.models.SparseGPRegression(X, Y, kernel=kern, num_inducing=self.num_inducing)

        # --- restrict variance if exact evaluations of the objective
        if self.exact_feval:
            self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False)
        else:
            # --- We make sure we do not get ridiculously small residual noise variance
            self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False) #constrain_positive(warning=False)

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """
        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.set_XY(X_all, Y_all)

        # WARNING: Even if self.max_iters=0, the hyperparameters are bit modified...
        if self.max_iters > 0:
            # --- update the model maximizing the marginal likelihood.
            if self.optimize_restarts==1:
                self.model.optimize(optimizer=self.optimizer, max_iters = self.max_iters, messages=False, ipython_notebook=False)
            else:
                self.model.optimize_restarts(num_restarts=self.optimize_restarts, optimizer=self.optimizer, max_iters = self.max_iters, verbose=self.verbose)

    def _predict(self, X, full_cov, include_likelihood):
        if X.ndim == 1:
            X = X[None,:]
        m, v = self.model.predict(X, full_cov=full_cov, include_likelihood=include_likelihood)
        v = np.clip(v, 1e-10, np.inf)
        return m, v

    def predict(self, X, with_noise=True):
        """
        Predictions with the model. Returns posterior means and standard deviations at X. Note that this is different in GPy where the variances are given.

        Parameters:
            X (np.ndarray) - points to run the prediction for.
            with_noise (bool) - whether to add noise to the prediction. Default is True.
        """
        m, v = self._predict(X, False, with_noise)
        # We can take the square root because v is just a diagonal matrix of variances
        return m, np.sqrt(v)

    def predict_covariance(self, X, with_noise=True):
        """
        Predicts the covariance matric for points in X.

        Parameters:
            X (np.ndarray) - points to run the prediction for.
            with_noise (bool) - whether to add noise to the prediction. Default is True.
        """
        _, v = self._predict(X, True, with_noise)
        return v

    def get_fmin(self):
        """
        Returns the location where the posterior mean is takes its minimal value.
        """
        return self.model.predict(self.model.X)[0].min()

    def predict_withGradients(self, X):
        """
        Returns the mean, standard deviation, mean gradient and standard deviation gradient at X.
        """
        if X.ndim==1: X = X[None,:]
        m, v = self.model.predict(X)
        v = np.clip(v, 1e-10, np.inf)
        dmdx, dvdx = self.model.predictive_gradients(X)
        dmdx = dmdx[:,:,0]
        dsdx = dvdx / (2*np.sqrt(v))

        return m, np.sqrt(v), dmdx, dsdx

    def copy(self):
        """
        Makes a safe copy of the model.
        """
        copied_model = GPModel(kernel = self.model.kern.copy(),
                            noise_var=self.noise_var,
                            exact_feval=self.exact_feval,
                            optimizer=self.optimizer,
                            max_iters=self.max_iters,
                            optimize_restarts=self.optimize_restarts,
                            verbose=self.verbose,
                            ARD=self.ARD)

        copied_model._create_model(self.model.X,self.model.Y)
        copied_model.updateModel(self.model.X,self.model.Y, None, None)
        return copied_model

    def get_model_parameters(self):
        """
        Returns a 2D numpy array with the parameters of the model
        """
        return np.atleast_2d(self.model[:])

    def get_model_parameters_names(self):
        """
        Returns a list with the names of the parameters of the model
        """
        return self.model.parameter_names_flat().tolist()

    def get_covariance_between_points(self, x1, x2):
        """
        Given the current posterior, computes the covariance between two sets of points.
        """
        return self.model.posterior_covariance_between_points(x1, x2)

class GPModel_MCMC(BOModel):
    """
    General class for handling a Gaussian Process in GPyOpt.

    :param kernel: GPy kernel to use in the GP model.
    :param noise_var: value of the noise variance if known.
    :param exact_feval: whether noiseless evaluations are available. IMPORTANT to make the optimization work well in noiseless scenarios (default, False).
    :param n_samples: number of MCMC samples.
    :param n_burnin: number of samples not used.
    :param subsample_interval: sub-sample interval in the MCMC.
    :param step_size: step-size in the MCMC.
    :param leapfrog_steps: ??
    :param verbose: print out the model messages (default, False).

    .. Note:: This model does MCMC over the hyperparameters.

    """

    MCMC_sampler = True
    analytical_gradient_prediction = True # --- Needed in all models to check is the gradients of acquisitions are computable.

    def __init__(self, kernel=None, noise_var=None, exact_feval=False, n_samples = 10, n_burnin = 100, subsample_interval = 10, step_size = 1e-1, leapfrog_steps=20, verbose=False):
        self.kernel = kernel
        self.noise_var = noise_var
        self.exact_feval = exact_feval
        self.verbose = verbose
        self.n_samples = n_samples
        self.subsample_interval = subsample_interval
        self.n_burnin = n_burnin
        self.step_size = step_size
        self.leapfrog_steps = leapfrog_steps
        self.model = None

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """

        # --- define kernel
        self.input_dim = X.shape[1]
        if self.kernel is None:
            kern = GPy.kern.RBF(self.input_dim, variance=1.)
        else:
            kern = self.kernel
            self.kernel = None

        # --- define model
        noise_var = Y.var()*0.01 if self.noise_var is None else self.noise_var
        self.model = GPy.models.GPRegression(X, Y, kernel=kern, noise_var=noise_var)

        # --- Define prior on the hyper-parameters for the kernel (for integrated acquisitions)
        self.model.kern.set_prior(GPy.priors.Gamma.from_EV(2.,4.))
        self.model.likelihood.variance.set_prior(GPy.priors.Gamma.from_EV(2.,4.))

        # --- Restrict variance if exact evaluations of the objective
        if self.exact_feval:
            self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False)
        else:
            self.model.Gaussian_noise.constrain_positive(warning=False)

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """

        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.set_XY(X_all, Y_all)

        # update the model generating hmc samples
        self.model.optimize(max_iters = 200)
        self.model.param_array[:] = self.model.param_array * (1.+np.random.randn(self.model.param_array.size)*0.01)
        self.hmc = GPy.inference.mcmc.HMC(self.model, stepsize=self.step_size)
        ss = self.hmc.sample(num_samples=self.n_burnin + self.n_samples* self.subsample_interval, hmc_iters=self.leapfrog_steps)
        self.hmc_samples = ss[self.n_burnin::self.subsample_interval]

    def predict(self, X):
        """
        Predictions with the model for all the MCMC samples. Returns posterior means and standard deviations at X. Note that this is different in GPy where the variances are given.
        """

        if X.ndim==1: X = X[None,:]
        ps = self.model.param_array.copy()
        means = []
        stds = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            m, v = self.model.predict(X)
            means.append(m)
            stds.append(np.sqrt(np.clip(v, 1e-10, np.inf)))
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()
        return means, stds

    def get_fmin(self):
        """
        Returns the location where the posterior mean is takes its minimal value.
        """
        ps = self.model.param_array.copy()
        fmins = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            fmins.append(self.model.predict(self.model.X)[0].min())
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()

        return fmins

    def predict_withGradients(self, X):
        """
        Returns the mean, standard deviation, mean gradient and standard deviation gradient at X for all the MCMC samples.
        """
        if X.ndim==1: X = X[None,:]
        ps = self.model.param_array.copy()
        means = []
        stds = []
        dmdxs = []
        dsdxs = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            m, v = self.model.predict(X)
            std = np.sqrt(np.clip(v, 1e-10, np.inf))
            dmdx, dvdx = self.model.predictive_gradients(X)
            dmdx = dmdx[:,:,0]
            dsdx = dvdx / (2*std)
            means.append(m)
            stds.append(std)
            dmdxs.append(dmdx)
            dsdxs.append(dsdx)
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()
        return means, stds, dmdxs, dsdxs

    def copy(self):
        """
        Makes a safe copy of the model.
        """

        copied_model = GPModel( kernel = self.model.kern.copy(),
                                noise_var= self.noise_var ,
                                exact_feval= self.exact_feval,
                                n_samples = self.n_samples,
                                n_burnin = self.n_burnin,
                                subsample_interval = self.subsample_interval,
                                step_size = self.step_size,
                                leapfrog_steps= self.leapfrog_steps,
                                verbose= self.verbose)

        copied_model._create_model(self.model.X,self.model.Y)
        copied_model.updateModel(self.model.X,self.model.Y, None, None)
        return copied_model

    def get_model_parameters(self):
        """
        Returns a 2D numpy array with the parameters of the model
        """
        return np.atleast_2d(self.model[:])

    def get_model_parameters_names(self):
        """
        Returns a list with the names of the parameters of the model
        """
        return self.model.parameter_names()
<commit_msg>Added the ability to set the mean function (#268)<commit_after># Copyright (c) 2016, the GPyOpt Authors
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import numpy as np
import GPy

from .base import BOModel

class GPModel(BOModel):
    """
    General class for handling a Gaussian Process in GPyOpt.

    :param kernel: GPy kernel to use in the GP model.
    :param noise_var: value of the noise variance if known.
    :param exact_feval: whether noiseless evaluations are available. IMPORTANT to make the optimization work well in noiseless scenarios (default, False).
    :param optimizer: optimizer of the model. Check GPy for details.
    :param max_iters: maximum number of iterations used to optimize the parameters of the model.
    :param optimize_restarts: number of restarts in the optimization.
    :param sparse: whether to use a sparse GP (default, False). This is useful when many observations are available.
    :param num_inducing: number of inducing points if a sparse GP is used.
    :param verbose: print out the model messages (default, False).
    :param ARD: whether ARD is used in the kernel (default, False).
    :param mean_function: GPy Mapping to use as the mean function for the GP model (default, None).

    .. Note:: This model does Maximum likelihood estimation of the hyper-parameters.

    """

    analytical_gradient_prediction = True  # --- Needed in all models to check is the gradients of acquisitions are computable.

    def __init__(self, kernel=None, noise_var=None, exact_feval=False, optimizer='bfgs', max_iters=1000, optimize_restarts=5, sparse = False, num_inducing = 10,  verbose=True, ARD=False, mean_function=None):
        self.kernel = kernel
        self.noise_var = noise_var
        self.exact_feval = exact_feval
        self.optimize_restarts = optimize_restarts
        self.optimizer = optimizer
        self.max_iters = max_iters
        self.verbose = verbose
        self.sparse = sparse
        self.num_inducing = num_inducing
        self.model = None
        self.ARD = ARD
        self.mean_function = mean_function

    @staticmethod
    def fromConfig(config):
        return GPModel(**config)

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """

        # --- define kernel
        self.input_dim = X.shape[1]
        if self.kernel is None:
            kern = GPy.kern.Matern52(self.input_dim, variance=1., ARD=self.ARD) #+ GPy.kern.Bias(self.input_dim)
        else:
            kern = self.kernel
            self.kernel = None

        # --- define model
        noise_var = Y.var()*0.01 if self.noise_var is None else self.noise_var

        if not self.sparse:
            self.model = GPy.models.GPRegression(X, Y, kernel=kern, noise_var=noise_var, mean_function=self.mean_function)
        else:
            self.model = GPy.models.SparseGPRegression(X, Y, kernel=kern, num_inducing=self.num_inducing, mean_function=self.mean_function)

        # --- restrict variance if exact evaluations of the objective
        if self.exact_feval:
            self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False)
        else:
            # --- We make sure we do not get ridiculously small residual noise variance
            self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False) #constrain_positive(warning=False)

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """
        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.set_XY(X_all, Y_all)

        # WARNING: Even if self.max_iters=0, the hyperparameters are bit modified...
        if self.max_iters > 0:
            # --- update the model maximizing the marginal likelihood.
            if self.optimize_restarts==1:
                self.model.optimize(optimizer=self.optimizer, max_iters = self.max_iters, messages=False, ipython_notebook=False)
            else:
                self.model.optimize_restarts(num_restarts=self.optimize_restarts, optimizer=self.optimizer, max_iters = self.max_iters, verbose=self.verbose)

    def _predict(self, X, full_cov, include_likelihood):
        if X.ndim == 1:
            X = X[None,:]
        m, v = self.model.predict(X, full_cov=full_cov, include_likelihood=include_likelihood)
        v = np.clip(v, 1e-10, np.inf)
        return m, v

    def predict(self, X, with_noise=True):
        """
        Predictions with the model. Returns posterior means and standard deviations at X. Note that this is different in GPy where the variances are given.

        Parameters:
            X (np.ndarray) - points to run the prediction for.
            with_noise (bool) - whether to add noise to the prediction. Default is True.
        """
        m, v = self._predict(X, False, with_noise)
        # We can take the square root because v is just a diagonal matrix of variances
        return m, np.sqrt(v)

    def predict_covariance(self, X, with_noise=True):
        """
        Predicts the covariance matric for points in X.

        Parameters:
            X (np.ndarray) - points to run the prediction for.
            with_noise (bool) - whether to add noise to the prediction. Default is True.
        """
        _, v = self._predict(X, True, with_noise)
        return v

    def get_fmin(self):
        """
        Returns the location where the posterior mean is takes its minimal value.
        """
        return self.model.predict(self.model.X)[0].min()

    def predict_withGradients(self, X):
        """
        Returns the mean, standard deviation, mean gradient and standard deviation gradient at X.
        """
        if X.ndim==1: X = X[None,:]
        m, v = self.model.predict(X)
        v = np.clip(v, 1e-10, np.inf)
        dmdx, dvdx = self.model.predictive_gradients(X)
        dmdx = dmdx[:,:,0]
        dsdx = dvdx / (2*np.sqrt(v))

        return m, np.sqrt(v), dmdx, dsdx

    def copy(self):
        """
        Makes a safe copy of the model.
        """
        copied_model = GPModel(kernel = self.model.kern.copy(),
                            noise_var=self.noise_var,
                            exact_feval=self.exact_feval,
                            optimizer=self.optimizer,
                            max_iters=self.max_iters,
                            optimize_restarts=self.optimize_restarts,
                            verbose=self.verbose,
                            ARD=self.ARD)

        copied_model._create_model(self.model.X,self.model.Y)
        copied_model.updateModel(self.model.X,self.model.Y, None, None)
        return copied_model

    def get_model_parameters(self):
        """
        Returns a 2D numpy array with the parameters of the model
        """
        return np.atleast_2d(self.model[:])

    def get_model_parameters_names(self):
        """
        Returns a list with the names of the parameters of the model
        """
        return self.model.parameter_names_flat().tolist()

    def get_covariance_between_points(self, x1, x2):
        """
        Given the current posterior, computes the covariance between two sets of points.
        """
        return self.model.posterior_covariance_between_points(x1, x2)

class GPModel_MCMC(BOModel):
    """
    General class for handling a Gaussian Process in GPyOpt.

    :param kernel: GPy kernel to use in the GP model.
    :param noise_var: value of the noise variance if known.
    :param exact_feval: whether noiseless evaluations are available. IMPORTANT to make the optimization work well in noiseless scenarios (default, False).
    :param n_samples: number of MCMC samples.
    :param n_burnin: number of samples not used.
    :param subsample_interval: sub-sample interval in the MCMC.
    :param step_size: step-size in the MCMC.
    :param leapfrog_steps: ??
    :param verbose: print out the model messages (default, False).

    .. Note:: This model does MCMC over the hyperparameters.

    """

    MCMC_sampler = True
    analytical_gradient_prediction = True # --- Needed in all models to check is the gradients of acquisitions are computable.

    def __init__(self, kernel=None, noise_var=None, exact_feval=False, n_samples = 10, n_burnin = 100, subsample_interval = 10, step_size = 1e-1, leapfrog_steps=20, verbose=False):
        self.kernel = kernel
        self.noise_var = noise_var
        self.exact_feval = exact_feval
        self.verbose = verbose
        self.n_samples = n_samples
        self.subsample_interval = subsample_interval
        self.n_burnin = n_burnin
        self.step_size = step_size
        self.leapfrog_steps = leapfrog_steps
        self.model = None

    def _create_model(self, X, Y):
        """
        Creates the model given some input data X and Y.
        """

        # --- define kernel
        self.input_dim = X.shape[1]
        if self.kernel is None:
            kern = GPy.kern.RBF(self.input_dim, variance=1.)
        else:
            kern = self.kernel
            self.kernel = None

        # --- define model
        noise_var = Y.var()*0.01 if self.noise_var is None else self.noise_var
        self.model = GPy.models.GPRegression(X, Y, kernel=kern, noise_var=noise_var)

        # --- Define prior on the hyper-parameters for the kernel (for integrated acquisitions)
        self.model.kern.set_prior(GPy.priors.Gamma.from_EV(2.,4.))
        self.model.likelihood.variance.set_prior(GPy.priors.Gamma.from_EV(2.,4.))

        # --- Restrict variance if exact evaluations of the objective
        if self.exact_feval:
            self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False)
        else:
            self.model.Gaussian_noise.constrain_positive(warning=False)

    def updateModel(self, X_all, Y_all, X_new, Y_new):
        """
        Updates the model with new observations.
        """

        if self.model is None:
            self._create_model(X_all, Y_all)
        else:
            self.model.set_XY(X_all, Y_all)

        # update the model generating hmc samples
        self.model.optimize(max_iters = 200)
        self.model.param_array[:] = self.model.param_array * (1.+np.random.randn(self.model.param_array.size)*0.01)
        self.hmc = GPy.inference.mcmc.HMC(self.model, stepsize=self.step_size)
        ss = self.hmc.sample(num_samples=self.n_burnin + self.n_samples* self.subsample_interval, hmc_iters=self.leapfrog_steps)
        self.hmc_samples = ss[self.n_burnin::self.subsample_interval]

    def predict(self, X):
        """
        Predictions with the model for all the MCMC samples. Returns posterior means and standard deviations at X. Note that this is different in GPy where the variances are given.
        """

        if X.ndim==1: X = X[None,:]
        ps = self.model.param_array.copy()
        means = []
        stds = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            m, v = self.model.predict(X)
            means.append(m)
            stds.append(np.sqrt(np.clip(v, 1e-10, np.inf)))
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()
        return means, stds

    def get_fmin(self):
        """
        Returns the location where the posterior mean is takes its minimal value.
        """
        ps = self.model.param_array.copy()
        fmins = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            fmins.append(self.model.predict(self.model.X)[0].min())
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()

        return fmins

    def predict_withGradients(self, X):
        """
        Returns the mean, standard deviation, mean gradient and standard deviation gradient at X for all the MCMC samples.
        """
        if X.ndim==1: X = X[None,:]
        ps = self.model.param_array.copy()
        means = []
        stds = []
        dmdxs = []
        dsdxs = []
        for s in self.hmc_samples:
            if self.model._fixes_ is None:
                self.model[:] = s
            else:
                self.model[self.model._fixes_] = s
            self.model._trigger_params_changed()
            m, v = self.model.predict(X)
            std = np.sqrt(np.clip(v, 1e-10, np.inf))
            dmdx, dvdx = self.model.predictive_gradients(X)
            dmdx = dmdx[:,:,0]
            dsdx = dvdx / (2*std)
            means.append(m)
            stds.append(std)
            dmdxs.append(dmdx)
            dsdxs.append(dsdx)
        self.model.param_array[:] = ps
        self.model._trigger_params_changed()
        return means, stds, dmdxs, dsdxs

    def copy(self):
        """
        Makes a safe copy of the model.
        """

        copied_model = GPModel( kernel = self.model.kern.copy(),
                                noise_var= self.noise_var ,
                                exact_feval= self.exact_feval,
                                n_samples = self.n_samples,
                                n_burnin = self.n_burnin,
                                subsample_interval = self.subsample_interval,
                                step_size = self.step_size,
                                leapfrog_steps= self.leapfrog_steps,
                                verbose= self.verbose)

        copied_model._create_model(self.model.X,self.model.Y)
        copied_model.updateModel(self.model.X,self.model.Y, None, None)
        return copied_model

    def get_model_parameters(self):
        """
        Returns a 2D numpy array with the parameters of the model
        """
        return np.atleast_2d(self.model[:])

    def get_model_parameters_names(self):
        """
        Returns a list with the names of the parameters of the model
        """
        return self.model.parameter_names()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from todolist import *
import glob
import os
import sys

import pygtk
pygtk.require('2.0')
import gtk
import pango

json_folder = os.path.expanduser('~') + '/.todopy/'

class TodoGUI:

	lists = {}

	def __init__(self):
		"""
		Creates the GUI display.
		"""

		# Initialise default list.
		# TODO: Support multiple lists
		self.todolist = TodoList(json_folder + 'todo.json')

		# Start of multiple lists support. Currently unused
		for infile in glob.glob( os.path.join(json_folder, '*.json') ):
			listname = infile.replace(json_folder, '').replace('.json', '')
			self.lists[listname] = TodoList(infile)
			print 'Found:', infile.replace(json_folder, '').replace('.json', '')

		self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
		self.window.set_title('Simple Python Todo')

		send_button = gtk.Button(None, gtk.STOCK_ADD)
		send_button.connect('clicked', self.send_button_clicked)

		vbox = gtk.VBox(False, 10)
		vbox.set_border_width(10)

		hbox_note_area = gtk.HBox(False, 0)
		hbox_send_area = gtk.HBox(False, 0)

		self.current_list = ListPanel(self.todolist)
		sw_display = self.current_list.sw
		hbox_note_area.pack_start(sw_display)

		# Set up the text view for adding new notes
		sw_add = gtk.ScrolledWindow()
		sw_add.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)

		self.textview_add = self.create_text_view(230, 80)
		self.textview_add.connect('key_press_event', self.textbox_key_pressed)
		sw_add.add(self.textview_add)

		hbox_send_area.pack_start(sw_add)
		hbox_send_area.pack_start(send_button)

		# Organise the boxes
		vbox.pack_start(hbox_note_area)
		hbox_note_area.show()
		vbox.pack_start(hbox_send_area)
		hbox_send_area.show()
		self.window.add(vbox)

		self.window.connect('delete_event', self.delete_event)
		self.window.show_all()

	def main(self):
		"""
		Starts the Todo GUI
		"""
		gtk.main()

	def delete_event(self, widget, event, Data=None):
		"""
		Saves the list and closes the app.
		"""
		self.todolist.save()
		gtk.main_quit()
		return False

	def textbox_key_pressed(self, widget, event, Data=None):
		"""
		Checks if enter was pressed, and if so, adds a new item
		"""
		if event.keyval == gtk.gdk.keyval_from_name('Return') or \
		event.keyval == gtk.gdk.keyval_from_name('KP_Enter'):
			self.add_item_to_list(self.current_list)
			return True

	def send_button_clicked(self, widget, event, Data=None):
		"""
		Adds a new item when the send button is clicked.
		"""
		self.add_item_to_list(self.current_list)
		return True

	def add_item_to_list(self, todolist):
		"""
		Copies the text from the entry box to a todolist.
		"""
		note = self.get_all_text_view_text(self.textview_add)
		todolist.add_item(note)
		self.textview_add.get_buffer().set_text('')

	def create_text_view(self, width=200, height=200, editable=True, wrap=True):
		"""
		Creates a text view with certain settings.
		"""
		textview = gtk.TextView()
		textview.set_size_request(width, height)

		if wrap is True:
			textview.set_wrap_mode(gtk.WRAP_WORD)

		textview.set_editable(editable)
		# Don't show cursor in uneditable text views.
		textview.set_cursor_visible(editable)

		return textview

	def get_all_text_view_text(self, textview):
		"""
		Gets all the text from a text view.
		"""
		textbuffer = textview.get_buffer()
		startiter, enditer = textbuffer.get_bounds()
		text = startiter.get_slice(enditer)
		return text

class ListPanel:

	def __init__(self, todolist):
		"""
		Set up the tree view for showing a todo list.
		"""
		self.todolist = todolist
		self.selection_id = None;

		self.store = self.create_model(todolist)
		self.display_view = gtk.TreeView(self.store)
		self.display_view.set_size_request(280, 200)
		self.display_view.set_rules_hint(True)

		self.display_view.connect('key_press_event', self.key_pressed)

		selection = self.display_view.get_selection()
		selection.connect("changed", self.selection_changed)
		selection.set_mode(gtk.SELECTION_SINGLE)

		self.create_columns(self.display_view)

		self.sw = gtk.ScrolledWindow()
		self.sw.add(self.display_view)
		self.sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)

	def create_model(self, todolist):
		"""
		Sets up the store for the TreeView
		"""
		store = gtk.ListStore(int, str)

		for todo in todolist:
			new_row = (todo.id, todo.text)
			print new_row
			store.append(new_row)

		return store

	def create_columns(self, tree_view):
		"""
		Sets up the columns for the TreeView
		"""

		renderer_text = gtk.CellRendererText()
		renderer_text.props.wrap_width = 280
		renderer_text.props.wrap_mode = pango.WRAP_WORD
		column = gtk.TreeViewColumn('Text', renderer_text, text=1)
		column.set_sizing(gtk.TREE_VIEW_COLUMN_AUTOSIZE)
		column.set_sort_column_id(0)
		tree_view.append_column(column)

	def add_item(self, text):
		"""
		Adds an item to the todo list.
		"""
		new_todo = self.todolist.add(text)
		self.store.append((new_todo.id, text))

	def remove_item(self, item_id):
		"""
		Remove an item from the list, and the display.
		"""
		self.todolist.remove(item_id)
		store = self.store
		store.foreach(self.find_item_to_remove, item_id)
		#titer = store.get_iter_root()
		#while True:
		#	if store.get_path(titer)[0] == item_id:
		#		store.remove(titer)
		#		break
		#	else:
		#		titer = store.iter_next(titer)
		#		if titer is None:
		#			break

	def find_item_to_remove(self, model, path, iter, user_data):
		"""
		Finds and removes the correct item from the ListStore.
		"""
		if path[0] == user_data:
			model.remove(iter)
			return True

	def key_pressed(self, widget, event, Data=None):
		"""
		Deletes the item from the list if the delete key is pressed.
		"""
		if event.keyval == gtk.gdk.keyval_from_name('Delete') and self.selection_id is not None:
			self.remove_item(self.selection_id)
			self.selection_id = None;
			return True

	def selection_changed(self, selection):
		"""
		Sets the currently selected items ID
		"""
		store, titer = selection.get_selected()
		if titer is not None:
			print store.get_path(titer)
			for item in store:
				print item[0], item[1]
			self.selection_id = store.get_value(titer, 0)

TodoGUI().main()
<commit_msg>Fix deleting items based on position in the list (which sometimes caused IndexError's) and deleted them based on actual ID.<commit_after>#!/usr/bin/env python
from todolist import *
import glob
import os
import sys

import pygtk
pygtk.require('2.0')
import gtk
import pango

json_folder = os.path.expanduser('~') + '/.todopy/'

class TodoGUI:

	lists = {}

	def __init__(self):
		"""
		Creates the GUI display.
		"""

		# Initialise default list.
		# TODO: Support multiple lists
		self.todolist = TodoList(json_folder + 'todo.json')

		# Start of multiple lists support. Currently unused
		for infile in glob.glob( os.path.join(json_folder, '*.json') ):
			listname = infile.replace(json_folder, '').replace('.json', '')
			self.lists[listname] = TodoList(infile)
			print 'Found:', infile.replace(json_folder, '').replace('.json', '')

		self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
		self.window.set_title('Simple Python Todo')

		send_button = gtk.Button(None, gtk.STOCK_ADD)
		send_button.connect('clicked', self.send_button_clicked)

		vbox = gtk.VBox(False, 10)
		vbox.set_border_width(10)

		hbox_note_area = gtk.HBox(False, 0)
		hbox_send_area = gtk.HBox(False, 0)

		self.current_list = ListPanel(self.todolist)
		sw_display = self.current_list.sw
		hbox_note_area.pack_start(sw_display)

		# Set up the text view for adding new notes
		sw_add = gtk.ScrolledWindow()
		sw_add.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)

		self.textview_add = self.create_text_view(230, 80)
		self.textview_add.connect('key_press_event', self.textbox_key_pressed)
		sw_add.add(self.textview_add)

		hbox_send_area.pack_start(sw_add)
		hbox_send_area.pack_start(send_button)

		# Organise the boxes
		vbox.pack_start(hbox_note_area)
		hbox_note_area.show()
		vbox.pack_start(hbox_send_area)
		hbox_send_area.show()
		self.window.add(vbox)

		self.window.connect('delete_event', self.delete_event)
		self.window.show_all()

	def main(self):
		"""
		Starts the Todo GUI
		"""
		gtk.main()

	def delete_event(self, widget, event, Data=None):
		"""
		Saves the list and closes the app.
		"""
		self.todolist.save()
		gtk.main_quit()
		return False

	def textbox_key_pressed(self, widget, event, Data=None):
		"""
		Checks if enter was pressed, and if so, adds a new item
		"""
		if event.keyval == gtk.gdk.keyval_from_name('Return') or \
		event.keyval == gtk.gdk.keyval_from_name('KP_Enter'):
			self.add_item_to_list(self.current_list)
			return True

	def send_button_clicked(self, widget, event, Data=None):
		"""
		Adds a new item when the send button is clicked.
		"""
		self.add_item_to_list(self.current_list)
		return True

	def add_item_to_list(self, todolist):
		"""
		Copies the text from the entry box to a todolist.
		"""
		note = self.get_all_text_view_text(self.textview_add)
		todolist.add_item(note)
		self.textview_add.get_buffer().set_text('')

	def create_text_view(self, width=200, height=200, editable=True, wrap=True):
		"""
		Creates a text view with certain settings.
		"""
		textview = gtk.TextView()
		textview.set_size_request(width, height)

		if wrap is True:
			textview.set_wrap_mode(gtk.WRAP_WORD)

		textview.set_editable(editable)
		# Don't show cursor in uneditable text views.
		textview.set_cursor_visible(editable)

		return textview

	def get_all_text_view_text(self, textview):
		"""
		Gets all the text from a text view.
		"""
		textbuffer = textview.get_buffer()
		startiter, enditer = textbuffer.get_bounds()
		text = startiter.get_slice(enditer)
		return text

class ListPanel:

	def __init__(self, todolist):
		"""
		Set up the tree view for showing a todo list.
		"""
		self.todolist = todolist
		self.selection_id = None;

		self.store = self.create_model(todolist)
		self.display_view = gtk.TreeView(self.store)
		self.display_view.set_size_request(280, 200)
		self.display_view.set_rules_hint(True)

		self.display_view.connect('key_press_event', self.key_pressed)

		selection = self.display_view.get_selection()
		selection.connect("changed", self.selection_changed)
		selection.set_mode(gtk.SELECTION_SINGLE)

		self.create_columns(self.display_view)

		self.sw = gtk.ScrolledWindow()
		self.sw.add(self.display_view)
		self.sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)

	def create_model(self, todolist):
		"""
		Sets up the store for the TreeView
		"""
		store = gtk.ListStore(int, str)

		for todo in todolist:
			new_row = (todo.id, todo.text)
			print new_row
			store.append(new_row)

		return store

	def create_columns(self, tree_view):
		"""
		Sets up the columns for the TreeView
		"""

		renderer_text = gtk.CellRendererText()
		renderer_text.props.wrap_width = 280
		renderer_text.props.wrap_mode = pango.WRAP_WORD
		column = gtk.TreeViewColumn('Text', renderer_text, text=1)
		column.set_sizing(gtk.TREE_VIEW_COLUMN_AUTOSIZE)
		column.set_sort_column_id(0)
		tree_view.append_column(column)

	def add_item(self, text):
		"""
		Adds an item to the todo list.
		"""
		new_todo = self.todolist.add(text)
		self.store.append((new_todo.id, text))

	def remove_item(self, item_id):
		"""
		Remove an item from the list, and the display.
		"""
		self.todolist.remove(item_id)
		store = self.store
		for row in store:
			if row[0] == item_id:
				store.remove(row.iter)
				break

	def key_pressed(self, widget, event, Data=None):
		"""
		Deletes the item from the list if the delete key is pressed.
		"""
		if event.keyval == gtk.gdk.keyval_from_name('Delete') and self.selection_id is not None:
			self.remove_item(self.selection_id)
			self.selection_id = None;
			return True

	def selection_changed(self, selection):
		"""
		Sets the currently selected items ID
		"""
		store, titer = selection.get_selected()
		if titer is not None:
			print store.get_path(titer)
			for item in store:
				print item[0], item[1]
			self.selection_id = store.get_value(titer, 0)

TodoGUI().main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.util.frozenutils import freeze
from synapse.util.caches import intern_dict

# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
# bugs where we accidentally share e.g. signature dicts. However, converting
# a dict to frozen_dicts is expensive.
USE_FROZEN_DICTS = True

class _EventInternalMetadata(object):
    def __init__(self, internal_metadata_dict):
        self.__dict__ = dict(internal_metadata_dict)

    def get_dict(self):
        return dict(self.__dict__)

    def is_outlier(self):
        return getattr(self, "outlier", False)

    def is_invite_from_remote(self):
        return getattr(self, "invite_from_remote", False)

    def get_send_on_behalf_of(self):
        """Whether this server should send the event on behalf of another server.
        This is used by the federation "send_join" API to forward the initial join
        event for a server in the room.

        returns a str with the name of the server this event is sent on behalf of.
        """
        return getattr(self, "get_send_on_behalf_of", None)

def _event_dict_property(key):
    def getter(self):
        return self._event_dict[key]

    def setter(self, v):
        self._event_dict[key] = v

    def delete(self):
        del self._event_dict[key]

    return property(
        getter,
        setter,
        delete,
    )

class EventBase(object):
    def __init__(self, event_dict, signatures={}, unsigned={},
                 internal_metadata_dict={}, rejected_reason=None):
        self.signatures = signatures
        self.unsigned = unsigned
        self.rejected_reason = rejected_reason

        self._event_dict = event_dict

        self.internal_metadata = _EventInternalMetadata(
            internal_metadata_dict
        )

    auth_events = _event_dict_property("auth_events")
    depth = _event_dict_property("depth")
    content = _event_dict_property("content")
    event_id = _event_dict_property("event_id")
    hashes = _event_dict_property("hashes")
    origin = _event_dict_property("origin")
    origin_server_ts = _event_dict_property("origin_server_ts")
    prev_events = _event_dict_property("prev_events")
    prev_state = _event_dict_property("prev_state")
    redacts = _event_dict_property("redacts")
    room_id = _event_dict_property("room_id")
    sender = _event_dict_property("sender")
    state_key = _event_dict_property("state_key")
    type = _event_dict_property("type")
    user_id = _event_dict_property("sender")

    @property
    def membership(self):
        return self.content["membership"]

    def is_state(self):
        return hasattr(self, "state_key") and self.state_key is not None

    def get_dict(self):
        d = dict(self._event_dict)
        d.update({
            "signatures": self.signatures,
            "unsigned": dict(self.unsigned),
        })

        return d

    def get(self, key, default=None):
        return self._event_dict.get(key, default)

    def get_internal_metadata_dict(self):
        return self.internal_metadata.get_dict()

    def get_pdu_json(self, time_now=None):
        pdu_json = self.get_dict()

        if time_now is not None and "age_ts" in pdu_json["unsigned"]:
            age = time_now - pdu_json["unsigned"]["age_ts"]
            pdu_json.setdefault("unsigned", {})["age"] = int(age)
            del pdu_json["unsigned"]["age_ts"]

        # This may be a frozen event
        pdu_json["unsigned"].pop("redacted_because", None)

        return pdu_json

    def __set__(self, instance, value):
        raise AttributeError("Unrecognized attribute %s" % (instance,))

    def __getitem__(self, field):
        return self._event_dict[field]

    def __contains__(self, field):
        return field in self._event_dict

    def items(self):
        return self._event_dict.items()

class FrozenEvent(EventBase):
    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
        event_dict = dict(event_dict)

        # Signatures is a dict of dicts, and this is faster than doing a
        # copy.deepcopy
        signatures = {
            name: {sig_id: sig for sig_id, sig in sigs.items()}
            for name, sigs in event_dict.pop("signatures", {}).items()
        }

        unsigned = dict(event_dict.pop("unsigned", {}))

        # We intern these strings because they turn up a lot (especially when
        # caching).
        event_dict = intern_dict(event_dict)

        if USE_FROZEN_DICTS:
            frozen_dict = freeze(event_dict)
        else:
            frozen_dict = event_dict

        super(FrozenEvent, self).__init__(
            frozen_dict,
            signatures=signatures,
            unsigned=unsigned,
            internal_metadata_dict=internal_metadata_dict,
            rejected_reason=rejected_reason,
        )

    @staticmethod
    def from_event(event):
        e = FrozenEvent(
            event.get_pdu_json()
        )

        e.internal_metadata = event.internal_metadata

        return e

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "<FrozenEvent event_id='%s', type='%s', state_key='%s'>" % (
            self.get("event_id", None),
            self.get("type", None),
            self.get("state_key", None),
        )
<commit_msg>fix typo breaking the fix to #1753<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.util.frozenutils import freeze
from synapse.util.caches import intern_dict

# Whether we should use frozen_dict in FrozenEvent. Using frozen_dicts prevents
# bugs where we accidentally share e.g. signature dicts. However, converting
# a dict to frozen_dicts is expensive.
USE_FROZEN_DICTS = True

class _EventInternalMetadata(object):
    def __init__(self, internal_metadata_dict):
        self.__dict__ = dict(internal_metadata_dict)

    def get_dict(self):
        return dict(self.__dict__)

    def is_outlier(self):
        return getattr(self, "outlier", False)

    def is_invite_from_remote(self):
        return getattr(self, "invite_from_remote", False)

    def get_send_on_behalf_of(self):
        """Whether this server should send the event on behalf of another server.
        This is used by the federation "send_join" API to forward the initial join
        event for a server in the room.

        returns a str with the name of the server this event is sent on behalf of.
        """
        return getattr(self, "send_on_behalf_of", None)

def _event_dict_property(key):
    def getter(self):
        return self._event_dict[key]

    def setter(self, v):
        self._event_dict[key] = v

    def delete(self):
        del self._event_dict[key]

    return property(
        getter,
        setter,
        delete,
    )

class EventBase(object):
    def __init__(self, event_dict, signatures={}, unsigned={},
                 internal_metadata_dict={}, rejected_reason=None):
        self.signatures = signatures
        self.unsigned = unsigned
        self.rejected_reason = rejected_reason

        self._event_dict = event_dict

        self.internal_metadata = _EventInternalMetadata(
            internal_metadata_dict
        )

    auth_events = _event_dict_property("auth_events")
    depth = _event_dict_property("depth")
    content = _event_dict_property("content")
    event_id = _event_dict_property("event_id")
    hashes = _event_dict_property("hashes")
    origin = _event_dict_property("origin")
    origin_server_ts = _event_dict_property("origin_server_ts")
    prev_events = _event_dict_property("prev_events")
    prev_state = _event_dict_property("prev_state")
    redacts = _event_dict_property("redacts")
    room_id = _event_dict_property("room_id")
    sender = _event_dict_property("sender")
    state_key = _event_dict_property("state_key")
    type = _event_dict_property("type")
    user_id = _event_dict_property("sender")

    @property
    def membership(self):
        return self.content["membership"]

    def is_state(self):
        return hasattr(self, "state_key") and self.state_key is not None

    def get_dict(self):
        d = dict(self._event_dict)
        d.update({
            "signatures": self.signatures,
            "unsigned": dict(self.unsigned),
        })

        return d

    def get(self, key, default=None):
        return self._event_dict.get(key, default)

    def get_internal_metadata_dict(self):
        return self.internal_metadata.get_dict()

    def get_pdu_json(self, time_now=None):
        pdu_json = self.get_dict()

        if time_now is not None and "age_ts" in pdu_json["unsigned"]:
            age = time_now - pdu_json["unsigned"]["age_ts"]
            pdu_json.setdefault("unsigned", {})["age"] = int(age)
            del pdu_json["unsigned"]["age_ts"]

        # This may be a frozen event
        pdu_json["unsigned"].pop("redacted_because", None)

        return pdu_json

    def __set__(self, instance, value):
        raise AttributeError("Unrecognized attribute %s" % (instance,))

    def __getitem__(self, field):
        return self._event_dict[field]

    def __contains__(self, field):
        return field in self._event_dict

    def items(self):
        return self._event_dict.items()

class FrozenEvent(EventBase):
    def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
        event_dict = dict(event_dict)

        # Signatures is a dict of dicts, and this is faster than doing a
        # copy.deepcopy
        signatures = {
            name: {sig_id: sig for sig_id, sig in sigs.items()}
            for name, sigs in event_dict.pop("signatures", {}).items()
        }

        unsigned = dict(event_dict.pop("unsigned", {}))

        # We intern these strings because they turn up a lot (especially when
        # caching).
        event_dict = intern_dict(event_dict)

        if USE_FROZEN_DICTS:
            frozen_dict = freeze(event_dict)
        else:
            frozen_dict = event_dict

        super(FrozenEvent, self).__init__(
            frozen_dict,
            signatures=signatures,
            unsigned=unsigned,
            internal_metadata_dict=internal_metadata_dict,
            rejected_reason=rejected_reason,
        )

    @staticmethod
    def from_event(event):
        e = FrozenEvent(
            event.get_pdu_json()
        )

        e.internal_metadata = event.internal_metadata

        return e

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "<FrozenEvent event_id='%s', type='%s', state_key='%s'>" % (
            self.get("event_id", None),
            self.get("type", None),
            self.get("state_key", None),
        )
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

import glyphsLib.glyphdata
from .common import to_ufo_time, from_loose_ufo_time
from .constants import GLYPHLIB_PREFIX, GLYPHS_COLORS, PUBLIC_PREFIX

logger = logging.getLogger(__name__)

SCRIPT_LIB_KEY = GLYPHLIB_PREFIX + "script"
ORIGINAL_WIDTH_KEY = GLYPHLIB_PREFIX + "originalWidth"
BACKGROUND_WIDTH_KEY = GLYPHLIB_PREFIX + "backgroundWidth"

def to_ufo_glyph(self, ufo_glyph, layer, glyph):
    """Add .glyphs metadata, paths, components, and anchors to a glyph."""
    ufo_font = self._sources[layer.associatedMasterId or layer.layerId].font

    ufo_glyph.unicodes = [int(uval, 16) for uval in glyph.unicodes]

    note = glyph.note
    if note is not None:
        ufo_glyph.note = note

    # Optimization: profiling glyphs2ufo of NotoSans-MM.glyphs (6000 glyphs) on a Mac
    # mini late 2014, Python 3.6.8, revealed that a whopping 17% of the time was spent
    # converting lastChange to UFO timestamps. I could not reproduce this on a Windows
    # 10/Python 3.7 setup, so this might be a platform thing. If-guarding anyway
    # because these timestamps are useless in a UFO scenario if you use Git.
    if (
        self.minimize_glyphs_diffs
        and self.font.customParameters["Disable Last Change"] is not True
        and glyph.lastChange is not None
    ):
        ufo_glyph.lib[GLYPHLIB_PREFIX + "lastChange"] = to_ufo_time(glyph.lastChange)

    color_index = glyph.color
    if color_index is not None:
        # .3f is enough precision to round-trip uint8 to float losslessly.
        # https://github.com/unified-font-object/ufo-spec/issues/61
        # #issuecomment-389759127
        if (
            isinstance(color_index, list)
            and len(color_index) == 4
            and all(0 <= v < 256 for v in color_index)
        ):
            ufo_glyph.markColor = ",".join(
                "0" if v == 0 else "1" if v == 255 else "{:.3f}".format(v / 255)
                for v in color_index
            )
        elif isinstance(color_index, int) and color_index in range(len(GLYPHS_COLORS)):
            ufo_glyph.markColor = GLYPHS_COLORS[color_index]
        else:
            logger.warning(
                "Glyph {}, layer {}: Invalid color index/tuple {}".format(
                    glyph.name, layer.name, color_index
                )
            )

    export = glyph.export
    if not export:
        if self.write_skipexportglyphs:
            if "public.skipExportGlyphs" not in self._designspace.lib:
                self._designspace.lib["public.skipExportGlyphs"] = []
            self._designspace.lib["public.skipExportGlyphs"].append(glyph.name)
        else:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "Export"] = export

    # FIXME: (jany) next line should be an API of GSGlyph?
    glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)
    production_name = glyph.production or glyphinfo.production_name
    if production_name != ufo_glyph.name:
        postscriptNamesKey = PUBLIC_PREFIX + "postscriptNames"
        if postscriptNamesKey not in ufo_font.lib:
            ufo_font.lib[postscriptNamesKey] = dict()
        ufo_font.lib[postscriptNamesKey][ufo_glyph.name] = production_name

    for key in ["leftMetricsKey", "rightMetricsKey", "widthMetricsKey"]:
        value = getattr(layer, key, None)
        if value:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "layer." + key] = value
        value = getattr(glyph, key, None)
        if value:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "glyph." + key] = value

    if glyph.script is not None:
        ufo_glyph.lib[SCRIPT_LIB_KEY] = glyph.script

    # if glyph contains custom 'category' and 'subCategory' overrides, store
    # them in the UFO glyph's lib
    category = glyph.category
    if category is None:
        category = glyphinfo.category
    else:
        ufo_glyph.lib[GLYPHLIB_PREFIX + "category"] = category
    subCategory = glyph.subCategory
    if subCategory is None:
        subCategory = glyphinfo.subCategory
    else:
        ufo_glyph.lib[GLYPHLIB_PREFIX + "subCategory"] = subCategory

    # load width before background, which is loaded with lib data
    width = layer.width
    if width is None:
        pass
    elif category == "Mark" and subCategory == "Nonspacing" and width > 0:
        # zero the width of Nonspacing Marks like Glyphs.app does on export
        # TODO: (jany) check for customParameter DisableAllAutomaticBehaviour
        # FIXME: (jany) also don't do that when rt UFO -> glyphs -> UFO
        ufo_glyph.lib[ORIGINAL_WIDTH_KEY] = width
        ufo_glyph.width = 0
    else:
        ufo_glyph.width = width

    self.to_ufo_background_image(ufo_glyph, layer)
    self.to_ufo_guidelines(ufo_glyph, layer)
    self.to_ufo_glyph_background(ufo_glyph, layer)
    self.to_ufo_annotations(ufo_glyph, layer)
    self.to_ufo_hints(ufo_glyph, layer)
    self.to_ufo_glyph_user_data(ufo_font, glyph)
    self.to_ufo_layer_user_data(ufo_glyph, layer)
    self.to_ufo_smart_component_axes(ufo_glyph, glyph)

    self.to_ufo_paths(ufo_glyph, layer)
    self.to_ufo_components(ufo_glyph, layer)
    self.to_ufo_glyph_anchors(ufo_glyph, layer.anchors)
    if self.is_vertical:
        self.to_ufo_glyph_height_and_vertical_origin(ufo_glyph, layer)

def to_glyphs_glyph(self, ufo_glyph, ufo_layer, master):  # noqa: C901
    """Add UFO glif metadata, paths, components, and anchors to a GSGlyph.
    If the matching GSGlyph does not exist, then it is created,
    else it is updated with the new data.
    In all cases, a matching GSLayer is created in the GSGlyph to hold paths.
    """

    # FIXME: (jany) split between glyph and layer attributes
    #        have a write the first time, compare the next times for glyph
    #        always write for the layer

    # NOTE: This optimizes around the performance drain that is glyph name lookup
    #       without replacing the actual data structure. Ideally, FontGlyphsProxy
    #       provides O(1) lookup for all the ways you can use strings to look up
    #       glyphs.
    ufo_glyph_name = ufo_glyph.name  # Avoid method lookup in hot loop.
    glyph = None
    for glyph_object in self.font._glyphs:  # HOT LOOP. Avoid FontGlyphsProxy for speed!
        if glyph_object.name == ufo_glyph_name:  # HOT HOT HOT
            glyph = glyph_object
            break
    if glyph is None:
        glyph = self.glyphs_module.GSGlyph(name=ufo_glyph_name)
        # FIXME: (jany) ordering?
        self.font.glyphs.append(glyph)

    if ufo_glyph.unicodes:
        glyph.unicodes = [f"{c:04X}" for c in ufo_glyph.unicodes]
    glyph.note = ufo_glyph.note or ""
    if GLYPHLIB_PREFIX + "lastChange" in ufo_glyph.lib:
        last_change = ufo_glyph.lib[GLYPHLIB_PREFIX + "lastChange"]
        # We cannot be strict about the dateformat because it's not an official
        # UFO field mentioned in the spec so it could happen to have a timezone
        glyph.lastChange = from_loose_ufo_time(last_change)
    if ufo_glyph.markColor:
        glyph.color = _to_glyphs_color(ufo_glyph.markColor)

    # The export flag can be stored in the glyph's lib key (for upgrading legacy
    # sources) or the Designspace-level public.skipExportGlyphs lib key (canonical
    # place to store the information). The UFO level lib key is ignored.
    if GLYPHLIB_PREFIX + "Export" in ufo_glyph.lib:
        glyph.export = ufo_glyph.lib[GLYPHLIB_PREFIX + "Export"]
    if ufo_glyph.name in self.skip_export_glyphs:
        glyph.export = False

    ufo_font = self._sources[master.id].font
    ps_names_key = PUBLIC_PREFIX + "postscriptNames"
    if ps_names_key in ufo_font.lib and ufo_glyph.name in ufo_font.lib[ps_names_key]:
        glyph.production = ufo_font.lib[ps_names_key][ufo_glyph.name]
        # FIXME: (jany) maybe put something in glyphinfo? No, it's readonly
        #        maybe don't write in glyph.production if glyphinfo already
        #        has something
        # glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)
        # production_name = glyph.production or glyphinfo.production_name

    glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)

    layer = self.to_glyphs_layer(ufo_layer, glyph, master)

    for key in ["leftMetricsKey", "rightMetricsKey", "widthMetricsKey"]:
        # Also read the old version of the key that didn't have a prefix and
        # store it on the layer (because without the "glyph"/"layer" prefix we
        # didn't know whether it originally came from the layer of the glyph,
        # so it's easier to put it back on the most specific level, i.e. the
        # layer)
        for prefix, glyphs_object in (
            ("glyph.", glyph),
            ("", layer),
            ("layer.", layer),
        ):
            full_key = GLYPHLIB_PREFIX + prefix + key
            if full_key in ufo_glyph.lib:
                value = ufo_glyph.lib[full_key]
                setattr(glyphs_object, key, value)

    if SCRIPT_LIB_KEY in ufo_glyph.lib:
        glyph.script = ufo_glyph.lib[SCRIPT_LIB_KEY]

    if GLYPHLIB_PREFIX + "category" in ufo_glyph.lib:
        # TODO: (jany) store category only if different from glyphinfo?
        category = ufo_glyph.lib[GLYPHLIB_PREFIX + "category"]
        glyph.category = category
    else:
        category = glyphinfo.category
    if GLYPHLIB_PREFIX + "subCategory" in ufo_glyph.lib:
        sub_category = ufo_glyph.lib[GLYPHLIB_PREFIX + "subCategory"]
        glyph.subCategory = sub_category
    else:
        sub_category = glyphinfo.subCategory

    # load width before background, which is loaded with lib data
    if hasattr(layer, "foreground"):
        if ufo_glyph.width:
            # Don't store "0", it's the default in UFO.
            # Store in userData because the background's width is not relevant
            # in Glyphs.
            layer.userData[BACKGROUND_WIDTH_KEY] = ufo_glyph.width
    else:
        layer.width = ufo_glyph.width
    if category == "Mark" and sub_category == "Nonspacing" and layer.width == 0:
        # Restore originalWidth
        if ORIGINAL_WIDTH_KEY in ufo_glyph.lib:
            layer.width = ufo_glyph.lib[ORIGINAL_WIDTH_KEY]
            # TODO: (jany) check for customParam DisableAllAutomaticBehaviour?

    self.to_glyphs_background_image(ufo_glyph, layer)
    self.to_glyphs_guidelines(ufo_glyph, layer)
    self.to_glyphs_annotations(ufo_glyph, layer)
    self.to_glyphs_hints(ufo_glyph, layer)
    self.to_glyphs_glyph_user_data(ufo_font, glyph)
    self.to_glyphs_layer_user_data(ufo_glyph, layer)
    self.to_glyphs_smart_component_axes(ufo_glyph, glyph)

    self.to_glyphs_paths(ufo_glyph, layer)
    self.to_glyphs_components(ufo_glyph, layer)
    self.to_glyphs_glyph_anchors(ufo_glyph, layer)
    self.to_glyphs_glyph_height_and_vertical_origin(ufo_glyph, master, layer)

def to_ufo_glyph_height_and_vertical_origin(self, ufo_glyph, layer):
    # implentation based on:
    # https://github.com/googlefonts/glyphsLib/issues/557#issuecomment-667074856
    assert self.is_vertical

    ascender, descender = _get_typo_ascender_descender(layer.master)

    if layer.vertWidth:
        ufo_glyph.height = layer.vertWidth
    else:
        ufo_glyph.height = ascender - descender

    if layer.vertOrigin:
        ufo_glyph.verticalOrigin = ascender - layer.vertOrigin
    else:
        ufo_glyph.verticalOrigin = ascender

def _get_typo_ascender_descender(master):
    # Glyphsapp will use the typo metrics to set the verOrigin and
    # vertWidth. If typo metrics are not present, the master
    # ascender and descender are used instead.
    if "typoAscender" in master.customParameters:
        ascender = master.customParameters["typoAscender"]
    else:
        ascender = master.ascender
    if "typoDescender" in master.customParameters:
        descender = master.customParameters["typoDescender"]
    else:
        descender = master.descender
    return ascender, descender

def to_ufo_glyph_background(self, glyph, layer):
    """Set glyph background."""

    if not layer.hasBackground:
        return

    background = layer.background
    ufo_layer = self.to_ufo_background_layer(layer)
    new_glyph = ufo_layer.newGlyph(glyph.name)

    width = background.userData[BACKGROUND_WIDTH_KEY]
    if width is not None:
        new_glyph.width = width

    self.to_ufo_background_image(new_glyph, background)
    self.to_ufo_paths(new_glyph, background)
    self.to_ufo_components(new_glyph, background)
    self.to_ufo_glyph_anchors(new_glyph, background.anchors)
    self.to_ufo_guidelines(new_glyph, background)

def _to_glyphs_color(color):
    # If the color matches one of Glyphs's predefined colors, return that
    # index.
    for index, glyphs_color in enumerate(GLYPHS_COLORS):
        if str(color) == glyphs_color:
            return index

    # Otherwise, make a Glyphs-formatted RGBA color list: [u8, u8, u8, u8].
    # Glyphs up to version 2.5.1 always set the alpha channel to 1. It should
    # round-trip the actual value in later versions.
    # https://github.com/googlefonts/glyphsLib/pull/363#issuecomment-390418497
    return [round(float(component) * 255) for component in color.split(",")]

def to_glyphs_glyph_height_and_vertical_origin(self, ufo_glyph, master, layer):
    if ufo_glyph.height:
        layer.vertWidth = ufo_glyph.height

    if ufo_glyph.verticalOrigin:
        ascender, _ = _get_typo_ascender_descender(master)
        layer.vertOrigin = ascender - ufo_glyph.verticalOrigin
<commit_msg>flake8: ignore C901 for func to_ufo_glyph<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

import glyphsLib.glyphdata
from .common import to_ufo_time, from_loose_ufo_time
from .constants import GLYPHLIB_PREFIX, GLYPHS_COLORS, PUBLIC_PREFIX

logger = logging.getLogger(__name__)

SCRIPT_LIB_KEY = GLYPHLIB_PREFIX + "script"
ORIGINAL_WIDTH_KEY = GLYPHLIB_PREFIX + "originalWidth"
BACKGROUND_WIDTH_KEY = GLYPHLIB_PREFIX + "backgroundWidth"

def to_ufo_glyph(self, ufo_glyph, layer, glyph):  # noqa: C901
    """Add .glyphs metadata, paths, components, and anchors to a glyph."""
    ufo_font = self._sources[layer.associatedMasterId or layer.layerId].font

    ufo_glyph.unicodes = [int(uval, 16) for uval in glyph.unicodes]

    note = glyph.note
    if note is not None:
        ufo_glyph.note = note

    # Optimization: profiling glyphs2ufo of NotoSans-MM.glyphs (6000 glyphs) on a Mac
    # mini late 2014, Python 3.6.8, revealed that a whopping 17% of the time was spent
    # converting lastChange to UFO timestamps. I could not reproduce this on a Windows
    # 10/Python 3.7 setup, so this might be a platform thing. If-guarding anyway
    # because these timestamps are useless in a UFO scenario if you use Git.
    if (
        self.minimize_glyphs_diffs
        and self.font.customParameters["Disable Last Change"] is not True
        and glyph.lastChange is not None
    ):
        ufo_glyph.lib[GLYPHLIB_PREFIX + "lastChange"] = to_ufo_time(glyph.lastChange)

    color_index = glyph.color
    if color_index is not None:
        # .3f is enough precision to round-trip uint8 to float losslessly.
        # https://github.com/unified-font-object/ufo-spec/issues/61
        # #issuecomment-389759127
        if (
            isinstance(color_index, list)
            and len(color_index) == 4
            and all(0 <= v < 256 for v in color_index)
        ):
            ufo_glyph.markColor = ",".join(
                "0" if v == 0 else "1" if v == 255 else "{:.3f}".format(v / 255)
                for v in color_index
            )
        elif isinstance(color_index, int) and color_index in range(len(GLYPHS_COLORS)):
            ufo_glyph.markColor = GLYPHS_COLORS[color_index]
        else:
            logger.warning(
                "Glyph {}, layer {}: Invalid color index/tuple {}".format(
                    glyph.name, layer.name, color_index
                )
            )

    export = glyph.export
    if not export:
        if self.write_skipexportglyphs:
            if "public.skipExportGlyphs" not in self._designspace.lib:
                self._designspace.lib["public.skipExportGlyphs"] = []
            self._designspace.lib["public.skipExportGlyphs"].append(glyph.name)
        else:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "Export"] = export

    # FIXME: (jany) next line should be an API of GSGlyph?
    glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)
    production_name = glyph.production or glyphinfo.production_name
    if production_name != ufo_glyph.name:
        postscriptNamesKey = PUBLIC_PREFIX + "postscriptNames"
        if postscriptNamesKey not in ufo_font.lib:
            ufo_font.lib[postscriptNamesKey] = dict()
        ufo_font.lib[postscriptNamesKey][ufo_glyph.name] = production_name

    for key in ["leftMetricsKey", "rightMetricsKey", "widthMetricsKey"]:
        value = getattr(layer, key, None)
        if value:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "layer." + key] = value
        value = getattr(glyph, key, None)
        if value:
            ufo_glyph.lib[GLYPHLIB_PREFIX + "glyph." + key] = value

    if glyph.script is not None:
        ufo_glyph.lib[SCRIPT_LIB_KEY] = glyph.script

    # if glyph contains custom 'category' and 'subCategory' overrides, store
    # them in the UFO glyph's lib
    category = glyph.category
    if category is None:
        category = glyphinfo.category
    else:
        ufo_glyph.lib[GLYPHLIB_PREFIX + "category"] = category
    subCategory = glyph.subCategory
    if subCategory is None:
        subCategory = glyphinfo.subCategory
    else:
        ufo_glyph.lib[GLYPHLIB_PREFIX + "subCategory"] = subCategory

    # load width before background, which is loaded with lib data
    width = layer.width
    if width is None:
        pass
    elif category == "Mark" and subCategory == "Nonspacing" and width > 0:
        # zero the width of Nonspacing Marks like Glyphs.app does on export
        # TODO: (jany) check for customParameter DisableAllAutomaticBehaviour
        # FIXME: (jany) also don't do that when rt UFO -> glyphs -> UFO
        ufo_glyph.lib[ORIGINAL_WIDTH_KEY] = width
        ufo_glyph.width = 0
    else:
        ufo_glyph.width = width

    self.to_ufo_background_image(ufo_glyph, layer)
    self.to_ufo_guidelines(ufo_glyph, layer)
    self.to_ufo_glyph_background(ufo_glyph, layer)
    self.to_ufo_annotations(ufo_glyph, layer)
    self.to_ufo_hints(ufo_glyph, layer)
    self.to_ufo_glyph_user_data(ufo_font, glyph)
    self.to_ufo_layer_user_data(ufo_glyph, layer)
    self.to_ufo_smart_component_axes(ufo_glyph, glyph)

    self.to_ufo_paths(ufo_glyph, layer)
    self.to_ufo_components(ufo_glyph, layer)
    self.to_ufo_glyph_anchors(ufo_glyph, layer.anchors)
    if self.is_vertical:
        self.to_ufo_glyph_height_and_vertical_origin(ufo_glyph, layer)

def to_glyphs_glyph(self, ufo_glyph, ufo_layer, master):  # noqa: C901
    """Add UFO glif metadata, paths, components, and anchors to a GSGlyph.
    If the matching GSGlyph does not exist, then it is created,
    else it is updated with the new data.
    In all cases, a matching GSLayer is created in the GSGlyph to hold paths.
    """

    # FIXME: (jany) split between glyph and layer attributes
    #        have a write the first time, compare the next times for glyph
    #        always write for the layer

    # NOTE: This optimizes around the performance drain that is glyph name lookup
    #       without replacing the actual data structure. Ideally, FontGlyphsProxy
    #       provides O(1) lookup for all the ways you can use strings to look up
    #       glyphs.
    ufo_glyph_name = ufo_glyph.name  # Avoid method lookup in hot loop.
    glyph = None
    for glyph_object in self.font._glyphs:  # HOT LOOP. Avoid FontGlyphsProxy for speed!
        if glyph_object.name == ufo_glyph_name:  # HOT HOT HOT
            glyph = glyph_object
            break
    if glyph is None:
        glyph = self.glyphs_module.GSGlyph(name=ufo_glyph_name)
        # FIXME: (jany) ordering?
        self.font.glyphs.append(glyph)

    if ufo_glyph.unicodes:
        glyph.unicodes = [f"{c:04X}" for c in ufo_glyph.unicodes]
    glyph.note = ufo_glyph.note or ""
    if GLYPHLIB_PREFIX + "lastChange" in ufo_glyph.lib:
        last_change = ufo_glyph.lib[GLYPHLIB_PREFIX + "lastChange"]
        # We cannot be strict about the dateformat because it's not an official
        # UFO field mentioned in the spec so it could happen to have a timezone
        glyph.lastChange = from_loose_ufo_time(last_change)
    if ufo_glyph.markColor:
        glyph.color = _to_glyphs_color(ufo_glyph.markColor)

    # The export flag can be stored in the glyph's lib key (for upgrading legacy
    # sources) or the Designspace-level public.skipExportGlyphs lib key (canonical
    # place to store the information). The UFO level lib key is ignored.
    if GLYPHLIB_PREFIX + "Export" in ufo_glyph.lib:
        glyph.export = ufo_glyph.lib[GLYPHLIB_PREFIX + "Export"]
    if ufo_glyph.name in self.skip_export_glyphs:
        glyph.export = False

    ufo_font = self._sources[master.id].font
    ps_names_key = PUBLIC_PREFIX + "postscriptNames"
    if ps_names_key in ufo_font.lib and ufo_glyph.name in ufo_font.lib[ps_names_key]:
        glyph.production = ufo_font.lib[ps_names_key][ufo_glyph.name]
        # FIXME: (jany) maybe put something in glyphinfo? No, it's readonly
        #        maybe don't write in glyph.production if glyphinfo already
        #        has something
        # glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)
        # production_name = glyph.production or glyphinfo.production_name

    glyphinfo = glyphsLib.glyphdata.get_glyph(ufo_glyph.name)

    layer = self.to_glyphs_layer(ufo_layer, glyph, master)

    for key in ["leftMetricsKey", "rightMetricsKey", "widthMetricsKey"]:
        # Also read the old version of the key that didn't have a prefix and
        # store it on the layer (because without the "glyph"/"layer" prefix we
        # didn't know whether it originally came from the layer of the glyph,
        # so it's easier to put it back on the most specific level, i.e. the
        # layer)
        for prefix, glyphs_object in (
            ("glyph.", glyph),
            ("", layer),
            ("layer.", layer),
        ):
            full_key = GLYPHLIB_PREFIX + prefix + key
            if full_key in ufo_glyph.lib:
                value = ufo_glyph.lib[full_key]
                setattr(glyphs_object, key, value)

    if SCRIPT_LIB_KEY in ufo_glyph.lib:
        glyph.script = ufo_glyph.lib[SCRIPT_LIB_KEY]

    if GLYPHLIB_PREFIX + "category" in ufo_glyph.lib:
        # TODO: (jany) store category only if different from glyphinfo?
        category = ufo_glyph.lib[GLYPHLIB_PREFIX + "category"]
        glyph.category = category
    else:
        category = glyphinfo.category
    if GLYPHLIB_PREFIX + "subCategory" in ufo_glyph.lib:
        sub_category = ufo_glyph.lib[GLYPHLIB_PREFIX + "subCategory"]
        glyph.subCategory = sub_category
    else:
        sub_category = glyphinfo.subCategory

    # load width before background, which is loaded with lib data
    if hasattr(layer, "foreground"):
        if ufo_glyph.width:
            # Don't store "0", it's the default in UFO.
            # Store in userData because the background's width is not relevant
            # in Glyphs.
            layer.userData[BACKGROUND_WIDTH_KEY] = ufo_glyph.width
    else:
        layer.width = ufo_glyph.width
    if category == "Mark" and sub_category == "Nonspacing" and layer.width == 0:
        # Restore originalWidth
        if ORIGINAL_WIDTH_KEY in ufo_glyph.lib:
            layer.width = ufo_glyph.lib[ORIGINAL_WIDTH_KEY]
            # TODO: (jany) check for customParam DisableAllAutomaticBehaviour?

    self.to_glyphs_background_image(ufo_glyph, layer)
    self.to_glyphs_guidelines(ufo_glyph, layer)
    self.to_glyphs_annotations(ufo_glyph, layer)
    self.to_glyphs_hints(ufo_glyph, layer)
    self.to_glyphs_glyph_user_data(ufo_font, glyph)
    self.to_glyphs_layer_user_data(ufo_glyph, layer)
    self.to_glyphs_smart_component_axes(ufo_glyph, glyph)

    self.to_glyphs_paths(ufo_glyph, layer)
    self.to_glyphs_components(ufo_glyph, layer)
    self.to_glyphs_glyph_anchors(ufo_glyph, layer)
    self.to_glyphs_glyph_height_and_vertical_origin(ufo_glyph, master, layer)

def to_ufo_glyph_height_and_vertical_origin(self, ufo_glyph, layer):
    # implentation based on:
    # https://github.com/googlefonts/glyphsLib/issues/557#issuecomment-667074856
    assert self.is_vertical

    ascender, descender = _get_typo_ascender_descender(layer.master)

    if layer.vertWidth:
        ufo_glyph.height = layer.vertWidth
    else:
        ufo_glyph.height = ascender - descender

    if layer.vertOrigin:
        ufo_glyph.verticalOrigin = ascender - layer.vertOrigin
    else:
        ufo_glyph.verticalOrigin = ascender

def _get_typo_ascender_descender(master):
    # Glyphsapp will use the typo metrics to set the verOrigin and
    # vertWidth. If typo metrics are not present, the master
    # ascender and descender are used instead.
    if "typoAscender" in master.customParameters:
        ascender = master.customParameters["typoAscender"]
    else:
        ascender = master.ascender
    if "typoDescender" in master.customParameters:
        descender = master.customParameters["typoDescender"]
    else:
        descender = master.descender
    return ascender, descender

def to_ufo_glyph_background(self, glyph, layer):
    """Set glyph background."""

    if not layer.hasBackground:
        return

    background = layer.background
    ufo_layer = self.to_ufo_background_layer(layer)
    new_glyph = ufo_layer.newGlyph(glyph.name)

    width = background.userData[BACKGROUND_WIDTH_KEY]
    if width is not None:
        new_glyph.width = width

    self.to_ufo_background_image(new_glyph, background)
    self.to_ufo_paths(new_glyph, background)
    self.to_ufo_components(new_glyph, background)
    self.to_ufo_glyph_anchors(new_glyph, background.anchors)
    self.to_ufo_guidelines(new_glyph, background)

def _to_glyphs_color(color):
    # If the color matches one of Glyphs's predefined colors, return that
    # index.
    for index, glyphs_color in enumerate(GLYPHS_COLORS):
        if str(color) == glyphs_color:
            return index

    # Otherwise, make a Glyphs-formatted RGBA color list: [u8, u8, u8, u8].
    # Glyphs up to version 2.5.1 always set the alpha channel to 1. It should
    # round-trip the actual value in later versions.
    # https://github.com/googlefonts/glyphsLib/pull/363#issuecomment-390418497
    return [round(float(component) * 255) for component in color.split(",")]

def to_glyphs_glyph_height_and_vertical_origin(self, ufo_glyph, master, layer):
    if ufo_glyph.height:
        layer.vertWidth = ufo_glyph.height

    if ufo_glyph.verticalOrigin:
        ascender, _ = _get_typo_ascender_descender(master)
        layer.vertOrigin = ascender - ufo_glyph.verticalOrigin
<|endoftext|>


<commit_before>from sanic import Sanic
from sanic.response import json

import aiohttp

app = Sanic(__name__)

async def fetch(session, url):
    """
    Use session object to perform 'get' request on url
    """
    async with session.get(url) as response:
        return await response.json()

@app.route("/")
async def test(request):
    """
    Download and serve example JSON
    """
    url = "https://api.github.com/repos/channelcat/sanic"

    async with aiohttp.ClientSession() as session:
        response = await fetch(session, url)
        return json(response)

app.run(host="0.0.0.0", port=8000, workers=2)
<commit_msg>Updated aiohttp example.<commit_after>from sanic import Sanic
from sanic import response

import aiohttp

app = Sanic()

async def fetch(session, url):
    """
    Use session object to perform 'get' request on url
    """
    async with session.get(url) as result:
        return await result.text()

@app.route('/')
async def handle_request(request):
    url = "https://api.github.com/repos/channelcat/sanic"

    async with aiohttp.ClientSession() as session:
        result = await fetch(session, url)
        return response.json(result)

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=8000, workers=2)
<|endoftext|>


<commit_before>import onifile as oni
import struct
from collections import defaultdict

def copy(args,a,b):
    r = oni.Reader(a)
    print "firstheader",r.h0
    w = oni.Writer(b,r.h0)
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            print dict(nid=h["nid"],ps=h["ps"],fs=h["fs"],frameid=hh["frameid"],timestamp=hh["timestamp"])
            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
        else:
        	w.copyblock(h,a)

    w.finalize()

def strip(args,action,a,b):
    # scan all and keep pre and last
    r = oni.Reader(a)
    w = oni.Writer(b)

    ignoredtype = dict(stripcolor=oni.NODE_TYPE_IMAGE,stripdepth=oni.NODE_TYPE_DEPTH,stripir=oni.NODE_TYPE_IR)[action]
    ignoredid = -1
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_NODE_ADDED:
            hh = oni.parseadded(a,h)
            if hh["nodetype"] == ignoredtype:
            	ignoredid = h["nid"]
            else:
	        	w.copyblock(h,a)
        elif h["nid"] == ignoredid:
        	continue
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            print dict(nid=h["nid"],ps=h["ps"],fs=h["fs"],frameid=hh["frameid"],timestamp=hh["timestamp"])
            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
        else:
        	w.copyblock(h,a)
    w.finalize()
"""
def cut(args,action,a,b):
    while True:
            h = oni.readrechead(a)
            if h is None:
                    break
            if h["rt"] == oni.RECORD_NEW_DATA:
                t = oni.parsedatahead(a,h)["timestamp"]
                q = stats[h["nid"]]
                good = False
                if target[0] == "frame":
                    good = q.oldframes >= target[1][0] and q.oldframes <= target[1][1]
                else:
                    good = t >= target[1][0] and t <= target[1][1]
                if good:
                    # retime
                    if q.newframes == 0:
                        q.oldbasetime = t
                    t2 = t-q.oldbasetime

                    #q.newtime(t2)
                    #q.addframe()
                    #q.copyblock()
                    #self.framesoffset.append((b.tell(),q.newframe))

                q.oldframes += 1
            elif h["rt"] == oni.RECORD_NODE_ADDED:
                hh = oni.parseadded(a,h)
                q = stats[h["nid"]]
                self.headerblock = h
                self.headerdata = hh
                # append block
            elif h["rt"] == oni.RECORD_SEEK_TABLE:
                # append new seektable
                pass
            else:
                # append block
                pass

            # next record
            a.seek(h["nextheader"]) #h["fs"]-HEADER_SIZE+h["ps"]+pt,0)
    needclose = True
"""

def skip(args,a,b):
    r = oni.Reader(a)
    w = oni.Writer(b)
    qf = defaultdict(lambda: 0)
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            oldframe = qf[h["nid"]]
            if (oldframe % args.skipframes) == 0:
	            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
            qf[h["nid"]] += 1
        else:
        	w.copyblock(h,a)
    w.finalize()

"""
def dupframes(args,a):
    while True:
            h = oni.readrechead(a)
            if h is None:
                    break
            prelast = last
            last = h
            if h["nid"] > mid:
                    mid = h["nid"]
            if h["rt"] == oni.RECORD_SEEK_TABLE:
                # TODO gen seek
                pass
            elif h["rt"] == oni.RECORD_NEW_DATA:
                hh = oni.parsedatahead(a,h)
                q = stats[h["nid"]]
                for i in range(0,args.dupframes):
                    q.addframe(h,hh,b)
                    oni.copyblock(a,h,b,frame=q.newframe-1,timestamp=q.timestamp)
            a.seek(h["nextheader"])
"""<commit_msg>refactored in skip frames<commit_after>import onifile as oni
import struct
from collections import defaultdict

def copy(args,a,b):
    r = oni.Reader(a)
    print "firstheader",r.h0
    w = oni.Writer(b,r.h0)
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            print dict(nid=h["nid"],ps=h["ps"],fs=h["fs"],frameid=hh["frameid"],timestamp=hh["timestamp"])
            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
        else:
        	w.copyblock(h,a)

    w.finalize()

def strip(args,action,a,b):
    # scan all and keep pre and last
    r = oni.Reader(a)
    w = oni.Writer(b)

    ignoredtype = dict(stripcolor=oni.NODE_TYPE_IMAGE,stripdepth=oni.NODE_TYPE_DEPTH,stripir=oni.NODE_TYPE_IR)[action]
    ignoredid = -1
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_NODE_ADDED:
            hh = oni.parseadded(a,h)
            if hh["nodetype"] == ignoredtype:
            	ignoredid = h["nid"]
            else:
	        	w.copyblock(h,a)
        elif h["nid"] == ignoredid:
        	continue
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            print dict(nid=h["nid"],ps=h["ps"],fs=h["fs"],frameid=hh["frameid"],timestamp=hh["timestamp"])
            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
        else:
        	w.copyblock(h,a)
    w.finalize()
"""
def cut(args,action,a,b):
    while True:
            h = oni.readrechead(a)
            if h is None:
                    break
            if h["rt"] == oni.RECORD_NEW_DATA:
                t = oni.parsedatahead(a,h)["timestamp"]
                q = stats[h["nid"]]
                good = False
                if target[0] == "frame":
                    good = q.oldframes >= target[1][0] and q.oldframes <= target[1][1]
                else:
                    good = t >= target[1][0] and t <= target[1][1]
                if good:
                    # retime
                    if q.newframes == 0:
                        q.oldbasetime = t
                    t2 = t-q.oldbasetime

                    #q.newtime(t2)
                    #q.addframe()
                    #q.copyblock()
                    #self.framesoffset.append((b.tell(),q.newframe))

                q.oldframes += 1
            elif h["rt"] == oni.RECORD_NODE_ADDED:
                hh = oni.parseadded(a,h)
                q = stats[h["nid"]]
                self.headerblock = h
                self.headerdata = hh
                # append block
            elif h["rt"] == oni.RECORD_SEEK_TABLE:
                # append new seektable
                pass
            else:
                # append block
                pass

            # next record
            a.seek(h["nextheader"]) #h["fs"]-HEADER_SIZE+h["ps"]+pt,0)
    needclose = True
"""

def skip(args,a,b):
    r = oni.Reader(a)
    w = oni.Writer(b)
    qf = defaultdict(lambda: 0)
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            oldframe = qf[h["nid"]]
            if (oldframe % args.skipframes) == 0:
	            w.addframe(h["nid"],hh["frameid"],hh["timestamp"],a.read(h["ps"]))
            qf[h["nid"]] += 1
        else:
        	w.copyblock(h,a)
    w.finalize()

def dupframes(args,a):
    r = oni.Reader(a)
    w = oni.Writer(b)
    dt = 13 #ms
    qf = dict()
    while True:
        h = r.next()
        if h is None:
            break
        elif h["rt"] == oni.RECORD_SEEK_TABLE:
        	w.emitseek(h["nid"])
        elif h["rt"] == oni.RECORD_NEW_DATA:
            hh = oni.parsedatahead(a,h)
            dd = a.read(h["ps"])
            qff = qf.get(h["nid"])
            if qff is None:
            	qff = [hh["frameid"],hh["timestamp"]]
            	qf[h["nid"]] = qff
            for i in range(0,args.dupframes):
	            w.addframe(h["nid"],qf["frameid"],qf["timestamp"],dd)
	            qf["frameid"] += 1
	            qf["timestamp"] += dt
        else:
        	w.copyblock(h,a)
    w.finalize()

<|endoftext|>


<commit_before>#!/usr/bin/env python2

'''
- validate disk image,
	- parse disk image log file and searching for line that starts with "Md5 checksum"
	- run md5 against disk image and make sure it's the same

- read DPX metadata file
- for each line
	- poll TMS object API endpoint, and get the correlating component include
	- make directory named with component number --- component id --- object id (get from file name)

- run fiwalk on disk image to get DFXML
- parse DPX CSV to find dir names of components
- search DFXML for files that include these folder names in their paths
– grab the MD5
- write "data/"+[ filename path from DFXML ], a tab space, and then the MD5 (one line per file)

- subprocess: tsk_recover -v -a [disk image] [destination]

- move DPX folders to their appropriate component dir
- continue ?

'''<commit_msg>added bit about mmls<commit_after>#!/usr/bin/env python2

'''
- validate disk image,
	- parse disk image log file and searching for line that starts with "Md5 checksum"
	- run md5 against disk image and make sure it's the same

- read DPX metadata file
- for each line
	- poll TMS object API endpoint, and get the correlating component include
	- make directory named with component number --- component id --- object id (get from file name)

- run fiwalk on disk image to get DFXML
- parse DPX CSV to find dir names of components
- search DFXML for files that include these folder names in their paths
– grab the MD5
- write "data/"+[ filename path from DFXML ], a tab space, and then the MD5 (one line per file)

- run 'mmls' (tsk tool) to see if there is a volume map
	- if there is, get the byte offset
- subprocess: tsk_recover -v -a [disk image] [destination]

two min per 1,000 files

- move DPX folders to their appropriate component dir
- continue ?

'''<|endoftext|>


<commit_before># Copyright 2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import httplib
import json
import logging
import os
import subprocess
import sys
import time
import urllib
import urllib2

def is_port_listening(port):
  import socket
  s = socket.socket()
  try:
    s.connect(('localhost', port))
  except socket.error:
    return False
  s.close()
  return True

class FrontendDaemonHost(object):
  def __init__(self, port, basedir):
    # If a daemon is running, try killing it via /kill
    if is_port_listening(port):
      for i in range(2):
        logging.warning("Existing daemon found. Asking it to exit")
        try:
          conn = httplib.HTTPConnection('localhost', port, True)
          conn.request('GET', '/exit')
        except:
          break
        res = conn.getresponse()
        if res.status != 200:
          break
        else:
          time.sleep(0.2)

    if is_port_listening(port):
      raise Exception("Daemon running")
    self._port = port

    self._daemon_proc = subprocess.Popen([
        sys.executable, "-m", "src.frontend_daemon", str(port), basedir])
    try:
      self._wait_for_daemon_start()
    except Exception, ex:
      import traceback
      traceback.print_exc()
      self.close()
      raise ex

  def _wait_for_daemon_start(self):
    ok = False
    for i in range(10):
      try:
        conn = httplib.HTTPConnection('localhost', self._port, True)
        conn.request('GET', '/ping')
      except:
        time.sleep(0.05)
        continue

      res = conn.getresponse()
      if res.status != 200:
        ok = False
        break
      if res.read() != 'OK':
        ok = False
        break
      ok = True
      break
    if not ok:
      raise Exception("Daemon did not come up")

    self._conn = None
    self._db_proxy = None

  def close(self):
    try:
      pass
#      conn = httplib.HTTPConnection('localhost', self._port, True)
#      conn.request('GET', '/exit')
    except:
      pass
    t_beginning = time.time()
    while True:
      if self._daemon_proc.poll() is not None:
        break
      seconds_passed = time.time() - t_beginning
      if seconds_passed > 0.5:
        self._daemon_proc.terminate()
        self._daemon_proc.wait()
        break
      time.sleep(0.1)

  @property
  def host(self):
    return 'localhost'

  @property
  def port(self):
    return self._port

  @property
  def baseurl(self):
    return 'http://localhost:%i' % self._port

  def urlread(self, path):
    return urllib2.urlopen(urllib.basejoin(self.baseurl, path)).read()

<commit_msg>Bugfix for webkit-1.0-2 failing to load urls that use http://localhost/<commit_after># Copyright 2011 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import httplib
import json
import logging
import os
import socket
import subprocess
import sys
import time
import urllib
import urllib2

def is_port_listening(port):
  import socket
  s = socket.socket()
  try:
    s.connect(('localhost', port))
  except socket.error:
    return False
  s.close()
  return True

class FrontendDaemonHost(object):
  def __init__(self, port, basedir):
    # If a daemon is running, try killing it via /kill
    if is_port_listening(port):
      for i in range(2):
        logging.warning("Existing daemon found. Asking it to exit")
        try:
          conn = httplib.HTTPConnection('localhost', port, True)
          conn.request('GET', '/exit')
        except:
          break
        res = conn.getresponse()
        if res.status != 200:
          break
        else:
          time.sleep(0.2)

    if is_port_listening(port):
      raise Exception("Daemon running")
    self._port = port

    self._daemon_proc = subprocess.Popen([
        sys.executable, "-m", "src.frontend_daemon", str(port), basedir])
    try:
      self._wait_for_daemon_start()
    except Exception, ex:
      import traceback
      traceback.print_exc()
      self.close()
      raise ex

  def _wait_for_daemon_start(self):
    ok = False
    for i in range(10):
      try:
        conn = httplib.HTTPConnection('localhost', self._port, True)
        conn.request('GET', '/ping')
      except:
        time.sleep(0.05)
        continue

      res = conn.getresponse()
      if res.status != 200:
        ok = False
        break
      if res.read() != 'OK':
        ok = False
        break
      ok = True
      break
    if not ok:
      raise Exception("Daemon did not come up")

    self._conn = None
    self._db_proxy = None

  def close(self):
    try:
      pass
#      conn = httplib.HTTPConnection('localhost', self._port, True)
#      conn.request('GET', '/exit')
    except:
      pass
    t_beginning = time.time()
    while True:
      if self._daemon_proc.poll() is not None:
        break
      seconds_passed = time.time() - t_beginning
      if seconds_passed > 0.5:
        self._daemon_proc.terminate()
        self._daemon_proc.wait()
        break
      time.sleep(0.1)

  @property
  def host(self):
    # Report as an IP address because on older webkit builds that use soup, localhost
    # does not work as a URL.
    return socket.gethostbyname('localhost')

  @property
  def port(self):
    return self._port

  @property
  def baseurl(self):
    return 'http://%s:%i' % (self.host, self._port)

  def urlread(self, path):
    return urllib2.urlopen(urllib.basejoin(self.baseurl, path)).read()
<|endoftext|>