<commit_before>"""
The :mod:`tslearn.neighbors` module gathers nearest neighbor algorithms using time series metrics.
"""

import numpy
from sklearn.neighbors import KNeighborsClassifier, NearestNeighbors
from sklearn.neighbors.base import KNeighborsMixin, _get_weights
from scipy import stats
from sklearn.utils.extmath import weighted_mode

from tslearn.metrics import cdist_dtw, cdist_lr_dtw
from tslearn.utils import to_time_series_dataset

class KNeighborsTimeSeriesMixin(KNeighborsMixin):
    """Mixin for k-neighbors searches on Time Series."""

    def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
        """Finds the K-neighbors of a point.

        Returns indices of and distances to the neighbors of each point.

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            The query time series.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.
        n_neighbors : int
            Number of neighbors to get (default is the value passed to the constructor).
        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array
            Array representing the distance to points, only present if return_distance=True
        ind : array
            Indices of the nearest points in the population matrix.
        """
        self_neighbors = False
        if n_neighbors is None:
            n_neighbors = self.n_neighbors
        if X is None:
            X = self._fit_X
            self_neighbors = True
        if self.metric == "dtw":
            cdist_fun = cdist_dtw
        else:
            raise ValueError("Unrecognized time series metric string: %s (should be 'dtw')" % self.metric)
        full_dist_matrix = cdist_fun(X, self._fit_X)
        ind = numpy.argsort(full_dist_matrix, axis=1)

        if self_neighbors:
            ind = ind[:, 1:]
        if n_neighbors > full_dist_matrix.shape[1]:
            n_neighbors = full_dist_matrix.shape[1]
        ind = ind[:, :n_neighbors]

        n_ts = X.shape[0]
        sample_range = numpy.arange(n_ts)[:, None]
        dist = full_dist_matrix[sample_range, ind]

        if return_distance:
            return dist, ind
        else:
            return ind

class KNeighborsTimeSeries(KNeighborsTimeSeriesMixin, NearestNeighbors):
    """Unsupervised learner for implementing neighbor searches for Time Series.

    Parameters
    ----------
    n_neighbors : int (default: 5)
        Number of nearest neighbors to be considered for the decision.
    metric : {'dtw'} (default: 'dtw')
        Metric to be used at the core of the nearest neighbor procedure
    metric_params : dict or None (default: None)
        Dictionnary of metric parameters.

    Examples
    --------
    >>> time_series = [[1, 2, 3, 4], [3, 3, 2, 0], [1, 2, 2, 4]]
    >>> knn = KNeighborsTimeSeries(n_neighbors=1).fit(time_series)
    >>> dist, ind = knn.kneighbors([[1, 1, 2, 2, 2, 3, 4]], return_distance=True)
    >>> dist
    array([[ 0.]])
    >>> ind
    array([[0]])

    """
    def __init__(self, n_neighbors=5, metric="dtw", metric_params=None):
        NearestNeighbors.__init__(self, n_neighbors=n_neighbors, algorithm='brute')
        self.metric = metric
        self.metric_params = metric_params

    def fit(self, X, y=None):
        """Fit the model using X as training data

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Training data.
        """
        self._fit_X = X
        return self

    def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
        """Finds the K-neighbors of a point.

        Returns indices of and distances to the neighbors of each point.

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            The query time series.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.
        n_neighbors : int
            Number of neighbors to get (default is the value passed to the constructor).
        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array
            Array representing the distance to points, only present if return_distance=True
        ind : array
            Indices of the nearest points in the population matrix.
        """
        X_ = to_time_series_dataset(X)
        return KNeighborsTimeSeriesMixin.kneighbors(self,
                                                    X=X_,
                                                    n_neighbors=n_neighbors,
                                                    return_distance=return_distance)

class KNeighborsTimeSeriesClassifier(KNeighborsClassifier, KNeighborsTimeSeriesMixin):
    """Classifier implementing the k-nearest neighbors vote for Time Series.

    Parameters
    ----------
    n_neighbors : int (default: 5)
        Number of nearest neighbors to be considered for the decision.
    weights : str or callable, optional (default: 'uniform')
        Weight function used in prediction. Possible values:

        - 'uniform' : uniform weights. All points in each neighborhood are weighted equally.
        - 'distance' : weight points by the inverse of their distance. in this case, closer neighbors of a query point
          will have a greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an array of distances, and returns an array of the same
          shape containing the weights.
    metric : {'dtw' or 'lr_dtw'} (default: 'dtw')
        Metric to be used at the core of the nearest neighbor procedure
    metric_params : dict or None (default: None)
        Dictionnary of metric parameters. Recognized keys are `"gamma"` (which has default value 0.) for LR-DTW.

    Examples
    --------
    >>> clf = KNeighborsTimeSeriesClassifier(n_neighbors=2, metric="dtw")
    >>> clf.fit([[1, 2, 3], [1, 1.2, 3.2], [3, 2, 1]], y=[0, 0, 1])
    >>> clf.predict([1, 2.2, 3.5])
    array([0])
    """
    def __init__(self, n_neighbors=5, weights='uniform', metric="dtw", metric_params=None):
        KNeighborsClassifier.__init__(self, n_neighbors=n_neighbors, weights=weights, algorithm='brute')
        self.metric = metric
        self.metric_params = metric_params

    def fit(self, X, y):
        """Fit the model using X as training data and y as target values

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Training data.
        y : array-like, shape (n_ts, )
            Target values.
        """
        self._fit_X = to_time_series_dataset(X)
        self._fit_y = numpy.array(y)

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Test samples.
        """
        X_ = to_time_series_dataset(X)
        neigh_dist, neigh_ind = self.kneighbors(X_)

        weights = _get_weights(neigh_dist, self.weights)

        if weights is None:
            mode, _ = stats.mode(self._fit_y[neigh_ind], axis=1)
        else:
            mode, _ = weighted_mode(self._fit_y[neigh_ind], weights, axis=1)

        return mode[:, 0]
<commit_msg>Update neighbors.py<commit_after>"""
The :mod:`tslearn.neighbors` module gathers nearest neighbor algorithms using time series metrics.
"""

import numpy
from sklearn.neighbors import KNeighborsClassifier, NearestNeighbors
from sklearn.neighbors.base import KNeighborsMixin, _get_weights
from scipy import stats
from sklearn.utils.extmath import weighted_mode

from tslearn.metrics import cdist_dtw
from tslearn.utils import to_time_series_dataset

class KNeighborsTimeSeriesMixin(KNeighborsMixin):
    """Mixin for k-neighbors searches on Time Series."""

    def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
        """Finds the K-neighbors of a point.

        Returns indices of and distances to the neighbors of each point.

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            The query time series.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.
        n_neighbors : int
            Number of neighbors to get (default is the value passed to the constructor).
        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array
            Array representing the distance to points, only present if return_distance=True
        ind : array
            Indices of the nearest points in the population matrix.
        """
        self_neighbors = False
        if n_neighbors is None:
            n_neighbors = self.n_neighbors
        if X is None:
            X = self._fit_X
            self_neighbors = True
        if self.metric == "dtw":
            cdist_fun = cdist_dtw
        else:
            raise ValueError("Unrecognized time series metric string: %s (should be 'dtw')" % self.metric)
        full_dist_matrix = cdist_fun(X, self._fit_X)
        ind = numpy.argsort(full_dist_matrix, axis=1)

        if self_neighbors:
            ind = ind[:, 1:]
        if n_neighbors > full_dist_matrix.shape[1]:
            n_neighbors = full_dist_matrix.shape[1]
        ind = ind[:, :n_neighbors]

        n_ts = X.shape[0]
        sample_range = numpy.arange(n_ts)[:, None]
        dist = full_dist_matrix[sample_range, ind]

        if return_distance:
            return dist, ind
        else:
            return ind

class KNeighborsTimeSeries(KNeighborsTimeSeriesMixin, NearestNeighbors):
    """Unsupervised learner for implementing neighbor searches for Time Series.

    Parameters
    ----------
    n_neighbors : int (default: 5)
        Number of nearest neighbors to be considered for the decision.
    metric : {'dtw'} (default: 'dtw')
        Metric to be used at the core of the nearest neighbor procedure
    metric_params : dict or None (default: None)
        Dictionnary of metric parameters.

    Examples
    --------
    >>> time_series = [[1, 2, 3, 4], [3, 3, 2, 0], [1, 2, 2, 4]]
    >>> knn = KNeighborsTimeSeries(n_neighbors=1).fit(time_series)
    >>> dist, ind = knn.kneighbors([[1, 1, 2, 2, 2, 3, 4]], return_distance=True)
    >>> dist
    array([[ 0.]])
    >>> ind
    array([[0]])

    """
    def __init__(self, n_neighbors=5, metric="dtw", metric_params=None):
        NearestNeighbors.__init__(self, n_neighbors=n_neighbors, algorithm='brute')
        self.metric = metric
        self.metric_params = metric_params

    def fit(self, X, y=None):
        """Fit the model using X as training data

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Training data.
        """
        self._fit_X = X
        return self

    def kneighbors(self, X=None, n_neighbors=None, return_distance=True):
        """Finds the K-neighbors of a point.

        Returns indices of and distances to the neighbors of each point.

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            The query time series.
            If not provided, neighbors of each indexed point are returned.
            In this case, the query point is not considered its own neighbor.
        n_neighbors : int
            Number of neighbors to get (default is the value passed to the constructor).
        return_distance : boolean, optional. Defaults to True.
            If False, distances will not be returned

        Returns
        -------
        dist : array
            Array representing the distance to points, only present if return_distance=True
        ind : array
            Indices of the nearest points in the population matrix.
        """
        X_ = to_time_series_dataset(X)
        return KNeighborsTimeSeriesMixin.kneighbors(self,
                                                    X=X_,
                                                    n_neighbors=n_neighbors,
                                                    return_distance=return_distance)

class KNeighborsTimeSeriesClassifier(KNeighborsClassifier, KNeighborsTimeSeriesMixin):
    """Classifier implementing the k-nearest neighbors vote for Time Series.

    Parameters
    ----------
    n_neighbors : int (default: 5)
        Number of nearest neighbors to be considered for the decision.
    weights : str or callable, optional (default: 'uniform')
        Weight function used in prediction. Possible values:

        - 'uniform' : uniform weights. All points in each neighborhood are weighted equally.
        - 'distance' : weight points by the inverse of their distance. in this case, closer neighbors of a query point
          will have a greater influence than neighbors which are further away.
        - [callable] : a user-defined function which accepts an array of distances, and returns an array of the same
          shape containing the weights.
    metric : {'dtw'} (default: 'dtw')
        Metric to be used at the core of the nearest neighbor procedure
    metric_params : dict or None (default: None)
        Dictionnary of metric parameters. Recognized keys are `"gamma"` (which has default value 0.) for LR-DTW.

    Examples
    --------
    >>> clf = KNeighborsTimeSeriesClassifier(n_neighbors=2, metric="dtw")
    >>> clf.fit([[1, 2, 3], [1, 1.2, 3.2], [3, 2, 1]], y=[0, 0, 1])
    >>> clf.predict([1, 2.2, 3.5])
    array([0])
    """
    def __init__(self, n_neighbors=5, weights='uniform', metric="dtw", metric_params=None):
        KNeighborsClassifier.__init__(self, n_neighbors=n_neighbors, weights=weights, algorithm='brute')
        self.metric = metric
        self.metric_params = metric_params

    def fit(self, X, y):
        """Fit the model using X as training data and y as target values

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Training data.
        y : array-like, shape (n_ts, )
            Target values.
        """
        self._fit_X = to_time_series_dataset(X)
        self._fit_y = numpy.array(y)

    def predict(self, X):
        """Predict the class labels for the provided data

        Parameters
        ----------
        X : array-like, shape (n_ts, sz, d)
            Test samples.
        """
        X_ = to_time_series_dataset(X)
        neigh_dist, neigh_ind = self.kneighbors(X_)

        weights = _get_weights(neigh_dist, self.weights)

        if weights is None:
            mode, _ = stats.mode(self._fit_y[neigh_ind], axis=1)
        else:
            mode, _ = weighted_mode(self._fit_y[neigh_ind], weights, axis=1)

        return mode[:, 0]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
command-line: tunasimu

executes numerical simulations from command-line. Usage:

    [python] tunasimu [[--dir <path>]
                       [-l|--label <exp-name>]
                       [-s|--seed <seed>]
                       [-f|--force]
                       [-s|--samples <number-of-samples>]
                       [--alpha <growth-rate-target>]
                       [--alpha_sd <standard-deviation-for-alpha]
                       [--alpha_autocorr <autocorrelation time for alpha>]
                       [--div_lambda <lambda division parameter>]
                       [--div_size_cutoff <target size for division>]
                       [--div_use_alpha {'parameter', 'birth'}]
                       [--div_fluctuation_mode {'gamma', 'none'}]
                       [--div_relative_fluctuations <ratio sd/mean>]
                       [--initial_birth_size_mode {'fixed', 'lognormal'}]
                       [--initial_birth_size_sigma <sigma for initial birth size>]
                       [--start <starting-time-value>]
                       [--stop <stoping-time-value>]
                       [--period <time-interval-between-acquisitions>]
                      ]

All parameters have default values (see code).
"""
from __future__ import print_function

import argparse
import os
import yaml

import numpy as np

from tunacell.simu.ou import (OUParams,
                              SimuParams, DivisionParams, SampleInitialSize,
                              run_ou_simulation)

def setup_args():
    """Configure argparse parser to get all parameters for numerical simulation
    """
    # Arguments
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # files
    parser.add_argument('--dir', type=str,
                        help='Parent directory in which simulation is stored',
                        default='~/tmptunacell')
    parser.add_argument('-l', '--label', type=str,
                        help='Label of the experiment/simulation',
                        default='simutest')
    parser.add_argument('-c', '--conf', type=str,
                        help='Configuration file (e.g. "metadata.yml")',
                        default=None)
    parser.add_argument('-s', '--seed', type=int,
                        help='Random generator seed (use a specific number for reproducibility)',
                        default=None)
    parser.add_argument('-f', '--force',
                        help='Force writting new data (erase old simulations)',
                        action='store_true')
    parser.add_argument('-N', '--N_containers', type=int,
                        help='Number of simulated container samples',
                        default=100)
    parser.add_argument('-t', '--tree_per_container', type=int,
                        help='Number of trees per container',
                        default=2)
    # OU parameters
    parser.add_argument('--alpha', type=float,
                        help='Growth rate target (in [/min])',
                        default=np.log(2.)/60.)  # corresponding to size doubling time = 1 hour, in /min
    parser.add_argument('--alpha_sd', type=float,
                        help='Equilibrium standard deviation for alpha (in [/min])',
                        default=np.log(2.)/(10.*60.))  #1/10 of default target, in /min
    parser.add_argument('--alpha_autocorr', type=float,
                        help='Autocorrelation time for alpha (in [min]',
                        default=30.)  # in mins
    # division parameters
    parser.add_argument('--div_lambda', type=float,
                        help='Lambda parameter for interdivision timing (between 0 and 1)',
                        default=0)  # timer
    parser.add_argument('--div_size_cutoff', type=float,
                        help='Division size cutoff (in arbitrary units)',
                        default=2.)
    parser.add_argument('--div_use_alpha', type=str,
                        help='Use parameter, or birth value growth rate',
                        default='parameter')
    parser.add_argument('--div_fluctuation_mode', type=str,
                        help='[gamma, none] whether to add or not fluctuations to interdivision time',
                        default='gamma')
    parser.add_argument('--div_relative_fluctuations', type=float,
                        help='Ratio sd/predicted linear interdivision time',
                        default=.1)

    # initial birth size sampling
    parser.add_argument('--birth_size_mean', type=float,
                        help='Initial mean birth size', default=1.)
    parser.add_argument('--birth_size_mode', type=str,
                        help='[fixed, lognormal]', default='fixed')
    parser.add_argument('--birth_size_relative_fluctuations', type=float,
                        help='Relative fluctuations in the lognormal mode',
                        default=0.1)

    # simulation parameters
    parser.add_argument('--start', type=float,
                        help='Time at which simulation starts',
                        default=0.)
    parser.add_argument('--stop', type=float,
                        help='Time at which simulation stops',
                        default=180.)
    parser.add_argument('--period', type=float,
                        help=('Time period between two consecutive time-lapse'
                              ' acquisitions'),
                        default=5.)
    args = parser.parse_args()
#    print(vars(args).keys())
    return args

def main(args):
    """Setup numerical simulation parameters from command line arguments

    Parameters
    ----------
    args : :class:`argparse.Namespace` instance
        any Python object is valid as it contains following attributes:
        ['dir', 'label', 'random_seed', 'force', 'samples',
        'alpha', 'alpha_sd', 'alpha_autocorr', 'div_lambda', 'div_size_cutoff',
        'div_use_alpha', 'div_fluctuation_mode', 'div_relative_fluctuations',
        'initial_birth_size_mode', 'initial_birth_size_sigma', 'start', 'stop',
        'period']
    """

    path = os.path.abspath(os.path.expanduser(args.dir))
    if not os.path.exists(path):
        os.makedirs(path)
    print('Path: {}'.format(path))
    print('Label: {}'.format(args.label))
    # SET PARAMETERS FOR NUMERICAL SIMULATION

    # check that a configuration file is provided
    if args.conf is not None:
        with open(args.conf, 'r') as f:
            y = yaml.load(f)
        simuParams = SimuParams(**y['simu_params'])
        ouParams = OUParams(**y['ornstein_uhlenbeck_params'])
        divParams = DivisionParams(**y['division_params'])
        birthsizeParams = SampleInitialSize(**y['birth_size_params'])
    # otherwise read default
    else:
        # general simulation parameters
        tstart = args.start  # initial time of simulation
        tstop = args.stop  # final time of simulation (data is not recorded beyond this)
        dt = args.period  # time interval for data recording (sampling of process)
        ncontainers = args.N_containers  # number of containers that will be created by simulation
        ntrees = args.tree_per_container  # number of trees per container
        seed = args.seed

        simuParams = SimuParams(nbr_container=ncontainers,
                                nbr_colony_per_container=ntrees,
                                start=tstart, stop=tstop, period=dt,
                                seed=seed)

        # OU parameters
        target_value = args.alpha  # average value of the random process
        spring = 1./args.alpha_autocorr  # spring constant (brings back random walker to target value)
        noise_intensity = 2. * spring * (args.alpha_sd)**2   # noise intensity for random walk

        ouParams = OUParams(target=target_value, spring=spring, noise=noise_intensity)

        # DIVISION TIMING
        divParams = DivisionParams(div_lambda=args.div_lambda,
                                   div_size_target=args.div_size_cutoff,
                                   div_mode=args.div_fluctuation_mode,
                                   div_sd_to_mean=args.div_relative_fluctuations,
                                   use_growth_rate=args.div_use_alpha)

        # INITIAL BIRTH SIZE
        birthsizeParams = SampleInitialSize(birth_size_mean=args.birth_size_mean,
                                            birth_size_mode=args.birth_size_mode,
                                            birth_size_sd_to_mean=args.birth_size_relative_fluctuations)

    run_ou_simulation(simuParams, divParams, birthsizeParams, ouParams,
                      path, args.label, force=args.force)

if __name__ == '__main__':
    args = setup_args()
    main(args)
<commit_msg>Edit module docstring<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This module indirectly defines the command-line: tunasimu (see entry point in setup.py)

It executes numerical simulations from command-line. Usage:

    tunasimu [[--dir <path>]
               [-l|--label <exp-name>]
               [-s|--seed <seed>]
               [-f|--force]
               [-s|--samples <number-of-samples>]
               [--alpha <growth-rate-target>]
               [--alpha_sd <standard-deviation-for-alpha]
               [--alpha_autocorr <autocorrelation time for alpha>]
               [--div_lambda <lambda division parameter>]
               [--div_size_cutoff <target size for division>]
               [--div_use_alpha {'parameter', 'birth'}]
               [--div_fluctuation_mode {'gamma', 'none'}]
               [--div_relative_fluctuations <ratio sd/mean>]
               [--initial_birth_size_mode {'fixed', 'lognormal'}]
               [--initial_birth_size_sigma <sigma for initial birth size>]
               [--start <starting-time-value>]
               [--stop <stoping-time-value>]
               [--period <time-interval-between-acquisitions>]
              ]

All parameters have default values (see code).

(the execution can be stated as:

    python -m tunacell.simu.run [options...]
"""
from __future__ import print_function

import argparse
import os
import yaml

import numpy as np

from tunacell.simu.ou import (OUParams,
                              SimuParams, DivisionParams, SampleInitialSize,
                              run_ou_simulation)

def setup_args():
    """Configure argparse parser to get all parameters for numerical simulation
    """
    # Arguments
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # files
    parser.add_argument('--dir', type=str,
                        help='Parent directory in which simulation is stored',
                        default='~/tmptunacell')
    parser.add_argument('-l', '--label', type=str,
                        help='Label of the experiment/simulation',
                        default='simutest')
    parser.add_argument('-c', '--conf', type=str,
                        help='Configuration file (e.g. "metadata.yml")',
                        default=None)
    parser.add_argument('-s', '--seed', type=int,
                        help='Random generator seed (use a specific number for reproducibility)',
                        default=None)
    parser.add_argument('-f', '--force',
                        help='Force writting new data (erase old simulations)',
                        action='store_true')
    parser.add_argument('-N', '--N_containers', type=int,
                        help='Number of simulated container samples',
                        default=100)
    parser.add_argument('-t', '--tree_per_container', type=int,
                        help='Number of trees per container',
                        default=2)
    # OU parameters
    parser.add_argument('--alpha', type=float,
                        help='Growth rate target (in [/min])',
                        default=np.log(2.)/60.)  # corresponding to size doubling time = 1 hour, in /min
    parser.add_argument('--alpha_sd', type=float,
                        help='Equilibrium standard deviation for alpha (in [/min])',
                        default=np.log(2.)/(10.*60.))  #1/10 of default target, in /min
    parser.add_argument('--alpha_autocorr', type=float,
                        help='Autocorrelation time for alpha (in [min]',
                        default=30.)  # in mins
    # division parameters
    parser.add_argument('--div_lambda', type=float,
                        help='Lambda parameter for interdivision timing (between 0 and 1)',
                        default=0)  # timer
    parser.add_argument('--div_size_cutoff', type=float,
                        help='Division size cutoff (in arbitrary units)',
                        default=2.)
    parser.add_argument('--div_use_alpha', type=str,
                        help='Use parameter, or birth value growth rate',
                        default='parameter')
    parser.add_argument('--div_fluctuation_mode', type=str,
                        help='[gamma, none] whether to add or not fluctuations to interdivision time',
                        default='gamma')
    parser.add_argument('--div_relative_fluctuations', type=float,
                        help='Ratio sd/predicted linear interdivision time',
                        default=.1)

    # initial birth size sampling
    parser.add_argument('--birth_size_mean', type=float,
                        help='Initial mean birth size', default=1.)
    parser.add_argument('--birth_size_mode', type=str,
                        help='[fixed, lognormal]', default='fixed')
    parser.add_argument('--birth_size_relative_fluctuations', type=float,
                        help='Relative fluctuations in the lognormal mode',
                        default=0.1)

    # simulation parameters
    parser.add_argument('--start', type=float,
                        help='Time at which simulation starts',
                        default=0.)
    parser.add_argument('--stop', type=float,
                        help='Time at which simulation stops',
                        default=180.)
    parser.add_argument('--period', type=float,
                        help=('Time period between two consecutive time-lapse'
                              ' acquisitions'),
                        default=5.)
    args = parser.parse_args()
#    print(vars(args).keys())
    return args

def main(args):
    """Setup numerical simulation parameters from command line arguments

    Parameters
    ----------
    args : :class:`argparse.Namespace` instance
        any Python object is valid as it contains following attributes:
        ['dir', 'label', 'random_seed', 'force', 'samples',
        'alpha', 'alpha_sd', 'alpha_autocorr', 'div_lambda', 'div_size_cutoff',
        'div_use_alpha', 'div_fluctuation_mode', 'div_relative_fluctuations',
        'initial_birth_size_mode', 'initial_birth_size_sigma', 'start', 'stop',
        'period']
    """

    path = os.path.abspath(os.path.expanduser(args.dir))
    if not os.path.exists(path):
        os.makedirs(path)
    print('Path: {}'.format(path))
    print('Label: {}'.format(args.label))
    # SET PARAMETERS FOR NUMERICAL SIMULATION

    # check that a configuration file is provided
    if args.conf is not None:
        with open(args.conf, 'r') as f:
            y = yaml.load(f)
        simuParams = SimuParams(**y['simu_params'])
        ouParams = OUParams(**y['ornstein_uhlenbeck_params'])
        divParams = DivisionParams(**y['division_params'])
        birthsizeParams = SampleInitialSize(**y['birth_size_params'])
    # otherwise read default
    else:
        # general simulation parameters
        tstart = args.start  # initial time of simulation
        tstop = args.stop  # final time of simulation (data is not recorded beyond this)
        dt = args.period  # time interval for data recording (sampling of process)
        ncontainers = args.N_containers  # number of containers that will be created by simulation
        ntrees = args.tree_per_container  # number of trees per container
        seed = args.seed

        simuParams = SimuParams(nbr_container=ncontainers,
                                nbr_colony_per_container=ntrees,
                                start=tstart, stop=tstop, period=dt,
                                seed=seed)

        # OU parameters
        target_value = args.alpha  # average value of the random process
        spring = 1./args.alpha_autocorr  # spring constant (brings back random walker to target value)
        noise_intensity = 2. * spring * (args.alpha_sd)**2   # noise intensity for random walk

        ouParams = OUParams(target=target_value, spring=spring, noise=noise_intensity)

        # DIVISION TIMING
        divParams = DivisionParams(div_lambda=args.div_lambda,
                                   div_size_target=args.div_size_cutoff,
                                   div_mode=args.div_fluctuation_mode,
                                   div_sd_to_mean=args.div_relative_fluctuations,
                                   use_growth_rate=args.div_use_alpha)

        # INITIAL BIRTH SIZE
        birthsizeParams = SampleInitialSize(birth_size_mean=args.birth_size_mean,
                                            birth_size_mode=args.birth_size_mode,
                                            birth_size_sd_to_mean=args.birth_size_relative_fluctuations)

    run_ou_simulation(simuParams, divParams, birthsizeParams, ouParams,
                      path, args.label, force=args.force)

if __name__ == '__main__':
    args = setup_args()
    main(args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import division, print_function, unicode_literals

__all__ = ["K2Data"]

import h5py
import fitsio
import numpy as np
from scipy.linalg import cho_solve, cho_factor

from .pipeline import Pipeline

class K2Data(Pipeline):

    query_parameters = {
        "light_curve_file": (None, True),
        "basis_file": (None, True),
    }

    def get_result(self, query, parent_response):
        return dict(model_light_curves=[K2LightCurve(query["light_curve_file"],
                                                     query["basis_file"])])

class K2LightCurve(object):

    def __init__(self, fn, basis_file):
        data = fitsio.read(fn)
        aps = fitsio.read(fn, 2)

        # Choose the photometry with the smallest variance.
        var = aps["cdpp6"]
        var[var < 0.0] = np.inf
        i = np.argmin(var)

        # Load the data.
        self.time = data["time"]
        self.flux = data["flux"][:, i]
        q = data["quality"]

        # Drop the bad data.
        m = np.isfinite(self.time) * np.isfinite(self.flux) * (q == 0)
        self.time = np.ascontiguousarray(self.time[m], dtype=np.float64)
        self.flux = np.ascontiguousarray(self.flux[m], dtype=np.float64)

        # Normalize the data.
        self.flux = self.flux / np.median(self.flux) - 1.0
        self.flux *= 1e3  # Convert to ppt.

        # Estimate the uncertainties.
        self.ivar = 1.0 / np.median(np.diff(self.flux) ** 2)
        self.ferr = np.ones_like(self.flux) / np.sqrt(self.ivar)

        # Load the prediction basis.
        with h5py.File(basis_file, "r") as f:
            basis = f["basis"][...]
        self.basis = np.concatenate((basis[:, m], np.ones((1, m.sum()))))

        self.ll0 = self.lnlike()

    def lnlike(self, model=None):
        if model is None:
            return -0.5 * np.sum((self.predict() - self.flux)**2) * self.ivar

        # Evaluate the transit model.
        m = model(self.time)
        if m[0] != 0.0 or m[-1] != 0.0 or np.all(m == 0.0):
            return 0.0, 0.0, 0.0

        # Build the design matrix and do the linear fit.
        A = np.concatenate(([m], self.basis))
        ATA = np.dot(A, A.T * self.ivar)
        ATA[np.diag_indices_from(ATA)] += 1e-10

        # This is the depth inverse variance.
        s = 1.0 / np.linalg.inv(ATA)[0, 0]

        # And the linear weights (depth is the first).
        factor = cho_factor(ATA, overwrite_a=True)
        y = self.ivar * (self.flux - model)
        w = cho_solve(factor, np.dot(A, y), overwrite_b=True)

        # Compute the lnlikelihood.
        ll = -0.5 * np.sum((self.flux - np.dot(w, A))**2) * self.ivar
        return ll - self.ll0, w[0], s

    def predict(self, y=None):
        if y is None:
            y = self.flux
        ATA = np.dot(self.basis, self.basis.T)
        ATA[np.diag_indices_from(ATA)] += 1e-10
        w = np.linalg.solve(ATA, np.dot(self.basis, y))
        return np.dot(w, self.basis)
<commit_msg>nevermind<commit_after># -*- coding: utf-8 -*-

from __future__ import division, print_function, unicode_literals

__all__ = ["K2Data"]

import h5py
import fitsio
import numpy as np
from scipy.linalg import cho_solve, cho_factor

from .pipeline import Pipeline

class K2Data(Pipeline):

    query_parameters = {
        "light_curve_file": (None, True),
        "basis_file": (None, True),
    }

    def get_result(self, query, parent_response):
        return dict(model_light_curves=[K2LightCurve(query["light_curve_file"],
                                                     query["basis_file"])])

class K2LightCurve(object):

    def __init__(self, fn, basis_file):
        data = fitsio.read(fn)
        aps = fitsio.read(fn, 2)

        # Choose the photometry with the smallest variance.
        var = aps["cdpp6"]
        var[var < 0.0] = np.inf
        i = np.argmin(var)

        # Load the data.
        self.time = data["time"]
        self.flux = data["flux"][:, i]
        q = data["quality"]

        # Drop the bad data.
        m = np.isfinite(self.time) * np.isfinite(self.flux) * (q == 0)
        self.time = np.ascontiguousarray(self.time[m], dtype=np.float64)
        self.flux = np.ascontiguousarray(self.flux[m], dtype=np.float64)

        # Normalize the data.
        self.flux = self.flux / np.median(self.flux) - 1.0
        self.flux *= 1e3  # Convert to ppt.

        # Estimate the uncertainties.
        self.ivar = 1.0 / np.median(np.diff(self.flux) ** 2)
        self.ferr = np.ones_like(self.flux) / np.sqrt(self.ivar)
        self.scaled = self.flux * self.ivar

        # Load the prediction basis.
        with h5py.File(basis_file, "r") as f:
            basis = f["basis"][...]
        self.basis = np.concatenate((basis[:, m], np.ones((1, m.sum()))))

        self.ll0 = self.lnlike()

    def lnlike(self, model=None):
        if model is None:
            return -0.5 * np.sum((self.predict() - self.flux)**2) * self.ivar

        # Evaluate the transit model.
        m = model(self.time)
        if m[0] != 0.0 or m[-1] != 0.0 or np.all(m == 0.0):
            return 0.0, 0.0, 0.0

        # Build the design matrix and do the linear fit.
        A = np.concatenate(([m], self.basis))
        ATA = np.dot(A, A.T * self.ivar)
        ATA[np.diag_indices_from(ATA)] += 1e-10

        # This is the depth inverse variance.
        s = 1.0 / np.linalg.inv(ATA)[0, 0]

        # And the linear weights (depth is the first).
        factor = cho_factor(ATA, overwrite_a=True)
        w = cho_solve(factor, np.dot(A, self.scaled), overwrite_b=True)

        # Compute the lnlikelihood.
        ll = -0.5 * np.sum((self.flux - np.dot(w, A))**2) * self.ivar
        return ll - self.ll0, w[0], s

    def predict(self, y=None):
        if y is None:
            y = self.flux
        ATA = np.dot(self.basis, self.basis.T)
        ATA[np.diag_indices_from(ATA)] += 1e-10
        w = np.linalg.solve(ATA, np.dot(self.basis, y))
        return np.dot(w, self.basis)
<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import

import sys
from random import randint, random, shuffle
from turtle import TurtleScreen, RawTurtle, TK

if sys.version_info[0] < 3:
    from Tkinter import Tk, mainloop
else:
    from tkinter import Tk
    mainloop = False

DEBUG = False

def noisy(value, variance=0.01):
    size = value * variance
    return value + (random() * size * 2) - size

def wrap(turtle, screen_width, screen_height):
    """wrap a turtle coords around"""
    x, y = turtle.pos()
    new_x = new_y = None
    if x > screen_width / 2:
        new_x = x - screen_width
    elif x < -screen_width / 2:
        new_x = x + screen_width

    if y > screen_height / 2:
        new_y = y - screen_height
    elif y < -screen_height / 2:
        new_y = y + screen_height

    was_down = turtle.isdown()
    if new_x is not None:
        turtle.penup()
        turtle.setx(new_x)
    if new_y is not None:
        turtle.penup()
        turtle.sety(new_y)
    if was_down:
        turtle.pendown()

def clamp(turtle, screen_width, screen_height):
    """Clamp turtle to window"""
    x, y = turtle.pos()
    new_x = new_y = None
    if x > screen_width / 2:
        new_x = screen_width / 2
    elif x < -screen_width / 2:
        new_x = -screen_width / 2

    if y > screen_height / 2:
        new_y = screen_height / 2
    elif y < -screen_height / 2:
        new_y = -screen_height / 2

    if new_x is not None:
        turtle.setx(new_x)
    if new_y is not None:
        turtle.sety(new_y)

class TurtleWorld(object):

    def __init__(self, width, height, borders=wrap, title="TurtlePower"):
        self.width = width
        self.half_width = width // 2
        self.height = height
        self.half_height = height // 2
        self.borders = borders
        self.window_title = title

        self.init_screen()

        self.fps = 0
        self.done = True
        self.turtles = []

    def init_screen(self):
        # intialise screen and turn off auto-render
        root = Tk()
        root.wm_title(self.window_title)
        window = TK.Canvas(master=root, width=self.width, height=self.height)
        window.pack()
        self.screen = TurtleScreen(window)
        self.screen.tracer(0, 0)

    def position_turtle(self, t, pos=None, angle=None):
        # move to location
        t.hideturtle()
        t.penup()
        if pos is None:
            pos = (randint(-self.half_width, self.half_width),
                   randint(-self.half_height, self.half_height))
        x, y = pos
        t.goto(x, y)
        if angle is None:
            angle = random() * 360
        t.setheading(angle)
        # ready to go
        t.showturtle()
        t.pendown()
        return t

    def random_position(self, turtle):
        return self.position_turtle(turtle)

    def _print_fps(self):
        if not self.done:
            print(self.fps)
            self.screen.ontimer(self._print_fps, 1000)
        self.fps = 0

    def create_turtle(self, callback, pos=None, angle=None):
        t = PowerTurtle(self)
        t.set_callback(callback)
        self.position_turtle(t, pos, angle)
        self.turtles.append(t)
        return t

    def add_turtle(self, turtle):
        turtle.clear()
        self.turtles.append(turtle)

    def remove_turtle(self, turtle):
        turtle.hideturtle()
        turtle.clear()
        self.turtles.remove(turtle)

    def run(self, ticks=1000):
        # run for 1000 ticks
        self.done = False
        if DEBUG:
            self.screen.ontimer(self._print_fps, 1000)
        self.ticks = ticks
        self.screen.ontimer(self.tick, 33)
        if mainloop:
            mainloop()
        else:
            self.screen.mainloop()

    def tick(self):
        shuffle(self.turtles)
        for t in self.turtles:
            t.callback(self)
            self.borders(t, self.width, self.height)
        self.screen.update()
        self.fps += 1
        self.ticks -= 1
        if self.ticks == 0:
            self.done = True
        else:
            self.screen.ontimer(self.tick, 33)

class PowerTurtleMixin(object):
    """A set of useful extra methods for a turtle"""
    type = "turtle"

    def __init__(self, world):
        self.world = world
        super(PowerTurtleMixin, self).__init__(world.screen)
        self.setup()

    def setup(self):
        """Initialisation function, called once"""
        super(PowerTurtleMixin, self).setup()

    def set_callback(self, callback):
        """Set the callback to a function, for classes usage"""
        self.callback = lambda world: callback(self, world)

    def turn_towards(self, desired, amount):
        """Helper to to turn a small amount towards a heading"""
        heading = self.heading()
        angle = desired - heading
        angle = (angle + 180) % 360 - 180
        if angle >= 0:
            amount = min(amount, angle)
        else:
            amount = max(-amount, angle)
        self.left(amount)
        return amount

    def get_neighbours(self, distance=60, angle=120):
        """Other turtles you can see that are with distance and angle of your
        current heading"""
        neighbours = []
        for t in self.world.turtles:
            if t is not self:
                a = abs(self.heading() - self.towards(t))
                if a < angle and self.distance(t) < distance:
                    neighbours.append(t)
        return neighbours

class PowerTurtle(PowerTurtleMixin, RawTurtle):

    def setup(self):
        """PowerTurtle based on stdlib turtle module"""
        pass
<commit_msg>Don't cover _print_fps.<commit_after>from __future__ import division, print_function, absolute_import

import sys
from random import randint, random, shuffle
from turtle import TurtleScreen, RawTurtle, TK

if sys.version_info[0] < 3:
    from Tkinter import Tk, mainloop
else:
    from tkinter import Tk
    mainloop = False

DEBUG = False

def noisy(value, variance=0.01):
    size = value * variance
    return value + (random() * size * 2) - size

def wrap(turtle, screen_width, screen_height):
    """wrap a turtle coords around"""
    x, y = turtle.pos()
    new_x = new_y = None
    if x > screen_width / 2:
        new_x = x - screen_width
    elif x < -screen_width / 2:
        new_x = x + screen_width

    if y > screen_height / 2:
        new_y = y - screen_height
    elif y < -screen_height / 2:
        new_y = y + screen_height

    was_down = turtle.isdown()
    if new_x is not None:
        turtle.penup()
        turtle.setx(new_x)
    if new_y is not None:
        turtle.penup()
        turtle.sety(new_y)
    if was_down:
        turtle.pendown()

def clamp(turtle, screen_width, screen_height):
    """Clamp turtle to window"""
    x, y = turtle.pos()
    new_x = new_y = None
    if x > screen_width / 2:
        new_x = screen_width / 2
    elif x < -screen_width / 2:
        new_x = -screen_width / 2

    if y > screen_height / 2:
        new_y = screen_height / 2
    elif y < -screen_height / 2:
        new_y = -screen_height / 2

    if new_x is not None:
        turtle.setx(new_x)
    if new_y is not None:
        turtle.sety(new_y)

class TurtleWorld(object):

    def __init__(self, width, height, borders=wrap, title="TurtlePower"):
        self.width = width
        self.half_width = width // 2
        self.height = height
        self.half_height = height // 2
        self.borders = borders
        self.window_title = title

        self.init_screen()

        self.fps = 0
        self.done = True
        self.turtles = []

    def init_screen(self):
        # intialise screen and turn off auto-render
        root = Tk()
        root.wm_title(self.window_title)
        window = TK.Canvas(master=root, width=self.width, height=self.height)
        window.pack()
        self.screen = TurtleScreen(window)
        self.screen.tracer(0, 0)

    def position_turtle(self, t, pos=None, angle=None):
        # move to location
        t.hideturtle()
        t.penup()
        if pos is None:
            pos = (randint(-self.half_width, self.half_width),
                   randint(-self.half_height, self.half_height))
        x, y = pos
        t.goto(x, y)
        if angle is None:
            angle = random() * 360
        t.setheading(angle)
        # ready to go
        t.showturtle()
        t.pendown()
        return t

    def random_position(self, turtle):
        return self.position_turtle(turtle)

    def _print_fps(self):  # pragma: no cover
        if not self.done:
            print(self.fps)
            self.screen.ontimer(self._print_fps, 1000)
        self.fps = 0

    def create_turtle(self, callback, pos=None, angle=None):
        t = PowerTurtle(self)
        t.set_callback(callback)
        self.position_turtle(t, pos, angle)
        self.turtles.append(t)
        return t

    def add_turtle(self, turtle):
        turtle.clear()
        self.turtles.append(turtle)

    def remove_turtle(self, turtle):
        turtle.hideturtle()
        turtle.clear()
        self.turtles.remove(turtle)

    def run(self, ticks=1000):
        # run for 1000 ticks
        self.done = False
        if DEBUG:
            self.screen.ontimer(self._print_fps, 1000)
        self.ticks = ticks
        self.screen.ontimer(self.tick, 33)
        if mainloop:
            mainloop()
        else:
            self.screen.mainloop()

    def tick(self):
        shuffle(self.turtles)
        for t in self.turtles:
            t.callback(self)
            self.borders(t, self.width, self.height)
        self.screen.update()
        self.fps += 1
        self.ticks -= 1
        if self.ticks == 0:
            self.done = True
        else:
            self.screen.ontimer(self.tick, 33)

class PowerTurtleMixin(object):
    """A set of useful extra methods for a turtle"""
    type = "turtle"

    def __init__(self, world):
        self.world = world
        super(PowerTurtleMixin, self).__init__(world.screen)
        self.setup()

    def setup(self):
        """Initialisation function, called once"""
        super(PowerTurtleMixin, self).setup()

    def set_callback(self, callback):
        """Set the callback to a function, for classes usage"""
        self.callback = lambda world: callback(self, world)

    def turn_towards(self, desired, amount):
        """Helper to to turn a small amount towards a heading"""
        heading = self.heading()
        angle = desired - heading
        angle = (angle + 180) % 360 - 180
        if angle >= 0:
            amount = min(amount, angle)
        else:
            amount = max(-amount, angle)
        self.left(amount)
        return amount

    def get_neighbours(self, distance=60, angle=120):
        """Other turtles you can see that are with distance and angle of your
        current heading"""
        neighbours = []
        for t in self.world.turtles:
            if t is not self:
                a = abs(self.heading() - self.towards(t))
                if a < angle and self.distance(t) < distance:
                    neighbours.append(t)
        return neighbours

class PowerTurtle(PowerTurtleMixin, RawTurtle):

    def setup(self):
        """PowerTurtle based on stdlib turtle module"""
        pass
<|endoftext|>


<commit_before>"""
Django settings for twitcher project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

import configparser
import os
from os.path import dirname, join

config = configparser.ConfigParser()
config.read('config.ini.test')

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

SECRET_KEY = config.get('DJANGO', 'SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'twitterapi',
    'twitcher',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'twitcher.urls'

SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"

WSGI_APPLICATION = 'twitcher.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_URL = '/assets/'

STATIC_ROOT = join(dirname(dirname(__file__)), "assets")

STATICFILES_DIRS = (join(dirname(dirname(__file__)), "static"),)

# TEMPLATES SETTINGS
TEMPLATE_DIRS = (
    join(dirname(dirname(__file__)), "templates"),
)

# LOGIN SETTINGS
LOGIN_URL = '/login/'
LOGIN_ERROR_URL = '/login/'
LOGIN_REDIRECT_URL = '/home/'
LOGOUT_REDIRECT_URL = '/'

# TWITTER API SETTINGS
TWITTER_APP_KEY = config.get('TWITTER_API', 'TWITTER_APP_KEY')
TWITTER_APP_SECRET = config.get('TWITTER_API', 'TWITTER_APP_SECRET')
TWITTER_OAUTH_TOKEN = config.get('TWITTER_API', 'TWITTER_OAUTH_TOKEN')
TWITTER_OAUTH_TOKEN_SECRET = config.get('TWITTER_API', 'TWITTER_OAUTH_TOKEN_SECRET')
<commit_msg>Change config file name<commit_after>"""
Django settings for twitcher project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

import configparser
import os
from os.path import dirname, join

config = configparser.ConfigParser()
config.read('config.ini')

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

SECRET_KEY = config.get('DJANGO', 'SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'twitterapi',
    'twitcher',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'twitcher.urls'

SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"

WSGI_APPLICATION = 'twitcher.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_URL = '/assets/'

STATIC_ROOT = join(dirname(dirname(__file__)), "assets")

STATICFILES_DIRS = (join(dirname(dirname(__file__)), "static"),)

# TEMPLATES SETTINGS
TEMPLATE_DIRS = (
    join(dirname(dirname(__file__)), "templates"),
)

# LOGIN SETTINGS
LOGIN_URL = '/login/'
LOGIN_ERROR_URL = '/login/'
LOGIN_REDIRECT_URL = '/home/'
LOGOUT_REDIRECT_URL = '/'

# TWITTER API SETTINGS
TWITTER_APP_KEY = config.get('TWITTER_API', 'TWITTER_APP_KEY')
TWITTER_APP_SECRET = config.get('TWITTER_API', 'TWITTER_APP_SECRET')
TWITTER_OAUTH_TOKEN = config.get('TWITTER_API', 'TWITTER_OAUTH_TOKEN')
TWITTER_OAUTH_TOKEN_SECRET = config.get('TWITTER_API', 'TWITTER_OAUTH_TOKEN_SECRET')
<|endoftext|>


<commit_before>import math
import os
import xml.etree.ElementTree as et

HEADERS = b'''<?xml version=\"1.0\" standalone=\"no\"?>
<?xml-stylesheet href="../wheel.css" type="text/css"?>
<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"
\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">
'''

PATH_TEMPLATE = '''
M {start_outer_x},{start_outer_y}
A{outer_radius},{outer_radius} 0 0 1 {end_outer_x},{end_outer_y}
L {start_inner_x},{start_inner_y}
A{inner_radius},{inner_radius} 0 0 0 {end_inner_x},{end_inner_y}
Z
'''

FRACTION_LINE = 80
OFFSET = 20
PADDING = 10
RADIUS = 180
CENTER = PADDING + RADIUS
TAU = 2 * math.pi

def annular_sector_path(start, stop):
    inner_radius = RADIUS // 2
    outer_radius = RADIUS
    cos_stop = math.cos(stop)
    cos_start = math.cos(start)
    sin_stop = math.sin(stop)
    sin_start = math.sin(start)

    points = {
        'inner_radius': inner_radius,
        'outer_radius': outer_radius,
        'start_outer_x': CENTER + outer_radius * cos_start,
        'start_outer_y': CENTER + outer_radius * sin_start,
        'end_outer_x': CENTER + outer_radius * cos_stop,
        'end_outer_y': CENTER + outer_radius * sin_stop,
        'start_inner_x': CENTER + inner_radius * cos_stop,
        'start_inner_y': CENTER + inner_radius * sin_stop,
        'end_inner_x': CENTER + inner_radius * cos_start,
        'end_inner_y': CENTER + inner_radius * sin_start,
    }
    return PATH_TEMPLATE.format(**points)

def add_annular_sector(wheel, start, stop, style_class):
    return et.SubElement(
        wheel, 'path',
        d=annular_sector_path(start=start, stop=stop),
        attrib={'class': style_class},
    )

def angles(index, total):
    start = index * TAU / total
    stop = (index + 1) * TAU / total

    return (start - TAU / 4, stop - TAU / 4)

def add_fraction(wheel, packages, total, version):
    text_attributes = {
        'text-anchor': 'middle',
        'dominant-baseline': 'central',
        'font-size': str(2 * OFFSET),
        'font-family': '"Helvetica Neue",Helvetica,Arial,sans-serif',
        'fill': '#333333',
    }

    # Packages with some sort of wheel
    wheel_packages = sum(
        1 if package[version]['dropped_support'] == "yes" else
        0 for package in packages)

    packages_with_wheels = et.SubElement(
        wheel, 'text',
        x=str(CENTER), y=str(CENTER - OFFSET),
        attrib=text_attributes,
    )
    packages_with_wheels.text = '{0}'.format(wheel_packages)

    title = et.SubElement(packages_with_wheels, 'title')
    title.text = '{:.0%}'.format(wheel_packages/total)

    # Dividing line
    et.SubElement(
        wheel, 'line',
        x1=str(CENTER - FRACTION_LINE // 2), y1=str(CENTER),
        x2=str(CENTER + FRACTION_LINE // 2), y2=str(CENTER),
        attrib={'stroke': '#333333', 'stroke-width': '2'},
    )

    # Total packages
    total_packages = et.SubElement(
        wheel, 'text',
        x=str(CENTER), y=str(CENTER + OFFSET),
        attrib=text_attributes,
    )
    total_packages.text = '{0}'.format(total)

    title = et.SubElement(total_packages, 'title')
    title.text = '{:.0%}'.format(wheel_packages/float(total))

def generate_svg_wheel(packages, total, versions):
    for version in versions:

        wheel = et.Element(
            'svg',
            viewBox='0 0 {0} {0}'.format(2 * CENTER),
            version='1.1',
            xmlns='http://www.w3.org/2000/svg',
        )

        for index, result in enumerate(packages):

            start, stop = angles(index, total)
            sector = add_annular_sector(
                wheel,
                start=start, stop=stop,
                style_class=result[version]['css_class'],
            )
            title = et.SubElement(sector, 'title')
            title.text = u'{0} {1}'.format(
                result['name'], result[version]['icon'])

        add_fraction(wheel, packages, total, version)

        wheel_svg = os.path.join(version, 'wheel.svg')
        wheel_png = os.path.join(version, 'wheel.png')
        with open(wheel_svg, 'wb') as svg:
            svg.write(HEADERS)
            svg.write(et.tostring(wheel))

        # Install with: npm install svgexport -g
        os.system('svgexport {svg} {png} 32:32'.format(
            svg=wheel_svg, png=wheel_png))
<commit_msg>Python 2.7 fix so not 0%<commit_after>import math
import os
import xml.etree.ElementTree as et

HEADERS = b'''<?xml version=\"1.0\" standalone=\"no\"?>
<?xml-stylesheet href="../wheel.css" type="text/css"?>
<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"
\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">
'''

PATH_TEMPLATE = '''
M {start_outer_x},{start_outer_y}
A{outer_radius},{outer_radius} 0 0 1 {end_outer_x},{end_outer_y}
L {start_inner_x},{start_inner_y}
A{inner_radius},{inner_radius} 0 0 0 {end_inner_x},{end_inner_y}
Z
'''

FRACTION_LINE = 80
OFFSET = 20
PADDING = 10
RADIUS = 180
CENTER = PADDING + RADIUS
TAU = 2 * math.pi

def annular_sector_path(start, stop):
    inner_radius = RADIUS // 2
    outer_radius = RADIUS
    cos_stop = math.cos(stop)
    cos_start = math.cos(start)
    sin_stop = math.sin(stop)
    sin_start = math.sin(start)

    points = {
        'inner_radius': inner_radius,
        'outer_radius': outer_radius,
        'start_outer_x': CENTER + outer_radius * cos_start,
        'start_outer_y': CENTER + outer_radius * sin_start,
        'end_outer_x': CENTER + outer_radius * cos_stop,
        'end_outer_y': CENTER + outer_radius * sin_stop,
        'start_inner_x': CENTER + inner_radius * cos_stop,
        'start_inner_y': CENTER + inner_radius * sin_stop,
        'end_inner_x': CENTER + inner_radius * cos_start,
        'end_inner_y': CENTER + inner_radius * sin_start,
    }
    return PATH_TEMPLATE.format(**points)

def add_annular_sector(wheel, start, stop, style_class):
    return et.SubElement(
        wheel, 'path',
        d=annular_sector_path(start=start, stop=stop),
        attrib={'class': style_class},
    )

def angles(index, total):
    start = index * TAU / total
    stop = (index + 1) * TAU / total

    return (start - TAU / 4, stop - TAU / 4)

def add_fraction(wheel, packages, total, version):
    text_attributes = {
        'text-anchor': 'middle',
        'dominant-baseline': 'central',
        'font-size': str(2 * OFFSET),
        'font-family': '"Helvetica Neue",Helvetica,Arial,sans-serif',
        'fill': '#333333',
    }

    # Packages with some sort of wheel
    wheel_packages = sum(
        1 if package[version]['dropped_support'] == "yes" else
        0 for package in packages)

    packages_with_wheels = et.SubElement(
        wheel, 'text',
        x=str(CENTER), y=str(CENTER - OFFSET),
        attrib=text_attributes,
    )
    packages_with_wheels.text = '{0}'.format(wheel_packages)

    title = et.SubElement(packages_with_wheels, 'title')
    percentage = '{:.0%}'.format(wheel_packages/float(total))
    title.text = percentage

    # Dividing line
    et.SubElement(
        wheel, 'line',
        x1=str(CENTER - FRACTION_LINE // 2), y1=str(CENTER),
        x2=str(CENTER + FRACTION_LINE // 2), y2=str(CENTER),
        attrib={'stroke': '#333333', 'stroke-width': '2'},
    )

    # Total packages
    total_packages = et.SubElement(
        wheel, 'text',
        x=str(CENTER), y=str(CENTER + OFFSET),
        attrib=text_attributes,
    )
    total_packages.text = '{0}'.format(total)

    title = et.SubElement(total_packages, 'title')
    title.text = percentage

def generate_svg_wheel(packages, total, versions):
    for version in versions:

        wheel = et.Element(
            'svg',
            viewBox='0 0 {0} {0}'.format(2 * CENTER),
            version='1.1',
            xmlns='http://www.w3.org/2000/svg',
        )

        for index, result in enumerate(packages):

            start, stop = angles(index, total)
            sector = add_annular_sector(
                wheel,
                start=start, stop=stop,
                style_class=result[version]['css_class'],
            )
            title = et.SubElement(sector, 'title')
            title.text = u'{0} {1}'.format(
                result['name'], result[version]['icon'])

        add_fraction(wheel, packages, total, version)

        wheel_svg = os.path.join(version, 'wheel.svg')
        wheel_png = os.path.join(version, 'wheel.png')
        with open(wheel_svg, 'wb') as svg:
            svg.write(HEADERS)
            svg.write(et.tostring(wheel))

        # Install with: npm install svgexport -g
        os.system('svgexport {svg} {png} 32:32'.format(
            svg=wheel_svg, png=wheel_png))
<|endoftext|>


<commit_before>import os
import subprocess
import edalize

from toolchain import Toolchain
from utils import Timed

class VPR(Toolchain):
    '''VPR using Yosys for synthesis'''
    carries = (False, )

    def __init__(self, rootdir):
        Toolchain.__init__(self, rootdir)
        self.toolchain = 'vpr'
        self.files = []

    def run(self):
        with Timed(self, 'prepare'):
            os.makedirs(self.out_dir, exist_ok=True)

            for f in self.srcs:
                self.files.append(
                    {
                        'name': os.path.realpath(f),
                        'file_type': 'verilogSource'
                    }
                )

            if self.pcf:
                self.files.append(
                    {
                        'name': os.path.realpath(self.pcf),
                        'file_type': 'PCF'
                    }
                )
            if self.sdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.sdc),
                        'file_type': 'SDC'
                    }
                )

            if self.xdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.xdc),
                        'file_type': 'xdc'
                    }
                )

            chip = self.family + self.device

            self.edam = {
                'files': self.files,
                'name': self.project_name,
                'toplevel': self.top,
                'tool_options':
                    {
                        'symbiflow':
                            {
                                'part': chip,
                                'package': self.package,
                                'vendor': 'xilinx',
                                'builddir': '.',
                                'pnr': 'vpr'
                            }
                    }
            }
            self.backend = edalize.get_edatool('symbiflow')(
                edam=self.edam, work_root=self.out_dir
            )
            self.backend.configure("")
        with Timed(self, 'synthesis'):
            self.backend.build_main(self.top + '.eblif')
        with Timed(self, 'pack'):
            self.backend.build_main(self.top + '.net')
        with Timed(self, 'place'):
            self.backend.build_main(self.top + '.place')
        with Timed(self, 'route'):
            self.backend.build_main(self.top + '.route')
        with Timed(self, 'fasm'):
            self.backend.build_main(self.top + '.fasm')
        with Timed(self, 'bitstream'):
            self.backend.build_main(self.top + '.bit')

    def get_critical_paths(self, clocks, timing):

        report = self.out_dir + '/report_timing.{}.rpt'.format(timing)
        processing = False
        in_block = False
        critical_paths = None
        with open(report, 'r') as fp:
            for l in fp:
                l = l.strip('\n')
                if l.startswith('#Path'):
                    in_block = True
                    continue

                if in_block is True:
                    if l.startswith('data arrival time'):
                        processing = True
                        continue

                if processing is True:
                    if l.startswith('clock') and not l.startswith(
                        ('clock source latency', 'clock uncertainty')):
                        clock = l.split()[1]
                        if clock in clocks:
                            if critical_paths is None:
                                critical_paths = dict()

                            if clock not in critical_paths:
                                critical_paths[clock] = dict()
                                critical_paths[clock]['requested'] = float(
                                    l.split()[-1]
                                )

                    if l.startswith('slack'):
                        if critical_paths is None:
                            in_block = False
                            processing = False
                            continue
                        if clock in clocks and not 'met' in critical_paths[
                                clock]:
                            critical_paths[clock]['met'] = '(MET)' in l
                            critical_paths[clock]['violation'] = float(
                                l.split()[-1]
                            ) if not critical_paths[clock]['met'] else 0.0
                        in_block = False
                        processing = False

        return critical_paths

    def max_freq(self):

        freqs = dict()
        clocks = dict()
        route_log = self.out_dir + '/route.log'

        processing = False

        with open(route_log, 'r') as fp:
            for l in fp:

                if l == "Intra-domain critical path delays (CPDs):\n":
                    processing = True
                    continue

                if processing is True:
                    if len(l.strip('\n')) == 0:
                        processing = False
                        continue

                    fields = l.split(':')
                    group = fields[0].split()[0]
                    freqs[group] = 1e9 / float(fields[1].split()[0].strip())

        for clk in freqs:
            criticals = self.get_critical_paths(clk, 'setup')
            clocks[clk] = dict()
            clocks[clk]['actual'] = freqs[clk]
            if criticals is not None:
                clocks[clk]['requested'] = 1e9 / criticals[clk]['requested']
                clocks[clk]['met'] = criticals[clk]['met']
                clocks[clk]['setup_violation'] = criticals[clk]['violation']
            criticals = self.get_critical_paths(clk, 'hold')
            if criticals is not None:
                clocks[clk]['hold_violation'] = criticals[clk]['violation']
                if 'requested' not in clocks[clk]:
                    clocks[clk]['requested'
                                ] = 1e9 / criticals[clk]['requested']
                if 'met' not in clocks[clk]:
                    clocks[clk]['met'] = criticals[clk]['met']
            for v in ['requested', 'setup_violation', 'hold_violation']:
                if v not in clocks[clk]:
                    clocks[clk][v] = 0.0
            if 'met' not in clocks[clk]:
                clocks[clk]['met'] = 'unknown'

        return clocks

    def get_resources(self):
        """
        pack.log:
        (...)
        Pb types usage...
          GND          : 1
          BLK-TL-IOPAD : 2
          lut          : 1
          IOB33_MODES  : 2
          COMMON_LUT_AND_F78MUX : 1
          ALUT         : 1
          COMMON_SLICE : 6
          FDRE         : 24
          REG_FDSE_or_FDRE : 24
          SYN-GND      : 1
          BLK-TL-SLICEL : 6
          CEUSEDMUX    : 6
          CARRY4_VPR   : 6
          outpad       : 1
          A5LUT        : 1
          SLICEL0      : 6
          IOB33        : 2
          CE_VCC       : 24
          SRUSEDMUX    : 6
          inpad        : 1
          SLICE_FF     : 6
          SR_GND       : 24
        (...)
        """
        pack_logfile = self.out_dir + "/pack.log"
        resources = {}
        with open(pack_logfile, 'r') as fp:
            processing = False
            for l in fp:
                l = l.strip()
                if l == "Pb types usage...":
                    processing = True
                    continue

                if not processing:
                    continue
                else:
                    if len(l) == 0:
                        break
                    res = l.split(":")
                    restype = res[0].strip()
                    rescount = int(res[1].strip())
                    resources[restype] = rescount

        return resources

    def resources(self):
        lut = 0
        dff = 0
        carry = 0
        iob = 0
        pll = 0
        bram = 0

        res = self.get_vpr_resources()

        if 'lut' in res:
            lut = res['lut']
        if 'REG_FDSE_or_FDRE' in res:
            dff = dff = res['REG_FDSE_or_FDRE']
        if 'CARRY4_VPR' in res:
            carry = carry + res['CARRY4_VPR']
        if 'outpad' in res:
            iob = iob + res['outpad']
        if 'inpad' in res:
            iob = iob + res['inpad']
        if 'RAMB18E1_Y0' in res:
            bram += res['RAMB18E1_Y0']
        if 'RAMB18E1_Y1' in res:
            bram += res['RAMB18E1_Y1']
        if 'PLLE2_ADV' in res:
            pll = res['PLLE2_ADV']

        ret = {
            "LUT": str(lut),
            "DFF": str(dff),
            "BRAM": str(bram),
            "CARRY": str(carry),
            "GLB": "unsupported",
            "PLL": str(pll),
            "IOB": str(iob),
        }
        return ret

    @staticmethod
    def yosys_ver():
        # Yosys 0.7+352 (git sha1 baddb017, clang 3.8.1-24 -fPIC -Os)
        return subprocess.check_output(
            "yosys -V", shell=True, universal_newlines=True
        ).strip()

    @staticmethod
    def vpr_version():
        '''
        vpr  --version

        VPR FPGA Placement and Routing.
        Version: 8.0.0-dev+vpr-7.0.5-6027-g94a747729
        Revision: vpr-7.0.5-6027-g94a747729
        Compiled: 2018-06-21T16:45:11 (release build)
        Compiler: GNU 6.3.0 on Linux-4.9.0-5-amd64 x86_64
        University of Toronto
        vtr-users@googlegroups.com
        This is free open source code under MIT license.
        '''
        out = subprocess.check_output(
            "vpr --version", shell=True, universal_newlines=True
        ).strip()
        version = None
        revision = None
        for l in out.split('\n'):
            l = l.strip()
            if l.find('Version:') == 0:
                version = l
            if l.find('Revision:') == 0:
                revision = l
        assert version is not None
        assert revision is not None
        return version + ', ' + revision

    def versions(self):
        return {
            'yosys': VPR.yosys_ver(),
            'vpr': VPR.vpr_version(),
        }

    @staticmethod
    def seedable():
        return True

    @staticmethod
    def check_env():
        return {
            'yosys': have_exec('yosys'),
            'vpr': have_exec(VPR.vpr_bin()),
        }

class Nextpnr(Toolchain):
    '''nextpnr using Yosys for synthesis'''
    carries = (False, )

    def __init__(self, rootdir):
        Toolchain.__init__(self, rootdir)
        self.toolchain = 'nextpnr'
        self.files = []

    def run(self):
        with Timed(self, 'prepare'):
            os.makedirs(self.out_dir, exist_ok=True)

            for f in self.srcs:
                self.files.append(
                    {
                        'name': os.path.realpath(f),
                        'file_type': 'verilogSource'
                    }
                )

            if self.xdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.xdc),
                        'file_type': 'xdc'
                    }
                )

            chip = self.family + self.device

            chipdb = os.path.join(
                self.rootdir, 'third_party', 'install', 'share', 'nextpnr',
                '{}{}.bin'.format(self.family, self.part)
            )
            self.files.append(
                {
                    'name': os.path.realpath(chipdb),
                    'file_type': 'bba'
                }
            )

            self.edam = {
                'files': self.files,
                'name': self.project_name,
                'toplevel': self.top,
                'tool_options':
                    {
                        'symbiflow':
                            {
                                'part': chip,
                                'package': self.package,
                                'vendor': 'xilinx',
                                'builddir': '.',
                                'pnr': 'nextpnr'
                            }
                    }
            }
            self.backend = edalize.get_edatool('symbiflow')(
                edam=self.edam, work_root=self.out_dir
            )
            self.backend.configure("")
        with Timed(self, 'synthesis + implementation'):
            self.backend.build_main(self.project_name + '.fasm')
        with Timed(self, 'bitstream'):
            self.backend.build_main(self.project_name + '.bit')

    def get_critical_paths(self, clocks, timing):
        #TODO critical paths
        return None

    def max_freq(self):
        #TODO max freq
        clocks = dict()

        return clocks

    def get_resources(self):
        #TODO resources
        resources = {}

        return resources

    def resources(self):
        lut = 0
        dff = 0
        carry = 0
        iob = 0
        pll = 0
        bram = 0

        res = self.get_resources()

        if 'lut' in res:
            lut = res['lut']
        if 'REG_FDSE_or_FDRE' in res:
            dff = dff = res['REG_FDSE_or_FDRE']
        if 'CARRY4_VPR' in res:
            carry = carry + res['CARRY4_VPR']
        if 'outpad' in res:
            iob = iob + res['outpad']
        if 'inpad' in res:
            iob = iob + res['inpad']
        if 'RAMB18E1_Y0' in res:
            bram += res['RAMB18E1_Y0']
        if 'RAMB18E1_Y1' in res:
            bram += res['RAMB18E1_Y1']
        if 'PLLE2_ADV' in res:
            pll = res['PLLE2_ADV']

        ret = {
            "LUT": "NA",  #str(lut),
            "DFF": "NA",  #str(dff),
            "BRAM": "NA",  #str(bram),
            "CARRY": "NA",  #str(carry),
            "GLB": "unsupported",
            "PLL": "NA",  #str(pll),
            "IOB": "NA",  #str(iob),
        }
        return ret

    @staticmethod
    def yosys_ver():
        # Yosys 0.7+352 (git sha1 baddb017, clang 3.8.1-24 -fPIC -Os)
        return subprocess.check_output(
            "yosys -V", shell=True, universal_newlines=True
        ).strip()

    @staticmethod
    def nextpnr_version():
        '''
        nextpnr-xilinx  --version
        '''
        return subprocess.check_output(
            "nextpnr-xilinx --version", shell=True, universal_newlines=True
        ).strip()

    def versions(self):
        return {
            'yosys': Nextpnr.yosys_ver(),
            'vpr': Nextpnr.nextpnr_version(),
        }

    @staticmethod
    def seedable():
        return True

    @staticmethod
    def check_env():
        return {
            'yosys': have_exec('yosys'),
            'nextpnr': have_exec(Nextpnr.nextpnr_bin()),
        }
<commit_msg>symbiflow: add safe division by zero<commit_after>import os
import subprocess
import edalize

from toolchain import Toolchain
from utils import Timed

class VPR(Toolchain):
    '''VPR using Yosys for synthesis'''
    carries = (False, )

    def __init__(self, rootdir):
        Toolchain.__init__(self, rootdir)
        self.toolchain = 'vpr'
        self.files = []

    def run(self):
        with Timed(self, 'prepare'):
            os.makedirs(self.out_dir, exist_ok=True)

            for f in self.srcs:
                self.files.append(
                    {
                        'name': os.path.realpath(f),
                        'file_type': 'verilogSource'
                    }
                )

            if self.pcf:
                self.files.append(
                    {
                        'name': os.path.realpath(self.pcf),
                        'file_type': 'PCF'
                    }
                )
            if self.sdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.sdc),
                        'file_type': 'SDC'
                    }
                )

            if self.xdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.xdc),
                        'file_type': 'xdc'
                    }
                )

            chip = self.family + self.device

            self.edam = {
                'files': self.files,
                'name': self.project_name,
                'toplevel': self.top,
                'tool_options':
                    {
                        'symbiflow':
                            {
                                'part': chip,
                                'package': self.package,
                                'vendor': 'xilinx',
                                'builddir': '.',
                                'pnr': 'vpr'
                            }
                    }
            }
            self.backend = edalize.get_edatool('symbiflow')(
                edam=self.edam, work_root=self.out_dir
            )
            self.backend.configure("")
        with Timed(self, 'synthesis'):
            self.backend.build_main(self.top + '.eblif')
        with Timed(self, 'pack'):
            self.backend.build_main(self.top + '.net')
        with Timed(self, 'place'):
            self.backend.build_main(self.top + '.place')
        with Timed(self, 'route'):
            self.backend.build_main(self.top + '.route')
        with Timed(self, 'fasm'):
            self.backend.build_main(self.top + '.fasm')
        with Timed(self, 'bitstream'):
            self.backend.build_main(self.top + '.bit')

    def get_critical_paths(self, clocks, timing):

        report = self.out_dir + '/report_timing.{}.rpt'.format(timing)
        processing = False
        in_block = False
        critical_paths = None
        with open(report, 'r') as fp:
            for l in fp:
                l = l.strip('\n')
                if l.startswith('#Path'):
                    in_block = True
                    continue

                if in_block is True:
                    if l.startswith('data arrival time'):
                        processing = True
                        continue

                if processing is True:
                    if l.startswith('clock') and not l.startswith(
                        ('clock source latency', 'clock uncertainty')):
                        clock = l.split()[1]
                        if clock in clocks:
                            if critical_paths is None:
                                critical_paths = dict()

                            if clock not in critical_paths:
                                critical_paths[clock] = dict()
                                critical_paths[clock]['requested'] = float(
                                    l.split()[-1]
                                )

                    if l.startswith('slack'):
                        if critical_paths is None:
                            in_block = False
                            processing = False
                            continue
                        if clock in clocks and not 'met' in critical_paths[
                                clock]:
                            critical_paths[clock]['met'] = '(MET)' in l
                            critical_paths[clock]['violation'] = float(
                                l.split()[-1]
                            ) if not critical_paths[clock]['met'] else 0.0
                        in_block = False
                        processing = False

        return critical_paths

    def max_freq(self):
        def safe_division_by_zero(n, d):
            return n / d if d else 0.0

        freqs = dict()
        clocks = dict()
        route_log = self.out_dir + '/route.log'

        processing = False

        with open(route_log, 'r') as fp:
            for l in fp:

                if l == "Intra-domain critical path delays (CPDs):\n":
                    processing = True
                    continue

                if processing is True:
                    if len(l.strip('\n')) == 0:
                        processing = False
                        continue

                    fields = l.split(':')
                    group = fields[0].split()[0]
                    freqs[group] = safe_division_by_zero(
                        1e9, float(fields[1].split()[0].strip())
                    )

        for clk in freqs:
            criticals = self.get_critical_paths(clk, 'setup')
            clocks[clk] = dict()
            clocks[clk]['actual'] = freqs[clk]
            if criticals is not None:
                clocks[clk]['requested'] = safe_division_by_zero(
                    1e9, criticals[clk]['requested']
                )
                clocks[clk]['met'] = criticals[clk]['met']
                clocks[clk]['setup_violation'] = criticals[clk]['violation']
            criticals = self.get_critical_paths(clk, 'hold')
            if criticals is not None:
                clocks[clk]['hold_violation'] = criticals[clk]['violation']
                if 'requested' not in clocks[clk]:
                    clocks[clk]['requested'] = safe_division_by_zero(
                        1e9, criticals[clk]['requested']
                    )
                if 'met' not in clocks[clk]:
                    clocks[clk]['met'] = criticals[clk]['met']
            for v in ['requested', 'setup_violation', 'hold_violation']:
                if v not in clocks[clk]:
                    clocks[clk][v] = 0.0
            if 'met' not in clocks[clk]:
                clocks[clk]['met'] = 'unknown'

        return clocks

    def get_resources(self):
        """
        pack.log:
        (...)
        Pb types usage...
          GND          : 1
          BLK-TL-IOPAD : 2
          lut          : 1
          IOB33_MODES  : 2
          COMMON_LUT_AND_F78MUX : 1
          ALUT         : 1
          COMMON_SLICE : 6
          FDRE         : 24
          REG_FDSE_or_FDRE : 24
          SYN-GND      : 1
          BLK-TL-SLICEL : 6
          CEUSEDMUX    : 6
          CARRY4_VPR   : 6
          outpad       : 1
          A5LUT        : 1
          SLICEL0      : 6
          IOB33        : 2
          CE_VCC       : 24
          SRUSEDMUX    : 6
          inpad        : 1
          SLICE_FF     : 6
          SR_GND       : 24
        (...)
        """
        pack_logfile = self.out_dir + "/pack.log"
        resources = {}
        with open(pack_logfile, 'r') as fp:
            processing = False
            for l in fp:
                l = l.strip()
                if l == "Pb types usage...":
                    processing = True
                    continue

                if not processing:
                    continue
                else:
                    if len(l) == 0:
                        break
                    res = l.split(":")
                    restype = res[0].strip()
                    rescount = int(res[1].strip())
                    resources[restype] = rescount

        return resources

    def resources(self):
        lut = 0
        dff = 0
        carry = 0
        iob = 0
        pll = 0
        bram = 0

        res = self.get_resources()

        if 'lut' in res:
            lut = res['lut']
        if 'REG_FDSE_or_FDRE' in res:
            dff = dff = res['REG_FDSE_or_FDRE']
        if 'CARRY4_VPR' in res:
            carry = carry + res['CARRY4_VPR']
        if 'outpad' in res:
            iob = iob + res['outpad']
        if 'inpad' in res:
            iob = iob + res['inpad']
        if 'RAMB18E1_Y0' in res:
            bram += res['RAMB18E1_Y0']
        if 'RAMB18E1_Y1' in res:
            bram += res['RAMB18E1_Y1']
        if 'PLLE2_ADV' in res:
            pll = res['PLLE2_ADV']

        ret = {
            "LUT": str(lut),
            "DFF": str(dff),
            "BRAM": str(bram),
            "CARRY": str(carry),
            "GLB": "unsupported",
            "PLL": str(pll),
            "IOB": str(iob),
        }
        return ret

    @staticmethod
    def yosys_ver():
        # Yosys 0.7+352 (git sha1 baddb017, clang 3.8.1-24 -fPIC -Os)
        return subprocess.check_output(
            "yosys -V", shell=True, universal_newlines=True
        ).strip()

    @staticmethod
    def vpr_version():
        '''
        vpr  --version

        VPR FPGA Placement and Routing.
        Version: 8.0.0-dev+vpr-7.0.5-6027-g94a747729
        Revision: vpr-7.0.5-6027-g94a747729
        Compiled: 2018-06-21T16:45:11 (release build)
        Compiler: GNU 6.3.0 on Linux-4.9.0-5-amd64 x86_64
        University of Toronto
        vtr-users@googlegroups.com
        This is free open source code under MIT license.
        '''
        out = subprocess.check_output(
            "vpr --version", shell=True, universal_newlines=True
        ).strip()
        version = None
        revision = None
        for l in out.split('\n'):
            l = l.strip()
            if l.find('Version:') == 0:
                version = l
            if l.find('Revision:') == 0:
                revision = l
        assert version is not None
        assert revision is not None
        return version + ', ' + revision

    def versions(self):
        return {
            'yosys': VPR.yosys_ver(),
            'vpr': VPR.vpr_version(),
        }

    @staticmethod
    def seedable():
        return True

    @staticmethod
    def check_env():
        return {
            'yosys': have_exec('yosys'),
            'vpr': have_exec(VPR.vpr_bin()),
        }

class Nextpnr(Toolchain):
    '''nextpnr using Yosys for synthesis'''
    carries = (False, )

    def __init__(self, rootdir):
        Toolchain.__init__(self, rootdir)
        self.toolchain = 'nextpnr'
        self.files = []

    def run(self):
        with Timed(self, 'prepare'):
            os.makedirs(self.out_dir, exist_ok=True)

            for f in self.srcs:
                self.files.append(
                    {
                        'name': os.path.realpath(f),
                        'file_type': 'verilogSource'
                    }
                )

            if self.xdc:
                self.files.append(
                    {
                        'name': os.path.realpath(self.xdc),
                        'file_type': 'xdc'
                    }
                )

            chip = self.family + self.device

            chipdb = os.path.join(
                self.rootdir, 'third_party', 'install', 'share', 'nextpnr',
                '{}{}.bin'.format(self.family, self.part)
            )
            self.files.append(
                {
                    'name': os.path.realpath(chipdb),
                    'file_type': 'bba'
                }
            )

            self.edam = {
                'files': self.files,
                'name': self.project_name,
                'toplevel': self.top,
                'tool_options':
                    {
                        'symbiflow':
                            {
                                'part': chip,
                                'package': self.package,
                                'vendor': 'xilinx',
                                'builddir': '.',
                                'pnr': 'nextpnr'
                            }
                    }
            }
            self.backend = edalize.get_edatool('symbiflow')(
                edam=self.edam, work_root=self.out_dir
            )
            self.backend.configure("")
        with Timed(self, 'synthesis + implementation'):
            self.backend.build_main(self.project_name + '.fasm')
        with Timed(self, 'bitstream'):
            self.backend.build_main(self.project_name + '.bit')

    def get_critical_paths(self, clocks, timing):
        #TODO critical paths
        return None

    def max_freq(self):
        #TODO max freq
        clocks = dict()

        return clocks

    def get_resources(self):
        #TODO resources
        resources = {}

        return resources

    def resources(self):
        lut = 0
        dff = 0
        carry = 0
        iob = 0
        pll = 0
        bram = 0

        res = self.get_resources()

        if 'lut' in res:
            lut = res['lut']
        if 'REG_FDSE_or_FDRE' in res:
            dff = dff = res['REG_FDSE_or_FDRE']
        if 'CARRY4_VPR' in res:
            carry = carry + res['CARRY4_VPR']
        if 'outpad' in res:
            iob = iob + res['outpad']
        if 'inpad' in res:
            iob = iob + res['inpad']
        if 'RAMB18E1_Y0' in res:
            bram += res['RAMB18E1_Y0']
        if 'RAMB18E1_Y1' in res:
            bram += res['RAMB18E1_Y1']
        if 'PLLE2_ADV' in res:
            pll = res['PLLE2_ADV']

        ret = {
            "LUT": "NA",  #str(lut),
            "DFF": "NA",  #str(dff),
            "BRAM": "NA",  #str(bram),
            "CARRY": "NA",  #str(carry),
            "GLB": "unsupported",
            "PLL": "NA",  #str(pll),
            "IOB": "NA",  #str(iob),
        }
        return ret

    @staticmethod
    def yosys_ver():
        # Yosys 0.7+352 (git sha1 baddb017, clang 3.8.1-24 -fPIC -Os)
        return subprocess.check_output(
            "yosys -V", shell=True, universal_newlines=True
        ).strip()

    @staticmethod
    def nextpnr_version():
        '''
        nextpnr-xilinx  --version
        '''
        return subprocess.check_output(
            "nextpnr-xilinx --version", shell=True, universal_newlines=True
        ).strip()

    def versions(self):
        return {
            'yosys': Nextpnr.yosys_ver(),
            'vpr': Nextpnr.nextpnr_version(),
        }

    @staticmethod
    def seedable():
        return True

    @staticmethod
    def check_env():
        return {
            'yosys': have_exec('yosys'),
            'nextpnr': have_exec(Nextpnr.nextpnr_bin()),
        }
<|endoftext|>


<commit_before>
DEFAULT_CONF = """
# Available styles keys:
#
# - bold, underline, hidden
# - grey, red, green, yellow, blue, magenta, cyan, white
# - on-grey, on-red, on-green, on-yellow, on-blue, on-magenta, on-cyan, on-white
#
# The order of definitions matters, with highest precedence at the top.
#
# If a stlye definition starts with '!' then the whole line will be styled
# if it matches the given regex, and not just the match itself.
#

[Style="example"]
# Print in red any line containing the word 'error'
!red: "error"
red: "evil\.org"
# Date
green: "\d{4}-\d\d-\d\d"
# Time
green bold: "\d\d:\d\d:\d\d"
# IP address (and port number if present)
yellow underline: "\d+\.\d+\.\d+\.\d+(:\d+)?"
magenta: "\[samplesession\]"
# Catch-all for anything else inside [square brackets]
blue: "\[[^\]]+\]"
# Catch-all for any remaining standalone numbers
cyan bold: "\b\d+\b"

[Style="syslog"]
!red bold: "<warn>"
on-magenta: "^\w\w\w \d\d\s?"
bold on-blue: "\d+:\d\d:\d+"
yellow: "\d+\.\d+\.\d+\.\d+"
red on-white: "\([^\)]+\)"
grey: "\[[^\]]+\]"

[Style="java"]
!red: "Exception"
#green: "INFO|DEBUG|WARN"
white on-129 bold: "INFO|DEBUG"
green bold: "\(\w+\.java:\d+\)"
grey : "\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d(,\d\d\d)?"
yellow: "[a-zA-Z]+:\d+"
yellow: "com\.[\w+|\.]+"
cyan: "org\.[\w+|\.]+"
blue: "\[[^\]]+\]"

"""
<commit_msg>Updated default conf.<commit_after>
DEFAULT_CONF = """
# Available styles keys:
#
# - bold, underline, hidden
# - grey, red, green, yellow, blue, magenta, cyan, white
# - on-grey, on-red, on-green, on-yellow, on-blue, on-magenta, on-cyan, on-white
#
# The order of definitions matters, with highest precedence at the top.
#
# If a stlye definition starts with '!' then the whole line will be styled
# if it matches the given regex, and not just the match itself.
#

[Style="example"]
# Print in red any line containing the word 'error'
!red: "error"
red: "evil\.org"
# Date
green: "\d{4}-\d\d-\d\d"
# Time
green bold: "\d\d:\d\d:\d\d"
# IP address (and port number if present)
yellow underline: "\d+\.\d+\.\d+\.\d+(:\d+)?"
magenta: "\[samplesession\]"
# Catch-all for anything else inside [square brackets]
blue: "\[[^\]]+\]"
# Catch-all for any remaining standalone numbers
cyan bold: "\b\d+\b"

[Style="ifconfig"]
yellow bold: "\d+\.\d+\.\d+\.\d+(:\d+)?"
green bold: "(eth|wlan|lo)\d?"
blue bold: "(\d\d:?){6}"
red: "errors:\d+"
magenta: "[a-zA-Z]+:\d+"
cyan bold: "RX|TX|Link|inet\d?"

[Style="calendar"]
bold: "\d{4}"
172 bold underline: "Jan\w+|Feb\w+|Mar\w+|Apr\w+|May|Jun\w|Jul\w|Aug\w+|Sep\w+|Oct\w+|Nov\w+|Dec\w+"
229: "\d\d?"
160 bold: "Su"
220 bold: "\w+"

[Style="syslog"]
!red bold: "<warn>"
on-magenta: "^\w\w\w \d\d\s?"
bold on-blue: "\d+:\d\d:\d+"
yellow: "\d+\.\d+\.\d+\.\d+"
red on-white: "\([^\)]+\)"
grey: "\[[^\]]+\]"

[Style="java"]
!red: "Exception"
#green: "INFO|DEBUG|WARN"
white on-129 bold: "INFO|DEBUG"
green bold: "\(\w+\.java:\d+\)"
grey : "\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d(,\d\d\d)?"
yellow: "[a-zA-Z]+:\d+"
yellow: "com\.[\w+|\.]+"
cyan: "org\.[\w+|\.]+"
blue: "\[[^\]]+\]"

"""
<|endoftext|>