<commit_before>"""
Filename:     remove_drift.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Remove drift from a data series

"""

# Import general Python modules

import sys, os, pdb
import argparse, re
import numpy
import iris
import cf_units
from iris.experimental.equalise_cubes import equalise_attributes

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import timeseries
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')

# Define functions

def apply_polynomial(x_data, coefficient_a_data, coefficient_b_data, coefficient_c_data, coefficient_d_data, control_start_data, chunk=False):
    """Evaluate cubic polynomial.

    Args:
      x_data (numpy.ndarray): One dimensional x-axis data
      coefficient_data (numpy.ndarray): Multi-dimensional coefficient array (e.g. lat, lon, depth)
      chunk (bool): Chunk the polynomial calculation to avoid memory errors

    """

    x_data = x_data.astype(numpy.float32)
    coefficient_dict = {}
    coefficient_dict['a'] = coefficient_a_data
    coefficient_dict['b'] = coefficient_b_data
    coefficient_dict['c'] = coefficient_c_data
    coefficient_dict['d'] = coefficient_d_data

    while x_data.ndim < coefficient_a_data.ndim + 1:
        x_data = x_data[..., numpy.newaxis]
    for letter in ['a', 'b', 'c', 'd']:
        coef = coefficient_dict[letter]
        coef = numpy.repeat(coef[numpy.newaxis, ...], x_data.shape[0], axis=0)
        assert x_data.ndim == coef.ndim
        coefficient_dict[letter] = coef

    if chunk:
        result = numpy.zeros(coefficient_dict['b'].shape, dtype='float32')
        for index in range(0, coefficient_dict['b'].shape[-1]):
            # loop to avoid memory error with large arrays
            result[..., index] = coefficient_dict['a'] + coefficient_dict['b'][..., index] * x_data[..., 0] + coefficient_dict['c'][..., index] * x_data[..., 0]**2 + coefficient_dict['d'][..., index] * x_data[..., 0]**3 - control_start_data
    else:
        result = coefficient_dict['a'] + coefficient_dict['b'] * x_data + coefficient_dict['c'] * x_data**2 + coefficient_dict['d'] * x_data**3 - control_start_data

    return result

def check_attributes(data_attrs, control_attrs):
    """Make sure the correct control run has been used."""

    assert data_attrs['parent_experiment_id'] in [control_attrs['experiment_id'], 'N/A']

    control_rip = 'r%si%sp%s' %(control_attrs['realization'],
                                control_attrs['initialization_method'],
                                control_attrs['physics_version'])
    assert data_attrs['parent_experiment_rip'] in [control_rip, 'N/A']

def check_data_units(data_cube, coefficient_cube):
    """Make sure the units of the data and coefficients match."""

    if data_cube.standard_name == 'sea_water_salinity':
         data_cube = gio.salinity_unit_check(data_cube)

    assert data_cube.units == coefficient_cube.units

    return data_cube

def coefficient_sanity_check(coefficient_a_cube, coefficient_b_cube, coefficient_c_cube, coefficient_d_cube, variable):
    """Sanity check the cubic polynomial coefficients.

    Polynomial is a + bx + cx^2 + dx^3. The telling sign of a poor
    fit is an 'a' value that does not represent a realistic value.

    """

    assert variable in ['sea_water_potential_temperature', 'sea_water_salinity']

    if variable == 'sea_water_potential_temperature':
        var_max = 330
        var_min = 250
    elif variable == 'sea_water_salinity':
        var_max = 55
        var_min = 2

    nmasked_original = numpy.sum(coefficient_a_cube.data.mask)

    a_data = coefficient_a_cube.data

    original_mask = a_data.mask

    crazy_mask_min = numpy.ma.where(a_data < var_min, True, False)
    crazy_mask_max = numpy.ma.where(a_data > var_max, True, False)
    new_mask = numpy.ma.mask_or(crazy_mask_min, crazy_mask_max)
    ncrazy_min = numpy.sum(crazy_mask_min)
    ncrazy_max = numpy.sum(crazy_mask_max)
    ncrazy = ncrazy_min + ncrazy_max

    nmasked_new = numpy.sum(new_mask)
    npoints = numpy.prod(a_data.shape)
    summary = "Masked %i of %i points because cubic fit was poor" %(ncrazy, npoints)
    #numpy.argwhere(x == np.min(x)) to see what those points are

    coefficient_a_cube.data.mask = new_mask
    coefficient_b_cube.data.mask = new_mask
    coefficient_c_cube.data.mask = new_mask
    coefficient_d_cube.data.mask = new_mask

    assert nmasked_new == ncrazy + nmasked_original

    return summary

def time_adjustment(first_data_cube, coefficient_cube):
    """Determine the adjustment that needs to be made to time axis."""

    branch_time_value = float(first_data_cube.attributes['branch_time'])
    branch_time_unit = coefficient_cube.attributes['time_unit']
    branch_time_calendar = coefficient_cube.attributes['time_calendar']
    data_time_coord = first_data_cube.coord('time')

    new_unit = cf_units.Unit(branch_time_unit, calendar=branch_time_calendar)
    data_time_coord.convert_units(new_unit)

    first_experiment_time = data_time_coord.points[0]
    time_diff = first_experiment_time - branch_time_value - coefficient_cube.attributes['time_start']

    return time_diff, branch_time_value, new_unit

def check_start_time(time_values, coefficient_cube, branch_time, annual=False):
    """Check that the time adjustment was correct

    The branch time given in CMIP5 metadata is for monthly timescale data.

    If the timescale is unchanged, then after the time adjustment the first time point
      of the first data file should be zero.

    If the timescale is annual, then after time adjustment the first point should
      be about 6 months, because iris sets the time axis values to the mid point of the year.

    """

    time_diff = time_values[0] - (coefficient_cube.attributes['time_start'] + branch_time)
    if annual:
        assert 100 < time_diff < 200
    else:
        assert time_diff == 0

def main(inargs):
    """Run the program."""

    first_data_cube = iris.load_cube(inargs.data_files[0], inargs.var)
    coefficient_a_cube = iris.load_cube(inargs.coefficient_file, 'coefficient a')
    coefficient_b_cube = iris.load_cube(inargs.coefficient_file, 'coefficient b')
    coefficient_c_cube = iris.load_cube(inargs.coefficient_file, 'coefficient c')
    coefficient_d_cube = iris.load_cube(inargs.coefficient_file, 'coefficient d')
    control_start_cube = iris.load_cube(inargs.coefficient_file, inargs.var)

    sanity_summary = coefficient_sanity_check(coefficient_a_cube, coefficient_b_cube, coefficient_c_cube, coefficient_d_cube, inargs.var)

    time_diff, branch_time, new_time_unit = time_adjustment(first_data_cube, coefficient_a_cube)
    del first_data_cube

    new_cubelist = []
    for fnum, filename in enumerate(inargs.data_files):
        data_cube = iris.load_cube(filename, inargs.var)
        data_cube = check_data_units(data_cube, coefficient_a_cube)
        data_cube = gio.check_time_units(data_cube)
        if not inargs.no_parent_check:
            check_attributes(data_cube.attributes, coefficient_a_cube.attributes)

        if inargs.annual:
            data_cube = timeseries.convert_to_annual(data_cube)
            data_cube.data = data_cube.data.astype(numpy.float32)

        # Sync the data time axis with the coefficient time axis
        time_coord = data_cube.coord('time')
        time_coord.convert_units(new_time_unit)

        time_values = time_coord.points.astype(numpy.float32) - time_diff
        if fnum == 0:
            check_start_time(time_values, coefficient_a_cube, branch_time, annual=inargs.annual)

        # Remove the drift
        drift_signal = apply_polynomial(time_values, coefficient_a_cube.data, coefficient_b_cube.data, coefficient_c_cube.data, coefficient_d_cube.data, control_start_cube.data, chunk=inargs.chunk)

        if not inargs.dummy:
            new_cube = data_cube - drift_signal
        else:
            print 'fake run - drift signal not subtracted'
            new_cube = data_cube
        new_cube.metadata = data_cube.metadata
        new_cube.attributes['drift_removal'] = sanity_summary

        assert (inargs.outfile[-3:] == '.nc') or (inargs.outfile[-1] == '/')

        if inargs.outfile[-3:] == '.nc':
            new_cubelist.append(new_cube)
        elif inargs.outfile[-1] == '/':
            infile = filename.split('/')[-1]
            if inargs.annual:
                infile = re.sub('Omon', 'Oyr', infile)
            outfile = inargs.outfile + infile
            metadata_dict = {infile: data_cube.attributes['history'],
                             inargs.coefficient_file: coefficient_a_cube.attributes['history']}
            new_cube.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

            assert new_cube.data.dtype == numpy.float32
            iris.save(new_cube, outfile, netcdf_format='NETCDF3_CLASSIC')
            print 'output:', outfile
            del new_cube
            del drift_signal

    if inargs.outfile[-3:] == '.nc':
        new_cubelist = iris.cube.CubeList(new_cubelist)
        equalise_attributes(new_cubelist)
        new_cubelist = new_cubelist.concatenate_cube()

        metadata_dict = {inargs.data_files[0]: data_cubelist[0].attributes['history'],
                        inargs.coefficient_file: coefficient_a_cube.attributes['history']}
        new_cubelist.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

        assert new_cubelist[0].data.dtype == numpy.float32
        iris.save(new_cubelist, inargs.outfile, netcdf_format='NETCDF3_CLASSIC')

if __name__ == '__main__':

    extra_info ="""
example:

author:
    Damien Irving, irving.damien@gmail.com
notes:
    Generate the polynomial coefficients first from calc_drift_coefficients.py

"""

    description='Remove drift from a data series'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("data_files", type=str, nargs='*', help="Input data files, in chronological order (needs to include whole experiment to get time axis correct)")
    parser.add_argument("var", type=str, help="Variable standard_name")
    parser.add_argument("coefficient_file", type=str, help="Input coefficient file")
    parser.add_argument("outfile", type=str, help="Give a path instead of a file name if you want an output file corresponding to each input file")

    parser.add_argument("--annual", action="store_true", default=False,
                        help="Convert data to annual timescale [default: False]")
    parser.add_argument("--no_parent_check", action="store_true", default=False,
                        help="Do not perform the parent experiment check [default: False]")
    parser.add_argument("--chunk", action="store_true", default=False,
                        help="Split the polynomial calculation up to avoid memory errors [default: False]")

    parser.add_argument("--dummy", action="store_true", default=False,
                        help="Do not actually subtract the drift [default: False]")

    args = parser.parse_args()

    main(args)
<commit_msg>Fixed time adjustment and polynomial anomaly<commit_after>"""
Filename:     remove_drift.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Remove drift from a data series

"""

# Import general Python modules

import sys, os, pdb
import argparse, re
import numpy
import iris
import cf_units
from iris.experimental.equalise_cubes import equalise_attributes

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import timeseries
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')

# Define functions

def apply_polynomial(x_data, coefficient_a_data, coefficient_b_data, coefficient_c_data, coefficient_d_data, control_start_data, chunk=False):
    """Evaluate cubic polynomial.

    Args:
      x_data (numpy.ndarray): One dimensional x-axis data
      coefficient_data (numpy.ndarray): Multi-dimensional coefficient array (e.g. lat, lon, depth)
      chunk (bool): Chunk the polynomial calculation to avoid memory errors

    """

    x_data = x_data.astype(numpy.float32)
    coefficient_dict = {}
    coefficient_dict['a'] = coefficient_a_data
    coefficient_dict['b'] = coefficient_b_data
    coefficient_dict['c'] = coefficient_c_data
    coefficient_dict['d'] = coefficient_d_data

    while x_data.ndim < coefficient_a_data.ndim + 1:
        x_data = x_data[..., numpy.newaxis]
    for letter in ['a', 'b', 'c', 'd']:
        coef = coefficient_dict[letter]
        coef = numpy.repeat(coef[numpy.newaxis, ...], x_data.shape[0], axis=0)
        assert x_data.ndim == coef.ndim
        coefficient_dict[letter] = coef

    if chunk:
        result = numpy.zeros(coefficient_dict['b'].shape, dtype='float32')
        for index in range(0, coefficient_dict['b'].shape[-1]):
            # loop to avoid memory error with large arrays
            polynomial = coefficient_dict['a'] + coefficient_dict['b'][..., index] * x_data[..., 0] + coefficient_dict['c'][..., index] * x_data[..., 0]**2 + coefficient_dict['d'][..., index] * x_data[..., 0]**3
            result[..., index] = polynomial - polynomial[0, ::]
    else:
        polynomial = coefficient_dict['a'] + coefficient_dict['b'] * x_data + coefficient_dict['c'] * x_data**2 + coefficient_dict['d'] * x_data**3
        result = polynomial - polynomial[0, ::]

    return result

def check_attributes(data_attrs, control_attrs):
    """Make sure the correct control run has been used."""

    assert data_attrs['parent_experiment_id'] in [control_attrs['experiment_id'], 'N/A']

    control_rip = 'r%si%sp%s' %(control_attrs['realization'],
                                control_attrs['initialization_method'],
                                control_attrs['physics_version'])
    assert data_attrs['parent_experiment_rip'] in [control_rip, 'N/A']

def check_data_units(data_cube, coefficient_cube):
    """Make sure the units of the data and coefficients match."""

    if data_cube.standard_name == 'sea_water_salinity':
         data_cube = gio.salinity_unit_check(data_cube)

    assert data_cube.units == coefficient_cube.units

    return data_cube

def coefficient_sanity_check(coefficient_a_cube, coefficient_b_cube, coefficient_c_cube, coefficient_d_cube, variable):
    """Sanity check the cubic polynomial coefficients.

    Polynomial is a + bx + cx^2 + dx^3. The telling sign of a poor
    fit is an 'a' value that does not represent a realistic value.

    """

    assert variable in ['sea_water_potential_temperature', 'sea_water_salinity']

    if variable == 'sea_water_potential_temperature':
        var_max = 330
        var_min = 250
    elif variable == 'sea_water_salinity':
        var_max = 55
        var_min = 2

    nmasked_original = numpy.sum(coefficient_a_cube.data.mask)

    a_data = coefficient_a_cube.data

    original_mask = a_data.mask

    crazy_mask_min = numpy.ma.where(a_data < var_min, True, False)
    crazy_mask_max = numpy.ma.where(a_data > var_max, True, False)
    new_mask = numpy.ma.mask_or(crazy_mask_min, crazy_mask_max)
    ncrazy_min = numpy.sum(crazy_mask_min)
    ncrazy_max = numpy.sum(crazy_mask_max)
    ncrazy = ncrazy_min + ncrazy_max

    nmasked_new = numpy.sum(new_mask)
    npoints = numpy.prod(a_data.shape)
    summary = "Masked %i of %i points because cubic fit was poor" %(ncrazy, npoints)
    #numpy.argwhere(x == np.min(x)) to see what those points are

    coefficient_a_cube.data.mask = new_mask
    coefficient_b_cube.data.mask = new_mask
    coefficient_c_cube.data.mask = new_mask
    coefficient_d_cube.data.mask = new_mask

    assert nmasked_new == ncrazy + nmasked_original

    return summary

def time_adjustment(first_data_cube, coefficient_cube):
    """Determine the adjustment that needs to be made to time axis.

    For monthly data, the branch time represents the start of the month (e.g. 1 Jan),
      while the first data time is mid-month. A factor of 15.5 is used to fix this.
      This means this function is hard wired for monthly time adjustment only.

    """

    assert first_data_cube.attributes['frequency'] == 'mon'
    assert coefficient_cube.attributes['frequency'] == 'mon'

    branch_time_value = float(first_data_cube.attributes['branch_time']) + 15.5
    branch_time_unit = coefficient_cube.attributes['time_unit']
    branch_time_calendar = coefficient_cube.attributes['time_calendar']
    data_time_coord = first_data_cube.coord('time')

    new_unit = cf_units.Unit(branch_time_unit, calendar=branch_time_calendar)
    data_time_coord.convert_units(new_unit)

    first_experiment_time = data_time_coord.points[0]
    time_diff = first_experiment_time - branch_time_value

    return time_diff, branch_time_value, new_unit

def check_time_adjustment(time_values, coefficient_cube, branch_time, fnum, annual=False):
    """Check that the time adjustment was correct

    The branch time given in CMIP5 metadata is for monthly timescale data.

    If the timescale is unchanged, then after the time adjustment the first time point
      of the first data file should be zero.

    If the timescale is annual, then after time adjustment the first point should
      be about 6 months, because iris sets the time axis values to the mid point of the year.

    """

    assert time_values[0] > coefficient_cube.attributes['time_start'] - 1.0
    assert time_values[-1] < coefficient_cube.attributes['time_end'] + 1.0
    # 1.0 allows wriggle room for time_adjustment

    if fnum == 0:
        time_diff = time_values[0] - branch_time
        if annual:
            assert 100 < time_diff < 200
        else:
            assert time_diff == 0

def main(inargs):
    """Run the program."""

    first_data_cube = iris.load_cube(inargs.data_files[0], inargs.var)
    coefficient_a_cube = iris.load_cube(inargs.coefficient_file, 'coefficient a')
    coefficient_b_cube = iris.load_cube(inargs.coefficient_file, 'coefficient b')
    coefficient_c_cube = iris.load_cube(inargs.coefficient_file, 'coefficient c')
    coefficient_d_cube = iris.load_cube(inargs.coefficient_file, 'coefficient d')
    control_start_cube = iris.load_cube(inargs.coefficient_file, inargs.var)

    coord_names = [coord.name() for coord in first_data_cube.coords(dim_coords=True)]
    assert coord_names[0] == 'time'
    sanity_summary = coefficient_sanity_check(coefficient_a_cube, coefficient_b_cube, coefficient_c_cube, coefficient_d_cube, inargs.var)

    time_diff, branch_time, new_time_unit = time_adjustment(first_data_cube, coefficient_a_cube)
    del first_data_cube

    new_cubelist = []
    for fnum, filename in enumerate(inargs.data_files):
        # Read data
        data_cube = iris.load_cube(filename, inargs.var)
        data_cube = check_data_units(data_cube, coefficient_a_cube)
        data_cube = gio.check_time_units(data_cube)
        if not inargs.no_parent_check:
            check_attributes(data_cube.attributes, coefficient_a_cube.attributes)

        # Convert timescale
        if inargs.annual:
            data_cube = timeseries.convert_to_annual(data_cube)
            data_cube.data = data_cube.data.astype(numpy.float32)

        # Sync the data time axis with the coefficient time axis
        time_coord = data_cube.coord('time')
        time_coord.convert_units(new_time_unit)

        time_values = time_coord.points.astype(numpy.float32) - time_diff
        check_time_adjustment(time_values, coefficient_a_cube, branch_time, fnum, annual=inargs.annual)

        # Remove the drift
        drift_signal = apply_polynomial(time_values, coefficient_a_cube.data, coefficient_b_cube.data, coefficient_c_cube.data, coefficient_d_cube.data, control_start_cube.data, chunk=inargs.chunk)

        if not inargs.dummy:
            new_cube = data_cube - drift_signal
        else:
            print 'fake run - drift signal not subtracted'
            new_cube = data_cube
        new_cube.metadata = data_cube.metadata
        new_cube.attributes['drift_removal'] = sanity_summary

        assert (inargs.outfile[-3:] == '.nc') or (inargs.outfile[-1] == '/')

        if inargs.outfile[-3:] == '.nc':
            new_cubelist.append(new_cube)
        elif inargs.outfile[-1] == '/':
            infile = filename.split('/')[-1]
            if inargs.annual:
                infile = re.sub('Omon', 'Oyr', infile)
            outfile = inargs.outfile + infile
            metadata_dict = {infile: data_cube.attributes['history'],
                             inargs.coefficient_file: coefficient_a_cube.attributes['history']}
            new_cube.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

            assert new_cube.data.dtype == numpy.float32
            iris.save(new_cube, outfile, netcdf_format='NETCDF3_CLASSIC')
            print 'output:', outfile
            del new_cube
            del drift_signal

    if inargs.outfile[-3:] == '.nc':
        new_cubelist = iris.cube.CubeList(new_cubelist)
        equalise_attributes(new_cubelist)
        new_cubelist = new_cubelist.concatenate_cube()

        metadata_dict = {inargs.data_files[0]: data_cubelist[0].attributes['history'],
                        inargs.coefficient_file: coefficient_a_cube.attributes['history']}
        new_cubelist.attributes['history'] = gio.write_metadata(file_info=metadata_dict)

        assert new_cubelist[0].data.dtype == numpy.float32
        iris.save(new_cubelist, inargs.outfile, netcdf_format='NETCDF3_CLASSIC')

if __name__ == '__main__':

    extra_info ="""
example:

author:
    Damien Irving, irving.damien@gmail.com
notes:
    Generate the polynomial coefficients first from calc_drift_coefficients.py

"""

    description='Remove drift from a data series'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("data_files", type=str, nargs='*', help="Input data files, in chronological order (needs to include whole experiment to get time axis correct)")
    parser.add_argument("var", type=str, help="Variable standard_name")
    parser.add_argument("coefficient_file", type=str, help="Input coefficient file")
    parser.add_argument("outfile", type=str, help="Give a path instead of a file name if you want an output file corresponding to each input file")

    parser.add_argument("--annual", action="store_true", default=False,
                        help="Convert data to annual timescale [default: False]")
    parser.add_argument("--no_parent_check", action="store_true", default=False,
                        help="Do not perform the parent experiment check [default: False]")
    parser.add_argument("--chunk", action="store_true", default=False,
                        help="Split the polynomial calculation up to avoid memory errors [default: False]")

    parser.add_argument("--dummy", action="store_true", default=False,
                        help="Do not actually subtract the drift [default: False]")

    args = parser.parse_args()

    main(args)
<|endoftext|>


<commit_before># This file is part of Fedora Community.
# Copyright (C) 2008-2010  Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from fedoracommunity.lib import app_globals
import fedoracommunity
import fedoracommunity.lib

from tg.configuration import AppConfig, Bunch
from paste.deploy.converters import asbool
from pylons.i18n import ugettext

class FedoraCommunityConfig(AppConfig):
    tw2_initialized = False

    def add_tosca2_middleware(self, app):
        if self.tw2_initialized:
            return app

        from tg import config
        from tw2.core.middleware import Config, TwMiddleware
        default_tw2_config = dict( default_engine=self.default_renderer,
                                   translator=ugettext,
                                   auto_reload_templates=asbool(self.get('templating.mako.reloadfromdisk', 'false'))
                                   )
        res_prefix = config.get('fedoracommunity.resource_path_prefix')
        if res_prefix:
            default_tw2_config['res_prefix'] = res_prefix
        if getattr(self, 'custom_tw2_config', None):
            default_tw2_config.update(self.custom_tw2_config)
        app = TwMiddleware(app, **default_tw2_config)
        self.tw2_initialized = True
        return app

#    def add_auth_middleware(self, app, *args):
#        """ Add our FAS authentication middleware """
#        from fedoracommunity.connectors.faswhoplugin import fas_make_who_middleware
#        #from repoze.what.plugins.pylonshq import booleanize_predicates
#        from copy import copy
#        import logging
#
#        # TODO: go through moksha.lib.helpers and clean up the Predicate usage.
#        # Eventually we want to be using this, because this is how TG2/Pylons
#        # does it, however it currently breaks things for us...
#        #booleanize_predicates()
#
#        # Configuring auth logging:
#        if 'log_stream' not in self.fas_auth:
#            self.fas_auth['log_stream'] = logging.getLogger('auth')
#
#        auth_args = copy(self.fas_auth)
#
#        app = fas_make_who_middleware(app, **auth_args)
#        return app

base_config = FedoraCommunityConfig()
#base_config = AppConfig()
base_config.renderers = []
base_config.use_dotted_templatenames = True

base_config.use_toscawidgets = False
base_config.use_toscawidgets2 = True

base_config.package = fedoracommunity

# Set the default renderer
base_config.default_renderer = 'mako'
base_config.renderers.append('mako')
base_config.renderers.append('genshi')
base_config.auto_reload_templates = True
base_config.use_legacy_renderer = False

# Configure the base SQLALchemy Setup
base_config.use_sqlalchemy = False # fix this later
# base_config.model = myfedora.model
# base_config.DBSession = myfedora.model.DBSession

# Configure the authentication backend

# Setting this to 'sqlalchemy' ensures that our
# FedoraCommunityConfig.add_auth_middleware will get called.
# This is very broken in the code
#base_config.auth_backend = 'sqlalchemy'
#base_config.fas_auth = Bunch()

#base_config.sa_auth = Bunch()
#base_config.sa_auth.dbsession = model.DBSession
#base_config.sa_auth.user_class = model.User
#base_config.sa_auth.group_class = model.Group
#base_config.sa_auth.permission_class = model.Permission
#base_config.sa_auth.user_criterion = model.User.user_name
#base_config.sa_auth.user_id_column = 'user_id'

# override this if you would like to provide a different who plugin for.
# managing login and logout of your application
#base_config.sa_auth.form_plugin = None

# Enable profiling middleware.
base_config.profile = False

# Enable squeeze middlware
base_config.squeeze = False
<commit_msg>Make the script_name configurable, for tw2<commit_after># This file is part of Fedora Community.
# Copyright (C) 2008-2010  Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from fedoracommunity.lib import app_globals
import fedoracommunity
import fedoracommunity.lib

from tg.configuration import AppConfig, Bunch
from paste.deploy.converters import asbool
from pylons.i18n import ugettext

class FedoraCommunityConfig(AppConfig):
    tw2_initialized = False

    def add_tosca2_middleware(self, app):
        if self.tw2_initialized:
            return app

        from tg import config
        from tw2.core.middleware import Config, TwMiddleware
        default_tw2_config = dict( default_engine=self.default_renderer,
                                   translator=ugettext,
                                   auto_reload_templates=asbool(self.get('templating.mako.reloadfromdisk', 'false'))
                                   )
        res_prefix = config.get('fedoracommunity.resource_path_prefix')
        if res_prefix:
            default_tw2_config['res_prefix'] = res_prefix
        script_name = config.get('fedoracommunity.script_name')
        if script_name:
            default_tw2_config['script_name'] = script_name
        if getattr(self, 'custom_tw2_config', None):
            default_tw2_config.update(self.custom_tw2_config)
        app = TwMiddleware(app, **default_tw2_config)
        self.tw2_initialized = True
        return app

#    def add_auth_middleware(self, app, *args):
#        """ Add our FAS authentication middleware """
#        from fedoracommunity.connectors.faswhoplugin import fas_make_who_middleware
#        #from repoze.what.plugins.pylonshq import booleanize_predicates
#        from copy import copy
#        import logging
#
#        # TODO: go through moksha.lib.helpers and clean up the Predicate usage.
#        # Eventually we want to be using this, because this is how TG2/Pylons
#        # does it, however it currently breaks things for us...
#        #booleanize_predicates()
#
#        # Configuring auth logging:
#        if 'log_stream' not in self.fas_auth:
#            self.fas_auth['log_stream'] = logging.getLogger('auth')
#
#        auth_args = copy(self.fas_auth)
#
#        app = fas_make_who_middleware(app, **auth_args)
#        return app

base_config = FedoraCommunityConfig()
#base_config = AppConfig()
base_config.renderers = []
base_config.use_dotted_templatenames = True

base_config.use_toscawidgets = False
base_config.use_toscawidgets2 = True

base_config.package = fedoracommunity

# Set the default renderer
base_config.default_renderer = 'mako'
base_config.renderers.append('mako')
base_config.renderers.append('genshi')
base_config.auto_reload_templates = True
base_config.use_legacy_renderer = False

# Configure the base SQLALchemy Setup
base_config.use_sqlalchemy = False # fix this later
# base_config.model = myfedora.model
# base_config.DBSession = myfedora.model.DBSession

# Configure the authentication backend

# Setting this to 'sqlalchemy' ensures that our
# FedoraCommunityConfig.add_auth_middleware will get called.
# This is very broken in the code
#base_config.auth_backend = 'sqlalchemy'
#base_config.fas_auth = Bunch()

#base_config.sa_auth = Bunch()
#base_config.sa_auth.dbsession = model.DBSession
#base_config.sa_auth.user_class = model.User
#base_config.sa_auth.group_class = model.Group
#base_config.sa_auth.permission_class = model.Permission
#base_config.sa_auth.user_criterion = model.User.user_name
#base_config.sa_auth.user_id_column = 'user_id'

# override this if you would like to provide a different who plugin for.
# managing login and logout of your application
#base_config.sa_auth.form_plugin = None

# Enable profiling middleware.
base_config.profile = False

# Enable squeeze middlware
base_config.squeeze = False
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
generating standard genome indices.

Usage:
import prepare_data as pd
.__doc__
.__doc__

Requirement:
    STAR aligner
"""

import os
import re
import sys
import shutil
import subprocess

def create_star_genome_index(fasta_file, out_dir, num_workers=1, genome_anno=None):
    """
    Creating STAR genome index with or without using genome annotation

    @args fasta_file: reference genome sequence file .fasta format
    @type fasta_file: str
    @args out_dir: genome index binary file storage place
    @type out_dir: str
    @args num_workers: number of threads to run (default value = 1)
    @type num_workers: int
    @args genome_anno: genome annotation file (optional)
    @type genome_anno: str
    """

    cli = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d' % (out_dir, fasta_file, num_workers)
    cli = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d --sjdbGTFfile %s' % (out_dir, fasta_file, num_workers, genome_anno)
    cli = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d --sjdbGTFfile %s --sjdbGTFtagExonParentTranscript Parent' % (out_dir, fasta_file, num_workers, genome_anno)
    process = subprocess.Popen(cli, shell=True)
    process.wait()

if __name__=="__main__":
    print __doc__

"""
"""
<commit_msg>create star index files based on user provided data<commit_after>#!/usr/bin/env python
"""
generating standard genome indices.

Usage:
import prepare_data as pd
.__doc__
.__doc__

Requirement:
    STAR aligner
"""

import os
import re
import sys
import shutil
import subprocess
from gfftools import helper
from gfftools import GFFParser

def create_star_genome_index(fasta_file, out_dir, genome_anno=None, num_workers=1):
    """
    Creating STAR genome index with or without using genome annotation

    @args fasta_file: reference genome sequence file .fasta format
    @type fasta_file: str
    @args out_dir: genome index binary file storage place
    @type out_dir: str
    @args genome_anno: genome annotation file (optional)
    @type genome_anno: str
    @args num_workers: number of threads to run (default value = 1)
    @type num_workers: int
    """

    if not genome_anno:
        cli_cmd = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d' % (out_dir, fasta_file, num_workers)
    else:
        ## check for the file type
        gff_hand = helper.open_file(genome_anno)

        for rec in gff_hand:
            rec = rec.strip('\n\r')

            # skip empty line fasta identifier and commented line
            if not rec or rec[0] in  ['#', '>']:
                continue
            # skip the genome sequence
            if not re.search('\t', rec):
                continue

            parts = rec.split('\t')
            assert len(parts) >= 8, rec

            ftype, tags = GFFParser.attribute_tags(parts[-1])
            break

        gff_hand.close()

        ## according to the file type
        if ftype:
            cli_cmd = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d --sjdbGTFfile %s --sjdbGTFtagExonParentTranscript Parent' % (out_dir, fasta_file, num_workers, genome_anno)
        else:
            cli_cmd = 'STAR --runMode genomeGenerate --genomeDir %s --genomeFastaFiles %s --runThreadN %d --sjdbGTFfile %s' % (out_dir, fasta_file, num_workers, genome_anno)

    ## start the index job
    process = subprocess.Popen(cli_cmd, shell=True)
    process.wait()

    print
    print "STAR genome index files are stored at %s" % out_dir
    print

if __name__=="__main__":
    print __doc__

"""
"""
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Test script to check for required functionality.

Execute this code at the command line by typing:

  python swc-installation-test-2.py

How to get a command line:

- On OSX run this with the Terminal application.

- On Windows, go to the Start menu, select 'Run' and type 'cmd'
(without the quotes) to run the 'cmd.exe' Windows Command Prompt.

- On Linux, either use your login shell directly, or run one of a
  number of graphical terminals (e.g. 'xterm', 'gnome-terminal', ...).

Run the script and follow the instructions it prints at the end.

This script requires at least Python 2.6.  You can check the version
of Python that you have installed with 'swc-installation-test-1.py'.
"""

import distutils.ccompiler as _distutils_ccompiler
import importlib as _importlib
import logging as _logging
import os as _os
import platform as _platform
import re as _re
import subprocess as _subprocess
import sys as _sys

__version__ = '0.1'

# Comment out any entries you don't need
CHECKS = [
# Shell
    'bash',
# Editors
    'virtual-editor',
# Browsers
    'virtual-browser',
# Version control
    'git',
    'hg',              # Command line tool
    'mercurial',       # Python package
# Build tools and packaging
    'make',
    'easy_install',
    'setuptools',
# Testing
    'nosetests',       # Command line tool
    'nose',            # Python package
# SQL
    'sqlite3',         # Command line tool
    'sqlite3-python',  # Python package
# Python
    'python',
    'IPython',
    'numpy',
    'scipy',
    'matplotlib',
    'sympy',
    'Cython',
    'networkx',
    'mayavi.mlab',
    ]

CHECKER = {}

class DependencyError (Exception):
    def __init__(self, checker, message):
        self.checker = checker
        self.message = message

    def __str__(self):
        return 'check for {0} failed:\n{1}'.format(
            self.checker.full_name(), self.message)

def check(checks=None):
    successes = []
    failures = []
    if not checks:
        checks = CHECKS
    for check in checks:
        checker = CHECKER[check]
        _sys.stdout.write('check {0}...\t'.format(checker.full_name()))
        try:
            version = checker.check()
        except DependencyError as e:
            failures.append(e)
            _sys.stdout.write('fail\n')
        else:
            _sys.stdout.write('pass\n')
            successes.append((checker, version))
    if successes:
        print('\nSuccesses:\n')
        for checker,version in successes:
            print('{0} {1}'.format(
                    checker.full_name(),
                    version or 'unknown'))
    if failures:
        print('\nFailures:')
        printed = []
        for failure in failures:
            if failure not in printed:
                print()
                print(failure)
                printed.append(failure)
        return False
    return True

class Dependency (object):
    def __init__(self, name, long_name=None, minimum_version=None,
                 version_delimiter='.', and_dependencies=None,
                 or_dependencies=None):
        self.name = name
        self.long_name = long_name or name
        self.minimum_version = minimum_version
        self.version_delimiter = version_delimiter
        if not and_dependencies:
            and_dependencies = []
        self.and_dependencies = and_dependencies
        if not or_dependencies:
            or_dependencies = []
        self.or_dependencies = or_dependencies
        self._check_error = None

    def __str__(self):
        return '<{0} {1}>'.format(type(self).__name__, self.name)

    def full_name(self):
        if self.name == self.long_name:
            return self.name
        else:
            return '{0} ({1})'.format(self.long_name, self.name)

    def check(self):
        if self._check_error:
            raise self._check_error
        try:
            self._check_dependencies()
            return self._check()
        except DependencyError as e:
            self._check_error = e  # cache for future calls
            raise

    def _check_dependencies(self):
        for dependency in self.and_dependencies:
            if not hasattr(dependency, 'check'):
                dependency = CHECKER[dependency]
            dependency.check()
        self.or_pass = or_error = None
        for dependency in self.or_dependencies:
            if not hasattr(dependency, 'check'):
                dependency = CHECKER[dependency]
            try:
                version = dependency.check()
            except DependencyError as e:
                or_error = e
            else:
                self.or_pass = {
                    'dependency': dependency,
                    'version': version,
                    }
        if self.or_dependencies and not self.or_pass:
            raise or_error

    def _check(self):
        version = self._get_version()
        parsed_version = None
        if hasattr(self, '_get_parsed_version'):
            parsed_version = self._get_parsed_version()
        if self.minimum_version:
            self._check_version(version=version, parsed_version=parsed_version)
        return version

    def _get_version(self):
        raise NotImplementedError(self)

    def _check_version(self, version, parsed_version=None):
        if not parsed_version:
            parsed_version = self._parse_version(version=version)
        if parsed_version < self.minimum_version:
            raise DependencyError(
                checker=self,
                message='outdated version of {0}: {1} (need >= {2})'.format(
                    self.full_name(), version,
                    self.version_delimiter.join(
                        str(part) for part in self.minimum_version)))

    def _parse_version(self, version):
        parsed_version = []
        for part in version.split(self.version_delimiter):
            try:
                parsed_version.append(int(part))
            except ValueError as e:
                raise NotImplementedError((version, part))# from e
        return tuple(parsed_version)

class PythonDependency (Dependency):
    def __init__(self, name='python', long_name='Python version',
                 minimum_version=(2, 6), **kwargs):
        super(PythonDependency, self).__init__(
            name=name, long_name=long_name, minimum_version=minimum_version,
            **kwargs)

    def _get_version(self):
        return _sys.version

    def _get_parsed_version(self):
        return _sys.version_info

CHECKER['python'] = PythonDependency()

class CommandDependency (Dependency):
    exe_extension = _distutils_ccompiler.new_compiler().exe_extension

    def __init__(self, command, version_option='--version',
                 version_regexp=None, version_stream='stdout', **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = command
        super(CommandDependency, self).__init__(**kwargs)
        self.command = command
        self.version_option = version_option
        if not version_regexp:
            regexp = r'([\d][\d{0}]*[\d])'.format(self.version_delimiter)
            version_regexp = _re.compile(regexp)
        self.version_regexp = version_regexp
        self.version_stream = version_stream

    def _get_version_stream(self):
        command = self.command + (self.exe_extension or '')
        try:
            p = _subprocess.Popen(
                [command, self.version_option],
                stdout=_subprocess.PIPE, stderr=_subprocess.PIPE,
                close_fds=True, shell=False, universal_newlines=True)
        except OSError as e:
            raise DependencyError(
                checker=self,
                message="could not find '{0}' executable".format(command),
                )# from e
        stdout,stderr = p.communicate()
        status = p.wait()
        if status:
            lines = [
                "failed to execute '{0} {1}':".format(
                    command, self.version_option),
                'status: {0}'.format(status),
                ]
            for name,string in [('stdout', stdout), ('stderr', stderr)]:
                if string:
                    lines.extend([name + ':', string])
            raise DependencyError(checker=self, message='\n'.join(lines))
        for name,string in [('stdout', stdout), ('stderr', stderr)]:
            if name == self.version_stream:
                return string
        raise NotImplementedError(self.version_stream)

    def _get_version(self):
        version_stream = self._get_version_stream()
        match = self.version_regexp.search(version_stream)
        if not match:
            raise DependencyError(
                checker=self,
                message='no version string in output:\n{0}'.format(
                    version_stream))
        return match.group(1)

for command,long_name,minimum_version in [
        ('bash', 'Bourne Again Shell', (4, 0)),
        ('easy_install', 'Setuptools easy_install', None),
        ('git', 'Git', (1, 8, 0)),
        ('hg', 'Mercurial', (2, 0, 0)),
        ('make', None, None),
        ('sqlite3', 'SQLite 3', None),
        ('nosetests', 'Nose', (1, 0, 0)),
        ('emacs', 'Emacs', None),
        ('xemacs', 'XEmacs', None),
        ('vim', 'Vim', None),
        ('vi', None, None),
        ('nano', 'Nano', None),
        ('kate', 'Kate', None),
        ('notepad++', 'Notepad++', None),
        ('firefox', 'Firefox', None),
        ('google-chrome', 'Google Chrome', None),
        ('chromium', 'Chromium', None),
        ]:
    if not long_name:
        long_name = command
    CHECKER[command] = CommandDependency(
        command=command, long_name=long_name, minimum_version=minimum_version)
del command, long_name, minimum_version  # cleanup namespace

class PythonPackageDependency (Dependency):
    def __init__(self, package, **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = package
        if 'and_dependencies' not in kwargs:
            kwargs['and_dependencies'] = []
        if 'python' not in kwargs['and_dependencies']:
            kwargs['and_dependencies'].append('python')
        super(PythonPackageDependency, self).__init__(**kwargs)
        self.package = package

    def _get_version(self):
        try:
            package = _importlib.import_module(self.package)
        except ImportError as e:
            raise DependencyError(
                checker=self,
                message="could not import the '{0}' package".format(
                    self.package),
                )# from e
        try:
            version = package.__version__
        except AttributeError:
            version = 'unknown'
        return version

for package,name,long_name,minimum_version in [
        ('mercurial', None, 'Mercurial Python package',
         CHECKER['hg'].minimum_version),
        ('nose', None, 'Nose Python package',
         CHECKER['nosetests'].minimum_version),
        ('sqlite3', 'sqlite3-python', 'SQLite Python package',
         CHECKER['sqlite3'].minimum_version),
        ('IPython', None, None, None),
        ('numpy', None, 'NumPy', None),
        ('scipy', None, 'SciPy', None),
        ('matplotlib', None, 'Matplotlib', None),
        ('sympy', None, 'SymPy', None),
        ('Cython', None, None, None),
        ('networkx', None, 'NetworkX', None),
        ('mayavi.mlab', None, 'MayaVi', None),
        ('setuptools', None, 'Setuptools', None),
        ]:
    if not name:
        name = package
    if not long_name:
        long_name = name
    CHECKER[name] = PythonPackageDependency(
        package=package, name=name, long_name=long_name,
        minimum_version=minimum_version)
del package, name, long_name, minimum_version  # cleanup namespace

class VirtualDependency (Dependency):
    def _check(self):
        return '{0} {1}'.format(
            self.or_pass['dependency'].full_name(),
            self.or_pass['version'])

for name,dependencies in [
        ('virtual-editor', (
            'emacs',
            'xemacs',
            'vim',
            'vi',
            'nano',
            'kate',
            'notepad++',
            )),
        ('virtual-browser', (
            'firefox',
            'google-chrome',
            'chromium',
            )),
        ]:
    CHECKER[name] = VirtualDependency(
        name=name, long_name=name, or_dependencies=dependencies)
del name, dependencies  # cleanup namespace

def print_system_info():
    print("If you do not understand why the above failures occurred,")
    print("copy and send the *entire* output (all info above and summary")
    print("below) to the instructor for help.")
    print()
    print('==================')
    print('System information')
    print('==================')
    print('os.name      : {0}'.format(_os.name))
    try:
        print('os.uname     : {0}'.format(_os.uname()))
    except:
        pass
    print('platform     : {0}'.format(_sys.platform))
    print('platform+    : {0}'.format(_platform.platform()))
    print('prefix       : {0}'.format(_sys.prefix))
    print('exec_prefix  : {0}'.format(_sys.exec_prefix))
    print('executable   : {0}'.format(_sys.executable))
    print('version_info : {0}'.format(_sys.version_info))
    print('version      : {0}'.format(_sys.version))
    print('environment  :')
    for key,value in sorted(_os.environ.items()):
        print('  {0}={1}'.format(key, value))
    print('==================')

if __name__ == '__main__':
    if not check(_sys.argv[1:]):
        print()
        print_system_info()
<commit_msg>swc-installation-test-2.py: import print_function for Python 2.6<commit_after>#!/usr/bin/env python

"""Test script to check for required functionality.

Execute this code at the command line by typing:

  python swc-installation-test-2.py

How to get a command line:

- On OSX run this with the Terminal application.

- On Windows, go to the Start menu, select 'Run' and type 'cmd'
(without the quotes) to run the 'cmd.exe' Windows Command Prompt.

- On Linux, either use your login shell directly, or run one of a
  number of graphical terminals (e.g. 'xterm', 'gnome-terminal', ...).

Run the script and follow the instructions it prints at the end.

This script requires at least Python 2.6.  You can check the version
of Python that you have installed with 'swc-installation-test-1.py'.
"""

from __future__ import print_function  # for Python 2.6 compatibility

import distutils.ccompiler as _distutils_ccompiler
import importlib as _importlib
import logging as _logging
import os as _os
import platform as _platform
import re as _re
import subprocess as _subprocess
import sys as _sys

__version__ = '0.1'

# Comment out any entries you don't need
CHECKS = [
# Shell
    'bash',
# Editors
    'virtual-editor',
# Browsers
    'virtual-browser',
# Version control
    'git',
    'hg',              # Command line tool
    'mercurial',       # Python package
# Build tools and packaging
    'make',
    'easy_install',
    'setuptools',
# Testing
    'nosetests',       # Command line tool
    'nose',            # Python package
# SQL
    'sqlite3',         # Command line tool
    'sqlite3-python',  # Python package
# Python
    'python',
    'IPython',
    'numpy',
    'scipy',
    'matplotlib',
    'sympy',
    'Cython',
    'networkx',
    'mayavi.mlab',
    ]

CHECKER = {}

class DependencyError (Exception):
    def __init__(self, checker, message):
        self.checker = checker
        self.message = message

    def __str__(self):
        return 'check for {0} failed:\n{1}'.format(
            self.checker.full_name(), self.message)

def check(checks=None):
    successes = []
    failures = []
    if not checks:
        checks = CHECKS
    for check in checks:
        checker = CHECKER[check]
        _sys.stdout.write('check {0}...\t'.format(checker.full_name()))
        try:
            version = checker.check()
        except DependencyError as e:
            failures.append(e)
            _sys.stdout.write('fail\n')
        else:
            _sys.stdout.write('pass\n')
            successes.append((checker, version))
    if successes:
        print('\nSuccesses:\n')
        for checker,version in successes:
            print('{0} {1}'.format(
                    checker.full_name(),
                    version or 'unknown'))
    if failures:
        print('\nFailures:')
        printed = []
        for failure in failures:
            if failure not in printed:
                print()
                print(failure)
                printed.append(failure)
        return False
    return True

class Dependency (object):
    def __init__(self, name, long_name=None, minimum_version=None,
                 version_delimiter='.', and_dependencies=None,
                 or_dependencies=None):
        self.name = name
        self.long_name = long_name or name
        self.minimum_version = minimum_version
        self.version_delimiter = version_delimiter
        if not and_dependencies:
            and_dependencies = []
        self.and_dependencies = and_dependencies
        if not or_dependencies:
            or_dependencies = []
        self.or_dependencies = or_dependencies
        self._check_error = None

    def __str__(self):
        return '<{0} {1}>'.format(type(self).__name__, self.name)

    def full_name(self):
        if self.name == self.long_name:
            return self.name
        else:
            return '{0} ({1})'.format(self.long_name, self.name)

    def check(self):
        if self._check_error:
            raise self._check_error
        try:
            self._check_dependencies()
            return self._check()
        except DependencyError as e:
            self._check_error = e  # cache for future calls
            raise

    def _check_dependencies(self):
        for dependency in self.and_dependencies:
            if not hasattr(dependency, 'check'):
                dependency = CHECKER[dependency]
            dependency.check()
        self.or_pass = or_error = None
        for dependency in self.or_dependencies:
            if not hasattr(dependency, 'check'):
                dependency = CHECKER[dependency]
            try:
                version = dependency.check()
            except DependencyError as e:
                or_error = e
            else:
                self.or_pass = {
                    'dependency': dependency,
                    'version': version,
                    }
        if self.or_dependencies and not self.or_pass:
            raise or_error

    def _check(self):
        version = self._get_version()
        parsed_version = None
        if hasattr(self, '_get_parsed_version'):
            parsed_version = self._get_parsed_version()
        if self.minimum_version:
            self._check_version(version=version, parsed_version=parsed_version)
        return version

    def _get_version(self):
        raise NotImplementedError(self)

    def _check_version(self, version, parsed_version=None):
        if not parsed_version:
            parsed_version = self._parse_version(version=version)
        if parsed_version < self.minimum_version:
            raise DependencyError(
                checker=self,
                message='outdated version of {0}: {1} (need >= {2})'.format(
                    self.full_name(), version,
                    self.version_delimiter.join(
                        str(part) for part in self.minimum_version)))

    def _parse_version(self, version):
        parsed_version = []
        for part in version.split(self.version_delimiter):
            try:
                parsed_version.append(int(part))
            except ValueError as e:
                raise NotImplementedError((version, part))# from e
        return tuple(parsed_version)

class PythonDependency (Dependency):
    def __init__(self, name='python', long_name='Python version',
                 minimum_version=(2, 6), **kwargs):
        super(PythonDependency, self).__init__(
            name=name, long_name=long_name, minimum_version=minimum_version,
            **kwargs)

    def _get_version(self):
        return _sys.version

    def _get_parsed_version(self):
        return _sys.version_info

CHECKER['python'] = PythonDependency()

class CommandDependency (Dependency):
    exe_extension = _distutils_ccompiler.new_compiler().exe_extension

    def __init__(self, command, version_option='--version',
                 version_regexp=None, version_stream='stdout', **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = command
        super(CommandDependency, self).__init__(**kwargs)
        self.command = command
        self.version_option = version_option
        if not version_regexp:
            regexp = r'([\d][\d{0}]*[\d])'.format(self.version_delimiter)
            version_regexp = _re.compile(regexp)
        self.version_regexp = version_regexp
        self.version_stream = version_stream

    def _get_version_stream(self):
        command = self.command + (self.exe_extension or '')
        try:
            p = _subprocess.Popen(
                [command, self.version_option],
                stdout=_subprocess.PIPE, stderr=_subprocess.PIPE,
                close_fds=True, shell=False, universal_newlines=True)
        except OSError as e:
            raise DependencyError(
                checker=self,
                message="could not find '{0}' executable".format(command),
                )# from e
        stdout,stderr = p.communicate()
        status = p.wait()
        if status:
            lines = [
                "failed to execute '{0} {1}':".format(
                    command, self.version_option),
                'status: {0}'.format(status),
                ]
            for name,string in [('stdout', stdout), ('stderr', stderr)]:
                if string:
                    lines.extend([name + ':', string])
            raise DependencyError(checker=self, message='\n'.join(lines))
        for name,string in [('stdout', stdout), ('stderr', stderr)]:
            if name == self.version_stream:
                return string
        raise NotImplementedError(self.version_stream)

    def _get_version(self):
        version_stream = self._get_version_stream()
        match = self.version_regexp.search(version_stream)
        if not match:
            raise DependencyError(
                checker=self,
                message='no version string in output:\n{0}'.format(
                    version_stream))
        return match.group(1)

for command,long_name,minimum_version in [
        ('bash', 'Bourne Again Shell', (4, 0)),
        ('easy_install', 'Setuptools easy_install', None),
        ('git', 'Git', (1, 8, 0)),
        ('hg', 'Mercurial', (2, 0, 0)),
        ('make', None, None),
        ('sqlite3', 'SQLite 3', None),
        ('nosetests', 'Nose', (1, 0, 0)),
        ('emacs', 'Emacs', None),
        ('xemacs', 'XEmacs', None),
        ('vim', 'Vim', None),
        ('vi', None, None),
        ('nano', 'Nano', None),
        ('kate', 'Kate', None),
        ('notepad++', 'Notepad++', None),
        ('firefox', 'Firefox', None),
        ('google-chrome', 'Google Chrome', None),
        ('chromium', 'Chromium', None),
        ]:
    if not long_name:
        long_name = command
    CHECKER[command] = CommandDependency(
        command=command, long_name=long_name, minimum_version=minimum_version)
del command, long_name, minimum_version  # cleanup namespace

class PythonPackageDependency (Dependency):
    def __init__(self, package, **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = package
        if 'and_dependencies' not in kwargs:
            kwargs['and_dependencies'] = []
        if 'python' not in kwargs['and_dependencies']:
            kwargs['and_dependencies'].append('python')
        super(PythonPackageDependency, self).__init__(**kwargs)
        self.package = package

    def _get_version(self):
        try:
            package = _importlib.import_module(self.package)
        except ImportError as e:
            raise DependencyError(
                checker=self,
                message="could not import the '{0}' package".format(
                    self.package),
                )# from e
        try:
            version = package.__version__
        except AttributeError:
            version = 'unknown'
        return version

for package,name,long_name,minimum_version in [
        ('mercurial', None, 'Mercurial Python package',
         CHECKER['hg'].minimum_version),
        ('nose', None, 'Nose Python package',
         CHECKER['nosetests'].minimum_version),
        ('sqlite3', 'sqlite3-python', 'SQLite Python package',
         CHECKER['sqlite3'].minimum_version),
        ('IPython', None, None, None),
        ('numpy', None, 'NumPy', None),
        ('scipy', None, 'SciPy', None),
        ('matplotlib', None, 'Matplotlib', None),
        ('sympy', None, 'SymPy', None),
        ('Cython', None, None, None),
        ('networkx', None, 'NetworkX', None),
        ('mayavi.mlab', None, 'MayaVi', None),
        ('setuptools', None, 'Setuptools', None),
        ]:
    if not name:
        name = package
    if not long_name:
        long_name = name
    CHECKER[name] = PythonPackageDependency(
        package=package, name=name, long_name=long_name,
        minimum_version=minimum_version)
del package, name, long_name, minimum_version  # cleanup namespace

class VirtualDependency (Dependency):
    def _check(self):
        return '{0} {1}'.format(
            self.or_pass['dependency'].full_name(),
            self.or_pass['version'])

for name,dependencies in [
        ('virtual-editor', (
            'emacs',
            'xemacs',
            'vim',
            'vi',
            'nano',
            'kate',
            'notepad++',
            )),
        ('virtual-browser', (
            'firefox',
            'google-chrome',
            'chromium',
            )),
        ]:
    CHECKER[name] = VirtualDependency(
        name=name, long_name=name, or_dependencies=dependencies)
del name, dependencies  # cleanup namespace

def print_system_info():
    print("If you do not understand why the above failures occurred,")
    print("copy and send the *entire* output (all info above and summary")
    print("below) to the instructor for help.")
    print()
    print('==================')
    print('System information')
    print('==================')
    print('os.name      : {0}'.format(_os.name))
    try:
        print('os.uname     : {0}'.format(_os.uname()))
    except:
        pass
    print('platform     : {0}'.format(_sys.platform))
    print('platform+    : {0}'.format(_platform.platform()))
    print('prefix       : {0}'.format(_sys.prefix))
    print('exec_prefix  : {0}'.format(_sys.exec_prefix))
    print('executable   : {0}'.format(_sys.executable))
    print('version_info : {0}'.format(_sys.version_info))
    print('version      : {0}'.format(_sys.version))
    print('environment  :')
    for key,value in sorted(_os.environ.items()):
        print('  {0}={1}'.format(key, value))
    print('==================')

if __name__ == '__main__':
    if not check(_sys.argv[1:]):
        print()
        print_system_info()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Test script to check for required functionality.

Execute this code at the command line by typing:

  python swc-installation-test-2.py

How to get a command line:

- On OSX run this with the Terminal application.

- On Windows, go to the Start menu, select 'Run' and type 'cmd'
(without the quotes) to run the 'cmd.exe' Windows Command Prompt.

- On Linux, either use your login shell directly, or run one of a
  number of graphical terminals (e.g. 'xterm', 'gnome-terminal', ...).

Run the script and follow the instructions it prints at the end.

This script requires at least Python 2.6.  You can check the version
of Python that you have installed with 'swc-installation-test-1.py'.
"""

import importlib as _importlib
import logging as _logging
import os as _os
import platform as _platform
import re as _re
import subprocess as _subprocess
import sys as _sys

__version__ = '0.1'

# Comment out any entries you don't need
CHECKS = [
# Shell
    'bash',
# Version control
    'git',
    'hg',              # Command line tool
    'mercurial',       # Python package
# Build tools and packaging
    'make',
    'easy_install',
    'setuptools',
# Testing
    'nosetests',       # Command line tool
    'nose',            # Python package
# SQL
    'sqlite3',         # Command line tool
    'sqlite3-python',  # Python package
# Python
    'python',
    'IPython',
    'numpy',
    'scipy',
    'matplotlib',
    'sympy',
    'Cython',
    'networkx',
    'mayavi.mlab',
    ]

CHECKER = {}

class DependencyError (Exception):
    def __init__(self, checker, message):
        self.checker = checker
        self.message = message

    def __str__(self):
        return 'check for {0} failed:\n{1}'.format(
            self.checker.full_name(), self.message)

def check(checks=None):
    failures = []
    if not checks:
        checks = CHECKS
    for check in checks:
        checker = CHECKER[check]
        _sys.stdout.write('check {0}...\t'.format(checker.full_name()))
        try:
            checker.check()
        except DependencyError as e:
            failures.append(e)
            _sys.stdout.write('fail\n')
        else:
            _sys.stdout.write('pass\n')
    if failures:
        print('\nFailures:')
        for failure in failures:
            print()
            print(failure)
        return False
    return True

class Dependency (object):
    def __init__(self, name, long_name=None, minimum_version=None,
                 version_delimiter='.'):
        self.name = name
        self.long_name = long_name or name
        self.minimum_version = minimum_version
        self.version_delimiter = version_delimiter

    def __str__(self):
        return '<{0} {1}>'.format(type(self).__name__, self.name)

    def full_name(self):
        if self.name == self.long_name:
            return self.name
        else:
            return '{0} ({1})'.format(self.long_name, self.name)

    def check(self):
        version = self._get_version()
        parsed_version = None
        if hasattr(self, '_get_parsed_version'):
            parsed_version = self._get_parsed_version()
        if self.minimum_version:
            self._check_version(version=version, parsed_version=parsed_version)

    def _get_version(self):
        raise NotImplementedError(self)

    def _check_version(self, version, parsed_version=None):
        if not parsed_version:
            parsed_version = self._parse_version(version=version)
        if parsed_version < self.minimum_version:
            raise DependencyError(
                checker=self,
                message='outdated version of {0}: {1} (need >= {2})'.format(
                    self.full_name(), version,
                    self.version_delimiter.join(
                        str(part) for part in self.minimum_version)))

    def _parse_version(self, version):
        parsed_version = []
        for part in version.split(self.version_delimiter):
            try:
                parsed_version.append(int(part))
            except ValueError as e:
                raise NotImplementedError((version, part)) from e
        return tuple(parsed_version)

class PythonDependency (Dependency):
    def __init__(self, name='python', long_name='Python version',
                 minimum_version=(2, 6), **kwargs):
        super(PythonDependency, self).__init__(
            name=name, long_name=long_name, minimum_version=minimum_version,
            **kwargs)

    def _get_version(self):
        return _sys.version

    def _get_parsed_version(self):
        return _sys.version_info

CHECKER['python'] = PythonDependency()

class CommandDependency (Dependency):
    def __init__(self, command, version_option='--version',
                 version_regexp=None, version_stream='stdout', **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = command
        super(CommandDependency, self).__init__(**kwargs)
        self.command = command
        self.version_option = version_option
        if not version_regexp:
            regexp = r'([\d][\d{0}]*[\d])'.format(self.version_delimiter)
            version_regexp = _re.compile(regexp)
        self.version_regexp = version_regexp
        self.version_stream = version_stream

    def _get_version_stream(self):
        try:
            p = _subprocess.Popen(
                [self.command, self.version_option],
                stdout=_subprocess.PIPE, stderr=_subprocess.PIPE,
                close_fds=True, shell=False, universal_newlines=True)
        except OSError as e:
            raise DependencyError(
                checker=self,
                message="could not find '{0}' executable".format(self.command),
                ) from e
        stdout,stderr = p.communicate()
        status = p.wait()
        if status:
            lines = [
                "failed to execute '{0} {1}':".format(
                    self.command, self.version_option),
                'status: {0}'.format(status),
                ]
            for name,string in [('stdout', stdout), ('stderr', stderr)]:
                if string:
                    lines.extend([name + ':', string])
            raise DependencyError(checker=self, message='\n'.join(lines))
        for name,string in [('stdout', stdout), ('stderr', stderr)]:
            if name == self.version_stream:
                return string
        raise NotImplementedError(self.version_stream)

    def _get_version(self):
        version_stream = self._get_version_stream()
        match = self.version_regexp.search(version_stream)
        if not match:
            raise DependencyError(
                checker=self,
                message='no version string in output:\n{0}'.format(
                    version_stream))
        return match.group(1)

for command,long_name,minimum_version in [
        ('bash', 'Bourne Again Shell', (4, 0)),
        ('easy_install', 'Setuptools easy_install', None),
        ('git', 'Git', (1, 8, 0)),
        ('hg', 'Mercurial', (2, 0, 0)),
        ('make', None, None),
        ('sqlite3', 'SQLite 3', None),
        ('nosetests', 'Nose', (1, 0, 0)),
        ]:
    if not long_name:
        long_name = command
    CHECKER[command] = CommandDependency(
        command=command, long_name=long_name, minimum_version=minimum_version)
del command, long_name, minimum_version  # cleanup namespace

class PythonPackageDependency (Dependency):
    def __init__(self, package, **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = package
        super(PythonPackageDependency, self).__init__(**kwargs)
        self.package = package

    def _get_version(self):
        try:
            package = _importlib.import_module(self.package)
        except ImportError as e:
            raise DependencyError(
                checker=self,
                message="could not import the '{0}' package".format(
                    self.package),
                ) from e
        try:
            version = package.__version__
        except AttributeError:
            version = 'unknown'
        return version

for package,name,long_name,minimum_version in [
        ('mercurial', None, 'Mercurial Python package',
         CHECKER['hg'].minimum_version),
        ('nose', None, 'Nose Python package',
         CHECKER['nosetests'].minimum_version),
        ('sqlite3', 'sqlite3-python', 'SQLite Python package',
         CHECKER['sqlite3'].minimum_version),
        ('IPython', None, None, None),
        ('numpy', None, 'NumPy', None),
        ('scipy', None, 'SciPy', None),
        ('matplotlib', None, 'Matplotlib', None),
        ('sympy', None, 'SymPy', None),
        ('Cython', None, None, None),
        ('networkx', None, 'NetworkX', None),
        ('mayavi.mlab', None, 'MayaVi', None),
        ('setuptools', None, 'Setuptools', None),
        ]:
    if not name:
        name = package
    if not long_name:
        long_name = name
    CHECKER[name] = PythonPackageDependency(
        package=package, name=name, long_name=long_name,
        minimum_version=minimum_version)
del package, name, long_name, minimum_version  # cleanup namespace

def print_system_info():
    print("If you do not understand why the above failures occurred,")
    print("copy and send the *entire* output (all info above and summary")
    print("below) to the instructor for help.")
    print()
    print('==================')
    print('System information')
    print('==================')
    print('os.name      : {0}'.format(_os.name))
    try:
        print('os.uname     : {0}'.format(_os.uname()))
    except:
        pass
    print('platform     : {0}'.format(_sys.platform))
    print('platform+    : {0}'.format(_platform.platform()))
    print('prefix       : {0}'.format(_sys.prefix))
    print('exec_prefix  : {0}'.format(_sys.exec_prefix))
    print('executable   : {0}'.format(_sys.executable))
    print('version_info : {0}'.format(_sys.version_info))
    print('version      : {0}'.format(_sys.version))
    print('environment  :')
    for key,value in sorted(_os.environ.items()):
        print('  {0}={1}'.format(key, value))
    print('==================')

if __name__ == '__main__':
    if not check(_sys.argv[1:]):
        print()
        print_system_info()
<commit_msg>swc-installation-test-2.py: Add some editors and browsers<commit_after>#!/usr/bin/env python

"""Test script to check for required functionality.

Execute this code at the command line by typing:

  python swc-installation-test-2.py

How to get a command line:

- On OSX run this with the Terminal application.

- On Windows, go to the Start menu, select 'Run' and type 'cmd'
(without the quotes) to run the 'cmd.exe' Windows Command Prompt.

- On Linux, either use your login shell directly, or run one of a
  number of graphical terminals (e.g. 'xterm', 'gnome-terminal', ...).

Run the script and follow the instructions it prints at the end.

This script requires at least Python 2.6.  You can check the version
of Python that you have installed with 'swc-installation-test-1.py'.
"""

import importlib as _importlib
import logging as _logging
import os as _os
import platform as _platform
import re as _re
import subprocess as _subprocess
import sys as _sys

__version__ = '0.1'

# Comment out any entries you don't need
CHECKS = [
# Shell
    'bash',
# Version control
    'git',
    'hg',              # Command line tool
    'mercurial',       # Python package
# Build tools and packaging
    'make',
    'easy_install',
    'setuptools',
# Testing
    'nosetests',       # Command line tool
    'nose',            # Python package
# SQL
    'sqlite3',         # Command line tool
    'sqlite3-python',  # Python package
# Python
    'python',
    'IPython',
    'numpy',
    'scipy',
    'matplotlib',
    'sympy',
    'Cython',
    'networkx',
    'mayavi.mlab',
    ]

CHECKER = {}

class DependencyError (Exception):
    def __init__(self, checker, message):
        self.checker = checker
        self.message = message

    def __str__(self):
        return 'check for {0} failed:\n{1}'.format(
            self.checker.full_name(), self.message)

def check(checks=None):
    failures = []
    if not checks:
        checks = CHECKS
    for check in checks:
        checker = CHECKER[check]
        _sys.stdout.write('check {0}...\t'.format(checker.full_name()))
        try:
            checker.check()
        except DependencyError as e:
            failures.append(e)
            _sys.stdout.write('fail\n')
        else:
            _sys.stdout.write('pass\n')
    if failures:
        print('\nFailures:')
        for failure in failures:
            print()
            print(failure)
        return False
    return True

class Dependency (object):
    def __init__(self, name, long_name=None, minimum_version=None,
                 version_delimiter='.'):
        self.name = name
        self.long_name = long_name or name
        self.minimum_version = minimum_version
        self.version_delimiter = version_delimiter

    def __str__(self):
        return '<{0} {1}>'.format(type(self).__name__, self.name)

    def full_name(self):
        if self.name == self.long_name:
            return self.name
        else:
            return '{0} ({1})'.format(self.long_name, self.name)

    def check(self):
        version = self._get_version()
        parsed_version = None
        if hasattr(self, '_get_parsed_version'):
            parsed_version = self._get_parsed_version()
        if self.minimum_version:
            self._check_version(version=version, parsed_version=parsed_version)

    def _get_version(self):
        raise NotImplementedError(self)

    def _check_version(self, version, parsed_version=None):
        if not parsed_version:
            parsed_version = self._parse_version(version=version)
        if parsed_version < self.minimum_version:
            raise DependencyError(
                checker=self,
                message='outdated version of {0}: {1} (need >= {2})'.format(
                    self.full_name(), version,
                    self.version_delimiter.join(
                        str(part) for part in self.minimum_version)))

    def _parse_version(self, version):
        parsed_version = []
        for part in version.split(self.version_delimiter):
            try:
                parsed_version.append(int(part))
            except ValueError as e:
                raise NotImplementedError((version, part)) from e
        return tuple(parsed_version)

class PythonDependency (Dependency):
    def __init__(self, name='python', long_name='Python version',
                 minimum_version=(2, 6), **kwargs):
        super(PythonDependency, self).__init__(
            name=name, long_name=long_name, minimum_version=minimum_version,
            **kwargs)

    def _get_version(self):
        return _sys.version

    def _get_parsed_version(self):
        return _sys.version_info

CHECKER['python'] = PythonDependency()

class CommandDependency (Dependency):
    def __init__(self, command, version_option='--version',
                 version_regexp=None, version_stream='stdout', **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = command
        super(CommandDependency, self).__init__(**kwargs)
        self.command = command
        self.version_option = version_option
        if not version_regexp:
            regexp = r'([\d][\d{0}]*[\d])'.format(self.version_delimiter)
            version_regexp = _re.compile(regexp)
        self.version_regexp = version_regexp
        self.version_stream = version_stream

    def _get_version_stream(self):
        try:
            p = _subprocess.Popen(
                [self.command, self.version_option],
                stdout=_subprocess.PIPE, stderr=_subprocess.PIPE,
                close_fds=True, shell=False, universal_newlines=True)
        except OSError as e:
            raise DependencyError(
                checker=self,
                message="could not find '{0}' executable".format(self.command),
                ) from e
        stdout,stderr = p.communicate()
        status = p.wait()
        if status:
            lines = [
                "failed to execute '{0} {1}':".format(
                    self.command, self.version_option),
                'status: {0}'.format(status),
                ]
            for name,string in [('stdout', stdout), ('stderr', stderr)]:
                if string:
                    lines.extend([name + ':', string])
            raise DependencyError(checker=self, message='\n'.join(lines))
        for name,string in [('stdout', stdout), ('stderr', stderr)]:
            if name == self.version_stream:
                return string
        raise NotImplementedError(self.version_stream)

    def _get_version(self):
        version_stream = self._get_version_stream()
        match = self.version_regexp.search(version_stream)
        if not match:
            raise DependencyError(
                checker=self,
                message='no version string in output:\n{0}'.format(
                    version_stream))
        return match.group(1)

for command,long_name,minimum_version in [
        ('bash', 'Bourne Again Shell', (4, 0)),
        ('easy_install', 'Setuptools easy_install', None),
        ('git', 'Git', (1, 8, 0)),
        ('hg', 'Mercurial', (2, 0, 0)),
        ('make', None, None),
        ('sqlite3', 'SQLite 3', None),
        ('nosetests', 'Nose', (1, 0, 0)),
        ('emacs', 'Emacs', None),
        ('xemacs', 'XEmacs', None),
        ('vim', 'Vim', None),
        ('vi', None, None),
        ('nano', 'Nano', None),
        ('kate', 'Kate', None),
        ('notepad++', 'Notepad++', None),
        ('firefox', 'Firefox', None),
        ('google-chrome', 'Google Chrome', None),
        ('chromium', 'Chromium', None),
        ]:
    if not long_name:
        long_name = command
    CHECKER[command] = CommandDependency(
        command=command, long_name=long_name, minimum_version=minimum_version)
del command, long_name, minimum_version  # cleanup namespace

class PythonPackageDependency (Dependency):
    def __init__(self, package, **kwargs):
        if 'name' not in kwargs:
            kwargs['name'] = package
        super(PythonPackageDependency, self).__init__(**kwargs)
        self.package = package

    def _get_version(self):
        try:
            package = _importlib.import_module(self.package)
        except ImportError as e:
            raise DependencyError(
                checker=self,
                message="could not import the '{0}' package".format(
                    self.package),
                ) from e
        try:
            version = package.__version__
        except AttributeError:
            version = 'unknown'
        return version

for package,name,long_name,minimum_version in [
        ('mercurial', None, 'Mercurial Python package',
         CHECKER['hg'].minimum_version),
        ('nose', None, 'Nose Python package',
         CHECKER['nosetests'].minimum_version),
        ('sqlite3', 'sqlite3-python', 'SQLite Python package',
         CHECKER['sqlite3'].minimum_version),
        ('IPython', None, None, None),
        ('numpy', None, 'NumPy', None),
        ('scipy', None, 'SciPy', None),
        ('matplotlib', None, 'Matplotlib', None),
        ('sympy', None, 'SymPy', None),
        ('Cython', None, None, None),
        ('networkx', None, 'NetworkX', None),
        ('mayavi.mlab', None, 'MayaVi', None),
        ('setuptools', None, 'Setuptools', None),
        ]:
    if not name:
        name = package
    if not long_name:
        long_name = name
    CHECKER[name] = PythonPackageDependency(
        package=package, name=name, long_name=long_name,
        minimum_version=minimum_version)
del package, name, long_name, minimum_version  # cleanup namespace

def print_system_info():
    print("If you do not understand why the above failures occurred,")
    print("copy and send the *entire* output (all info above and summary")
    print("below) to the instructor for help.")
    print()
    print('==================')
    print('System information')
    print('==================')
    print('os.name      : {0}'.format(_os.name))
    try:
        print('os.uname     : {0}'.format(_os.uname()))
    except:
        pass
    print('platform     : {0}'.format(_sys.platform))
    print('platform+    : {0}'.format(_platform.platform()))
    print('prefix       : {0}'.format(_sys.prefix))
    print('exec_prefix  : {0}'.format(_sys.exec_prefix))
    print('executable   : {0}'.format(_sys.executable))
    print('version_info : {0}'.format(_sys.version_info))
    print('version      : {0}'.format(_sys.version))
    print('environment  :')
    for key,value in sorted(_os.environ.items()):
        print('  {0}={1}'.format(key, value))
    print('==================')

if __name__ == '__main__':
    if not check(_sys.argv[1:]):
        print()
        print_system_info()
<|endoftext|>


<commit_before>
import base64
import os.path

import pygments
import pygments.util
import pygments.lexers
import pygments.formatters

from django.conf import settings
from django.core.mail import send_mail
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.http import HttpResponseForbidden
from django.shortcuts import redirect
from django.template.loader import render_to_string

from review.models import CodeReview, ChangeStatusLog, get_all_reviewers
from extensions import models

from decorators import ajax_view, model_view, post_only_view
from utils import render

IMAGE_TYPES = {
    '.png':  'image/png',
    '.jpg':  'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.gif':  'image/gif',
    '.bmp':  'image/bmp',
    '.svg':  'image/svg+xml',
}

def can_review_extension(user, extension):
    if user == extension.creator:
        return True

    if user.has_perm("review.can-review-extensions"):
        return True

    return False

def can_approve_extension(user, extension):
    if user.is_superuser:
        return True

    if user.has_perm("review.can-review-extensions"):
        return user != extension.creator

    return False

@ajax_view
@model_view(models.ExtensionVersion)
def ajax_get_files_view(request, obj):
    formatter = pygments.formatters.HtmlFormatter(style="borland", cssclass="code")

    if not can_review_extension(request.user, obj.extension):
        return HttpResponseForbidden()

    zipfile = obj.get_zipfile('r')

    # filename => { raw, html, filename }
    files = []
    for filename in sorted(zipfile.namelist()):
        raw = zipfile.open(filename, 'r').read()

        base, extension = os.path.splitext(filename)

        file_ = dict(filename=filename)

        if extension in IMAGE_TYPES:
            mime = IMAGE_TYPES[extension]
            raw_base64 = base64.standard_b64encode(raw)

            file_.update(html='' % (mime, raw_base64,),
                         num_lines=0)

        else:
            try:
                lexer = pygments.lexers.guess_lexer_for_filename(filename, raw)
            except pygments.util.ClassNotFound:
                # released pygments doesn't yet have .json
                # so hack around it here.
                if filename.endswith('.json'):
                    lexer = pygments.lexers.get_lexer_by_name('js')
                else:
                    lexer = pygments.lexers.get_lexer_by_name('text')

            file_.update(html=pygments.highlight(raw, lexer, formatter),
                         num_lines=len(raw.strip().splitlines()))

        files.append(file_)

    return files

@post_only_view
@model_view(models.ExtensionVersion)
def submit_review_view(request, obj):
    extension, version = obj.extension, obj

    if not can_review_extension(request.user, extension):
        return HttpResponseForbidden()

    review = CodeReview(version=version,
                        reviewer=request.user,
                        comments=request.POST.get('comments'))

    messages.info(request, "Thank you for reviewing %s" % (extension.name,))

    if can_approve_extension(request.user, obj.extension):
        status_string = request.POST.get('status')
        newstatus = dict(approve=models.STATUS_ACTIVE,
                         reject=models.STATUS_REJECTED).get(status_string, None)

        if newstatus is not None:
            log = ChangeStatusLog(user=request.user,
                                  version=obj,
                                  newstatus=newstatus)
            log.save()

            obj.status = newstatus
            obj.save()

            review.changelog = log

            models.status_changed.send(sender=request, version=obj, log=log)

    review.save()

    models.reviewed.send(sender=request, request=request,
                         version=version, review=review)

    return redirect('review-list')

@model_view(models.ExtensionVersion)
def review_version_view(request, obj):
    extension, version = obj.extension, obj

    # Reviews on previous versions of the same extension.
    previous_versions = extension.versions.order_by('-version')

    # Exclude this version...
    previous_versions = previous_versions.exclude(version=version.version)

    # Other reviews on the same version
    previous_reviews = version.reviews.all()

    can_approve = can_approve_extension(request.user, extension)

    context = dict(extension=extension,
                   version=version,
                   previous_versions=previous_versions,
                   previous_reviews=previous_reviews,
                   can_approve=can_approve)

    if can_review_extension(request.user, extension):
        template_name = "review/review_reviewer.html"
    else:
        template_name = "review/review.html"

    return render(request, template_name, context)

def send_email_on_submitted(sender, request, version, **kwargs):
    extension = version.extension

    url = request.build_absolute_uri(reverse('review-version',
                                             kwargs=dict(pk=version.pk)))

    data = dict(version=version,
                extension=extension,
                url=url)

    reviewers = get_all_reviewers().values_list('email', flat=True)

    subject = render_to_string('review/submitted_mail_subject.txt', data).strip()
    subject = subject.replace('\n', '')
    subject = subject.replace('\r', '')

    message = render_to_string('review/submitted_mail.txt', data).strip()

    send_mail(subject=subject,
              message=message,
              from_email=settings.DEFAULT_FROM_EMAIL,
              recipient_list=reviewers)

models.submitted_for_review.connect(send_email_on_submitted)

def send_email_on_reviewed(sender, request, version, review, **kwargs):
    extension = version.extension

    url = request.build_absolute_uri(reverse('review-version',
                                             kwargs=dict(pk=version.pk)))

    data = dict(version=version,
                extension=extension,
                url=url)

    subject = render_to_string('review/reviewed_mail_subject.txt', data).strip()
    subject = subject.replace('\n', '')
    subject = subject.replace('\r', '')

    message = render_to_string('review/reviewed_mail.txt', data).strip()

    recipient_list = list(version.reviews.values_list('reviewer__email', flat=True).distinct())
    recipient_list.append(extension.creator.email)

    if review.reviewer.email in recipient_list:
        # Don't spam the reviewer with his own review.
        recipient_list.remove(review.reviewer.email)

    send_mail(subject=subject,
              message=message,
              from_email=settings.DEFAULT_FROM_EMAIL,
              recipient_list=recipient_list)

models.reviewed.connect(send_email_on_reviewed)
<commit_msg>Cache formatter<commit_after>
import base64
import os.path

import pygments
import pygments.util
import pygments.lexers
import pygments.formatters

from django.conf import settings
from django.core.mail import send_mail
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.http import HttpResponseForbidden
from django.shortcuts import redirect
from django.template.loader import render_to_string

from review.models import CodeReview, ChangeStatusLog, get_all_reviewers
from extensions import models

from decorators import ajax_view, model_view, post_only_view
from utils import render

IMAGE_TYPES = {
    '.png':  'image/png',
    '.jpg':  'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.gif':  'image/gif',
    '.bmp':  'image/bmp',
    '.svg':  'image/svg+xml',
}

FORMATTER = pygments.formatters.HtmlFormatter(style="borland", cssclass="code")

def can_review_extension(user, extension):
    if user == extension.creator:
        return True

    if user.has_perm("review.can-review-extensions"):
        return True

    return False

def can_approve_extension(user, extension):
    if user.is_superuser:
        return True

    if user.has_perm("review.can-review-extensions"):
        return user != extension.creator

    return False

@ajax_view
@model_view(models.ExtensionVersion)
def ajax_get_files_view(request, obj):
    if not can_review_extension(request.user, obj.extension):
        return HttpResponseForbidden()

    zipfile = obj.get_zipfile('r')

    # filename => { raw, html, filename }
    files = []
    for filename in sorted(zipfile.namelist()):
        raw = zipfile.open(filename, 'r').read()

        base, extension = os.path.splitext(filename)

        file_ = dict(filename=filename)

        if extension in IMAGE_TYPES:
            mime = IMAGE_TYPES[extension]
            raw_base64 = base64.standard_b64encode(raw)

            file_.update(html='' % (mime, raw_base64,),
                         num_lines=0)

        else:
            try:
                lexer = pygments.lexers.guess_lexer_for_filename(filename, raw)
            except pygments.util.ClassNotFound:
                # released pygments doesn't yet have .json
                # so hack around it here.
                if filename.endswith('.json'):
                    lexer = pygments.lexers.get_lexer_by_name('js')
                else:
                    lexer = pygments.lexers.get_lexer_by_name('text')

            file_.update(html=pygments.highlight(raw, lexer, FORMATTER),
                         num_lines=len(raw.strip().splitlines()))

        files.append(file_)

    return files

@post_only_view
@model_view(models.ExtensionVersion)
def submit_review_view(request, obj):
    extension, version = obj.extension, obj

    if not can_review_extension(request.user, extension):
        return HttpResponseForbidden()

    review = CodeReview(version=version,
                        reviewer=request.user,
                        comments=request.POST.get('comments'))

    messages.info(request, "Thank you for reviewing %s" % (extension.name,))

    if can_approve_extension(request.user, obj.extension):
        status_string = request.POST.get('status')
        newstatus = dict(approve=models.STATUS_ACTIVE,
                         reject=models.STATUS_REJECTED).get(status_string, None)

        if newstatus is not None:
            log = ChangeStatusLog(user=request.user,
                                  version=obj,
                                  newstatus=newstatus)
            log.save()

            obj.status = newstatus
            obj.save()

            review.changelog = log

            models.status_changed.send(sender=request, version=obj, log=log)

    review.save()

    models.reviewed.send(sender=request, request=request,
                         version=version, review=review)

    return redirect('review-list')

@model_view(models.ExtensionVersion)
def review_version_view(request, obj):
    extension, version = obj.extension, obj

    # Reviews on previous versions of the same extension.
    previous_versions = extension.versions.order_by('-version')

    # Exclude this version...
    previous_versions = previous_versions.exclude(version=version.version)

    # Other reviews on the same version
    previous_reviews = version.reviews.all()

    can_approve = can_approve_extension(request.user, extension)

    context = dict(extension=extension,
                   version=version,
                   previous_versions=previous_versions,
                   previous_reviews=previous_reviews,
                   can_approve=can_approve)

    if can_review_extension(request.user, extension):
        template_name = "review/review_reviewer.html"
    else:
        template_name = "review/review.html"

    return render(request, template_name, context)

def send_email_on_submitted(sender, request, version, **kwargs):
    extension = version.extension

    url = request.build_absolute_uri(reverse('review-version',
                                             kwargs=dict(pk=version.pk)))

    data = dict(version=version,
                extension=extension,
                url=url)

    reviewers = get_all_reviewers().values_list('email', flat=True)

    subject = render_to_string('review/submitted_mail_subject.txt', data).strip()
    subject = subject.replace('\n', '')
    subject = subject.replace('\r', '')

    message = render_to_string('review/submitted_mail.txt', data).strip()

    send_mail(subject=subject,
              message=message,
              from_email=settings.DEFAULT_FROM_EMAIL,
              recipient_list=reviewers)

models.submitted_for_review.connect(send_email_on_submitted)

def send_email_on_reviewed(sender, request, version, review, **kwargs):
    extension = version.extension

    url = request.build_absolute_uri(reverse('review-version',
                                             kwargs=dict(pk=version.pk)))

    data = dict(version=version,
                extension=extension,
                url=url)

    subject = render_to_string('review/reviewed_mail_subject.txt', data).strip()
    subject = subject.replace('\n', '')
    subject = subject.replace('\r', '')

    message = render_to_string('review/reviewed_mail.txt', data).strip()

    recipient_list = list(version.reviews.values_list('reviewer__email', flat=True).distinct())
    recipient_list.append(extension.creator.email)

    if review.reviewer.email in recipient_list:
        # Don't spam the reviewer with his own review.
        recipient_list.remove(review.reviewer.email)

    send_mail(subject=subject,
              message=message,
              from_email=settings.DEFAULT_FROM_EMAIL,
              recipient_list=recipient_list)

models.reviewed.connect(send_email_on_reviewed)
<|endoftext|>