<commit_before>from __future__ import print_function
from __future__ import absolute_import

import numpy as np

def set_data_dir(path=None, store=None):
    """
    Set the top level directory pysat uses to look for data and reload.

    Parameters
    ----------
    path : string
        valid path to directory pysat uses to look for data
    store : bool
        if True, store data directory for future runs
    """

    import os
    import sys
    import pysat
    if sys.version_info[0] >= 3:
        import importlib
        re_load = importlib.reload
    else:
        re_load = reload

    if store is None:
        store = True

    if os.path.isdir(path):
        if store:
            with open(os.path.join(os.path.expanduser('~'), '.pysat',
                                   'data_path.txt'), 'w') as f:
                f.write(path)
        pysat.data_dir = path
        pysat._files = re_load(pysat._files)
        pysat._instrument = re_load(pysat._instrument)
    else:
        raise ValueError('Path %s does not lead to a valid directory.' % path)

def computational_form(data):
    """
    Input Series of numbers, Series, or DataFrames repackaged
    for calculation.

    Parameters
    ----------
    data : pandas.Series
        Series of numbers, Series, DataFrames

    Returns
    -------
    pandas.Series, DataFrame, or Panel
        repacked data, aligned by indices, ready for calculation
    """

    from pysat import DataFrame, Series, datetime, Panel

    if isinstance(data.iloc[0], DataFrame):
        dslice = Panel.from_dict(dict([(i, data.iloc[i])
                                       for i in xrange(len(data))]))
    elif isinstance(data.iloc[0], Series):
        dslice = DataFrame(data.tolist())
        dslice.index = data.index
    else:
        dslice = data
    return dslice

def load_netcdf4(fnames=None, strict_meta=False, file_format=None,
                 epoch_name='Epoch', units_label='units',
                 name_label='long_name', notes_label='notes',
                 desc_label='desc', plot_label='label', axis_label='axis',
                 scale_label='scale', min_label='value_min',
                 max_label='value_max', fill_label='fill'):
    # unix_time=False, **kwargs):
    """Load netCDF-3/4 file produced by pysat.

    Parameters
    ----------
    fnames : string or array_like of strings (None)
        filenames to load
    strict_meta : boolean (False)
        check if metadata across fnames is the same
    file_format : string (None)
        file_format keyword passed to netCDF4 routine
        NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, and NETCDF4
    epoch_name : string ('Epoch')
    units_label : string ('units')
        keyword for unit information
    name_label : string ('long_name')
        keyword for informative name label
    notes_label : string ('notes')
        keyword for file notes
    desc_label : string ('desc')
        keyword for data descriptions
    plot_label : string ('label')
        keyword for name to use on plot labels
    axis_label : string ('axis')
        keyword for axis labels
    scale_label : string ('scale')
        keyword for plot scaling
    min_label : string ('value_min')
        keyword for minimum in allowable value range
    max_label : string ('value_max')
        keyword for maximum in allowable value range
    fill_label : string ('fill')
        keyword for fill values

    Returns
    --------
    out : pandas.core.frame.DataFrame
        DataFrame output
    mdata : pysat._meta.Meta
        Meta data
    """

    import copy
    import netCDF4
    import pandas as pds
    import string
    import pysat
    try:
        basestring
    except NameError:
        basestring = str

    if fnames is None:
        raise ValueError("Must supply a filename/list of filenames")
    if isinstance(fnames, basestring):
        fnames = [fnames]

    if file_format is None:
        file_format = 'NETCDF4'
    else:
        file_format = file_format.upper()

    saved_mdata = None
    running_idx = 0
    running_store = []
    two_d_keys = []
    two_d_dims = []
    three_d_keys = []
    three_d_dims = []

    for fname in fnames:
        with netCDF4.Dataset(fname, mode='r', format=file_format) as data:
            # build up dictionary with all global ncattrs
            # and add those attributes to a pysat meta object
            ncattrsList = data.ncattrs()
            mdata = pysat.Meta(units_label=units_label, name_label=name_label,
                               notes_label=notes_label, desc_label=desc_label,
                               plot_label=plot_label, axis_label=axis_label,
                               scale_label=scale_label,
                               min_label=min_label, max_label=max_label,
                               fill_label=fill_label)
            for d in ncattrsList:
                if hasattr(mdata, d):
                    mdata.__setattr__(d+'_', data.getncattr(d))
                else:
                    mdata.__setattr__(d, data.getncattr(d))

            # loadup all of the variables in the netCDF
            loadedVars = {}
            for key in data.variables.keys():
                # load up metadata.  From here group unique
                # dimensions and act accordingly, 1D, 2D, 3D
                if len(data.variables[key].dimensions) == 1:
                    # load 1D data variable
                    # assuming basic time dimension
                    loadedVars[key] = data.variables[key][:]
                    # if key != epoch_name:
                    # load up metadata
                    meta_dict = {}
                    for nc_key in data.variables[key].ncattrs():
                        meta_dict[nc_key] = \
                                data.variables[key].getncattr(nc_key)
                    mdata[key] = meta_dict
                if len(data.variables[key].dimensions) == 2:
                    # part of dataframe within dataframe
                    two_d_keys.append(key)
                    two_d_dims.append(data.variables[key].dimensions)

                if len(data.variables[key].dimensions) == 3:
                    # part of full/dedicated dataframe within dataframe
                    three_d_keys.append(key)
                    three_d_dims.append(data.variables[key].dimensions)

            # we now have a list of keys that need to go into a dataframe,
            # could be more than one, collect unique dimensions for 2D keys
            for dim in set(two_d_dims):
                # first dimension should be epoch
                # second dimension name used as variable name
                obj_key_name = dim[1]
                # collect variable names associated with dimension
                idx_bool = [dim == i for i in two_d_dims]
                idx, = np.where(np.array(idx_bool))
                obj_var_keys = []
                clean_var_keys = []
                for i in idx:
                    obj_var_keys.append(two_d_keys[i])
                    clean_var_keys.append(
                            two_d_keys[i].split(obj_key_name + '_')[-1])

                # figure out how to index this data, it could provide its own
                # index - or we may have to create simple integer based
                # DataFrame access. If the dimension is stored as its own
                # variable then use that info for index
                if obj_key_name in obj_var_keys:
                    # string used to indentify dimension also in data.variables
                    # will be used as an index
                    index_key_name = obj_key_name
                    # if the object index uses UNIX time, process into datetime
                    # index
                    if data.variables[obj_key_name].getncattr(name_label) == \
                            epoch_name:
                        # name to be used in DataFrame index
                        index_name = epoch_name
                        time_index_flag = True
                    else:
                        time_index_flag = False
                        # label to be used in DataFrame index
                        index_name = \
                            data.variables[obj_key_name].getncattr(name_label)
                else:
                    # dimension is not itself a variable
                    index_key_name = None

                # iterate over the variables and grab metadata
                dim_meta_data = pysat.Meta(units_label=units_label,
                                           name_label=name_label,
                                           notes_label=notes_label,
                                           desc_label=desc_label,
                                           plot_label=plot_label,
                                           axis_label=axis_label,
                                           scale_label=scale_label,
                                           min_label=min_label,
                                           max_label=max_label,
                                           fill_label=fill_label)

                for key, clean_key in zip(obj_var_keys, clean_var_keys):
                    # store attributes in metadata, exept for dim name
                    meta_dict = {}
                    for nc_key in data.variables[key].ncattrs():
                        meta_dict[nc_key] = \
                            data.variables[key].getncattr(nc_key)
                    dim_meta_data[clean_key] = meta_dict

                # print (dim_meta_data)
                dim_meta_dict = {'meta': dim_meta_data}
                if index_key_name is not None:
                    # add top level meta
                    for nc_key in data.variables[obj_key_name].ncattrs():
                        dim_meta_dict[nc_key] = \
                            data.variables[obj_key_name].getncattr(nc_key)
                    mdata[obj_key_name] = dim_meta_dict

                # iterate over all variables with this dimension and store data
                # data storage, whole shebang
                loop_dict = {}
                # list holds a series of slices, parsed from dict above
                loop_list = []
                for key, clean_key in zip(obj_var_keys, clean_var_keys):
                    # data
                    loop_dict[clean_key] = \
                        data.variables[key][:, :].flatten(order='C')
                # number of values in time
                loop_lim = data.variables[obj_var_keys[0]].shape[0]
                # number of values per time
                step_size = len(data.variables[obj_var_keys[0]][0, :])
                # check if there is an index we should use
                if not (index_key_name is None):
                    # an index was found
                    time_var = loop_dict.pop(index_key_name)
                    if time_index_flag:
                        # create datetime index from data
                        if file_format == 'NETCDF4':
                            time_var = pds.to_datetime(1E6*time_var)
                        else:
                            time_var = pds.to_datetime(1E6*time_var)
                    new_index = time_var
                    new_index_name = index_name
                else:
                    # using integer indexing
                    new_index = np.arange(loop_lim*step_size,
                                          dtype=int) % step_size
                    new_index_name = 'index'
                # load all data into frame
                if len(loop_dict.keys()) > 1:
                    loop_frame = pds.DataFrame(loop_dict,
                                               columns=clean_var_keys)
                    if obj_key_name in loop_frame:
                        del loop_frame[obj_key_name]
                    # break massive frame into bunch of smaller frames
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1), :])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name
                else:
                    loop_frame = pds.Series(loop_dict[clean_var_keys[0]],
                                            name=obj_var_keys[0])
                    # break massive series into bunch of smaller series
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1)])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name
                # print (loop_frame.columns)

                # add 2D object data, all based on a unique dimension within
                # netCDF, to loaded data dictionary
                loadedVars[obj_key_name] = loop_list
                del loop_list

            # we now have a list of keys that need to go into a dataframe,
            # could be more than one, collect unique dimensions for 2D keys
            for dim in set(three_d_dims):
                # collect variable names associated with dimension
                idx_bool = [dim == i for i in three_d_dims]
                idx, = np.where(np.array(idx_bool))
                obj_var_keys = []
                for i in idx:
                    obj_var_keys.append(three_d_keys[i])

                for obj_key_name in obj_var_keys:
                    # store attributes in metadata
                    meta_dict = {}
                    for nc_key in data.variables[obj_key_name].ncattrs():
                        meta_dict[nc_key] = \
                            data.variables[obj_key_name].getncattr(nc_key)
                    mdata[obj_key_name] = meta_dict

                    # iterate over all variables with this dimension and store data
                    # data storage, whole shebang
                    loop_dict = {}
                    # list holds a series of slices, parsed from dict above
                    loop_list = []
                    loop_dict[obj_key_name] = \
                        data.variables[obj_key_name][:, :, :]
                    # number of values in time
                    loop_lim = data.variables[obj_key_name].shape[0]
                    # number of values per time
                    step_size_x = len(data.variables[obj_key_name][0, :, 0])
                    step_size_y = len(data.variables[obj_key_name][0, 0, :])
                    step_size = step_size_x
                    loop_dict[obj_key_name] = \
                        loop_dict[obj_key_name].reshape((loop_lim*step_size_x,
                                                         step_size_y))
                    # check if there is an index we should use
                    if not (index_key_name is None):
                        # an index was found
                        time_var = loop_dict.pop(index_key_name)
                        if time_index_flag:
                            # create datetime index from data
                            if file_format == 'NETCDF4':
                                time_var = pds.to_datetime(1E6*time_var)
                            else:
                                time_var = pds.to_datetime(1E6*time_var)
                        new_index = time_var
                        new_index_name = index_name
                    else:
                        # using integer indexing
                        new_index = np.arange(loop_lim*step_size,
                                              dtype=int) % step_size
                        new_index_name = 'index'
                    # load all data into frame
                    loop_frame = pds.DataFrame(loop_dict[obj_key_name])
                    # del loop_frame['dimension_1']
                    # break massive frame into bunch of smaller frames
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1), :])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name

                    # add 2D object data, all based on a unique dimension within netCDF,
                    # to loaded data dictionary
                    loadedVars[obj_key_name] = loop_list
                    del loop_list

            # prepare dataframe index for this netcdf file
            time_var = loadedVars.pop(epoch_name)

            # convert from GPS seconds to seconds used in pandas (unix time,
            # no leap)
            # time_var = convert_gps_to_unix_seconds(time_var)
            if file_format == 'NETCDF4':
                loadedVars[epoch_name] = pds.to_datetime((1E6 *
                                                          time_var).astype(int))
            else:
                loadedVars[epoch_name] = pds.to_datetime((time_var *
                                                          1E6).astype(int))
            # loadedVars[epoch_name] = pds.to_datetime((time_var*1E6).astype(int))
            running_store.append(loadedVars)
            running_idx += len(loadedVars[epoch_name])

            if strict_meta:
                if saved_mdata is None:
                    saved_mdata = copy.deepcopy(mdata)
                elif (mdata != saved_mdata):
                    raise ValueError('Metadata across filenames is not the ' +
                                     'same.')

    # combine all of the data loaded across files together
    out = []
    for item in running_store:
        out.append(pds.DataFrame.from_records(item, index=epoch_name))
    out = pds.concat(out, axis=0)
    return out, mdata
<commit_msg>computational_form<commit_after>from __future__ import print_function
from __future__ import absolute_import

import numpy as np

def set_data_dir(path=None, store=None):
    """
    Set the top level directory pysat uses to look for data and reload.

    Parameters
    ----------
    path : string
        valid path to directory pysat uses to look for data
    store : bool
        if True, store data directory for future runs
    """

    import os
    import sys
    import pysat
    if sys.version_info[0] >= 3:
        import importlib
        re_load = importlib.reload
    else:
        re_load = reload

    if store is None:
        store = True

    if os.path.isdir(path):
        if store:
            with open(os.path.join(os.path.expanduser('~'), '.pysat',
                                   'data_path.txt'), 'w') as f:
                f.write(path)
        pysat.data_dir = path
        pysat._files = re_load(pysat._files)
        pysat._instrument = re_load(pysat._instrument)
    else:
        raise ValueError('Path %s does not lead to a valid directory.' % path)

def computational_form(data):
    """
    Repackages numbers, Series, or DataFrames

    Parameters
    ----------
    data : pandas.Series
        Series of numbers, Series, DataFrames

    Returns
    -------
    pandas.Series, DataFrame, or Panel
        repacked data, aligned by indices, ready for calculation
    """

    from pysat import DataFrame, Series, datetime, Panel

    if isinstance(data.iloc[0], DataFrame):
        dslice = Panel.from_dict(dict([(i, data.iloc[i])
                                       for i in xrange(len(data))]))
    elif isinstance(data.iloc[0], Series):
        dslice = DataFrame(data.tolist())
        dslice.index = data.index
    else:
        dslice = data
    return dslice

def load_netcdf4(fnames=None, strict_meta=False, file_format=None,
                 epoch_name='Epoch', units_label='units',
                 name_label='long_name', notes_label='notes',
                 desc_label='desc', plot_label='label', axis_label='axis',
                 scale_label='scale', min_label='value_min',
                 max_label='value_max', fill_label='fill'):
    # unix_time=False, **kwargs):
    """Load netCDF-3/4 file produced by pysat.

    Parameters
    ----------
    fnames : string or array_like of strings (None)
        filenames to load
    strict_meta : boolean (False)
        check if metadata across fnames is the same
    file_format : string (None)
        file_format keyword passed to netCDF4 routine
        NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, and NETCDF4
    epoch_name : string ('Epoch')
    units_label : string ('units')
        keyword for unit information
    name_label : string ('long_name')
        keyword for informative name label
    notes_label : string ('notes')
        keyword for file notes
    desc_label : string ('desc')
        keyword for data descriptions
    plot_label : string ('label')
        keyword for name to use on plot labels
    axis_label : string ('axis')
        keyword for axis labels
    scale_label : string ('scale')
        keyword for plot scaling
    min_label : string ('value_min')
        keyword for minimum in allowable value range
    max_label : string ('value_max')
        keyword for maximum in allowable value range
    fill_label : string ('fill')
        keyword for fill values

    Returns
    --------
    out : pandas.core.frame.DataFrame
        DataFrame output
    mdata : pysat._meta.Meta
        Meta data
    """

    import copy
    import netCDF4
    import pandas as pds
    import string
    import pysat
    try:
        basestring
    except NameError:
        basestring = str

    if fnames is None:
        raise ValueError("Must supply a filename/list of filenames")
    if isinstance(fnames, basestring):
        fnames = [fnames]

    if file_format is None:
        file_format = 'NETCDF4'
    else:
        file_format = file_format.upper()

    saved_mdata = None
    running_idx = 0
    running_store = []
    two_d_keys = []
    two_d_dims = []
    three_d_keys = []
    three_d_dims = []

    for fname in fnames:
        with netCDF4.Dataset(fname, mode='r', format=file_format) as data:
            # build up dictionary with all global ncattrs
            # and add those attributes to a pysat meta object
            ncattrsList = data.ncattrs()
            mdata = pysat.Meta(units_label=units_label, name_label=name_label,
                               notes_label=notes_label, desc_label=desc_label,
                               plot_label=plot_label, axis_label=axis_label,
                               scale_label=scale_label,
                               min_label=min_label, max_label=max_label,
                               fill_label=fill_label)
            for d in ncattrsList:
                if hasattr(mdata, d):
                    mdata.__setattr__(d+'_', data.getncattr(d))
                else:
                    mdata.__setattr__(d, data.getncattr(d))

            # loadup all of the variables in the netCDF
            loadedVars = {}
            for key in data.variables.keys():
                # load up metadata.  From here group unique
                # dimensions and act accordingly, 1D, 2D, 3D
                if len(data.variables[key].dimensions) == 1:
                    # load 1D data variable
                    # assuming basic time dimension
                    loadedVars[key] = data.variables[key][:]
                    # if key != epoch_name:
                    # load up metadata
                    meta_dict = {}
                    for nc_key in data.variables[key].ncattrs():
                        meta_dict[nc_key] = \
                                data.variables[key].getncattr(nc_key)
                    mdata[key] = meta_dict
                if len(data.variables[key].dimensions) == 2:
                    # part of dataframe within dataframe
                    two_d_keys.append(key)
                    two_d_dims.append(data.variables[key].dimensions)

                if len(data.variables[key].dimensions) == 3:
                    # part of full/dedicated dataframe within dataframe
                    three_d_keys.append(key)
                    three_d_dims.append(data.variables[key].dimensions)

            # we now have a list of keys that need to go into a dataframe,
            # could be more than one, collect unique dimensions for 2D keys
            for dim in set(two_d_dims):
                # first dimension should be epoch
                # second dimension name used as variable name
                obj_key_name = dim[1]
                # collect variable names associated with dimension
                idx_bool = [dim == i for i in two_d_dims]
                idx, = np.where(np.array(idx_bool))
                obj_var_keys = []
                clean_var_keys = []
                for i in idx:
                    obj_var_keys.append(two_d_keys[i])
                    clean_var_keys.append(
                            two_d_keys[i].split(obj_key_name + '_')[-1])

                # figure out how to index this data, it could provide its own
                # index - or we may have to create simple integer based
                # DataFrame access. If the dimension is stored as its own
                # variable then use that info for index
                if obj_key_name in obj_var_keys:
                    # string used to indentify dimension also in data.variables
                    # will be used as an index
                    index_key_name = obj_key_name
                    # if the object index uses UNIX time, process into datetime
                    # index
                    if data.variables[obj_key_name].getncattr(name_label) == \
                            epoch_name:
                        # name to be used in DataFrame index
                        index_name = epoch_name
                        time_index_flag = True
                    else:
                        time_index_flag = False
                        # label to be used in DataFrame index
                        index_name = \
                            data.variables[obj_key_name].getncattr(name_label)
                else:
                    # dimension is not itself a variable
                    index_key_name = None

                # iterate over the variables and grab metadata
                dim_meta_data = pysat.Meta(units_label=units_label,
                                           name_label=name_label,
                                           notes_label=notes_label,
                                           desc_label=desc_label,
                                           plot_label=plot_label,
                                           axis_label=axis_label,
                                           scale_label=scale_label,
                                           min_label=min_label,
                                           max_label=max_label,
                                           fill_label=fill_label)

                for key, clean_key in zip(obj_var_keys, clean_var_keys):
                    # store attributes in metadata, exept for dim name
                    meta_dict = {}
                    for nc_key in data.variables[key].ncattrs():
                        meta_dict[nc_key] = \
                            data.variables[key].getncattr(nc_key)
                    dim_meta_data[clean_key] = meta_dict

                # print (dim_meta_data)
                dim_meta_dict = {'meta': dim_meta_data}
                if index_key_name is not None:
                    # add top level meta
                    for nc_key in data.variables[obj_key_name].ncattrs():
                        dim_meta_dict[nc_key] = \
                            data.variables[obj_key_name].getncattr(nc_key)
                    mdata[obj_key_name] = dim_meta_dict

                # iterate over all variables with this dimension and store data
                # data storage, whole shebang
                loop_dict = {}
                # list holds a series of slices, parsed from dict above
                loop_list = []
                for key, clean_key in zip(obj_var_keys, clean_var_keys):
                    # data
                    loop_dict[clean_key] = \
                        data.variables[key][:, :].flatten(order='C')
                # number of values in time
                loop_lim = data.variables[obj_var_keys[0]].shape[0]
                # number of values per time
                step_size = len(data.variables[obj_var_keys[0]][0, :])
                # check if there is an index we should use
                if not (index_key_name is None):
                    # an index was found
                    time_var = loop_dict.pop(index_key_name)
                    if time_index_flag:
                        # create datetime index from data
                        if file_format == 'NETCDF4':
                            time_var = pds.to_datetime(1E6*time_var)
                        else:
                            time_var = pds.to_datetime(1E6*time_var)
                    new_index = time_var
                    new_index_name = index_name
                else:
                    # using integer indexing
                    new_index = np.arange(loop_lim*step_size,
                                          dtype=int) % step_size
                    new_index_name = 'index'
                # load all data into frame
                if len(loop_dict.keys()) > 1:
                    loop_frame = pds.DataFrame(loop_dict,
                                               columns=clean_var_keys)
                    if obj_key_name in loop_frame:
                        del loop_frame[obj_key_name]
                    # break massive frame into bunch of smaller frames
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1), :])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name
                else:
                    loop_frame = pds.Series(loop_dict[clean_var_keys[0]],
                                            name=obj_var_keys[0])
                    # break massive series into bunch of smaller series
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1)])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name
                # print (loop_frame.columns)

                # add 2D object data, all based on a unique dimension within
                # netCDF, to loaded data dictionary
                loadedVars[obj_key_name] = loop_list
                del loop_list

            # we now have a list of keys that need to go into a dataframe,
            # could be more than one, collect unique dimensions for 2D keys
            for dim in set(three_d_dims):
                # collect variable names associated with dimension
                idx_bool = [dim == i for i in three_d_dims]
                idx, = np.where(np.array(idx_bool))
                obj_var_keys = []
                for i in idx:
                    obj_var_keys.append(three_d_keys[i])

                for obj_key_name in obj_var_keys:
                    # store attributes in metadata
                    meta_dict = {}
                    for nc_key in data.variables[obj_key_name].ncattrs():
                        meta_dict[nc_key] = \
                            data.variables[obj_key_name].getncattr(nc_key)
                    mdata[obj_key_name] = meta_dict

                    # iterate over all variables with this dimension and store data
                    # data storage, whole shebang
                    loop_dict = {}
                    # list holds a series of slices, parsed from dict above
                    loop_list = []
                    loop_dict[obj_key_name] = \
                        data.variables[obj_key_name][:, :, :]
                    # number of values in time
                    loop_lim = data.variables[obj_key_name].shape[0]
                    # number of values per time
                    step_size_x = len(data.variables[obj_key_name][0, :, 0])
                    step_size_y = len(data.variables[obj_key_name][0, 0, :])
                    step_size = step_size_x
                    loop_dict[obj_key_name] = \
                        loop_dict[obj_key_name].reshape((loop_lim*step_size_x,
                                                         step_size_y))
                    # check if there is an index we should use
                    if not (index_key_name is None):
                        # an index was found
                        time_var = loop_dict.pop(index_key_name)
                        if time_index_flag:
                            # create datetime index from data
                            if file_format == 'NETCDF4':
                                time_var = pds.to_datetime(1E6*time_var)
                            else:
                                time_var = pds.to_datetime(1E6*time_var)
                        new_index = time_var
                        new_index_name = index_name
                    else:
                        # using integer indexing
                        new_index = np.arange(loop_lim*step_size,
                                              dtype=int) % step_size
                        new_index_name = 'index'
                    # load all data into frame
                    loop_frame = pds.DataFrame(loop_dict[obj_key_name])
                    # del loop_frame['dimension_1']
                    # break massive frame into bunch of smaller frames
                    for i in np.arange(loop_lim, dtype=int):
                        loop_list.append(loop_frame.iloc[step_size*i:step_size*(i+1), :])
                        loop_list[-1].index = new_index[step_size*i:step_size*(i+1)]
                        loop_list[-1].index.name = new_index_name

                    # add 2D object data, all based on a unique dimension within netCDF,
                    # to loaded data dictionary
                    loadedVars[obj_key_name] = loop_list
                    del loop_list

            # prepare dataframe index for this netcdf file
            time_var = loadedVars.pop(epoch_name)

            # convert from GPS seconds to seconds used in pandas (unix time,
            # no leap)
            # time_var = convert_gps_to_unix_seconds(time_var)
            if file_format == 'NETCDF4':
                loadedVars[epoch_name] = pds.to_datetime((1E6 *
                                                          time_var).astype(int))
            else:
                loadedVars[epoch_name] = pds.to_datetime((time_var *
                                                          1E6).astype(int))
            # loadedVars[epoch_name] = pds.to_datetime((time_var*1E6).astype(int))
            running_store.append(loadedVars)
            running_idx += len(loadedVars[epoch_name])

            if strict_meta:
                if saved_mdata is None:
                    saved_mdata = copy.deepcopy(mdata)
                elif (mdata != saved_mdata):
                    raise ValueError('Metadata across filenames is not the ' +
                                     'same.')

    # combine all of the data loaded across files together
    out = []
    for item in running_store:
        out.append(pds.DataFrame.from_records(item, index=epoch_name))
    out = pds.concat(out, axis=0)
    return out, mdata
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Sep  7 21:14:25 2017

@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import, unicode_literals
import h5py
import six
from warnings import warn
import numpy as np
from .hdf_utils import checkIfMain, getAuxData, get_attr, get_data_descriptor, get_formatted_labels, \
    get_dimensionality, get_sort_order, get_unit_values, reshape_to_Ndims

class PycroDataset(h5py.Dataset):

    def __init__(self, h5_ref, sort_dims=False):

        if not checkIfMain(h5_ref):
            raise TypeError('Supply a h5py.Dataset object that is a pycroscopy main dataset')

        super(PycroDataset, self).__init__(h5_ref.id)

        # User accessible properties
        # The required Position and Spectroscopic datasets
        self.h5_spec_vals = getAuxData(h5_ref, 'Spectroscopic_Values')[-1]
        self.h5_spec_inds = getAuxData(h5_ref, 'Spectroscopic_Indices')[-1]
        self.h5_pos_vals = getAuxData(h5_ref, 'Position_Values')[-1]
        self.h5_pos_inds = getAuxData(h5_ref, 'Position_Indices')[-1]

        # The dimension labels
        self.__pos_dim_labels = get_attr(self.h5_pos_inds, 'labels')
        self.__spec_dim_labels = get_attr(self.h5_spec_inds, 'labels')

        # Data desciptors
        self.data_descriptor = get_data_descriptor(self)
        self.pos_dim_descriptors = get_formatted_labels(self.h5_pos_inds)
        self.spec_dim_descriptors = get_formatted_labels(self.h5_spec_inds)

        # The size of each dimension
        self.__pos_dim_sizes = get_dimensionality(np.transpose(self.h5_pos_inds))
        self.__spec_dim_sizes = get_dimensionality(self.h5_spec_inds)

        # Sorted dimension order
        self.__pos_sort_order = get_sort_order(np.transpose(self.h5_pos_inds))
        self.__spec_sort_order = get_sort_order(self.h5_spec_inds)

        # iternal book-keeping / we don't want users to mess with these?
        self.__n_dim_sizes = np.append(self.__pos_dim_sizes, self.__spec_dim_sizes)
        self.__n_dim_labs = np.append(self.__pos_dim_labels, self.__spec_dim_labels)
        self.__n_dim_sort_order = np.append(self.__pos_sort_order, self.__spec_sort_order)
        self.__n_dim_data = None

        # Should the dimensions be sorted from fastest to slowest
        self.__sort_dims = sort_dims

    def __eq__(self, other):
        if isinstance(other, PycroDataset):
            if isinstance(other, h5py.Dataset):
                warn('Comparing PycroData object with h5py.Dataset')

            return super(PycroDataset, self).__eq__(other)

        return False

    def __repr__(self):
        h5_str = super(PycroDataset, self).__repr__()

        pos_str = ' \n'.join(['{} - size: {}'.format(dim_name, str(dim_size)) for dim_name, dim_size in
                              zip(self.__pos_dim_labels, self.__pos_dim_sizes)])
        spec_str = ' \n'.join(['{} - size: {}'.format(dim_name, str(dim_size)) for dim_name, dim_size in
                               zip(self.__spec_dim_labels, self.__spec_dim_sizes)])

        pycro_str = ' \n'.join(['located at:',
                                self.name,
                                'Data contains:', self.data_descriptor,
                                'Data dimensions and original shape:',
                                'Position Dimensions:',
                                pos_str,
                                'Spectroscopic Dimensions:',
                                spec_str])

        r = '\n'.join([h5_str, pycro_str])

        if six.PY2:
            pycro_str = pycro_str.encode('utf8')

        return '\n'.join([h5_str, pycro_str])

    def pos_dim_labels(self):
        if self.__sort_dims:
            return self.__pos_dim_labels[self.__pos_sort_order].tolist()
        else:
            return self.__pos_dim_labels.tolist()

    def spec_dim_labels(self):
        if self.__sort_dims:
            return self.__spec_dim_labels[self.__spec_sort_order].tolist()
        else:
            return self.__spec_dim_labels.tolist()

    def pos_dim_sizes(self):
        if self.__sort_dims:
            return self.__pos_dim_sizes[self.__pos_sort_order].tolist()
        else:
            return self.__pos_dim_sizes.tolist()

    def spec_dim_sizes(self):
        if self.__sort_dims:
            return self.__spec_dim_sizes[self.__spec_sort_order].tolist()
        else:
            return self.__spec_dim_sizes.tolist()

    def n_dim_labels(self):
        if self.__sort_dims:
            return self.__n_dim_labs[self.__n_dim_sort_order].tolist()
        else:
            return self.__n_dim_labs.tolist()

    def n_dim_sizes(self):
        if self.__sort_dims:
            return self.__n_dim_sizes[self.__n_dim_sort_order].tolist()
        else:
            return self.__n_dim_sizes.tolist()

    def get_pos_values(self, dim_name):
        return get_unit_values(self.h5_pos_inds, self.h5_pos_vals)[dim_name]

    def get_spec_values(self, dim_name):
        return get_unit_values(self.h5_spec_inds, self.h5_spec_vals)[dim_name]

    def current_sorting(self):
        if self.__sort_dims:
            print('Data dimensions are sorted in order from fastest changing dimension to slowest.')
        else:
            print('Data dimensions are in the order they occur in the file.')

    def toggle_sorting(self):
        if self.__n_dim_data is not None:
            if self.__sort_dims:
                nd_sort = self.__pos_sort_order[::-1] + self.__spec_sort_order[::-1]
            else:
                nd_sort = self.__n_dim_sort_order

            self.__n_dim_data = np.transpose(self.__n_dim_data, nd_sort)

        self.__sort_dims = not self.__sort_dims

    def get_n_dim_form(self):

        n_dim_data, success = reshape_to_Ndims(self, sort_dims=self.__sort_dims)

        if success is not True:
            raise ValueError('Unable to reshape data to N-dimensional form.')

        self.__n_dim_data = n_dim_data

        return self.__n_dim_data

    def slice(self, **kwargs):
        """

        Parameters
        ----------
        kwargs

        Returns
        -------

        """
        # Ensure that the n_dimensional data exists
        if self.__n_dim_data is None:
            warn('N-dimensional form of the dataset has not yet been extracted.  '
                 'This will be done before slicing.')
            _ = self.get_n_dim_form()

        # Create default slices that include the entire dimension
        n_dim_slices = [slice(None) for _ in self.__n_dim_labs]

        # Loop over all the keyword arguments and create slices for each.
        for key, val in kwargs.items():
            # Make sure the dimension is valid
            if key not in self.__n_dim_labs:

                raise KeyError('Cannot slice on dimension {}.  '
                               'Valid dimensions are {}.'.format(key, self.__n_dim_labs.tolist()))

            # Check the value and convert to a slice object if possible.
            # Use a list if not.
            if isinstance(val, slice) or isinstance(val, list):
                pass
            elif isinstance(val, np.ndarray):
                val = val.flatten().tolist()
            elif isinstance(val, tuple):
                val = list(val)
            else:
                raise TypeError('The slices must be array-likes or slice objects.')

            idim = self.n_dim_labels().index(key)

            n_dim_slices[idim] = val

        # Now that the slices are built, we just need to apply them to the data
        # This method is slow and memory intensive but shouldn't fail if multiple lists are given.
        # TODO: More elegant slicing method for PycroDataset objects
        data_slice = self.__n_dim_data[n_dim_slices[0]]

        for idim, this_slice in enumerate(n_dim_slices[1:]):
            idim += 1
            print(idim, this_slice)
            base_slice = [slice(None) for _ in self.__n_dim_labs]

            base_slice[idim] = this_slice
            print(base_slice)
            data_slice = data_slice[base_slice]

        return data_slice
<commit_msg>PycroDataset updates<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Sep  7 21:14:25 2017

@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import, unicode_literals
import h5py
import six
from warnings import warn
import numpy as np
from .hdf_utils import checkIfMain, getAuxData, get_attr, get_data_descriptor, get_formatted_labels, \
    get_dimensionality, get_sort_order, get_unit_values, reshape_to_Ndims

class PycroDataset(h5py.Dataset):

    def __init__(self, h5_ref, sort_dims=False):
        """
        New data object that extends the h5py.Dataset.

        Parameters
        ----------
        h5_ref : hdf5.Dataset
            The base dataset to be extended
        sort_dims : bool
            Should the dimensions be sorted internally from fastest changing to slowest.

        Methods
        -------

        Attributes
        ----------
        h5_spec_vals : h5py.Dataset
            Associated Spectroscopic Values dataset
        h5_spec_inds : h5py.Dataset
            Associated Spectroscopic Indices dataset
        h5_pos_vals : h5py.Dataset
            Associated Position Values dataset
        h5_pos_inds : h5py.Dataset
            Associated Position Indices dataset

        """

        if not checkIfMain(h5_ref):
            raise TypeError('Supply a h5py.Dataset object that is a pycroscopy main dataset')

        super(PycroDataset, self).__init__(h5_ref.id)

        # User accessible properties
        # The required Position and Spectroscopic datasets
        self.h5_spec_vals = getAuxData(h5_ref, 'Spectroscopic_Values')[-1]
        self.h5_spec_inds = getAuxData(h5_ref, 'Spectroscopic_Indices')[-1]
        self.h5_pos_vals = getAuxData(h5_ref, 'Position_Values')[-1]
        self.h5_pos_inds = getAuxData(h5_ref, 'Position_Indices')[-1]

        # The dimension labels
        self.__pos_dim_labels = get_attr(self.h5_pos_inds, 'labels')
        self.__spec_dim_labels = get_attr(self.h5_spec_inds, 'labels')

        # Data desciptors
        self.data_descriptor = get_data_descriptor(self)
        self.pos_dim_descriptors = get_formatted_labels(self.h5_pos_inds)
        self.spec_dim_descriptors = get_formatted_labels(self.h5_spec_inds)

        # The size of each dimension
        self.__pos_dim_sizes = get_dimensionality(np.transpose(self.h5_pos_inds))
        self.__spec_dim_sizes = get_dimensionality(self.h5_spec_inds)

        # Sorted dimension order
        self.__pos_sort_order = get_sort_order(np.transpose(self.h5_pos_inds))
        self.__spec_sort_order = get_sort_order(self.h5_spec_inds)

        # iternal book-keeping / we don't want users to mess with these?
        self.__n_dim_sizes = np.append(self.__pos_dim_sizes, self.__spec_dim_sizes)
        self.__n_dim_labs = np.append(self.__pos_dim_labels, self.__spec_dim_labels)
        self.__n_dim_sort_order = np.append(self.__pos_sort_order, self.__spec_sort_order)
        self.__n_dim_data = None

        # Should the dimensions be sorted from fastest to slowest
        self.__sort_dims = sort_dims

    def __eq__(self, other):
        if isinstance(other, PycroDataset):
            if isinstance(other, h5py.Dataset):
                warn('Comparing PycroData object with h5py.Dataset')

            return super(PycroDataset, self).__eq__(other)

        return False

    def __repr__(self):
        h5_str = super(PycroDataset, self).__repr__()

        pos_str = ' \n'.join(['{} - size: {}'.format(dim_name, str(dim_size)) for dim_name, dim_size in
                              zip(self.__pos_dim_labels, self.__pos_dim_sizes)])
        spec_str = ' \n'.join(['{} - size: {}'.format(dim_name, str(dim_size)) for dim_name, dim_size in
                               zip(self.__spec_dim_labels, self.__spec_dim_sizes)])

        pycro_str = ' \n'.join(['located at:',
                                self.name,
                                'Data contains:', self.data_descriptor,
                                'Data dimensions and original shape:',
                                'Position Dimensions:',
                                pos_str,
                                'Spectroscopic Dimensions:',
                                spec_str])

        r = '\n'.join([h5_str, pycro_str])

        if six.PY2:
            pycro_str = pycro_str.encode('utf8')

        return '\n'.join([h5_str, pycro_str])

    def pos_dim_labels(self):
        if self.__sort_dims:
            return self.__pos_dim_labels[self.__pos_sort_order].tolist()
        else:
            return self.__pos_dim_labels.tolist()

    def spec_dim_labels(self):
        if self.__sort_dims:
            return self.__spec_dim_labels[self.__spec_sort_order].tolist()
        else:
            return self.__spec_dim_labels.tolist()

    def pos_dim_sizes(self):
        if self.__sort_dims:
            return self.__pos_dim_sizes[self.__pos_sort_order].tolist()
        else:
            return self.__pos_dim_sizes.tolist()

    def spec_dim_sizes(self):
        if self.__sort_dims:
            return self.__spec_dim_sizes[self.__spec_sort_order].tolist()
        else:
            return self.__spec_dim_sizes.tolist()

    def n_dim_labels(self):
        if self.__sort_dims:
            return self.__n_dim_labs[self.__n_dim_sort_order].tolist()
        else:
            return self.__n_dim_labs.tolist()

    def n_dim_sizes(self):
        if self.__sort_dims:
            return self.__n_dim_sizes[self.__n_dim_sort_order].tolist()
        else:
            return self.__n_dim_sizes.tolist()

    def get_pos_values(self, dim_name):
        return get_unit_values(self.h5_pos_inds, self.h5_pos_vals)[dim_name]

    def get_spec_values(self, dim_name):
        return get_unit_values(self.h5_spec_inds, self.h5_spec_vals)[dim_name]

    def current_sorting(self):
        """
        Prints the current sorting method.

        """
        if self.__sort_dims:
            print('Data dimensions are sorted in order from fastest changing dimension to slowest.')
        else:
            print('Data dimensions are in the order they occur in the file.')

    def toggle_sorting(self):
        """
        Toggles between sorting from fastest changing dimension to slowest and sorting based on the
        order of the labels

        """
        if self.__n_dim_data is not None:
            if self.__sort_dims:
                nd_sort = self.__pos_sort_order[::-1] + self.__spec_sort_order[::-1]
            else:
                nd_sort = self.__n_dim_sort_order

            self.__n_dim_data = np.transpose(self.__n_dim_data, nd_sort)

        self.__sort_dims = not self.__sort_dims

    def get_n_dim_form(self):
        """
        Reshapes the dataset to an N-dimensional array

        Returns
        -------
        n_dim_data : numpy.ndarray
            N-dimensional form of the dataset

        """

        n_dim_data, success = reshape_to_Ndims(self, sort_dims=self.__sort_dims)

        if success is not True:
            raise ValueError('Unable to reshape data to N-dimensional form.')

        self.__n_dim_data = n_dim_data

        return self.__n_dim_data

    def slice(self, **slice_dict):
        """
        Slice the dataset based on an input dictionary of 'str': slice pairs.
        Each string should correspond to a dimension label.  The slices can be
        array-likes or slice objects.

        Parameters
        ----------
        slice_dict : dict
            Dictionary of array-likes.

        Returns
        -------
        data_slice : numpy.ndarray
            Slice of the dataset.  Dataset has been reshaped to N-dimensions if `success` is True, only
            by Position dimensions if `success` is 'Positions', or not reshape at all if `success`
            is False.
        success : str or bool
            Informs the user as to how the data_slice has been shaped.

        """
        # Ensure that the n_dimensional data exists
        if self.__n_dim_data is None:
            warn('N-dimensional form of the dataset has not yet been extracted.  '
                 'This will be done before slicing.')
            _ = self.get_n_dim_form()

        # Create default slices that include the entire dimension
        n_dim_slices = dict()
        n_dim_slices_sizes = dict()
        for dim_lab, dim_size in zip(self.n_dim_labels(), self.n_dim_sizes()):
            n_dim_slices[dim_lab] = list(range(dim_size))
            n_dim_slices_sizes[dim_lab] = len(n_dim_slices[dim_lab])

        # Loop over all the keyword arguments and create slices for each.
        for key, val in slice_dict.items():
            # Make sure the dimension is valid
            if key not in self.__n_dim_labs:
                raise KeyError('Cannot slice on dimension {}.  '
                               'Valid dimensions are {}.'.format(key, self.__n_dim_labs.tolist()))

            # Check the value and convert to a slice object if possible.
            # Use a list if not.
            if isinstance(val, slice):
                val = n_dim_slices[key][val]
            elif isinstance(val, list):
                pass
            elif isinstance(val, np.ndarray):
                val = val.flatten().tolist()
            elif isinstance(val, tuple):
                val = list(val)
            else:
                raise TypeError('The slices must be array-likes or slice objects.')

            n_dim_slices[key] = val

            n_dim_slices_sizes[key] = len(val)

        # Now that the slices are built, we just need to apply them to the data
        # This method is slow and memory intensive but shouldn't fail if multiple lists are given.
        # TODO: More elegant slicing method for PycroDataset objects
        for pos_ind, pos_lab in enumerate(self.__pos_dim_labels):
            n_dim_slices[pos_lab] = np.isin(self.h5_pos_inds[:, pos_ind], n_dim_slices[pos_lab])
            if pos_ind == 0:
                pos_slice = n_dim_slices[pos_lab]
            else:
                pos_slice = np.logical_and(pos_slice, n_dim_slices[pos_lab])

        pos_slice = np.argwhere(pos_slice)

        for spec_ind, spec_lab in enumerate(self.__spec_dim_labels):
            n_dim_slices[spec_lab] = np.isin(self.h5_spec_inds[spec_ind], n_dim_slices[spec_lab])
            if spec_ind == 0:
                spec_slice = n_dim_slices[spec_lab]
            else:
                spec_slice = np.logical_and(spec_slice, n_dim_slices[spec_lab])

        spec_slice = np.argwhere(spec_slice)

        if len(np.argwhere(pos_slice)) <= len(np.argwhere(spec_slice)):
            # Fewer final positions that spectra (Most common case)
            data_slice = self[pos_slice, :][:, spec_slice]
        else:
            data_slice = self[spec_slice, :][:, pos_slice]

        data_slice, success = reshape_to_Ndims(data_slice,
                                               h5_pos=self.h5_pos_inds[pos_slice, :],
                                               h5_spec=self.h5_spec_inds[:, spec_slice])

        return data_slice, success
<|endoftext|>


<commit_before>import sys
from . import check_numbers,match_spaces,split_conflict,wrap_sentences
import os
import gzip
import time
import subprocess
def errmsg():
    print r"""arguments are:
    num     :   check numbers in a latex catalog (e.g. of numbered notebook)
                of items of the form '\item[anything number.anything]'
    gensync :   use a compiled latex original (first arg) to generate a synctex
                file for a scanned document (second arg), e.g.  with
                handwritten markup
    wmatch  :   match whitespace
    gvr     :   git forward search, with arguments

                - file
                - line
    sc      :   split conflict
    wd      :   word diff
    wr      :   wrap with indented sentence format (for markdown or latex).
                Optional flag --cleanoo cleans latex exported from
                OpenOffice/LibreOffice
    xx      :   Convert xml to xlsx"""
    exit()
_ROOT = os.path.abspath(os.path.dirname(__file__))
def get_data(path):
    "return vbs and js scripts saved as package data"
    return os.path.join(_ROOT, path)
def main():
    if len(sys.argv) == 1:
        errmsg()
    command = sys.argv[1]
    arguments = sys.argv[2:]
    if command == 'num':
        check_numbers.run(arguments)
    elif command == 'gensync':
        with gzip.open(arguments[0].replace(
            '.pdf','.synctek.gz')) as fp:
            orig_synctex = fp.read()
            fp.close()
        # since the new synctex is in a new place, I need to tell it
        # how to get back to the original place
        relative_path = os.path.relpath(
                os.path.dir(arguments[0]),
                os.path.dir(arguments[1]))
        base_fname = arguments[0].replace('.pdf','')
        new_synctex = orig_synctex.replace(
                base_fname,
                relative_path + base_fname)
        new_synctex = orig_synctex
        with gzip.open(arguments[1].replace(
            '.pdf','.synctek.gz')) as fp:
            fp.write(new_synctex)
            fp.write(argument[1].replace())
            fp.close()
    elif command == 'wr':
        print "arguments are",arguments
        if len(arguments) == 1:
            wrap_sentences.run(arguments[0])
        elif len(arguments) == 2 and arguments[0] == '--cleanoo':
            wrap_sentences.run(arguments[1],stupid_strip = True)
            print "stripped stupid markup from LibreOffice"
        else:
            raise ValueError("I don't understand your arguments:"+repr(arguments))
    elif command == 'gvr':
        cmd = ['gvim']
        cmd.append('--remote-wait-silent')
        cmd.append('+'+arguments[1])
        cmd.append(arguments[0])
        subprocess.Popen(' '.join(cmd))# this is forked
        time.sleep(0.3)
        cmd = ['gvim']
        cmd.append('--remote-send')
        cmd.append('"zO"')
        time.sleep(0.3)
        cmd = ['gvim']
        cmd.append('--remote-send')
        cmd.append('":cd %:h\n"')
        subprocess.Popen(' '.join(cmd))
    elif command == 'wmatch':
        match_spaces.run(arguments)
    elif command == 'sc':
        split_conflict.run(arguments)
    elif command == 'wd':
        if arguments[0].find('Temp') > 0:
            #{{{ if it's a temporary file, I need to make a real copy to run pandoc on
            fp = open(arguments[0])
            contents = fp.read()
            fp.close()
            fp = open(arguments[1].replace('.md','_old.md'),'w')
            fp.write(contents)
            fp.close()
            arguments[0] = arguments[1].replace('.md','_old.md')
            #}}}
        word_files = [x.replace('.md','.docx') for x in arguments[:2]]
        local_dir = os.path.dirname(arguments[1])
        print "local directory:",local_dir
        for j in range(2):
            if arguments[0][-5:] == '.docx':
                print "the first argument has a docx extension, so I'm bypassing the pandoc step"
            else:
                cmd = ['pandoc']
                cmd += [arguments[j]]
                cmd += ['--csl=edited-pmid-format.csl']
                cmd += ['--bibliography library_abbrev_utf8.bib']
                cmd += ['-s --smart']
                if len(arguments) > 2:
                    if arguments[2][-5:] == '.docx':
                        cmd += ['--reference-docx='+arguments[2]]
                    else:
                        raise RuntimeError("if you pass three arguments to wd, then the third must be a template for the word document")
                elif os.path.isfile(local_dir + os.path.sep + "template.docx"):
                    # by default, use template.docx in the current directory
                    cmd += ['--reference-docx=' + local_dir + os.path.sep + "template.docx"]
                cmd += ['-o']
                cmd += [word_files[j]]
                print "about to run",' '.join(cmd)
                os.system(' '.join(cmd))
        cmd = ['start']
        cmd += [get_data('diff-doc.js')]
        print "word files are",word_files
        if word_files[0].find("C:") > -1:
            cmd += [word_files[0]]
        else:
            cmd += [os.getcwd() + os.path.sep + word_files[0]]
        cmd += [os.getcwd() + os.path.sep + word_files[1]]
        print "about to run",' '.join(cmd)
        os.system(' '.join(cmd))
    elif command == 'xx':
        format_codes = {'csv':6, 'xlsx':51, 'xml':46} # determined by microsoft vbs
        cmd = ['start']
        cmd += [get_data('xml2xlsx.vbs')]
        first_ext = arguments[0].split('.')[-1]
        second_ext = arguments[1].split('.')[-1]
        for j in arguments[0:2]:
            if j.find("C:") > -1:
                cmd += [j]
            else:
                cmd += [os.getcwd() + os.path.sep + j]
        cmd += [str(format_codes[j]) for j in [first_ext, second_ext]]
        print "about to run",' '.join(cmd)
        os.system(' '.join(cmd))
    else:
        errmsg()
    return
<commit_msg>added sumatrapdf forward search<commit_after>import sys
from . import check_numbers,match_spaces,split_conflict,wrap_sentences
import os
import gzip
import time
import subprocess
def errmsg():
    print r"""arguments are:
    fs      :   smart latex forward-search
                currently this works specifically for sumatra pdf located
                at "C:\Program Files\SumatraPDF\SumatraPDF.exe",
                but can easily be adapted based on os, etc.
                Add the following line (or something like it) to your vimrc:
                map <c-F>s :sil !pydifft fs %:p <c-r>=line(".")<cr><cr>
    num     :   check numbers in a latex catalog (e.g. of numbered notebook)
                of items of the form '\item[anything number.anything]'
    gensync :   use a compiled latex original (first arg) to generate a synctex
                file for a scanned document (second arg), e.g.  with
                handwritten markup
    wmatch  :   match whitespace
    gvr     :   git forward search, with arguments

                - file
                - line
    sc      :   split conflict
    wd      :   word diff
    wr      :   wrap with indented sentence format (for markdown or latex).
                Optional flag --cleanoo cleans latex exported from
                OpenOffice/LibreOffice
    xx      :   Convert xml to xlsx"""
    exit()
_ROOT = os.path.abspath(os.path.dirname(__file__))
def get_data(path):
    "return vbs and js scripts saved as package data"
    return os.path.join(_ROOT, path)
def main():
    if len(sys.argv) == 1:
        errmsg()
    command = sys.argv[1]
    arguments = sys.argv[2:]
    if command == 'num':
        check_numbers.run(arguments)
    elif command == 'gensync':
        with gzip.open(arguments[0].replace(
            '.pdf','.synctek.gz')) as fp:
            orig_synctex = fp.read()
            fp.close()
        # since the new synctex is in a new place, I need to tell it
        # how to get back to the original place
        relative_path = os.path.relpath(
                os.path.dir(arguments[0]),
                os.path.dir(arguments[1]))
        base_fname = arguments[0].replace('.pdf','')
        new_synctex = orig_synctex.replace(
                base_fname,
                relative_path + base_fname)
        new_synctex = orig_synctex
        with gzip.open(arguments[1].replace(
            '.pdf','.synctek.gz')) as fp:
            fp.write(new_synctex)
            fp.write(argument[1].replace())
            fp.close()
    elif command == 'wr':
        print "arguments are",arguments
        if len(arguments) == 1:
            wrap_sentences.run(arguments[0])
        elif len(arguments) == 2 and arguments[0] == '--cleanoo':
            wrap_sentences.run(arguments[1],stupid_strip = True)
            print "stripped stupid markup from LibreOffice"
        else:
            raise ValueError("I don't understand your arguments:"+repr(arguments))
    elif command == 'gvr':
        cmd = ['gvim']
        cmd.append('--remote-wait-silent')
        cmd.append('+'+arguments[1])
        cmd.append(arguments[0])
        subprocess.Popen(' '.join(cmd))# this is forked
        time.sleep(0.3)
        cmd = ['gvim']
        cmd.append('--remote-send')
        cmd.append('"zO"')
        time.sleep(0.3)
        cmd = ['gvim']
        cmd.append('--remote-send')
        cmd.append('":cd %:h\n"')
        subprocess.Popen(' '.join(cmd))
    elif command == 'wmatch':
        match_spaces.run(arguments)
    elif command == 'sc':
        split_conflict.run(arguments)
    elif command == 'wd':
        if arguments[0].find('Temp') > 0:
            #{{{ if it's a temporary file, I need to make a real copy to run pandoc on
            fp = open(arguments[0])
            contents = fp.read()
            fp.close()
            fp = open(arguments[1].replace('.md','_old.md'),'w')
            fp.write(contents)
            fp.close()
            arguments[0] = arguments[1].replace('.md','_old.md')
            #}}}
        word_files = [x.replace('.md','.docx') for x in arguments[:2]]
        local_dir = os.path.dirname(arguments[1])
        print "local directory:",local_dir
        for j in range(2):
            if arguments[0][-5:] == '.docx':
                print "the first argument has a docx extension, so I'm bypassing the pandoc step"
            else:
                cmd = ['pandoc']
                cmd += [arguments[j]]
                cmd += ['--csl=edited-pmid-format.csl']
                cmd += ['--bibliography library_abbrev_utf8.bib']
                cmd += ['-s --smart']
                if len(arguments) > 2:
                    if arguments[2][-5:] == '.docx':
                        cmd += ['--reference-docx='+arguments[2]]
                    else:
                        raise RuntimeError("if you pass three arguments to wd, then the third must be a template for the word document")
                elif os.path.isfile(local_dir + os.path.sep + "template.docx"):
                    # by default, use template.docx in the current directory
                    cmd += ['--reference-docx=' + local_dir + os.path.sep + "template.docx"]
                cmd += ['-o']
                cmd += [word_files[j]]
                print "about to run",' '.join(cmd)
                os.system(' '.join(cmd))
        cmd = ['start']
        cmd += [get_data('diff-doc.js')]
        print "word files are",word_files
        if word_files[0].find("C:") > -1:
            cmd += [word_files[0]]
        else:
            cmd += [os.getcwd() + os.path.sep + word_files[0]]
        cmd += [os.getcwd() + os.path.sep + word_files[1]]
        print "about to run",' '.join(cmd)
        os.system(' '.join(cmd))
    elif command == 'fs':
        texfile,lineno = arguments
        texfile = os.path.normpath(os.path.abspath(texfile))
        directory, texfile = texfile.rsplit(os.path.sep,1)
        assert texfile[-4:] == '.tex','needs to be called .tex'
        origbasename = texfile[:-4]
        cmd = ['"C:\\Program Files\\SumatraPDF\\SumatraPDF.exe" -reuse-instance']
        if os.path.exists(os.path.join(directory,origbasename+'.pdf')):
            cmd.append(os.path.join(directory,origbasename+'.pdf'))
        else:
            print "no pdf file for this guy, looking for one that has one"
            found = False
            for fname in os.listdir(directory):
                if fname[-4:] == '.tex':
                    basename = fname[:-4]
                    print "found tex file",basename
                    if os.path.exists(os.path.join(directory,basename + '.pdf')):
                        print "found matching tex/pdf pair",basename
                        with open(basename+'.tex','r') as fp:
                            alltxt = fp.read()
                            if '\input{'+origbasename+'}' in alltxt or '\include{'+origbasename+'}' in alltxt:
                                print 'this guy calls the guy you want -- just stop here and display it'
                                found = True
                                cmd.append(os.path.join(directory,basename+'.pdf'))
                                break
            if not found:
                raise IOError("This is not the PDF you are looking for!!!")
        cmd.append('-forward-search')
        cmd.append(origbasename+'.tex')
        cmd.append('%s -fwdsearch-color ff0000'%lineno)
        print "about to execute:\n\t",' '.join(cmd)
        os.system(' '.join(cmd))
    elif command == 'xx':
        format_codes = {'csv':6, 'xlsx':51, 'xml':46} # determined by microsoft vbs
        cmd = ['start']
        cmd += [get_data('xml2xlsx.vbs')]
        first_ext = arguments[0].split('.')[-1]
        second_ext = arguments[1].split('.')[-1]
        for j in arguments[0:2]:
            if j.find("C:") > -1:
                cmd += [j]
            else:
                cmd += [os.getcwd() + os.path.sep + j]
        cmd += [str(format_codes[j]) for j in [first_ext, second_ext]]
        print "about to run",' '.join(cmd)
        os.system(' '.join(cmd))
    else:
        errmsg()
    return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Author: Florian Mayer <florian.mayer@bitsrc.org>
#
# This module was developed with funding provided by
# the ESA Summer of Code (2011).
#
# pylint: disable=C0103,R0903

from __future__ import absolute_import

from sunpy.util.multimethod import MultiMethod

class Attr(object):
    """ This is the base for all attributes. It makes sure that attributes
    that are combined using the two logic operations AND (&) and OR (|) always
    are in disjunctive normal form, that is, there are only two levels -
    the first being disjunction and the second being conjunction. In other
    words, every combinations of attributes looks like this:
    (a AND b AND c) OR (d AND e). """
    def __and__(self, other):
        if isinstance(other, AttrOr):
            return AttrOr([elem & self for elem in other.attrs])
        if self.collides(other):
            return NotImplemented
        return AttrAnd([self, other])

    def __hash__(self):
        return hash(frozenset(vars(self).iteritems()))

    def __or__(self, other):
        # Optimization.
        if self == other:
            return self
        return AttrOr([self, other])

    def collides(self, other):
        raise NotImplementedError

    def __eq__(self, other):
        return dict(vars(self)) == dict(vars(other))

class DummyAttr(Attr):
    """ Empty attribute. Useful for building up queries. Returns other
    attribute when ORed or ANDed. """
    def __and__(self, other):
        return other

    def __or__(self, other):
        return other

    def collides(self, other):
        return False

    def __hash__(self):
        return hash(None)

    def __eq__(self, other):
        return isinstance(other, DummyAttr)

class AttrAnd(Attr):
    """ Attribute representing attributes ANDed together. """
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __and__(self, other):
        if any(other.collides(elem) for elem in self.attrs):
            return NotImplemented
        if isinstance(other, AttrAnd):
            return AttrAnd(self.attrs + other.attrs)
        if isinstance(other, AttrOr):
            return AttrOr([elem & self for elem in other.attrs])
        return AttrAnd(self.attrs + [other])

    __rand__ = __and__

    def __repr__(self):
        return "<AttrAnd(%r)>" % self.attrs

    def __eq__(self, other):
        if not isinstance(other, AttrAnd):
            return False
        return set(self.attrs) == set(other.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs))

    def collides(self, other):
        return any(elem.collides(other) for elem in self)

class AttrOr(Attr):
    """ Attribute representing attributes ORed together. """
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __or__(self, other):
        if isinstance(other, AttrOr):
            return AttrOr(self.attrs + other.attrs)
        return AttrOr(self.attrs + [other])

    __ror__ = __or__

    def __and__(self, other):
        return AttrOr([elem & other for elem in self.attrs])

    __rand__ = __and__

    def __xor__(self, other):
        new = AttrOr([])
        for elem in self.attrs:
            try:
                new |= elem ^ other
            except TypeError:
                pass
        return new

    def __contains__(self, other):
        for elem in self.attrs:
            try:
                if other in elem:
                    return True
            except TypeError:
                pass
        return False

    def __repr__(self):
        return "<AttrOr(%r)>" % self.attrs

    def __eq__(self, other):
        if not isinstance(other, AttrOr):
            return False
        return set(self.attrs) == set(other.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs))

    def collides(self, other):
        return all(elem.collides(other) for elem in self)

class ValueAttr(Attr):
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __repr__(self):
        return "<ValueAttr(%r)>" % (self.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs.iteritems()))

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.attrs == other.attrs

    def collides(self, other):
        if not isinstance(other, self.__class__):
            return False
        return any(k in other.attrs for k in self.attrs)

class AttrWalker(object):
    def __init__(self):
        self.applymm = MultiMethod(lambda *a, **kw: (a[1], ))
        self.createmm = MultiMethod(lambda *a, **kw: (a[1], ))

    def add_creator(self, *types):
        def _dec(fun):
            for type_ in types:
                self.createmm.add(fun, (type_, ))
            return fun
        return _dec

    def add_applier(self, *types):
        def _dec(fun):
            for type_ in types:
                self.applymm.add(fun, (type_, ))
            return fun
        return _dec

    def add_converter(self, *types):
        def _dec(fun):
            for type_ in types:
                self.applymm.add(self.cv_apply(fun), (type_, ))
                self.createmm.add(self.cv_create(fun), (type_, ))
            return fun
        return _dec

    def cv_apply(self, fun):
        def _fun(*args, **kwargs):
            args = list(args)
            args[1] = fun(args[1])
            return self.applymm(*args, **kwargs)
        return _fun

    def cv_create(self, fun):
        def _fun(*args, **kwargs):
            args = list(args)
            args[1] = fun(args[1])
            return self.createmm(*args, **kwargs)
        return _fun

    def create(self, *args, **kwargs):
        return self.createmm(self, *args, **kwargs)

    def apply(self, *args, **kwargs):
        return self.applymm(self, *args, **kwargs)

    def super_create(self, *args, **kwargs):
        return self.createmm.super(self, *args, **kwargs)

    def super_apply(self, *args, **kwargs):
        return self.applymm.super(self, *args, **kwargs)

def and_(*args):
    """ Trick operator precendence.

    and_(foo < bar, bar < baz)
    """
    value = DummyAttr()
    for elem in args:
        value &= elem
    return value

def or_(*args):
    """ Trick operator precendence.

    or_(foo < bar, bar < baz)
    """
    value = DummyAttr()
    for elem in args:
        value |= elem
    return value
<commit_msg>Reword.<commit_after># -*- coding: utf-8 -*-
# Author: Florian Mayer <florian.mayer@bitsrc.org>
#
# This module was developed with funding provided by
# the ESA Summer of Code (2011).
#
# pylint: disable=C0103,R0903

from __future__ import absolute_import

from sunpy.util.multimethod import MultiMethod

"""
Allow representation of queries as logic expressions. This module makes
sure that attributes that are combined using the two logic operations AND (&)
and OR (|) always are in disjunctive normal form, that is, there are only two
levels - the first being disjunction and the second being conjunction. In other
words, every combinations of attributes looks like this:
(a AND b AND c) OR (d AND e).
"""

class Attr(object):
    """ This is the base for all attributes. """
    def __and__(self, other):
        if isinstance(other, AttrOr):
            return AttrOr([elem & self for elem in other.attrs])
        if self.collides(other):
            return NotImplemented
        return AttrAnd([self, other])

    def __hash__(self):
        return hash(frozenset(vars(self).iteritems()))

    def __or__(self, other):
        # Optimization.
        if self == other:
            return self
        return AttrOr([self, other])

    def collides(self, other):
        raise NotImplementedError

    def __eq__(self, other):
        return dict(vars(self)) == dict(vars(other))

class DummyAttr(Attr):
    """ Empty attribute. Useful for building up queries. Returns other
    attribute when ORed or ANDed. """
    def __and__(self, other):
        return other

    def __or__(self, other):
        return other

    def collides(self, other):
        return False

    def __hash__(self):
        return hash(None)

    def __eq__(self, other):
        return isinstance(other, DummyAttr)

class AttrAnd(Attr):
    """ Attribute representing attributes ANDed together. """
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __and__(self, other):
        if any(other.collides(elem) for elem in self.attrs):
            return NotImplemented
        if isinstance(other, AttrAnd):
            return AttrAnd(self.attrs + other.attrs)
        if isinstance(other, AttrOr):
            return AttrOr([elem & self for elem in other.attrs])
        return AttrAnd(self.attrs + [other])

    __rand__ = __and__

    def __repr__(self):
        return "<AttrAnd(%r)>" % self.attrs

    def __eq__(self, other):
        if not isinstance(other, AttrAnd):
            return False
        return set(self.attrs) == set(other.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs))

    def collides(self, other):
        return any(elem.collides(other) for elem in self)

class AttrOr(Attr):
    """ Attribute representing attributes ORed together. """
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __or__(self, other):
        if isinstance(other, AttrOr):
            return AttrOr(self.attrs + other.attrs)
        return AttrOr(self.attrs + [other])

    __ror__ = __or__

    def __and__(self, other):
        return AttrOr([elem & other for elem in self.attrs])

    __rand__ = __and__

    def __xor__(self, other):
        new = AttrOr([])
        for elem in self.attrs:
            try:
                new |= elem ^ other
            except TypeError:
                pass
        return new

    def __contains__(self, other):
        for elem in self.attrs:
            try:
                if other in elem:
                    return True
            except TypeError:
                pass
        return False

    def __repr__(self):
        return "<AttrOr(%r)>" % self.attrs

    def __eq__(self, other):
        if not isinstance(other, AttrOr):
            return False
        return set(self.attrs) == set(other.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs))

    def collides(self, other):
        return all(elem.collides(other) for elem in self)

class ValueAttr(Attr):
    def __init__(self, attrs):
        Attr.__init__(self)
        self.attrs = attrs

    def __repr__(self):
        return "<ValueAttr(%r)>" % (self.attrs)

    def __hash__(self):
        return hash(frozenset(self.attrs.iteritems()))

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.attrs == other.attrs

    def collides(self, other):
        if not isinstance(other, self.__class__):
            return False
        return any(k in other.attrs for k in self.attrs)

class AttrWalker(object):
    def __init__(self):
        self.applymm = MultiMethod(lambda *a, **kw: (a[1], ))
        self.createmm = MultiMethod(lambda *a, **kw: (a[1], ))

    def add_creator(self, *types):
        def _dec(fun):
            for type_ in types:
                self.createmm.add(fun, (type_, ))
            return fun
        return _dec

    def add_applier(self, *types):
        def _dec(fun):
            for type_ in types:
                self.applymm.add(fun, (type_, ))
            return fun
        return _dec

    def add_converter(self, *types):
        def _dec(fun):
            for type_ in types:
                self.applymm.add(self.cv_apply(fun), (type_, ))
                self.createmm.add(self.cv_create(fun), (type_, ))
            return fun
        return _dec

    def cv_apply(self, fun):
        def _fun(*args, **kwargs):
            args = list(args)
            args[1] = fun(args[1])
            return self.applymm(*args, **kwargs)
        return _fun

    def cv_create(self, fun):
        def _fun(*args, **kwargs):
            args = list(args)
            args[1] = fun(args[1])
            return self.createmm(*args, **kwargs)
        return _fun

    def create(self, *args, **kwargs):
        return self.createmm(self, *args, **kwargs)

    def apply(self, *args, **kwargs):
        return self.applymm(self, *args, **kwargs)

    def super_create(self, *args, **kwargs):
        return self.createmm.super(self, *args, **kwargs)

    def super_apply(self, *args, **kwargs):
        return self.applymm.super(self, *args, **kwargs)

def and_(*args):
    """ Trick operator precendence.

    and_(foo < bar, bar < baz)
    """
    value = DummyAttr()
    for elem in args:
        value &= elem
    return value

def or_(*args):
    """ Trick operator precendence.

    or_(foo < bar, bar < baz)
    """
    value = DummyAttr()
    for elem in args:
        value |= elem
    return value
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4

__version__ = "0.3.5"
<commit_msg>Version bump. Let's go with 1.0<commit_after># -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4

__version__ = "1.0.0"
<|endoftext|>


<commit_before>import re
import cgi

class Template(object):
    tag_re = None
    otag, ctag = '{{', '}}'

    def __init__(self, template=None, context={}, **kwargs):
        from view import View

        self.template = template

        if kwargs:
            context.update(kwargs)

        self.view = context if isinstance(context, View) else View(context=context)
        self._compile_regexps()

    def _compile_regexps(self):
        tags = {'otag': re.escape(self.otag), 'ctag': re.escape(self.ctag)}
        tag = r"""
            (?P<content>[\s\S]*?)
            (?P<whitespace>[\ \t]*)
            %(otag)s \s*
            (?:
              (?P<change>=) \s* (?P<delims>.+?)   \s* = |
              (?P<raw>{)    \s* (?P<raw_name>.+?) \s* } |
              (?P<tag>\W?)  \s* (?P<name>[\s\S]+?)
            )
            \s* %(ctag)s
        """
        self.tag_re = re.compile(tag % tags, re.M | re.X)

    def _parse(self, template, section=None, index=0):
        """Parse a template into a syntax tree."""

        buffer = []
        pos = index

        while True:
            match = self.tag_re.search(template, pos)

            if match is None:
                break

            # Normalize the captures dictionary.
            captures = match.groupdict()
            if captures['change'] is not None:
                captures.update(tag='=', name=captures['delims'])
            elif captures['raw'] is not None:
                captures.update(tag='{', name=captures['raw_name'])

            # Save the literal text content.
            buffer.append(captures['content'])
            pos = match.end()

            # Standalone (non-interpolation) tags consume the entire line,
            # both leading whitespace and trailing newline.
            tagBeganLine = (not buffer[-1] or buffer[-1][-1] == '\n')
            tagEndedLine = (pos == len(template) or template[pos] == '\n')
            interpolationTag = captures['tag'] in ['', '&', '{']

            if (tagBeganLine and tagEndedLine and not interpolationTag):
                pos += 1
            elif captures['whitespace']:
                buffer.append(captures['whitespace'])
                captures['whitespace'] = ''

            # TODO: Process the remaining tag types.
            if captures['tag'] == '!':
                pass
            elif captures['tag'] in ['{', '&']:
                def unescapedTag(view):
                    return view.get(captures['name'])
                buffer.append(unescapedTag)
            elif captures['tag'] == '':
                def escapedTag(view):
                    return cgi.escape(view.get(captures['name']))
                buffer.append(escapedTag)

        # Save the rest of the template.
        buffer.append(template[pos:])

        return buffer

    def render(self, encoding=None):
        parsed = self._parse(self.template)

        def call(x):
            return unicode(callable(x) and x(self.view) or x)

        result = ''.join(map(call, parsed))

        if encoding is not None:
            result = result.encode(encoding)

        return result
<commit_msg>Refactoring the match handling logic.<commit_after>import re
import cgi

class Template(object):
    tag_re = None
    otag, ctag = '{{', '}}'

    def __init__(self, template=None, context={}, **kwargs):
        from view import View

        self.template = template

        if kwargs:
            context.update(kwargs)

        self.view = context if isinstance(context, View) else View(context=context)
        self._compile_regexps()

    def _compile_regexps(self):
        tags = {'otag': re.escape(self.otag), 'ctag': re.escape(self.ctag)}
        tag = r"""
            (?P<content>[\s\S]*?)
            (?P<whitespace>[\ \t]*)
            %(otag)s \s*
            (?:
              (?P<change>=) \s* (?P<delims>.+?)   \s* = |
              (?P<raw>{)    \s* (?P<raw_name>.+?) \s* } |
              (?P<tag>\W?)  \s* (?P<name>[\s\S]+?)
            )
            \s* %(ctag)s
        """
        self.tag_re = re.compile(tag % tags, re.M | re.X)

    def _parse(self, template, section=None, index=0):
        """Parse a template into a syntax tree."""

        buffer = []
        pos = index

        while True:
            match = self.tag_re.search(template, pos)

            if match is None:
                break

            pos = self._handle_match(template, match, buffer)

        # Save the rest of the template.
        buffer.append(template[pos:])

        return buffer

    def _handle_match(self, template, match, buffer):
        # Normalize the captures dictionary.
        captures = match.groupdict()
        if captures['change'] is not None:
            captures.update(tag='=', name=captures['delims'])
        elif captures['raw'] is not None:
            captures.update(tag='{', name=captures['raw_name'])

        # Save the literal text content.
        buffer.append(captures['content'])
        pos = match.end()

        # Standalone (non-interpolation) tags consume the entire line,
        # both leading whitespace and trailing newline.
        tagBeganLine = (not buffer[-1] or buffer[-1][-1] == '\n')
        tagEndedLine = (pos == len(template) or template[pos] == '\n')
        interpolationTag = captures['tag'] in ['', '&', '{']

        if (tagBeganLine and tagEndedLine and not interpolationTag):
            pos += 1
        elif captures['whitespace']:
            buffer.append(captures['whitespace'])
            captures['whitespace'] = ''

        # TODO: Process the remaining tag types.
        print captures['name']
        fetch = lambda view: unicode(view.get(captures['name']))
        if captures['tag'] == '!':
            pass
        elif captures['tag'] in ['{', '&']:
            buffer.append(fetch)
        elif captures['tag'] == '':
            buffer.append(lambda view: cgi.escape(fetch(view), True))
        else:
            print 'Error!'

        return pos

    def render(self, encoding=None):
        parsed = self._parse(self.template)

        def call(x):
            return unicode(callable(x) and x(self.view) or x)

        result = ''.join(map(call, parsed))

        if encoding is not None:
            result = result.encode(encoding)

        return result
<|endoftext|>


<commit_before>
#todo: everything

"""
This is a very crude script to find the information needed to
turn a Fortran codebase into modules which match the files.
"""

from re import sub as re_sub, findall
from sys import argv, stderr
from os.path import exists

"""
Get and check arguments.
"""
# if len(argv) < 2:
# 	stderr.write('no output target provided\n')
# 	exit(4)
# modname = argv[1]

targets = argv[1:]
quit = False

if not targets:
	stderr.write('no targets provided\n')
	exit(2)

for target in targets:
	if not exists(target):
		stderr.write('could not find "{0:s}"\n'.format(target))
		quit = True

if quit:
	exit(1)

"""
Read the source files.
"""
# iface_words = {'function', 'end function', 'subroutine', 'end subroutine', 'recursive', 'pure', 'elemental',
# 	'integer', 'real', 'double', 'complex', 'character', 'logical', 'parameter',}
# output = ['      !! automatically generated interface', '      !! pipe a .f90 file & compile with -ffixed-form',
# 	'', '      module {0:s}'.format(modname), '', '      interface', '']
defined = {}
called = {}
for target in targets:
	# stderr.write('.')
	stderr.write('{0:s}\n'.format(target))
	with open(target, 'r') as fh:
		content = fh.read().lower()
	defined = findall(r'(?:function|subroutine)\s+(\w+)\s*\(', content)
	print(defined)
	called = findall(r'call\s+(\w+)\s*\(', content)
	# functions are not included here
	print(called)

if False:
	next_too = False
	sub = []
	for line in lines:
		line = line.rstrip()
		code = line.lstrip()
		code = re_sub(r'"[^"]*"', '""', code)
		code = re_sub(r'\'[^\']*\'', '\'\'', code)
		if code.startswith('*') or code.startswith('c..'):
			code = ''
		code = code.split('!')[0]
		# if code.endswith(','):
		# 	line += '&'
		if next_too or any(code.startswith(word) for word in iface_words):
			if sub and 'end' not in code and ('subroutine' in code or 'function' in code):
				stderr.write('skipping the rest of "{0:s}"\n'.format(target))
				break
			sub.append(line)
			next_too = False
			if code.endswith(',') or code.endswith('&'):
				next_too = True
	if len(sub) and not 'end subroutine' in sub[-1] and not 'end function' in sub[-1]:
		strt = findall(r'(function|subroutine)\s+(\w+)\s*\(', sub[0])
		if not strt:
			stderr.write('could not determine appropriate ending for "{0:s}"'.format(sub[0]))
			# continue
		sub.append('      end {0:s} {1:s}'.format(strt[0][0], strt[0][1]))
		# if 'function' in sub[0]:
		# 	sub.append('      end function')
		# elif 'subroutine' in sub[0]:
		# 	sub.append('      end subroutine')
		# else:
		# 	stderr.write('invalid ending for "{0:s}" in "{1:s}"\n'.format(sub[0], target))
		# 	continue
	output.extend(sub)
	output.append('')
# output.extend(['      end interface', '', '      end module'])
# print('\n'.join(output))
# stderr.write('done\n')

<commit_msg>applying modulify automatically<commit_after>
#todo: everything

"""
This is a very crude script to find the information needed to
turn a Fortran codebase into modules which match the files.
"""

from collections import defaultdict
from os import getcwd

from re import findall
from sys import argv, stdout, stderr
from os.path import exists, basename, abspath

"""
Get and check arguments.
"""
targets = argv[1:]
quit = False

if not targets:
	stderr.write('no targets provided\n')
	exit(2)

for target in targets:
	if target.startswith('-'):
		continue
	if not exists(target):
		stderr.write('could not find "{0:s}"\n'.format(target))
		quit = True

if quit:
	exit(1)

"""
Set some ignored names that can be called without import.
"""
ignore = {'mpi_finalize', 'mpi_comm_size', 'mpi_comm_rank', 'mpi_recv', 'mpi_send', 'mpi_allreduce', 'flush', 'flush_', 'getenv'}

"""
Read the source files.
"""
defines = defaultdict(list)
defined_in = {}
called_in = defaultdict(set)
calls = defaultdict(set)
for target in targets:
	if target.startswith('-'):
		continue
	with open(target, 'r') as fh:
		content = fh.read().lower()
	for defi in findall(r'(?:function|subroutine)\s+(\w+)\s*\(', content):
		if defi in defined_in:
			stderr.write('{0:s} from {1:s} redefined in {2:s}\n'.format(defi, defined_in[defi], target))
		defined_in[defi] = target
		defines[target].append(defi)
	for calli in findall(r'call\s+(\w+)\s*\(', content):
		called_in[calli].add(target)
		calls[target].add(calli)

"""
Construct use statements.
"""
use = defaultdict(lambda: defaultdict(set))
for file, fcalls in calls.items():
	frm = defaultdict(list)
	for call in fcalls:
		if call in ignore:
			continue
		if call in defined_in:
			if file != defined_in[call]:
				name = basename(defined_in[call].split('.')[0])
				use[file][name].add(call)
		else:
			use[file][None].add(call)
			# stderr.write('don\'t know where {0:s} is defined (called from {1:s})\n'.format(call, file))

"""
Construct modules (including use statements).
"""
for file, routines in defines.items():
	stderr.write('{0:s}\n'.format(file))
	stdout.write('      module {0:s}\n'.format(file.split('.')[0]))
	for src, calls in use[file].items():
		if src is not None:
			stdout.write('      use {0:s}, only: {1:s}\n'.format(src, ', '.join(calls)))
	if None in use[file]:
		stdout.write('      !! unknown origin: {0:s}\n'.format(', '.join(use[file][None])))
	stdout.write('      private\n')
	public, internal = [], []
	for routine in routines:
		if {file} != called_in[routine]:
			public.append(routine)
		else:
			internal.append(routine)
	stdout.write('      public {0:s}\n'.format(', '.join(public)))
	if internal:
		stdout.write('      !! internal: {0:s}\n'.format(', '.join(internal)))
	stdout.write('      contains\n')
	stdout.write('      \n')
	stdout.write('      end module {0:s}\n'.format(file.split('.')[0]))

"""
Apply the changes if requested. This will probably need manual fixing after.
"""
if '--apply' in argv[1:]:
	for target in targets:
		if target.startswith('-') or getcwd() not in abspath(target):
			continue
		with open(target, 'r') as fh:
			content = fh.read()
		if 'module' in content.lower():
			stderr.write('{0:s} already has a module, not applying\n'.format(target))
			continue

<|endoftext|>


<commit_before>"""
Dynamic inheritance for class members.
"""

from _compat import *

class InheritMetaBase(type):

    def _kick_mro_update(cls):
        cls.__bases__ = cls.__bases__

    def mro(cls):
        meta = type(cls)
        for base in cls.__bases__:
            base_meta = type(base)
            if not issubclass(meta, base_meta):
                # Impose more strict requirements on changing __bases__,
                # see: http://bugs.python.org/issue21919
                raise TypeError("metaclass conflict: "
                                "the metaclass of a derived class "
                                "must be a (non-strict) subclass "
                                "of the metaclasses of all its bases")

        return super(InheritMetaBase, cls).mro()

class InheritOwnerMeta(InheritMetaBase):
    pass

class InheritValueMeta(InheritMetaBase):
    pass

<commit_msg>inherit: calculating new bases for an owned attr<commit_after>"""
Dynamic inheritance for class members.
"""

from _compat import *
from util.itertools import pop_iter
from util.itertools import unique

class InheritMetaBase(type):

    def _kick_mro_update(cls):
        cls.__bases__ = cls.__bases__

    def mro(cls):
        meta = type(cls)
        for base in cls.__bases__:
            base_meta = type(base)
            if not issubclass(meta, base_meta):
                # Impose more strict requirements on changing __bases__,
                # see: http://bugs.python.org/issue21919
                raise TypeError("metaclass conflict: "
                                "the metaclass of a derived class "
                                "must be a (non-strict) subclass "
                                "of the metaclasses of all its bases")

        return super(InheritMetaBase, cls).mro()

class InheritOwnerMeta(InheritMetaBase):

    def __new__(mcls, name, bases, attrs):
        cls = super(InheritOwnerMeta, mcls).__new__(mcls, name, bases, attrs)

        for attr, value in iteritems(cls.__dict__):
            if isinstance(value, InheritValueMeta):
                if value._inherit_owner is not None:
                    raise ValueError

                cls.__inherit_attr(attr, value)

        return cls

    def __inherit_attr(cls, attr, value):
        value._inherit_owner = cls
        value.__bases__ = cls.__bases_for_attr(attr)

    def __bases_for_attr(cls, attr):
        base_ids = set()
        todo = list(cls.__bases__)
        for base in unique(pop_iter(todo)):
            if attr in base.__dict__:
                base_ids.add(id(base))
            else:
                todo += base.__bases__

        return [base for base in cls.__mro__ if id(base) in base_ids]

class InheritValueMeta(InheritMetaBase):

    def __init__(cls, name, bases, attrs):
        super(InheritValueMeta, cls).__init__(name, bases, attrs)
        cls._inherit_owner = None

<|endoftext|>