<commit_before>"""Routines for batch processing of cells (v2)."""

import logging
import pathlib
import shutil
import warnings
import os

import pandas as pd

from cellpy import prms
from cellpy import log
from cellpy.utils.batch_tools.batch_exporters import CSVExporter
from cellpy.utils.batch_tools.batch_experiments import CyclingExperiment
from cellpy.utils.batch_tools.batch_plotters import CyclingSummaryPlotter
from cellpy.utils.batch_tools.batch_analyzers import OCVRelaxationAnalyzer
from cellpy.utils.batch_tools.dumpers import ram_dumper

logger = logging.getLogger(__name__)
logging.captureWarnings(True)

COLUMNS_SELECTED_FOR_VIEW = ["masses", "total_masses", "loadings"]

class Batch:
    def __init__(self, *args, **kwargs):
        default_log_level = kwargs.pop("log_level", None)
        if default_log_level is not None:
            log.setup_logging(
                custom_log_dir=prms.Paths.filelogdir, default_level=default_log_level
            )

        db_reader = kwargs.pop("db_reader", "default")

        file_name = kwargs.pop("file_name", None)

        logger.debug("creating CyclingExperiment")
        self.experiment = CyclingExperiment(db_reader=db_reader)

        if not file_name:
            if len(args) > 0:
                self.experiment.journal.name = args[0]

            if len(args) > 1:
                self.experiment.journal.project = args[1]

            for key in kwargs:
                if key == "name":
                    self.experiment.journal.name = kwargs[key]
                elif key == "project":
                    self.experiment.journal.project = kwargs[key]
                elif key == "batch_col":
                    self.experiment.journal.batch_col = kwargs[key]

        else:
            self.experiment.journal.from_file(file_name=file_name)

        self.exporter = CSVExporter()
        self.exporter._assign_dumper(ram_dumper)
        self.exporter.assign(self.experiment)
        self.plotter = CyclingSummaryPlotter()
        self.plotter.assign(self.experiment)
        self._journal_name = self.journal_name

    def __str__(self):
        return str(self.experiment)

    def show_pages(self, number_of_rows=5):
        warnings.warn("Deprecated - use pages.head() instead", DeprecationWarning)
        return self.experiment.journal.pages.head(number_of_rows)

    @property
    def view(self):
        warnings.warn("Deprecated - use report instead", DeprecationWarning)
        pages = self.experiment.journal.pages
        pages = pages[COLUMNS_SELECTED_FOR_VIEW]
        return pages

    @property
    def report(self):
        pages = self.experiment.journal.pages
        pages = pages[COLUMNS_SELECTED_FOR_VIEW]
        return pages

    @property
    def info_file(self):
        # renamed to journal_name
        warnings.warn("Deprecated - use journal_name instead", DeprecationWarning)
        return self.experiment.journal.file_name

    @property
    def journal_name(self):
        return self.experiment.journal.file_name

    @property
    def summaries(self):
        # should add link-mode?
        try:
            keys = [df.name for df in self.experiment.memory_dumped["summary_engine"]]
            return pd.concat(
                self.experiment.memory_dumped["summary_engine"], keys=keys, axis=1
            )
        except KeyError:
            logging.info("no summary exists")

    @property
    def summary_headers(self):
        try:
            return self.summaries.columns.get_level_values(0)
        except AttributeError:
            logging.info("can't get any columns")

    @property
    def cell_names(self):
        return self.experiment.cell_names

    @property
    def raw_headers(self):
        return self.experiment.data[0].cell.raw.columns

    @property
    def step_headers(self):
        return self.experiment.data[0].cell.steps.columns

    @property
    def pages(self):
        return self.experiment.journal.pages

    @pages.setter
    def pages(self, df):
        self.experiment.journal.pages = df

    def create_journal(self, description=None, from_db=True):
        """Create journal pages.

        This method is a wrapper for the different Journal methods for making
        journal pages (Batch.experiment.journal.xxx).

        Args:
            description: the information and meta-data needed to generate the journal
                pages.
                "empty": create an empty journal
                dictionary: create journal pages from a dictionary (not implemented yet)
                pd.DataFrame: create  journal pages from a pandas DataFrame
                    (not implemented yet)
                filename.xlxs: create journal pages from an excel file
                    (not implemented yet)
                filename.json: load cellpy batch file
                    (not implemented yet, use .experiment.journal.from_file() instead).
            from_db (bool): Deprecation Warning: this parameter will be removed as it is
                the default anyway. Generate the pages from a db (the default option).
                This will be over-ridden if description is given.
        """

        logging.debug("Creating a journal")
        logging.debug(f"description: {description}")
        logging.debug(f"from_db: {from_db}")
        logging.info(f"name: {self.experiment.journal.name}")
        logging.info(f"project: {self.experiment.journal.project}")

        if description is not None:
            from_db = False

        if from_db:
            self.experiment.journal.from_db()
            self.experiment.journal.to_file()

        else:
            is_file = False
            if isinstance(description, str):
                if description.lower() == "empty":
                    logging.debug("creating empty journal pages")

                if pathlib.Path(description).is_file():
                    is_file = True

            elif isinstance(description, pathlib.Path):
                logging.debug("pathlib.Path object given")
                is_file = True

            elif isinstance(description, pd.DataFrame):
                logging.debug("pandas DataFrame given")

            elif isinstance(description, dict):
                logging.debug("dictionary given")

            else:
                logging.debug(
                    "the option you provided seems to be either of "
                    "an unknown type or a file not found"
                )
                logging.info(
                    "did not understand the option - creating empty journal pages"
                )

            if is_file:
                logging.info(f"loading file {description}")
                logging.debug("not implemented yet")

            # empty journal pages (this might go further up)
            self.experiment.journal.pages = (
                self.experiment.journal._create_empty_pages()
            )

            # finally
            self.experiment.journal.generate_folder_names()
            self.experiment.journal.paginate()

    def create_folder_structure(self):
        warnings.warn("Deprecated - use paginate instead.", DeprecationWarning)
        self.experiment.journal.paginate()
        logging.info("created folders")

    def paginate(self):
        self.experiment.journal.paginate()
        logging.info("created folders")

    def save_journal(self):
        # rename to: save_journal
        # Remark! Got an recursive error when running on mac.
        self.experiment.journal.to_file()
        logging.info("saving journal pages")

    def duplicate_journal(self):
        journal_name = pathlib.Path(self.experiment.journal.file_name)
        if not journal_name.is_file():
            print("No journal saved")
            return
        new_journal_name = journal_name.name
        shutil.copy(journal_name, new_journal_name)

    def duplicate_cellpy_files(self, location="standard"):
        """Copy the cellpy files and make a journal with the new names available in
        the current folder.

        Args:
            location: where to copy the files. Either choose among the following
                options:
                "standard": data/interim folder
                "here": current directory
                "cellpydatadir": the stated cellpy data dir in your settings (prms)
            or if the location is not one of the above, use the actual value of the
                location argument.

        Returns:
            The updated journal pages.
        """
        pages = self.experiment.journal.pages
        cellpy_file_dir = pathlib.Path(prms.Paths.cellpydatadir)

        if location == "standard":
            batch_data_dir = pathlib.Path("data") / "interim"

        elif location == "here":
            batch_data_dir = pathlib.Path(".")

        elif location == "cellpydatadir":
            batch_data_dir = cellpy_file_dir

        else:
            batch_data_dir = location

        def _new_file_path(x):
            return str(batch_data_dir / pathlib.Path(x).name)

        # update the journal pages
        columns = pages.columns
        pages["new_cellpy_file_names"] = pages.cellpy_file_names.apply(_new_file_path)

        # copy the cellpy files
        for n, row in pages.iterrows():
            print(f"{row.cellpy_file_names} -> {row.new_cellpy_file_names}")
            try:
                from_file = row.cellpy_file_names
                to_file = row.new_cellpy_file_names
                os.makedirs(os.path.dirname(to_file), exist_ok=True)
                shutil.copy(from_file, to_file)
            except shutil.SameFileError:
                print("Same file! No point in copying")

        # save the journal pages
        pages["cellpy_file_names"] = pages["new_cellpy_file_names"]
        self.experiment.journal.pages = pages[columns]
        journal_file_name = pathlib.Path(self.experiment.journal.file_name).name
        print(f"saving journal to {journal_file_name}")
        self.experiment.journal.to_file(journal_file_name)

        # return pages

    # TODO: load_journal
    # TODO: list_journals?

    def link(self):
        self.experiment.link()

    def load(self):
        # does the same as update
        warnings.warn("Deprecated - use update instead.", DeprecationWarning)
        self.experiment.update()

    def update(self):
        self.experiment.update()

    def make_summaries(self):
        # rename to: combine_summaries
        # also: need a similar function that does not save to csv
        self.exporter.do()

    def plot_summaries(self):
        if prms.Batch.backend == "bokeh":

            try:
                import bokeh.plotting

                if prms.Batch.notebook:
                    bokeh.plotting.output_notebook()

            except ModuleNotFoundError:
                prms.Batch.backend = "matplotlib"
                logging.warning(
                    "could not find the bokeh " "module -> using matplotlib instead"
                )

        self.plotter.do()

def main():
    from pathlib import Path

    # Use these when working on my work PC:
    test_data_path = r"C:\Scripting\MyFiles\development_cellpy\testdata"
    out_data_path = r"C:\Scripting\Processing\Test\out"

    # Use these when working on my MacBook:
    test_data_path = "/Users/jepe/scripting/cellpy/testdata"
    out_data_path = "/Users/jepe/cellpy_data"

    test_data_path = Path(test_data_path)
    out_data_path = Path(out_data_path)

    print("---SETTING SOME PRMS---")
    prms.Paths["db_filename"] = "cellpy_db.xlsx"
    prms.Paths["cellpydatadir"] = test_data_path / "hdf5"
    prms.Paths["outdatadir"] = out_data_path
    prms.Paths["rawdatadir"] = test_data_path / "data"
    prms.Paths["db_path"] = test_data_path / "db"
    prms.Paths["filelogdir"] = test_data_path / "log"

    project = "prebens_experiment"
    name = "test"
    batch_col = "b01"

    print("---INITIALISATION OF BATCH---")
    b = init(name, project, batch_col=batch_col)
    b.experiment.export_raw = True
    b.experiment.export_cycles = True
    print("*creating info df*")
    b.create_journal()
    print("*creating folder structure*")
    b.create_folder_structure()
    print("*load and save*")
    b.update()
    print("*make summaries*")
    b.make_summaries()
    summaries = b.experiment.memory_dumped
    print("*plotting summaries*")
    b.plot_summaries()
    print("*using special features*")
    print(" - select_ocv_points")
    analyzer = OCVRelaxationAnalyzer()
    analyzer.assign(b.experiment)
    analyzer.do()
    ocv_df_list = analyzer.farms[0]
    for df in ocv_df_list:
        df_up = df.loc[df.type == "ocvrlx_up", :]
        df_down = df.loc[df.type == "ocvrlx_down", :]
        print(df_up)
    print("---FINISHED---")

def init(*args, **kwargs):
    """Returns an initialized instance of the Batch class.

    Args:
        *args: passed directly to Batch()
            name: name of batch
            project: name of project
            batch_col: batch column identifier
        **kwargs:
            file_name: json file if loading from pages
            default_log_level: "INFO" or "DEBUG"
            The rest is passed directly to Batch()

    Usage:
        >>> empty_batch = Batch.init(db_reader=None)
        >>> batch_from_file = Batch.init(file_name="cellpy_batch_my_experiment.json")
        >>> normal_init_of_batch = Batch.init()
    """
    # set up cellpy logger
    default_log_level = kwargs.pop("default_log_level", "INFO")
    file_name = kwargs.pop("file_name", None)

    log.setup_logging(
        custom_log_dir=prms.Paths["filelogdir"], default_level=default_log_level
    )
    logging.debug(f"returning Batch(kwargs: {kwargs})")
    if file_name is not None:
        kwargs.pop("db_reader", None)
        return Batch(*args, file_name=file_name, db_reader=None, **kwargs)
    return Batch(*args, **kwargs)

if __name__ == "__main__":
    print("---IN BATCH 2 MAIN---")
    main()
<commit_msg>some help txt<commit_after>"""Routines for batch processing of cells (v2)."""

import logging
import pathlib
import shutil
import warnings
import os

import pandas as pd

from cellpy import prms
from cellpy import log
from cellpy.utils.batch_tools.batch_exporters import CSVExporter
from cellpy.utils.batch_tools.batch_experiments import CyclingExperiment
from cellpy.utils.batch_tools.batch_plotters import CyclingSummaryPlotter
from cellpy.utils.batch_tools.batch_analyzers import OCVRelaxationAnalyzer
from cellpy.utils.batch_tools.dumpers import ram_dumper

logger = logging.getLogger(__name__)
logging.captureWarnings(True)

COLUMNS_SELECTED_FOR_VIEW = ["masses", "total_masses", "loadings"]

class Batch:
    def __init__(self, *args, **kwargs):
        default_log_level = kwargs.pop("log_level", None)
        if default_log_level is not None:
            log.setup_logging(
                custom_log_dir=prms.Paths.filelogdir, default_level=default_log_level
            )

        db_reader = kwargs.pop("db_reader", "default")

        file_name = kwargs.pop("file_name", None)

        logger.debug("creating CyclingExperiment")
        self.experiment = CyclingExperiment(db_reader=db_reader)

        if not file_name:
            if len(args) > 0:
                self.experiment.journal.name = args[0]

            if len(args) > 1:
                self.experiment.journal.project = args[1]

            for key in kwargs:
                if key == "name":
                    self.experiment.journal.name = kwargs[key]
                elif key == "project":
                    self.experiment.journal.project = kwargs[key]
                elif key == "batch_col":
                    self.experiment.journal.batch_col = kwargs[key]

        else:
            self.experiment.journal.from_file(file_name=file_name)

        self.exporter = CSVExporter()
        self.exporter._assign_dumper(ram_dumper)
        self.exporter.assign(self.experiment)
        self.plotter = CyclingSummaryPlotter()
        self.plotter.assign(self.experiment)
        self._journal_name = self.journal_name

    def __str__(self):
        return str(self.experiment)

    def show_pages(self, number_of_rows=5):
        warnings.warn("Deprecated - use pages.head() instead", DeprecationWarning)
        return self.experiment.journal.pages.head(number_of_rows)

    @property
    def view(self):
        warnings.warn("Deprecated - use report instead", DeprecationWarning)
        pages = self.experiment.journal.pages
        pages = pages[COLUMNS_SELECTED_FOR_VIEW]
        return pages

    @property
    def report(self):
        pages = self.experiment.journal.pages
        pages = pages[COLUMNS_SELECTED_FOR_VIEW]
        return pages

    @property
    def info_file(self):
        # renamed to journal_name
        warnings.warn("Deprecated - use journal_name instead", DeprecationWarning)
        return self.experiment.journal.file_name

    @property
    def journal_name(self):
        return self.experiment.journal.file_name

    @property
    def summaries(self):
        # should add link-mode?
        try:
            keys = [df.name for df in self.experiment.memory_dumped["summary_engine"]]
            return pd.concat(
                self.experiment.memory_dumped["summary_engine"], keys=keys, axis=1
            )
        except KeyError:
            logging.info("no summary exists")

    @property
    def summary_headers(self):
        try:
            return self.summaries.columns.get_level_values(0)
        except AttributeError:
            logging.info("can't get any columns")

    @property
    def cell_names(self):
        return self.experiment.cell_names

    @property
    def raw_headers(self):
        return self.experiment.data[0].cell.raw.columns

    @property
    def step_headers(self):
        return self.experiment.data[0].cell.steps.columns

    @property
    def pages(self):
        return self.experiment.journal.pages

    @pages.setter
    def pages(self, df):
        self.experiment.journal.pages = df

    def create_journal(self, description=None, from_db=True):
        """Create journal pages.

        This method is a wrapper for the different Journal methods for making
        journal pages (Batch.experiment.journal.xxx). It is under development. If you
        want to use 'advanced' options (i.e. not loading from a db), please consider
        using the methods available in Journal for now.

        Args:
            description: the information and meta-data needed to generate the journal
                pages.
                "empty": create an empty journal
                dictionary: create journal pages from a dictionary (not implemented yet)
                pd.DataFrame: create  journal pages from a pandas DataFrame
                    (not implemented yet)
                filename.xlxs: create journal pages from an excel file
                    (not implemented yet)
                filename.json: load cellpy batch file
                    (not implemented yet, use .experiment.journal.from_file() instead).
            from_db (bool): Deprecation Warning: this parameter will be removed as it is
                the default anyway. Generate the pages from a db (the default option).
                This will be over-ridden if description is given.
        """

        logging.debug("Creating a journal")
        logging.debug(f"description: {description}")
        logging.debug(f"from_db: {from_db}")
        logging.info(f"name: {self.experiment.journal.name}")
        logging.info(f"project: {self.experiment.journal.project}")

        if description is not None:
            from_db = False

        if from_db:
            self.experiment.journal.from_db()
            self.experiment.journal.to_file()

        else:
            is_file = False
            if isinstance(description, str):
                if description.lower() == "empty":
                    logging.debug("creating empty journal pages")

                if pathlib.Path(description).is_file():
                    is_file = True

            elif isinstance(description, pathlib.Path):
                logging.debug("pathlib.Path object given")
                is_file = True

            elif isinstance(description, pd.DataFrame):
                logging.debug("pandas DataFrame given")

            elif isinstance(description, dict):
                logging.debug("dictionary given")

            else:
                logging.debug(
                    "the option you provided seems to be either of "
                    "an unknown type or a file not found"
                )
                logging.info(
                    "did not understand the option - creating empty journal pages"
                )

            if is_file:
                logging.info(f"loading file {description}")
                logging.debug("not implemented yet")

            # empty journal pages (this might go further up)
            self.experiment.journal.pages = (
                self.experiment.journal._create_empty_pages()
            )

            # finally
            self.experiment.journal.generate_folder_names()
            self.experiment.journal.paginate()

    def create_folder_structure(self):
        warnings.warn("Deprecated - use paginate instead.", DeprecationWarning)
        self.experiment.journal.paginate()
        logging.info("created folders")

    def paginate(self):
        self.experiment.journal.paginate()
        logging.info("created folders")

    def save_journal(self):
        # rename to: save_journal
        # Remark! Got an recursive error when running on mac.
        self.experiment.journal.to_file()
        logging.info("saving journal pages")

    def duplicate_journal(self):
        journal_name = pathlib.Path(self.experiment.journal.file_name)
        if not journal_name.is_file():
            print("No journal saved")
            return
        new_journal_name = journal_name.name
        shutil.copy(journal_name, new_journal_name)

    def duplicate_cellpy_files(self, location="standard"):
        """Copy the cellpy files and make a journal with the new names available in
        the current folder.

        Args:
            location: where to copy the files. Either choose among the following
                options:
                "standard": data/interim folder
                "here": current directory
                "cellpydatadir": the stated cellpy data dir in your settings (prms)
            or if the location is not one of the above, use the actual value of the
                location argument.

        Returns:
            The updated journal pages.
        """
        pages = self.experiment.journal.pages
        cellpy_file_dir = pathlib.Path(prms.Paths.cellpydatadir)

        if location == "standard":
            batch_data_dir = pathlib.Path("data") / "interim"

        elif location == "here":
            batch_data_dir = pathlib.Path(".")

        elif location == "cellpydatadir":
            batch_data_dir = cellpy_file_dir

        else:
            batch_data_dir = location

        def _new_file_path(x):
            return str(batch_data_dir / pathlib.Path(x).name)

        # update the journal pages
        columns = pages.columns
        pages["new_cellpy_file_names"] = pages.cellpy_file_names.apply(_new_file_path)

        # copy the cellpy files
        for n, row in pages.iterrows():
            print(f"{row.cellpy_file_names} -> {row.new_cellpy_file_names}")
            try:
                from_file = row.cellpy_file_names
                to_file = row.new_cellpy_file_names
                os.makedirs(os.path.dirname(to_file), exist_ok=True)
                shutil.copy(from_file, to_file)
            except shutil.SameFileError:
                print("Same file! No point in copying")

        # save the journal pages
        pages["cellpy_file_names"] = pages["new_cellpy_file_names"]
        self.experiment.journal.pages = pages[columns]
        journal_file_name = pathlib.Path(self.experiment.journal.file_name).name
        print(f"saving journal to {journal_file_name}")
        self.experiment.journal.to_file(journal_file_name)

        # return pages

    # TODO: load_journal
    # TODO: list_journals?

    def link(self):
        self.experiment.link()

    def load(self):
        # does the same as update
        warnings.warn("Deprecated - use update instead.", DeprecationWarning)
        self.experiment.update()

    def update(self):
        self.experiment.update()

    def make_summaries(self):
        # rename to: combine_summaries
        # also: need a similar function that does not save to csv
        self.exporter.do()

    def plot_summaries(self):
        if prms.Batch.backend == "bokeh":

            try:
                import bokeh.plotting

                if prms.Batch.notebook:
                    bokeh.plotting.output_notebook()

            except ModuleNotFoundError:
                prms.Batch.backend = "matplotlib"
                logging.warning(
                    "could not find the bokeh " "module -> using matplotlib instead"
                )

        self.plotter.do()

def main():
    from pathlib import Path

    # Use these when working on my work PC:
    test_data_path = r"C:\Scripting\MyFiles\development_cellpy\testdata"
    out_data_path = r"C:\Scripting\Processing\Test\out"

    # Use these when working on my MacBook:
    test_data_path = "/Users/jepe/scripting/cellpy/testdata"
    out_data_path = "/Users/jepe/cellpy_data"

    test_data_path = Path(test_data_path)
    out_data_path = Path(out_data_path)

    print("---SETTING SOME PRMS---")
    prms.Paths["db_filename"] = "cellpy_db.xlsx"
    prms.Paths["cellpydatadir"] = test_data_path / "hdf5"
    prms.Paths["outdatadir"] = out_data_path
    prms.Paths["rawdatadir"] = test_data_path / "data"
    prms.Paths["db_path"] = test_data_path / "db"
    prms.Paths["filelogdir"] = test_data_path / "log"

    project = "prebens_experiment"
    name = "test"
    batch_col = "b01"

    print("---INITIALISATION OF BATCH---")
    b = init(name, project, batch_col=batch_col)
    b.experiment.export_raw = True
    b.experiment.export_cycles = True
    print("*creating info df*")
    b.create_journal()
    print("*creating folder structure*")
    b.create_folder_structure()
    print("*load and save*")
    b.update()
    print("*make summaries*")
    b.make_summaries()
    summaries = b.experiment.memory_dumped
    print("*plotting summaries*")
    b.plot_summaries()
    print("*using special features*")
    print(" - select_ocv_points")
    analyzer = OCVRelaxationAnalyzer()
    analyzer.assign(b.experiment)
    analyzer.do()
    ocv_df_list = analyzer.farms[0]
    for df in ocv_df_list:
        df_up = df.loc[df.type == "ocvrlx_up", :]
        df_down = df.loc[df.type == "ocvrlx_down", :]
        print(df_up)
    print("---FINISHED---")

def init(*args, **kwargs):
    """Returns an initialized instance of the Batch class.

    Args:
        *args: passed directly to Batch()
            name: name of batch
            project: name of project
            batch_col: batch column identifier
        **kwargs:
            file_name: json file if loading from pages
            default_log_level: "INFO" or "DEBUG"
            The rest is passed directly to Batch()

    Usage:
        >>> empty_batch = Batch.init(db_reader=None)
        >>> batch_from_file = Batch.init(file_name="cellpy_batch_my_experiment.json")
        >>> normal_init_of_batch = Batch.init()
    """
    # set up cellpy logger
    default_log_level = kwargs.pop("default_log_level", "INFO")
    file_name = kwargs.pop("file_name", None)

    log.setup_logging(
        custom_log_dir=prms.Paths["filelogdir"], default_level=default_log_level
    )
    logging.debug(f"returning Batch(kwargs: {kwargs})")
    if file_name is not None:
        kwargs.pop("db_reader", None)
        return Batch(*args, file_name=file_name, db_reader=None, **kwargs)
    return Batch(*args, **kwargs)

if __name__ == "__main__":
    print("---IN BATCH 2 MAIN---")
    main()
<|endoftext|>


<commit_before>import os, sys

class ModuleInterceptor(object):
    """
    This class is used to intercept an unset attribute of a module to perfrom a callback. The
    callback will only be performed if the attribute does not exist on the module. Any error raised
    in the callback will cause the original AttributeError to be raised.

        >>> def cb( module, attr):
        >>>     if attr == 'this':
        >>>         print "intercepted"
        >>>     else:
        >>>         raise ValueError
        >>> sys.modules[__name__] = ModuleInterceptor(__name__, cb)

    The class does not work when imported into the main namespace.
    """
    def __init__(self, moduleName, callback):
        self.module = __import__( moduleName , globals(), locals(), [''] )
        self.callback = callback
    def __getattr__(self, attr):
        try:
            return getattr(self.module, attr)
        except AttributeError, msg:
            try:
                self.callback( self.module, attr)
            except:
                raise AttributeError, msg
#-----------------------------------------------
#  Pymel Internals
#-----------------------------------------------
def pythonToMel(arg):
    if isinstance(arg,basestring):
        return '"%s"' % cmds.encodeString(arg)
    elif isIterable(arg):
        return '{%s}' % ','.join( map( pythonToMel, arg) )
    return unicode(arg)

def capitalize(s):
    return s[0].upper() + s[1:]

def uncapitalize(s):
    return s[0].lower() + s[1:]

def cacheProperty(getter, attr_name, fdel=None, doc=None):
    """a property type for getattr functions that only need to be called once per instance.
        future calls to getattr for this property will return the previous non-null value.
        attr_name is the name of an attribute in which to store the cached values"""
    def fget(obj):
        val = None

        if hasattr(obj,attr_name):
            val = getattr(obj, attr_name)
            #print "cacheProperty: retrieving cache: %s.%s = %s" % (obj, attr_name, val)

        if val is None:
            #print "cacheProperty: running getter: %s.%s" %  (obj, attr_name)
            val = getter(obj)
            #print "cacheProperty: caching: %s.%s = %s" % (obj, attr_name, val)
            setattr(obj, attr_name, val )
        return val

    def fset(obj, val):
        #print "cacheProperty: setting attr %s.%s=%s" % (obj, attr_name, val)
        setattr(obj, attr_name, val)

    return property( fget, fset, fdel, doc)

def moduleDir():
    return os.path.dirname( os.path.dirname( sys.modules[__name__].__file__ ) )
    #return os.path.split( sys.modules[__name__].__file__ )[0]

def timer( command='pass', number=10, setup='import pymel' ):
    import timeit
    t = timeit.Timer(command, setup)
    time = t.timeit(number=number)
    print "command took %.2f sec to execute" % time
    return time

def toZip( directory, zipFile ):
    """Sample for storing directory to a ZipFile"""
    import zipfile

    zipFile = path(zipFile)
    if zipFile.exists(): zipFile.remove()

    z = zipfile.ZipFile(
        zipFile, 'w', compression=zipfile.ZIP_DEFLATED
    )
    if not directory.endswith(os.sep):
        directory += os.sep

    directory = path(directory)

    for subdir in directory.dirs('[a-z]*') + [directory]:
        print "adding ", subdir
        for fname in subdir.files('[a-z]*'):
            archiveName = fname.replace( directory, '' )
            z.write( fname, archiveName, zipfile.ZIP_DEFLATED )
    z.close()
    return zipFile
<commit_msg>added escape and unescape to util.common, for increasing and decreasing string escape level<commit_after>import os, sys, re
from re import escape

class ModuleInterceptor(object):
    """
    This class is used to intercept an unset attribute of a module to perfrom a callback. The
    callback will only be performed if the attribute does not exist on the module. Any error raised
    in the callback will cause the original AttributeError to be raised.

        >>> def cb( module, attr):
        >>>     if attr == 'this':
        >>>         print "intercepted"
        >>>     else:
        >>>         raise ValueError
        >>> sys.modules[__name__] = ModuleInterceptor(__name__, cb)

    The class does not work when imported into the main namespace.
    """
    def __init__(self, moduleName, callback):
        self.module = __import__( moduleName , globals(), locals(), [''] )
        self.callback = callback
    def __getattr__(self, attr):
        try:
            return getattr(self.module, attr)
        except AttributeError, msg:
            try:
                self.callback( self.module, attr)
            except:
                raise AttributeError, msg
#-----------------------------------------------
#  Pymel Internals
#-----------------------------------------------
def pythonToMel(arg):
    if isinstance(arg,basestring):
        return '"%s"' % cmds.encodeString(arg)
        #return '"%s"' % re.escape(arg)
    elif isIterable(arg):
        return '{%s}' % ','.join( map( pythonToMel, arg) )
    return unicode(arg)

def capitalize(s):
    return s[0].upper() + s[1:]

def uncapitalize(s):
    return s[0].lower() + s[1:]

def unescape( string ):
    tokens = re.split( r'(\\*)"',  string )
    for i in range(1,len(tokens),2 ):
        if tokens[i]:
            tokens[i] = tokens[i][:-1]+'"'
    return ''.join(tokens)

def cacheProperty(getter, attr_name, fdel=None, doc=None):
    """a property type for getattr functions that only need to be called once per instance.
        future calls to getattr for this property will return the previous non-null value.
        attr_name is the name of an attribute in which to store the cached values"""
    def fget(obj):
        val = None

        if hasattr(obj,attr_name):
            val = getattr(obj, attr_name)
            #print "cacheProperty: retrieving cache: %s.%s = %s" % (obj, attr_name, val)

        if val is None:
            #print "cacheProperty: running getter: %s.%s" %  (obj, attr_name)
            val = getter(obj)
            #print "cacheProperty: caching: %s.%s = %s" % (obj, attr_name, val)
            setattr(obj, attr_name, val )
        return val

    def fset(obj, val):
        #print "cacheProperty: setting attr %s.%s=%s" % (obj, attr_name, val)
        setattr(obj, attr_name, val)

    return property( fget, fset, fdel, doc)

def moduleDir():
    return os.path.dirname( os.path.dirname( sys.modules[__name__].__file__ ) )
    #return os.path.split( sys.modules[__name__].__file__ )[0]

def timer( command='pass', number=10, setup='import pymel' ):
    import timeit
    t = timeit.Timer(command, setup)
    time = t.timeit(number=number)
    print "command took %.2f sec to execute" % time
    return time

def toZip( directory, zipFile ):
    """Sample for storing directory to a ZipFile"""
    import zipfile

    zipFile = path(zipFile)
    if zipFile.exists(): zipFile.remove()

    z = zipfile.ZipFile(
        zipFile, 'w', compression=zipfile.ZIP_DEFLATED
    )
    if not directory.endswith(os.sep):
        directory += os.sep

    directory = path(directory)

    for subdir in directory.dirs('[a-z]*') + [directory]:
        print "adding ", subdir
        for fname in subdir.files('[a-z]*'):
            archiveName = fname.replace( directory, '' )
            z.write( fname, archiveName, zipfile.ZIP_DEFLATED )
    z.close()
    return zipFile
<|endoftext|>


<commit_before>import numpy as np

def movingaverage(interval, window_size=14, pad=False):
    ''' padded moving average '''
    window = np.ones(int(window_size))/float(window_size)
    ma= np.convolve(interval, window, 'same')

    # pad the end properly
    if pad:
        w = window_size
        x = np.array(interval)
        n = len(ma)
        start = n-w

        for i in range(start, start+w):
            seq=x[i-w:i]
            ma[i]=seq.sum()/len(seq)
    return ma
<commit_msg>set padding by default<commit_after>import numpy as np

def movingaverage(interval, window_size=14, pad=True):
    ''' padded moving average '''
    window = np.ones(int(window_size))/float(window_size)
    ma= np.convolve(interval, window, 'same')

    # pad the end properly
    if pad:
        w = window_size
        x = np.array(interval)
        n = len(ma)
        start = n-w

        for i in range(start, start+w):
            seq=x[i-w:i]
            ma[i]=seq.sum()/len(seq)
    return ma
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 Jan-Philip Gehrcke. See LICENSE file for details.

import os
import sys
import time
from datetime import datetime
from itertools import chain
from random import randint, shuffle
import numpy as np
from matplotlib import pyplot as plt

from timegaps import FileSystemEntry, TimeFilter

import logging
logging.basicConfig(
    format='%(asctime)s,%(msecs)-6.1f [%(process)-5d]%(funcName)s# %(message)s',
    datefmt='%H:%M:%S')
log = logging.getLogger()
log.setLevel(logging.DEBUG)

def test_fixed_rules_8_per_cat_with_N_items(N):
    t0 = time.time()
    now = time.time()
    fses = list(fsegen(ref=now, N_per_cat=N, max_timecount=9))
    shuffle(fses)
    nbr_fses = len(fses)
    n = 8
    rules = {
        "years": n,
        "months": n,
        "weeks": n,
        "days": n,
        "hours": n,
        "recent": n
        }
    sduration = time.time() - t0
    log.info("Setup duration: %.3f s", sduration)
    t0 = time.time()
    a, r = TimeFilter(rules, now).filter(fses)
    duration = time.time() - t0
    return nbr_fses, duration

def bench(funcname, samplesize, Ns):
    log.info("Benching '%s' with Ns %s", funcname, Ns)
    durations = []
    nbrs_fses = []
    for N in Ns:
        dursamples = []
        nbrs_fses_samples = []
        durations.append(dursamples)
        nbrs_fses.append(nbrs_fses_samples)
        for _ in xrange(samplesize):
            nbr_fses, duration = globals()[funcname](N)
            dursamples.append(duration)
            nbrs_fses_samples.append(nbr_fses)
    log.info("Durations: %s", durations)
    log.info("FSE numbers: %s", nbrs_fses)
    duration_means = [np.mean(_) for _ in durations]
    duration_stddevs = [np.std(_) for _ in durations]

    # The nbrs_fses_samples should be boring, repetitive lists:
    for l in nbrs_fses:
        assert len(set(l)) == 1
    # Reduce.
    nbrs_fses = [l[0] for l in nbrs_fses]

    n_per_sec = nbrs_fses[-1]/duration_means[-1]
    linstring = "If linear, from last value pair: %i items/s" % n_per_sec
    log.info(linstring)
    log.info("Plotting durations vs. Ns.")
    plt.errorbar(
        x=nbrs_fses, y=duration_means, yerr=duration_stddevs, marker='o')
    plt.title("%s\n%s" % (funcname, linstring), fontsize=10)
    plt.xlabel("number of items")
    plt.ylabel("duration [s]")
    fname = "%s.png" % funcname
    log.info("Writing %s.", fname)
    plt.savefig(fname, dpi=200)

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] == "--short":
            bench("test_fixed_rules_8_per_cat_with_N_items", 2,
            (3*10**4, ))#, 10**5, int(1.5*10**5), 2*10**5, 3*10**5))
    else:
        bench("test_fixed_rules_8_per_cat_with_N_items", 10,
        (10**3, 10**4, 5*10**4, 10**5, int(1.5*10**5), 2*10**5, 3*10**5))

class FileSystemEntryMock(FileSystemEntry):
    def __init__(self, modtime):
        self.modtime = modtime

    def __str__(self):
        return "%s(moddate: %s)" % (self.__class__.__name__, self.moddate)

    def __repr__(self):
        return "%s(modtime=%s)" % (self.__class__.__name__, self.modtime)

def nrandint(n, min, max):
    for _ in xrange(n):
        yield randint(min, max)

def fsegen(ref, N_per_cat, max_timecount):
    N = N_per_cat
    c = max_timecount
    nowminusXyears =   (ref-60*60*24*365*i for i in nrandint(N, 1, c))
    nowminusXmonths =  (ref-60*60*24*30 *i for i in nrandint(N, 1, c))
    nowminusXweeks =   (ref-60*60*24*7  *i for i in nrandint(N, 1, c))
    nowminusXdays =    (ref-60*60*24    *i for i in nrandint(N, 1, c))
    nowminusXhours =   (ref-60*60       *i for i in nrandint(N, 1, c))
    nowminusXseconds = (ref-1           *i for i in nrandint(N, 1, c))
    times = chain(
        nowminusXyears,
        nowminusXmonths,
        nowminusXweeks,
        nowminusXdays,
        nowminusXhours,
        nowminusXseconds,
        )
    return (FileSystemEntryMock(modtime=t) for t in times)

if __name__ == "__main__":
    main()
<commit_msg>utils/bench.py: improve usability.<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 Jan-Philip Gehrcke. See LICENSE file for details.

import os
import sys
import time
from datetime import datetime
from itertools import chain
from random import randint, shuffle
import numpy as np
from matplotlib import pyplot as plt

sys.path.insert(0, os.path.abspath('..'))
from timegaps.timegaps import FileSystemEntry
from timegaps.timefilter import TimeFilter

import logging
logging.basicConfig(
    format='%(asctime)s,%(msecs)-6.1f [%(process)-5d]%(funcName)s# %(message)s',
    datefmt='%H:%M:%S')
log = logging.getLogger()
log.setLevel(logging.DEBUG)

def test_fixed_rules_8_per_cat_with_N_items(N):
    t0 = time.time()
    now = time.time()
    fses = list(fsegen(ref=now, N_per_cat=N, max_timecount=9))
    shuffle(fses)
    nbr_fses = len(fses)
    n = 8
    rules = {
        "years": n,
        "months": n,
        "weeks": n,
        "days": n,
        "hours": n,
        "recent": n
        }
    sduration = time.time() - t0
    log.info("Setup duration: %.3f s", sduration)
    t0 = time.time()
    a, r = TimeFilter(rules, now).filter(fses)
    duration = time.time() - t0
    return nbr_fses, duration

def bench(funcname, samplesize, Ns):
    log.info("Benching '%s' with Ns %s", funcname, Ns)
    durations = []
    nbrs_fses = []
    for N in Ns:
        dursamples = []
        nbrs_fses_samples = []
        durations.append(dursamples)
        nbrs_fses.append(nbrs_fses_samples)
        for _ in xrange(samplesize):
            nbr_fses, duration = globals()[funcname](N)
            dursamples.append(duration)
            nbrs_fses_samples.append(nbr_fses)
    log.info("Durations: %s", durations)
    log.info("FSE numbers: %s", nbrs_fses)
    duration_means = [np.mean(_) for _ in durations]
    duration_stddevs = [np.std(_) for _ in durations]

    # The nbrs_fses_samples should be boring, repetitive lists:
    for l in nbrs_fses:
        assert len(set(l)) == 1
    # Reduce.
    nbrs_fses = [l[0] for l in nbrs_fses]

    n_per_sec = nbrs_fses[-1]/duration_means[-1]
    linstring = "From last data point: (mean from n=%s): %i items/s" % (
        Ns, n_per_sec)
    log.info(linstring)
    log.info("Last data point std. dev. of duration: %.3f s", duration_stddevs[-1])
    log.info("Plotting durations vs. Ns.")
    plt.errorbar(
        x=nbrs_fses, y=duration_means, yerr=duration_stddevs, marker='o')
    plt.title("%s\n%s" % (funcname, linstring), fontsize=10)
    plt.xlabel("number of items")
    plt.ylabel("duration [s]")
    fname = "%s.png" % funcname
    log.info("Writing %s.", fname)
    plt.savefig(fname, dpi=200)

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] == "--short":
            bench("test_fixed_rules_8_per_cat_with_N_items", 3,
            (4*10**4, ))#, 10**5, int(1.5*10**5), 2*10**5, 3*10**5))
    else:
        bench("test_fixed_rules_8_per_cat_with_N_items", 10,
        (10**3, 10**4, 5*10**4, 10**5, int(1.5*10**5), 2*10**5, 3*10**5))

class FileSystemEntryMock(FileSystemEntry):
    def __init__(self, modtime):
        self.modtime = modtime

    def __str__(self):
        return "%s(moddate: %s)" % (self.__class__.__name__, self.moddate)

    def __repr__(self):
        return "%s(modtime=%s)" % (self.__class__.__name__, self.modtime)

def nrandint(n, min, max):
    for _ in xrange(n):
        yield randint(min, max)

def fsegen(ref, N_per_cat, max_timecount):
    N = N_per_cat
    c = max_timecount
    nowminusXyears =   (ref-60*60*24*365*i for i in nrandint(N, 1, c))
    nowminusXmonths =  (ref-60*60*24*30 *i for i in nrandint(N, 1, c))
    nowminusXweeks =   (ref-60*60*24*7  *i for i in nrandint(N, 1, c))
    nowminusXdays =    (ref-60*60*24    *i for i in nrandint(N, 1, c))
    nowminusXhours =   (ref-60*60       *i for i in nrandint(N, 1, c))
    nowminusXseconds = (ref-1           *i for i in nrandint(N, 1, c))
    times = chain(
        nowminusXyears,
        nowminusXmonths,
        nowminusXweeks,
        nowminusXdays,
        nowminusXhours,
        nowminusXseconds,
        )
    return (FileSystemEntryMock(modtime=t) for t in times)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
IRC Parser - parse.py
"""

import inspect
import traceback
import re
import subprocess
import sys

SED_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:(.+?)/)?s/(.+?)/([^/]+)/?(?:([gixs]{0,4}))?")

def parse(msg):
    """ Parse an IRC protocol message """
    if msg.startswith("PING"):
        info = {"method": "PING", "arg": msg.split()[-1]}
    else:
        splitmsg = msg.split(" ", 2)
        info = {"method": splitmsg[1], "host": splitmsg[0][1:], "arg":
                splitmsg[2]}
    return info

def evaluate(self, nick, chan, msg):
    """ Evaluate python code. """
    try:
        output = eval(msg, globals(), locals())
        if output is not None:
            self.leo = output
            if type(self.leo) == tuple:
                self.leo = list(self.leo)
                output = list(output)
            return output
    except (NameError, SyntaxError):
        try:
            exec(msg, globals())
        except:
            try:
                exec(msg, locals())
            except:
                try:
                    exec(msg,globals(),locals())
                except:
                    exec(msg,locals(),globals())

###
##
## Sed part
##
###

def populate(sedobject, groups):
    group_types = ("target", "qual", "to_replace", "replacement", "flags")
    for k, v in zip(group_types, groups):
        setattr(sedobject, k, v)

def set_flags(sedobject, flags):
    i = 0
    count = 1

    if not flags:
        setattr(sedobject, "flags", i)
        setattr(sedobject, "count", count)
        return

    for item in flags:
        if item == "i":
            i |= re.IGNORECASE
        if item == "x":
            i |= re.X
        if item == "s":
            i |= re.S
        if item == "g":
            count = 0

    setattr(sedobject, "flags", i)
    setattr(sedobject, "count", count)

def debug(sedobject):
    for k,v in sedobject.__dict__.items():
        if not k.startswith("_") or not k.endswith("_"):
            print("%s: %s" % (k,v))

def get_message(bot, sedregex, nick, chan, qual=None):
    if not nick in bot.state.messages:
        return ""
    for message in bot.state.messages[nick]:
        if not message.channel == bot.state.channels[chan]:
            continue
        msg = message.msg
        try:
            if qual:
                if re.search(sedregex, msg) and re.search(qual, msg) and not SED_REGEX.search(msg):
                    return msg
            else:
                if re.search(sedregex, msg) and not SED_REGEX.search(msg):
                    return msg
        except BaseException:
            pass
    return ""

def sed(bot, nick, chan, msg):
    """ Perform the sedding """
    s = type("SedObject", tuple(), {})
    #[print(i) for i in (bot, nick, chan, msg)]

    if not SED_REGEX.match(msg):
        return

    groups = SED_REGEX.match(msg).groups()
    populate(s, groups)
    set_flags(s, s.flags)

    if bot.debug:
        debug(s)

    if s.target:
        nick = s.target

    nick = nick.lower()

    if s.qual:
        s.msg = get_message(bot, s.to_replace, nick, chan, qual=s.qual)
    else:
        s.msg = get_message(bot, s.to_replace, nick, chan)

    if not s.msg: return

    try:

        new_msg = re.sub(s.to_replace, s.replacement, s.msg, s.count, s.flags)
        if '\x01' in new_msg:
            # probably a message with a CTCP ACTION in it.
            new_msg = new_msg.split(" ", 1)[1][:-1]
            return bot.msg(chan, "* %s %s" % (nick, new_msg))

        return bot.msg(chan, "<%s> %s" % (nick, new_msg))
    except BaseException:
        traceback.print_exc()

<commit_msg>Fix another problem with the sed regex<commit_after>"""
IRC Parser - parse.py
"""

import inspect
import traceback
import re
import subprocess
import sys

SED_REGEX = re.compile(r"^(?:(\S+)[:,]\s)?(?:(.+?)/)?s/(.+?)/([^/]*)/?(?:([gixs]{0,4}))?")

def parse(msg):
    """ Parse an IRC protocol message """
    if msg.startswith("PING"):
        info = {"method": "PING", "arg": msg.split()[-1]}
    else:
        splitmsg = msg.split(" ", 2)
        info = {"method": splitmsg[1], "host": splitmsg[0][1:], "arg":
                splitmsg[2]}
    return info

def evaluate(self, nick, chan, msg):
    """ Evaluate python code. """
    try:
        output = eval(msg, globals(), locals())
        if output is not None:
            self.leo = output
            if type(self.leo) == tuple:
                self.leo = list(self.leo)
                output = list(output)
            return output
    except (NameError, SyntaxError):
        try:
            exec(msg, globals())
        except:
            try:
                exec(msg, locals())
            except:
                try:
                    exec(msg,globals(),locals())
                except:
                    exec(msg,locals(),globals())

###
##
## Sed part
##
###

def populate(sedobject, groups):
    group_types = ("target", "qual", "to_replace", "replacement", "flags")
    for k, v in zip(group_types, groups):
        setattr(sedobject, k, v)

def set_flags(sedobject, flags):
    i = 0
    count = 1

    if not flags:
        setattr(sedobject, "flags", i)
        setattr(sedobject, "count", count)
        return

    for item in flags:
        if item == "i":
            i |= re.IGNORECASE
        if item == "x":
            i |= re.X
        if item == "s":
            i |= re.S
        if item == "g":
            count = 0

    setattr(sedobject, "flags", i)
    setattr(sedobject, "count", count)

def debug(sedobject):
    for k,v in sedobject.__dict__.items():
        if not k.startswith("_") or not k.endswith("_"):
            print("%s: %s" % (k,v))

def get_message(bot, sedregex, nick, chan, qual=None):
    if not nick in bot.state.messages:
        return ""
    for message in bot.state.messages[nick]:
        if not message.channel == bot.state.channels[chan]:
            continue
        msg = message.msg
        try:
            if qual:
                if re.search(sedregex, msg) and re.search(qual, msg) and not SED_REGEX.search(msg):
                    return msg
            else:
                if re.search(sedregex, msg) and not SED_REGEX.search(msg):
                    return msg
        except BaseException:
            pass
    return ""

def sed(bot, nick, chan, msg):
    """ Perform the sedding """
    s = type("SedObject", tuple(), {})
    #[print(i) for i in (bot, nick, chan, msg)]

    if not SED_REGEX.match(msg):
        return

    groups = SED_REGEX.match(msg).groups()
    populate(s, groups)
    set_flags(s, s.flags)

    if bot.debug:
        debug(s)

    if s.target:
        nick = s.target

    nick = nick.lower()

    if s.qual:
        s.msg = get_message(bot, s.to_replace, nick, chan, qual=s.qual)
    else:
        s.msg = get_message(bot, s.to_replace, nick, chan)

    if not s.msg: return

    try:

        new_msg = re.sub(s.to_replace, s.replacement, s.msg, s.count, s.flags)
        if '\x01' in new_msg:
            # probably a message with a CTCP ACTION in it.
            new_msg = new_msg.split(" ", 1)[1][:-1]
            return bot.msg(chan, "* %s %s" % (nick, new_msg))

        return bot.msg(chan, "<%s> %s" % (nick, new_msg))
    except BaseException:
        traceback.print_exc()

<|endoftext|>


<commit_before>from __future__ import division, print_function

def convert_edf_2_hdf5(edf_file, low_pass_pupil_f = 6.0, high_pass_pupil_f = 0.01):
    """converts the edf_file to hdf5 using hedfpy

    Requires hedfpy

    Parameters
    ----------
    edf_file : str
        absolute path to edf file.
    low_pass_pupil_f : float
        low pass cutoff frequency for band-pass filtering of the pupil signal
    high_pass_pupil_f : float
        high pass cutoff frequency for band-pass filtering of the pupil signal
    Returns
    -------
    hdf5_file : str
        absolute path to hdf5 file.
    """

    import os
    import os.path as op
    from hedfpy import HDFEyeOperator
    import tempfile

    tempdir = tempfile.mkdtemp()
    temp_edf = op.join(tempdir, op.split(edf_file)[-1])

    os.system('cp ' + edf_file + ' ' + temp_edf)

    hdf5_file = op.join(tempdir, op.split(op.splitext(edf_file)[0] + '.h5')[-1])
    alias = op.splitext(op.split(edf_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.add_edf_file(temp_edf)
    ho.edf_message_data_to_hdf(alias = alias)
    ho.edf_gaze_data_to_hdf(alias = alias, pupil_hp = high_pass_pupil_f, pupil_lp = low_pass_pupil_f)

    return hdf5_file

def convert_hdf_eye_to_tsv(hdf5_file, tsv_file = None, tr_key = 116.0, xy_intercepts = None, xy_slopes = None ):

    from hedfpy import HDFEyeOperator
    import tempfile
    import os

    alias = op.splitext(op.split(hdf5_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.open_hdf_file('a')
    node = ho.h5f.get_node('/' + alias)
    trial_data = node.trials.read()
    events = node.events.read()

    if tsv_file == None:
        tempdir = tempfile.mkdtemp()
        temp_tsv = op.join(tempdir, alias + '.tsv')
    else:
        temp_tsv = tsv_file

    # find first TR, and use it to define:
    # the duration of the datastream and the tr
    tr_timestamps = events['EL_timestamp'][(events['key'] == tr_key) & (events['up_down'] == 'Down')]
    tr = np.round(np.mean(np.diff(tr_timestamps)))

    eye = ho.eye_during_trial(0, alias)
    sample_rate = ho.sample_rate_during_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)

    raw_data = ho.data_from_time_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)
    selected_data = np.array(raw_data[['time', eye + '_gaze_x', eye + '_gaze_y', eye + '_pupil']])

    # some post-processing:
    # time in seconds
    selected_data[:,0] = (selected_data[:,0] - selected_data[0,0]) / float(sample_rate)
    # linear gaze position scaling
    if xy_intercepts != None and if xy_slopes != None:
        for i in [1,2]
            selected_data[:,i] = (selected_data[:,i] * xy_slopes[i-1]) + xy_intercepts[i-1]

    np.savetxt(temp_tsv, selected_data, fmt = '%3.2f', delimiter = '\t', header="time   X   Y   pupil")
    os.system('gzip ' + temp_tsv)

    return temp_tsv + '.gz'

def mask_nii_2_hdf5(in_files, mask_files, hdf5_file, folder_alias):
    """masks data in in_files with masks in mask_files,
    to be stored in an hdf5 file

    Takes a list of 4D fMRI nifti-files and masks the
    data with all masks in the list of nifti-files mask_files.
    These files are assumed to represent the same space, i.e.
    that of the functional acquisitions.
    These are saved in hdf5_file, in the folder folder_alias.

    Parameters
    ----------
    in_files : list
        list of absolute path to functional nifti-files.
        all nifti files are assumed to have the same ndim
    mask_files : list
        list of absolute path to mask nifti-files.
        mask_files are assumed to be 3D
    hdf5_file : str
    	absolute path to hdf5 file.
   	folder_alias : str
   		name of the to-be-created folder in the hdf5 file.

    Returns
    -------
    hdf5_file : str
        absolute path to hdf5 file.
    """

    import nibabel as nib
    import os.path as op
    import numpy as np
    import tables

    success = True

    mask_data = [nib.load(mf).get_data() for mf in mask_files]
    nifti_data = [nib.load(nf).get_data() for nf in in_files]

    mask_names = [op.split(mf)[-1].split('.nii.gz')[0] for mf in mask_files]
    nifti_names = [op.split(mf)[-1].split('.nii.gz')[0] for nf in in_files]

    data = nib.load(in_files[0])
    dims = data.shape
    n_dims = data.ndim

    h5file = tables.open_file(hdf5_file, mode = "w", title = hdf5_file)
    # get or make group for alias folder
    try:
        folder_alias_run_group = h5file.get_node("/", name = folder_alias, classname='Group')
    except NoSuchNodeError:
        print('Adding group ' + folder_alias + ' to this file')
        folder_alias_run_group = h5file.create_group("/", folder_alias, folder_alias)

    for (roi, roi_name) in zip(mask_data, mask_names):
        # get or make group for alias/roi
        try:
            run_group = h5file.get_node(where = "/" + folder_alias, name = roi_name, classname='Group')
        except NoSuchNodeError:
            print('Adding group ' + folder_alias + '_' + roi_name + ' to this file')
            run_group = h5file.create_group("/" + folder_alias, roi_name, folder_alias + '_' + roi_name)

        h5file.create_array(run_group, roi_name, roi, roi_name + ' mask file for reconstituting nii data from masked data')

        for (nii_d, nii_name) in zip(nifti_data, nifti_names):
            if n_dims == 3:
                these_roi_data = nii_d[roi]
            elif n_dims == 4:   # timeseries data, last dimension is time.
                these_roi_data = nii_d[roi,:]
            else:
                print("n_dims in data {nifti} do not fit with mask".format(nii_name))
                success = False

            h5file.create_array(run_group, nii_name, these_roi_data, roi_name + ' data from ' + nii_name)

    h5file.close()

    return hdf5_file

def roi_data_from_hdf(data_types_wildcards, roi_name_wildcard, hdf5_file, folder_alias):
    """takes data_type data from masks stored in hdf5_file

    Takes a list of 4D fMRI nifti-files and masks the
    data with all masks in the list of nifti-files mask_files.
    These files are assumed to represent the same space, i.e.
    that of the functional acquisitions.
    These are saved in hdf5_file, in the folder folder_alias.

    Parameters
    ----------
    data_types_wildcards : list
        list of data types to be loaded.
        correspond to nifti_names in mask_2_hdf5
    roi_name_wildcard : str
        wildcard for masks.
        corresponds to mask_name in mask_2_hdf5.
    hdf5_file : str
        absolute path to hdf5 file.
    folder_alias : str
        name of the folder in the hdf5 file from which data
        should be loaded.

    Returns
    -------
    output_data : list
        list of numpy arrays corresponding to data_types and roi_name_wildcards
    """
    import tables
    import itertools

    h5file = tables.open_file(hdf5_file, mode = "r")

    try:
        folder_alias_run_group = h5file.get_node(where = '/', name = folder_alias, classname='Group')
    except NoSuchNodeError:
        # import actual data
        print('No group ' + folder_alias + ' in this file')
        return None

    all_roi_names = h5file.list_nodes(where = '/' + folder_alias, classname = 'Group')
    roi_names = [rn for rn in all_roi_names if roi_name_wildcard in rn]
    if len(roi_names) == 0:
        print('No rois corresponding to ' + roi_wildcard + ' in group ' + folder_alias)
        return None

    data_arrays = []
    for rn in roi_names:
        try:
            roi_node = h5file.get_node(where = '/' + folder_alias, name = roi_name, classname='Group')
        except NoSuchNodeError:
            print('No data corresponding to ' + roi_name + ' in group ' + folder_alias)
            pass
        all_data_array_names = h5file.list_nodes(where = '/' + folder_alias + '/' + roi_name)
        data_array_names = [[dan for dan in all_data_array_names if dtwc in dan and dtwc + '_' not in dan] for dtwd in data_types_wildcards]
        data_array_names = list(itertools.chain(*data_array_names))

        if sort_data_types:
            data_array_names = sorted(data_array_names)

        if len(data_array_names) == 0:
            print('No data corresponding to ' + str(data_types_wildcards) + ' in group ' + folder_alias + '/' + rn)
            pass
        else:
            print('Taking data corresponding to ' + str(data_array_names) + ' from group ' + folder_alias + '/' + rn)
            data_arrays.append([])
            for dan in data_array_names:
                data_arrays[-1].append(eval('roi_node.' + dan + '.read()'))

    all_roi_data_np = np.hstack(all_roi_data).T

    return all_roi_data_np

<commit_msg>behavioral analysis, preprocessed events and times to tsv files<commit_after>from __future__ import division, print_function

def convert_edf_2_hdf5(edf_file, low_pass_pupil_f = 6.0, high_pass_pupil_f = 0.01):
    """converts the edf_file to hdf5 using hedfpy

    Requires hedfpy

    Parameters
    ----------
    edf_file : str
        absolute path to edf file.
    low_pass_pupil_f : float
        low pass cutoff frequency for band-pass filtering of the pupil signal
    high_pass_pupil_f : float
        high pass cutoff frequency for band-pass filtering of the pupil signal
    Returns
    -------
    hdf5_file : str
        absolute path to hdf5 file.
    """

    import os
    import os.path as op
    from hedfpy import HDFEyeOperator
    import tempfile

    tempdir = tempfile.mkdtemp()
    temp_edf = op.join(tempdir, op.split(edf_file)[-1])

    os.system('cp ' + edf_file + ' ' + temp_edf)

    hdf5_file = op.join(tempdir, op.split(op.splitext(edf_file)[0] + '.h5')[-1])
    alias = op.splitext(op.split(edf_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.add_edf_file(temp_edf)
    ho.edf_message_data_to_hdf(alias = alias)
    ho.edf_gaze_data_to_hdf(alias = alias, pupil_hp = high_pass_pupil_f, pupil_lp = low_pass_pupil_f)

    return hdf5_file

def convert_hdf_eye_to_tsv(hdf5_file, tsv_file = None, xy_intercepts = None, xy_slopes = None ):

    from hedfpy import HDFEyeOperator
    import tempfile
    import os

    tr_key = 116.0
    alias = op.splitext(op.split(hdf5_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.open_hdf_file('a')
    node = ho.h5f.get_node('/' + alias)
    trial_data = node.trials.read()
    events = node.events.read()

    if tsv_file == None:
        tempdir = tempfile.mkdtemp()
        temp_tsv = op.join(tempdir, alias + '.tsv')
    else:
        temp_tsv = tsv_file

    # find first TR, and use it to define:
    # the duration of the datastream and the tr
    tr_timestamps = events['EL_timestamp'][(events['key'] == tr_key) & (events['up_down'] == 'Down')]
    tr = np.round(np.mean(np.diff(tr_timestamps)))

    eye = ho.eye_during_trial(0, alias)
    sample_rate = ho.sample_rate_during_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)

    raw_data = ho.data_from_time_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)
    selected_data = np.array(raw_data[['time', eye + '_gaze_x', eye + '_gaze_y', eye + '_pupil']])

    # some post-processing:
    # time in seconds
    selected_data[:,0] = (selected_data[:,0] - selected_data[0,0]) / 1000.0,  # convert ms to s
    # linear gaze position scaling
    if (xy_intercepts != None) and (xy_slopes != None):
        for i in [1,2]:
            selected_data[:,i] = (selected_data[:,i] * xy_slopes[i-1]) + xy_intercepts[i-1]

    np.savetxt(temp_tsv, selected_data, fmt = '%3.3f', delimiter = '\t', header="time   X   Y   pupil")
    os.system('gzip ' + temp_tsv)

    return temp_tsv + '.gz'

def convert_unpredictable_trials_to_tsv(hdf5_file, tsv_file = None, reward_signal = 1.0 ):

    from hedfpy import HDFEyeOperator
    import tempfile
    import os
    import os.path as op
    import numpy as np
    import pandas as pd

    tr_key = 116.0

    alias = op.splitext(op.split(hdf5_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.open_hdf_file('a')
    node = ho.h5f.get_node('/' + alias)
    trial_phase_data = node.trial_phases.read()
    events = node.events.read()
    parameters = node.parameters.read()

    if tsv_file == None:
        tempdir = tempfile.mkdtemp()
        temp_tsv = op.join(tempdir, alias + '.tsv')
    else:
        temp_tsv = tsv_file

    # find first TR, and use it to define:
    # the duration of the datastream and the tr
    tr_timestamps = events['EL_timestamp'][(events['key'] == tr_key) & (events['up_down'] == 'Down')]
    tr = np.round(np.mean(np.diff(tr_timestamps)))
    sample_rate = ho.sample_rate_during_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)

    run_onset_EL_time = tr_timestamps[0]

    columns_df = ['sound', 'contrast', 'stim_eccentricity', 'orientation', 'mask_radius']
    tp_value = 2.0

    trial_phase_timestamps = np.array(trial_phase_data[trial_phase_data['trial_phase_index'] == tp_value]['trial_phase_EL_timestamp'])

    trial_parameters = pd.DataFrame(parameters[columns_df].T)

    # first, we re-code the parameters in terms of screen coordinates
    # convert stim position from proportion of screen (1920, 124 cm distance on BoldScreen32) to dva
    trial_parameters['stim_eccentricity'] = trial_parameters['stim_eccentricity'] * 11 # 11 degrees from fix to edge of screen
    trial_parameters['mask_radius'] = trial_parameters['mask_radius'] * 22 / 1920 # 11 degrees from fix to edge of screen

    # add the times at which stimuli and rewards were presented
    which_trials_reward = np.array(trial_parameters['sound'] == reward_signal)
    reward_times = np.zeros(len(trial_phase_timestamps))
    # the sound was 800 ms delayed wrt the stimulus onset
    reward_times[which_trials_reward] = \
                (800.0 + trial_phase_timestamps[which_trials_reward] - tr_timestamps[0] ) / 1000.0

    trial_parameters['reward_time'] = pd.Series(reward_times)
    trial_parameters['stim_onset_time'] = pd.Series((trial_phase_timestamps - tr_timestamps[0] ) / 1000.0)

    trial_parameters = trial_parameters.sort_values(by = 'stim_onset_time')

    trial_parameters.to_csv(temp_tsv, sep = '\t', float_format = '%3.2f', header = True, index = False)

    return temp_tsv

def convert_predictable_trials_to_tsv(hdf5_file, tsv_file = None ):

    from hedfpy import HDFEyeOperator
    import tempfile
    import os
    import os.path as op
    import numpy as np
    import pandas as pd
    import math

    tr_key = 116.0

    alias = op.splitext(op.split(hdf5_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.open_hdf_file('a')
    node = ho.h5f.get_node('/' + alias)
    trial_phase_data = node.trial_phases.read()
    events = node.events.read()
    parameters = node.parameters.read()

    if tsv_file == None:
        tempdir = tempfile.mkdtemp()
        temp_tsv = op.join(tempdir, alias + '.tsv')
    else:
        temp_tsv = tsv_file

    # find first TR, and use it to define:
    # the duration of the datastream and the tr
    tr_timestamps = events['EL_timestamp'][(events['key'] == tr_key) & (events['up_down'] == 'Down')]
    tr = np.round(np.mean(np.diff(tr_timestamps)))
    sample_rate = ho.sample_rate_during_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)

    run_onset_EL_time = tr_timestamps[0]

    columns_df = ['sound', 'contrast', 'x_position', 'orientation', 'mask_radius']
    tp_value = 2.0

    trial_phase_timestamps = np.array(trial_phase_data[trial_phase_data['trial_phase_index'] == tp_value]['trial_phase_EL_timestamp'])

    trial_parameters = pd.DataFrame(parameters[columns_df].T)

    # first, we re-code the parameters in terms of screen coordinates
    # convert stim position from proportion of screen (1920, 124 cm distance on BoldScreen32) to dva
    trial_parameters['x_position'] = trial_parameters['x_position'] * 11 # 11 degrees from fix to edge of screen
    trial_parameters['mask_radius'] = trial_parameters['mask_radius'] * 22 / 1920 # 11 degrees from fix to edge of screen

    # add the times at which stimuli and rewards were presented
    which_trials_reward = np.array(trial_parameters['sound'] == 1)
    reward_times = np.zeros(len(trial_phase_timestamps))
    # the sound was 800 ms delayed wrt the stimulus onset
    reward_times[which_trials_reward] = \
                (800.0 + trial_phase_timestamps[which_trials_reward] - tr_timestamps[0] ) / 1000.0

    trial_parameters['reward_time'] = pd.Series(reward_times)
    trial_parameters['stim_onset_time'] = pd.Series((trial_phase_timestamps - tr_timestamps[0] ) / 1000.0)

    trial_parameters = trial_parameters.sort_values(by = 'stim_onset_time')

    trial_parameters.to_csv(temp_tsv, sep = '\t', float_format = '%3.2f', header = True, index = False)

    return temp_tsv

def convert_variable_trials_to_tsv(hdf5_file, tsv_file = None, reward_signal = 1.0 ):

    from hedfpy import HDFEyeOperator
    import tempfile
    import os
    import os.path as op
    import numpy as np
    import pandas as pd
    import math

    tr_key = 116.0

    alias = op.splitext(op.split(hdf5_file)[-1])[0]

    ho = HDFEyeOperator(hdf5_file)
    ho.open_hdf_file('a')
    node = ho.h5f.get_node('/' + alias)
    trial_phase_data = node.trial_phases.read()
    events = node.events.read()
    parameters = node.parameters.read()

    if tsv_file == None:
        tempdir = tempfile.mkdtemp()
        temp_tsv = op.join(tempdir, alias + '.tsv')
    else:
        temp_tsv = tsv_file

    # find first TR, and use it to define:
    # the duration of the datastream and the tr
    tr_timestamps = events['EL_timestamp'][(events['key'] == tr_key) & (events['up_down'] == 'Down')]
    tr = np.round(np.mean(np.diff(tr_timestamps)))
    sample_rate = ho.sample_rate_during_period([tr_timestamps[0], tr_timestamps[-1] + tr], alias)

    run_onset_EL_time = tr_timestamps[0]

    columns_df = ['sound', 'contrast', 'stim_eccentricity', 'stim_orientation', 'mask_radius', 'reward_delay', 'which_stimulus_for_rgb_overlay']
    tp_value = 2.0

    trial_phase_timestamps = np.array(trial_phase_data[trial_phase_data['trial_phase_index'] == tp_value]['trial_phase_EL_timestamp'])

    trial_parameters = pd.DataFrame(parameters[columns_df].T)

    # first, we re-code the parameters in terms of screen coordinates
    # convert stim position from proportion of screen (1920, 124 cm distance on BoldScreen32) to dva
    trial_parameters['stim_eccentricity'] = trial_parameters['stim_eccentricity'] * 11 # 11 degrees from fix to edge of screen
    trial_parameters['mask_radius'] = trial_parameters['mask_radius'] * 22 / 1920 # 11 degrees from fix to edge of screen

    # add the times at which stimuli and rewards were presented
    which_trials_reward = np.array(trial_parameters['sound'] == reward_signal)
    reward_times = np.zeros(len(trial_phase_timestamps))
    # the sound was 800 ms delayed wrt the stimulus onset
    reward_times = trial_parameters['reward_delay'] + (trial_phase_timestamps - tr_timestamps[0] ) / 1000.0

    trial_parameters['reward_time'] = pd.Series(reward_times)
    trial_parameters['stim_onset_time'] = pd.Series((trial_phase_timestamps - tr_timestamps[0] ) / 1000.0)

    trial_parameters = trial_parameters.sort_values(by = 'stim_onset_time')

    trial_parameters.to_csv(temp_tsv, sep = '\t', float_format = '%3.2f', header = True, index = False)

    return temp_tsv

def mask_nii_2_hdf5(in_files, mask_files, hdf5_file, folder_alias):
    """masks data in in_files with masks in mask_files,
    to be stored in an hdf5 file

    Takes a list of 4D fMRI nifti-files and masks the
    data with all masks in the list of nifti-files mask_files.
    These files are assumed to represent the same space, i.e.
    that of the functional acquisitions.
    These are saved in hdf5_file, in the folder folder_alias.

    Parameters
    ----------
    in_files : list
        list of absolute path to functional nifti-files.
        all nifti files are assumed to have the same ndim
    mask_files : list
        list of absolute path to mask nifti-files.
        mask_files are assumed to be 3D
    hdf5_file : str
    	absolute path to hdf5 file.
   	folder_alias : str
   		name of the to-be-created folder in the hdf5 file.

    Returns
    -------
    hdf5_file : str
        absolute path to hdf5 file.
    """

    import nibabel as nib
    import os.path as op
    import numpy as np
    import tables

    success = True

    mask_data = [nib.load(mf).get_data() for mf in mask_files]
    nifti_data = [nib.load(nf).get_data() for nf in in_files]

    mask_names = [op.split(mf)[-1].split('.nii.gz')[0] for mf in mask_files]
    nifti_names = [op.split(mf)[-1].split('.nii.gz')[0] for nf in in_files]

    data = nib.load(in_files[0])
    dims = data.shape
    n_dims = data.ndim

    h5file = tables.open_file(hdf5_file, mode = "w", title = hdf5_file)
    # get or make group for alias folder
    try:
        folder_alias_run_group = h5file.get_node("/", name = folder_alias, classname='Group')
    except NoSuchNodeError:
        print('Adding group ' + folder_alias + ' to this file')
        folder_alias_run_group = h5file.create_group("/", folder_alias, folder_alias)

    for (roi, roi_name) in zip(mask_data, mask_names):
        # get or make group for alias/roi
        try:
            run_group = h5file.get_node(where = "/" + folder_alias, name = roi_name, classname='Group')
        except NoSuchNodeError:
            print('Adding group ' + folder_alias + '_' + roi_name + ' to this file')
            run_group = h5file.create_group("/" + folder_alias, roi_name, folder_alias + '_' + roi_name)

        h5file.create_array(run_group, roi_name, roi, roi_name + ' mask file for reconstituting nii data from masked data')

        for (nii_d, nii_name) in zip(nifti_data, nifti_names):
            if n_dims == 3:
                these_roi_data = nii_d[roi]
            elif n_dims == 4:   # timeseries data, last dimension is time.
                these_roi_data = nii_d[roi,:]
            else:
                print("n_dims in data {nifti} do not fit with mask".format(nii_name))
                success = False

            h5file.create_array(run_group, nii_name, these_roi_data, roi_name + ' data from ' + nii_name)

    h5file.close()

    return hdf5_file

def roi_data_from_hdf(data_types_wildcards, roi_name_wildcard, hdf5_file, folder_alias):
    """takes data_type data from masks stored in hdf5_file

    Takes a list of 4D fMRI nifti-files and masks the
    data with all masks in the list of nifti-files mask_files.
    These files are assumed to represent the same space, i.e.
    that of the functional acquisitions.
    These are saved in hdf5_file, in the folder folder_alias.

    Parameters
    ----------
    data_types_wildcards : list
        list of data types to be loaded.
        correspond to nifti_names in mask_2_hdf5
    roi_name_wildcard : str
        wildcard for masks.
        corresponds to mask_name in mask_2_hdf5.
    hdf5_file : str
        absolute path to hdf5 file.
    folder_alias : str
        name of the folder in the hdf5 file from which data
        should be loaded.

    Returns
    -------
    output_data : list
        list of numpy arrays corresponding to data_types and roi_name_wildcards
    """
    import tables
    import itertools

    h5file = tables.open_file(hdf5_file, mode = "r")

    try:
        folder_alias_run_group = h5file.get_node(where = '/', name = folder_alias, classname='Group')
    except NoSuchNodeError:
        # import actual data
        print('No group ' + folder_alias + ' in this file')
        return None

    all_roi_names = h5file.list_nodes(where = '/' + folder_alias, classname = 'Group')
    roi_names = [rn for rn in all_roi_names if roi_name_wildcard in rn]
    if len(roi_names) == 0:
        print('No rois corresponding to ' + roi_wildcard + ' in group ' + folder_alias)
        return None

    data_arrays = []
    for rn in roi_names:
        try:
            roi_node = h5file.get_node(where = '/' + folder_alias, name = roi_name, classname='Group')
        except NoSuchNodeError:
            print('No data corresponding to ' + roi_name + ' in group ' + folder_alias)
            pass
        all_data_array_names = h5file.list_nodes(where = '/' + folder_alias + '/' + roi_name)
        data_array_names = [[dan for dan in all_data_array_names if dtwc in dan and dtwc + '_' not in dan] for dtwd in data_types_wildcards]
        data_array_names = list(itertools.chain(*data_array_names))

        if sort_data_types:
            data_array_names = sorted(data_array_names)

        if len(data_array_names) == 0:
            print('No data corresponding to ' + str(data_types_wildcards) + ' in group ' + folder_alias + '/' + rn)
            pass
        else:
            print('Taking data corresponding to ' + str(data_array_names) + ' from group ' + folder_alias + '/' + rn)
            data_arrays.append([])
            for dan in data_array_names:
                data_arrays[-1].append(eval('roi_node.' + dan + '.read()'))

    all_roi_data_np = np.hstack(all_roi_data).T

    return all_roi_data_np

<|endoftext|>


<commit_before>#
#   Copyright (c) 2015 EUROGICIEL
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

from oslo.config import cfg
from pecan import hooks
from webob import exc

from cerberus.common import context
from cerberus.common import policy
from cerberus.db import api as dbapi

class ConfigHook(hooks.PecanHook):
    """Attach the config object to the request so controllers can get to it."""

    def before(self, state):
        state.request.cfg = cfg.CONF

class DBHook(hooks.PecanHook):
    """Attach the dbapi object to the request so controllers can get to it."""

    def before(self, state):
        state.request.dbapi = dbapi.get_instance()

class ContextHook(hooks.PecanHook):
    """Configures a request context and attaches it to the request.

    The following HTTP request headers are used:

    X-User-Id or X-User:
        Used for context.user_id.

    X-Tenant-Id or X-Tenant:
        Used for context.tenant.

    X-Auth-Token:
        Used for context.auth_token.

    X-Roles:
        Used for setting context.is_admin flag to either True or False.
        The flag is set to True, if X-Roles contains either an administrator
        or admin substring. Otherwise it is set to False.

    """
    def __init__(self, public_api_routes):
        self.public_api_routes = public_api_routes
        super(ContextHook, self).__init__()

    def before(self, state):
        user_id = state.request.headers.get('X-User-Id')
        user_id = state.request.headers.get('X-User', user_id)
        tenant_id = state.request.headers.get('X-Tenant-Id')
        tenant = state.request.headers.get('X-Tenant', tenant_id)
        domain_id = state.request.headers.get('X-User-Domain-Id')
        domain_name = state.request.headers.get('X-User-Domain-Name')
        auth_token = state.request.headers.get('X-Auth-Token')
        roles = state.request.headers.get('X-Roles', '').split(',')
        creds = {'roles': roles}

        is_public_api = state.request.environ.get('is_public_api', False)
        is_admin = policy.enforce('context_is_admin',
                                  state.request.headers,
                                  creds)

        state.request.context = context.RequestContext(
            auth_token=auth_token,
            user=user_id,
            tenant_id=tenant_id,
            tenant=tenant,
            domain_id=domain_id,
            domain_name=domain_name,
            is_admin=is_admin,
            is_public_api=is_public_api,
            roles=roles)

class AuthorizationHook(hooks.PecanHook):
    """Verify that the user has admin rights.

    Checks whether the request context is an admin context and
    rejects the request if the api is not public.

    """
    def __init__(self, member_routes):
        self.member_routes = member_routes
        super(AuthorizationHook, self).__init__()

    def before(self, state):
        ctx = state.request.context

        if not ctx.is_admin and not ctx.is_public_api and \
                state.request.path not in self.member_routes:
                raise exc.HTTPForbidden()

class NoExceptionTracebackHook(hooks.PecanHook):
    """Workaround rpc.common: deserialize_remote_exception.

    deserialize_remote_exception builds rpc exception traceback into error
    message which is then sent to the client. Such behavior is a security
    concern so this hook is aimed to cut-off traceback from the error message.

    """
    # NOTE(max_lobur): 'after' hook used instead of 'on_error' because
    # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator
    # catches and handles all the errors, so 'on_error' dedicated for unhandled
    # exceptions never fired.
    def after(self, state):
        # Omit empty body. Some errors may not have body at this level yet.
        if not state.response.body:
            return

        # Do nothing if there is no error.
        if 200 <= state.response.status_int < 400:
            return

        json_body = state.response.json
        # Do not remove traceback when server in debug mode (except 'Server'
        # errors when 'debuginfo' will be used for traces).
        if cfg.CONF.debug and json_body.get('faultcode') != 'Server':
            return

        faultsting = json_body.get('faultstring')
        traceback_marker = 'Traceback (most recent call last):'
        if faultsting and (traceback_marker in faultsting):
            # Cut-off traceback.
            faultsting = faultsting.split(traceback_marker, 1)[0]
            # Remove trailing newlines and spaces if any.
            json_body['faultstring'] = faultsting.rstrip()
            # Replace the whole json. Cannot change original one beacause it's
            # generated on the fly.
            state.response.json = json_body
<commit_msg>Check prefix path is member route<commit_after>#
#   Copyright (c) 2015 EUROGICIEL
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

from oslo.config import cfg
from pecan import hooks
from webob import exc

from cerberus.common import context
from cerberus.common import policy
from cerberus.db import api as dbapi

class ConfigHook(hooks.PecanHook):
    """Attach the config object to the request so controllers can get to it."""

    def before(self, state):
        state.request.cfg = cfg.CONF

class DBHook(hooks.PecanHook):
    """Attach the dbapi object to the request so controllers can get to it."""

    def before(self, state):
        state.request.dbapi = dbapi.get_instance()

class ContextHook(hooks.PecanHook):
    """Configures a request context and attaches it to the request.

    The following HTTP request headers are used:

    X-User-Id or X-User:
        Used for context.user_id.

    X-Tenant-Id or X-Tenant:
        Used for context.tenant.

    X-Auth-Token:
        Used for context.auth_token.

    X-Roles:
        Used for setting context.is_admin flag to either True or False.
        The flag is set to True, if X-Roles contains either an administrator
        or admin substring. Otherwise it is set to False.

    """
    def __init__(self, public_api_routes):
        self.public_api_routes = public_api_routes
        super(ContextHook, self).__init__()

    def before(self, state):
        user_id = state.request.headers.get('X-User-Id')
        user_id = state.request.headers.get('X-User', user_id)
        tenant_id = state.request.headers.get('X-Tenant-Id')
        tenant = state.request.headers.get('X-Tenant', tenant_id)
        domain_id = state.request.headers.get('X-User-Domain-Id')
        domain_name = state.request.headers.get('X-User-Domain-Name')
        auth_token = state.request.headers.get('X-Auth-Token')
        roles = state.request.headers.get('X-Roles', '').split(',')
        creds = {'roles': roles}

        is_public_api = state.request.environ.get('is_public_api', False)
        is_admin = policy.enforce('context_is_admin',
                                  state.request.headers,
                                  creds)

        state.request.context = context.RequestContext(
            auth_token=auth_token,
            user=user_id,
            tenant_id=tenant_id,
            tenant=tenant,
            domain_id=domain_id,
            domain_name=domain_name,
            is_admin=is_admin,
            is_public_api=is_public_api,
            roles=roles)

class AuthorizationHook(hooks.PecanHook):
    """Verify that the user has admin rights.

    Checks whether the request context is an admin context and
    rejects the request if the api is not public.

    """
    def __init__(self, member_routes):
        self.member_routes = member_routes
        super(AuthorizationHook, self).__init__()

    def is_path_in_routes(self, path):
        for p in self.member_routes:
            if path.startswith(p):
                return True
        return False

    def before(self, state):
        ctx = state.request.context

        if not ctx.is_admin and not ctx.is_public_api and \
                not self.is_path_in_routes(state.request.path):
                raise exc.HTTPForbidden()

class NoExceptionTracebackHook(hooks.PecanHook):
    """Workaround rpc.common: deserialize_remote_exception.

    deserialize_remote_exception builds rpc exception traceback into error
    message which is then sent to the client. Such behavior is a security
    concern so this hook is aimed to cut-off traceback from the error message.

    """
    # NOTE(max_lobur): 'after' hook used instead of 'on_error' because
    # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator
    # catches and handles all the errors, so 'on_error' dedicated for unhandled
    # exceptions never fired.
    def after(self, state):
        # Omit empty body. Some errors may not have body at this level yet.
        if not state.response.body:
            return

        # Do nothing if there is no error.
        if 200 <= state.response.status_int < 400:
            return

        json_body = state.response.json
        # Do not remove traceback when server in debug mode (except 'Server'
        # errors when 'debuginfo' will be used for traces).
        if cfg.CONF.debug and json_body.get('faultcode') != 'Server':
            return

        faultsting = json_body.get('faultstring')
        traceback_marker = 'Traceback (most recent call last):'
        if faultsting and (traceback_marker in faultsting):
            # Cut-off traceback.
            faultsting = faultsting.split(traceback_marker, 1)[0]
            # Remove trailing newlines and spaces if any.
            json_body['faultstring'] = faultsting.rstrip()
            # Replace the whole json. Cannot change original one beacause it's
            # generated on the fly.
            state.response.json = json_body
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys;
import os;

import json;

import collections;

from shapely.geometry import shape, mapping;
from shapely.ops import cascaded_union;

def recApply(d, func, condition=lambda x:True, isMap=lambda x:isinstance(x,collections.Mapping), isContainer=lambda x:isinstance(x,collections.Iterable) and not isinstance(x, basestring), applyToKey=True):
  """d に対して func を再帰的に適用する

  :param d: 任意の変数
  :param func: 適用する関数
  :param condition: 関数を適用するかどうか判定する関数
  :param isMap: 辞書系オブジェクトかどうか判定する関数
  :param isContainer: イテレータブル系かどうか判定する関数
  :param applyToKey: 辞書系オブジェクトのキーにも関数を適用するかどうかの真偽値
  """
  if isMap(d):
    if applyToKey:
      return d.__class__( ( (recApply(k,func,condition,isMap,isContainer,applyToKey),recApply(v,func,condition,isMap,isContainer,applyToKey)) for k,v in d.items() ) );
    return d.__class__( ( (k,recApply(v,func,condition,isMap,isContainer,applyToKey)) for k,v in d.items() ) );
  if isContainer(d):
    return d.__class__( ( recApply(v,func,condition,isMap,isContainer,applyToKey) for v in d ) );
  if condition(d):
    return func(d);
  return d;

def recDecode(d):
  """d と同じ構造で、 d の全ての str 要素に対して .decode('utf-8') を掛けたものを返す

  :param d: 任意の変数
  """
  return recApply(d, lambda x:x.decode('utf-8'), lambda x:isinstance(x, str));

def recEncode(d):
  """d と同じ構造で、 d の全ての unicode 要素に対して .encode('utf-8') を掛けたものを返す

  :param d: 任意の変数
  """
  return recApply(d, lambda x:x.encode('utf-8'), lambda x:isinstance(x, unicode));

class feature(object):
  def __init__(self, data):
    self.load(data);
  def load(self, data):
    self.__geometry = shape(data['geometry']);
    self.__properties = recApply(data['properties'],lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    self.__attributes = recApply({k:v for k,v in data.items() if k not in set(('geometry','properties','type'))},lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    return self;
  def dump(self):
    return dict({'type':'Feature','geometry':mapping(self.__geometry), 'properties':recApply(self.__properties,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode))},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
  def join(self, d):
    self.__properties = dict(self.__properties,**d);
    return self;
  @property
  def properties(self):
    return self.__properties;
  @property
  def geometry(self):
    return self.__geometry;
  @property
  def attributes(self):
    return self.__attributes;
  def isMatch(self, condition):
    return condition(self.__properties);

class features(object):
  def __init__(self, data):
    self.__data = None;
    self.load(data);
  def load(self, data):
    self.__data = [ feature(d) for d in data ];
    return self;
  def dump(self):
    if self.__data is None:
      return [];
    return [f.dump() for f in self.__data]
  def __iter__(self):
    if self.__data is None:
      raise StopIteration;
    return self.__data.__iter__();

class geoJson(object):
  def __init__(self, data):
    self.__features = None;
    self.__attributes = {};
    if isinstance(data, basestring):
      with open(data,'r') as f:
        self.load(json.loads(f.read()));
    elif hasattr(data, 'read'):
      self.load(json.loads(data.read()));
    else:
      self.load(data);
  def load(self, data):
    self.__attributes = recApply({k:v for k,v in data.items() if k not in set(('type','features'))},lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    self.__features = features(data['features']);
    return self;
  def dump(self):
    if self.__features is None:
      return dict({'type': 'FeaturesCollection','features':[]},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
    return dict({'type': 'FeaturesCollection','features':self.__features.dump()},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
  def __str__(self):
    return json.dumps(self.dump());
  def __iter__(self):
    if self.__features is None:
      raise StopIteration;
    return self.__features.__iter__();
<commit_msg>write class description of feature class<commit_after># -*- coding: utf-8 -*-

import sys;
import os;

import json;

import collections;

from shapely.geometry import shape, mapping;
from shapely.ops import cascaded_union;

def recApply(d, func, condition=lambda x:True, isMap=lambda x:isinstance(x,collections.Mapping), isContainer=lambda x:isinstance(x,collections.Iterable) and not isinstance(x, basestring), applyToKey=True):
  """d に対して func を再帰的に適用する

  :param d: 任意の変数
  :param func: 適用する関数
  :param condition: 関数を適用するかどうか判定する関数
  :param isMap: 辞書系オブジェクトかどうか判定する関数
  :param isContainer: イテレータブル系かどうか判定する関数
  :param applyToKey: 辞書系オブジェクトのキーにも関数を適用するかどうかの真偽値
  """
  if isMap(d):
    if applyToKey:
      return d.__class__( ( (recApply(k,func,condition,isMap,isContainer,applyToKey),recApply(v,func,condition,isMap,isContainer,applyToKey)) for k,v in d.items() ) );
    return d.__class__( ( (k,recApply(v,func,condition,isMap,isContainer,applyToKey)) for k,v in d.items() ) );
  if isContainer(d):
    return d.__class__( ( recApply(v,func,condition,isMap,isContainer,applyToKey) for v in d ) );
  if condition(d):
    return func(d);
  return d;

def recDecode(d):
  """d と同じ構造で、 d の全ての str 要素に対して .decode('utf-8') を掛けたものを返す

  :param d: 任意の変数
  """
  return recApply(d, lambda x:x.decode('utf-8'), lambda x:isinstance(x, str));

def recEncode(d):
  """d と同じ構造で、 d の全ての unicode 要素に対して .encode('utf-8') を掛けたものを返す

  :param d: 任意の変数
  """
  return recApply(d, lambda x:x.encode('utf-8'), lambda x:isinstance(x, unicode));

class feature(object):
  """単一の feature を扱うためのクラス
  """
  def __init__(self, data):
    """feature を構成する

    :param data: 'geometry' と 'properties' を要素に持ったオブジェクト
    """
    self.load(data);
  def load(self, data):
    self.__geometry = shape(data['geometry']);
    self.__properties = recApply(data['properties'],lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    self.__attributes = recApply({k:v for k,v in data.items() if k not in set(('geometry','properties','type'))},lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    return self;
  def dump(self):
    return dict({'type':'Feature','geometry':mapping(self.__geometry), 'properties':recApply(self.__properties,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode))},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
  def join(self, d):
    self.__properties = dict(self.__properties,**d);
    return self;
  @property
  def properties(self):
    return self.__properties;
  @property
  def geometry(self):
    return self.__geometry;
  @property
  def attributes(self):
    return self.__attributes;
  def isMatch(self, condition):
    return condition(self.__properties);

class features(object):
  def __init__(self, data):
    self.__data = None;
    self.load(data);
  def load(self, data):
    self.__data = [ feature(d) for d in data ];
    return self;
  def dump(self):
    if self.__data is None:
      return [];
    return [f.dump() for f in self.__data]
  def __iter__(self):
    if self.__data is None:
      raise StopIteration;
    return self.__data.__iter__();

class geoJson(object):
  def __init__(self, data):
    self.__features = None;
    self.__attributes = {};
    if isinstance(data, basestring):
      with open(data,'r') as f:
        self.load(json.loads(f.read()));
    elif hasattr(data, 'read'):
      self.load(json.loads(data.read()));
    else:
      self.load(data);
  def load(self, data):
    self.__attributes = recApply({k:v for k,v in data.items() if k not in set(('type','features'))},lambda x:x.decode('utf-8'),lambda x:isinstance(x,str));
    self.__features = features(data['features']);
    return self;
  def dump(self):
    if self.__features is None:
      return dict({'type': 'FeaturesCollection','features':[]},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
    return dict({'type': 'FeaturesCollection','features':self.__features.dump()},**recApply(self.__attributes,lambda x:x.encode('utf-8'),lambda x:isinstance(x,unicode)));
  def __str__(self):
    return json.dumps(self.dump());
  def __iter__(self):
    if self.__features is None:
      raise StopIteration;
    return self.__features.__iter__();
<|endoftext|>