<commit_before>#!/usr/bin/env
#Boa:Frame:ODMTools
import sys
import os

this_file = os.path.realpath(__file__)
directory = os.path.dirname(os.path.dirname(this_file))
sys.path.insert(0, directory)

import wx
import wx.grid
#import wx.aui

try:
    from agw import aui
except ImportError:  # if it's not there locally, try the wxPython lib.
    import wx.lib.agw.aui as aui

import wx.richtext
import wx.stc
import wx.lib.agw.aui as aui
from wx.lib.pubsub import pub as Publisher
import wx.py.crust
import frmDBConfiguration

from odmservices import ServiceManager
from odmservices import utilities as util
from pnlScript import pnlScript
import pnlSeriesSelector
import pnlPlot
import mnuRibbon
import pnlDataTable

from odmconsole import ConsoleTools
from common.logger import LoggerTool
import logging

def create(parent):
    return frmODMToolsMain(parent)

[
    wxID_ODMTOOLS, wxID_ODMTOOLSCHECKLISTBOX2, wxID_ODMTOOLSCOMBOBOX1,
    wxID_ODMTOOLSCOMBOBOX2, wxID_ODMTOOLSCOMBOBOX4, wxID_ODMTOOLSCOMBOBOX5,
    wxID_ODMTOOLSGRID1, wxID_ODMTOOLSPANEL1, wxID_ODMTOOLSPANEL2,
    wxID_ODMTOOLSTOOLBAR1, wxID_PNLSELECTOR, wxID_TXTPYTHONSCRIPT,
    wxID_TXTPYTHONCONSOLE,
] = [wx.NewId() for _init_ctrls in range(13)]

tool = LoggerTool()
logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG)

class frmODMToolsMain(wx.Frame):
    def __init__(self, parent):
        self.service_manager = ServiceManager()
        self.record_service = None
        conn_dict = self.service_manager.get_current_connection()
        #there is a connection but it is unsuccessful
        if (conn_dict == None
            or (conn_dict != None and not self.service_manager.test_connection(conn_dict))):
            # Create a DB form which will set a connection for the service manager
            db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
            db_config.ShowModal()
        if (conn_dict != None and self.service_manager.get_db_version(conn_dict) != u'1.1.1'):
            wx.MessageBox('The ODM database must be version 1.1.1 to use ODMToolsPython',
                          'Database Version Incompatible', wx.OK)
            db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
            db_config.ShowModal()

        self.createService()
        self._init_ctrls(parent)
        self.Refresh()

    #############Entire Form Sizers##########
    def _init_sizers(self):
        # generated method, don't edit
        self.s = wx.BoxSizer(wx.VERTICAL)
        self._init_s_Items(self.s)
        self.SetSizer(self.s)

    def _init_s_Items(self, parent):
        # generated method, don't edit
        parent.AddWindow(self._ribbon, 0, wx.EXPAND)
        parent.AddWindow(self.pnlDocking, 85, flag=wx.ALL | wx.EXPAND)

    ###################### Form ################
    def _init_ctrls(self, prnt):
        # generated method, don't edit
        wx.Frame.__init__(self, id=wxID_ODMTOOLS, name=u'ODMTools', parent=prnt,
                          size=wx.Size(1000, 900),
                          style=wx.DEFAULT_FRAME_STYLE, title=u'ODM Tools')
        self.SetFont(wx.Font(9, wx.SWISS, wx.NORMAL, wx.NORMAL,
                             False, u'Tahoma'))

        Publisher.subscribe(self.onDocking, ("adjust.Docking"))
        Publisher.subscribe(self.onPlotSelection, ("select.Plot"))
        Publisher.subscribe(self.onExecuteScript, ("execute.script"))

        Publisher.subscribe(self.onExecuteScript, ("execute.script"))
        Publisher.subscribe(self.onChangeDBConn, ("change.dbConfig"))
        Publisher.subscribe(self.onSetScriptTitle, ("script.title"))
        Publisher.subscribe(self.onClose, ("onClose"))

        ############### Ribbon ###################
        self._ribbon = mnuRibbon.mnuRibbon(parent=self, id=wx.ID_ANY, name='ribbon')

        ################ Docking Tools##############
        self.pnlDocking = wx.Panel(id=wxID_ODMTOOLSPANEL1, name='pnlDocking',
                                   parent=self, size=wx.Size(605, 458),
                                   style=wx.TAB_TRAVERSAL)

        ################ Series Selection Panel ##################
        self.pnlSelector = pnlSeriesSelector.pnlSeriesSelector(id=wxID_PNLSELECTOR, name=u'pnlSelector',
                                                               parent=self.pnlDocking, size=wx.Size(770, 388),
                                                               style=wx.TAB_TRAVERSAL, dbservice=self.sc)

        ####################grid Table View##################
        self.dataTable = pnlDataTable.pnlDataTable(id=wxID_ODMTOOLSGRID1, name='dataTable',
                                                   parent=self.pnlDocking, size=wx.Size(376, 280),
                                                   style=0)

        ############# Graph ###############
        self.pnlPlot = pnlPlot.pnlPlot(id=wxID_ODMTOOLSPANEL1, name='pnlPlot',
                                       parent=self.pnlDocking, size=wx.Size(605, 458),
                                       style=wx.TAB_TRAVERSAL)

        ############# Script & Console ###############
        self.txtPythonConsole = wx.py.crust.CrustFrame(id=wxID_TXTPYTHONCONSOLE,
                                                       size=wx.Size(500, 800), style=0)

        # Console tools object for usability
        self.console_tools = ConsoleTools(self._ribbon)
        self.txtPythonConsole.shell.run("Tools = app.TopWindow.console_tools", prompt=False, verbose=False)

        self.txtPythonScript = pnlScript(id=wxID_TXTPYTHONSCRIPT,
                                         name=u'txtPython', parent=self,
                                         size=wx.Size(200, 200))

        ############ Docking ###################
        self._mgr = aui.AuiManager()
        self._mgr.SetManagedWindow(self.pnlDocking)

        self._mgr.AddPane(self.dataTable, aui.AuiPaneInfo().Right().Name("Table").
                          Show(show=False).Caption('Table View').MinSize(wx.Size(200, 200)).Position(1).MinimizeButton(True))

        self._mgr.AddPane(self.pnlSelector, aui.AuiPaneInfo().Bottom().Name("Selector")
                         .Caption('Series Selector').Show(show=True).MinSize(wx.Size(100, 200)).Row(2).Position(0).MinimizeButton(True))

        self._mgr.AddPane(self.txtPythonScript, aui.AuiPaneInfo().Caption('Script').
                          Name("Script").Show(show=False).MinSize(wx.Size(200, 200)).Position(1).MinimizeButton(True), target=self._mgr.GetPane("Selector") )

        self._mgr.AddPane(self.txtPythonConsole, aui.AuiPaneInfo().Caption('Python Console').
                          Name("Console").Show(show=False).MinimizeButton(True).Position(2), target=self._mgr.GetPane("Selector"))

        #panedet = self._mgr.GetPane(self.pnlSelector)#.SetSelection(0)
        #print dir(panedet)

        self._mgr.AddPane(self.pnlPlot, aui.AuiPaneInfo().CenterPane().Name("Plot").Caption("Plot").MaximizeButton(True))

        self.loadDockingSettings()

        self._mgr.Update()

        self.Bind(wx.EVT_CLOSE, self.onClose)

        self._init_sizers()
        self._ribbon.Realize()

    def onDocking(self, value):
        panedet = self._mgr.GetPane(self.pnlPlot)
        logger.debug(panedet)

        if value == "Table":
            panedet = self._mgr.GetPane(self.dataTable)
        elif value == "Selector":
            panedet = self._mgr.GetPane(self.pnlSelector)
        elif value == "Script":
            panedet = self._mgr.GetPane(self.txtPythonScript)
        elif value == "Console":
            panedet = self._mgr.GetPane(self.txtPythonConsole)

        if panedet.IsShown():
            panedet.Show(show=False)
        else:
            panedet.Show(show=True)

        self._mgr.Update()

    def onPlotSelection(self, value):
        self.pnlPlot.selectPlot(value)

    def addPlot(self, memDB, seriesID):
        #Memory Database
        self.pnlPlot.addPlot(memDB, seriesID)
        Publisher.sendMessage("EnablePlotButton", plot=self.pnlPlot.getActivePlotID(), isActive=True)
        #self._ribbon.enableButtons(self.pnlPlot.getActivePlotID)

    def onSetScriptTitle(self, title):
        scriptPane = self._mgr.GetPane(self.txtPythonScript)
        scriptPane.Caption(title)
        if scriptPane.IsFloating():
            scriptPane.Restore()
        self._mgr.Update()

    def addEdit(self, seriesID, memDB):

        self.record_service = self.service_manager.get_record_service(self.txtPythonScript, seriesID,
                                                                      connection=memDB.conn)
        self.pnlPlot.addEditPlot(memDB, seriesID, self.record_service)
        self.dataTable.init(memDB, self.record_service)
        self._ribbon.toggleEditButtons(True)

        # set record service for console
        self.console_tools.set_record_service(self.record_service)

    def stopEdit(self):
        self.pnlPlot.stopEdit()
        self.dataTable.stopEdit()
        self.pnlSelector.stopEdit()
        self.record_service = None
        self._ribbon.toggleEditButtons(False)

    def getRecordService(self):
        return self.record_service

    def onChangeDBConn(self, event):
        db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
        value = db_config.ShowModal()

        print "Value: ", value
        #print "wxID_FRMDBCONFIGBTNSAVE: ", db_config._init_ctrls[2] #wxID_FRMDBCONFIGBTNSAVE
        #print "wxID_FRMDBCONFIGBTNCANCEL: ", db_config._init_ctrls[3] #wxID_FRMDBCONFIGBTNCANCEL
        #print "wxID_FRMDBCONFIGBTNTEST: ", db_config._init_ctrls[4] #wxID_FRMDBCONFIGBTNTEST

        if value == wx.ID_OK:
            #reset Series Selector
            self.createService()
            self.pnlSelector.resetDB(self.sc)
            #clear all plots
            self.pnlPlot.clear()
            #clear table
            self.dataTable.clear()

    def createService(self):
        self.sc = self.service_manager.get_series_service()

    def getDBService(self):
        return self.service_manager

    def toggleConsoleTools(self):
        self.txtPythonConsole.ToggleTools()

    def onExecuteScript(self, value):
        for i in ('red', 'blue', 'green', 'magenta', 'gold', 'cyan', 'brown', 'lime', 'purple', 'navy'):
            self.txtPythonScript('This is a test\n', i)

    def loadDockingSettings(self):
        #test if there is a perspective to load
        try:
            f = open(util.resource_path('ODMTools.config'), 'r')
        except:
            # Create the file if it doesn't exist
            open(util.resource_path('ODMTools.config'), 'w').close()
            f = open(util.resource_path('ODMTools.config'), 'r')

        self._mgr.LoadPerspective(f.read(), True)

    def onClose(self, event):
        # deinitialize the frame manager
        self.pnlPlot.Close()
        try:
            f = open(util.resource_path('ODMTools.config'), 'w')
            f.write(self._mgr.SavePerspective())
        except:
            print "error saving docking data"
        self._mgr.UnInit()
        # delete the frame
        self.Destroy()

if __name__ == '__main__':
    app = wx.App(False)
    frame = create(None)
    frame.Show()

    app.MainLoop()<commit_msg>Worked on Layout issues and also console bugs. Getting closer on Layout issues. We may need to use AuiNoteBook to hold the different elements as it appears to be more stable for what we're doing<commit_after>#!/usr/bin/env
#Boa:Frame:ODMTools
import sys
import os

this_file = os.path.realpath(__file__)
directory = os.path.dirname(os.path.dirname(this_file))
sys.path.insert(0, directory)

import wx
import wx.grid
#import wx.aui

try:
    from agw import aui
except ImportError:  # if it's not there locally, try the wxPython lib.
    import wx.lib.agw.aui as aui

import wx.richtext
import wx.stc
import wx.lib.agw.aui as aui
from wx.lib.pubsub import pub as Publisher
import wx.py.crust
import frmDBConfiguration

from odmservices import ServiceManager
from odmservices import utilities as util
from pnlScript import pnlScript
import pnlSeriesSelector
import pnlPlot
import mnuRibbon
import pnlDataTable

from odmconsole import ConsoleTools
from common.logger import LoggerTool
import logging

def create(parent):
    return frmODMToolsMain(parent)

[
    wxID_ODMTOOLS, wxID_ODMTOOLSCHECKLISTBOX2, wxID_ODMTOOLSCOMBOBOX1,
    wxID_ODMTOOLSCOMBOBOX2, wxID_ODMTOOLSCOMBOBOX4, wxID_ODMTOOLSCOMBOBOX5,
    wxID_ODMTOOLSGRID1, wxID_ODMTOOLSPANEL1, wxID_ODMTOOLSPANEL2,
    wxID_ODMTOOLSTOOLBAR1, wxID_PNLSELECTOR, wxID_TXTPYTHONSCRIPT,
    wxID_TXTPYTHONCONSOLE,
] = [wx.NewId() for _init_ctrls in range(13)]

tool = LoggerTool()
logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG)

class frmODMToolsMain(wx.Frame):
    def __init__(self, parent):
        self.service_manager = ServiceManager()
        self.record_service = None
        conn_dict = self.service_manager.get_current_connection()
        #there is a connection but it is unsuccessful
        if (conn_dict == None
            or (conn_dict != None and not self.service_manager.test_connection(conn_dict))):
            # Create a DB form which will set a connection for the service manager
            db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
            db_config.ShowModal()
        if (conn_dict != None and self.service_manager.get_db_version(conn_dict) != u'1.1.1'):
            wx.MessageBox('The ODM database must be version 1.1.1 to use ODMToolsPython',
                          'Database Version Incompatible', wx.OK)
            db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
            db_config.ShowModal()

        self.createService()
        self._init_ctrls(parent)
        self.Refresh()

    #############Entire Form Sizers##########
    def _init_sizers(self):
        # generated method, don't edit
        self.s = wx.BoxSizer(wx.VERTICAL)
        self._init_s_Items(self.s)
        self.SetSizer(self.s)

    def _init_s_Items(self, parent):
        # generated method, don't edit
        parent.AddWindow(self._ribbon, 0, wx.EXPAND)
        parent.AddWindow(self.pnlDocking, 85, flag=wx.ALL | wx.EXPAND)

    ###################### Form ################
    def _init_ctrls(self, prnt):
        # generated method, don't edit
        wx.Frame.__init__(self, id=wxID_ODMTOOLS, name=u'ODMTools', parent=prnt,
                          size=wx.Size(1000, 900),
                          style=wx.DEFAULT_FRAME_STYLE, title=u'ODM Tools')
        self.SetFont(wx.Font(9, wx.SWISS, wx.NORMAL, wx.NORMAL,
                             False, u'Tahoma'))

        Publisher.subscribe(self.onDocking, ("adjust.Docking"))
        Publisher.subscribe(self.onPlotSelection, ("select.Plot"))
        Publisher.subscribe(self.onExecuteScript, ("execute.script"))

        Publisher.subscribe(self.onExecuteScript, ("execute.script"))
        Publisher.subscribe(self.onChangeDBConn, ("change.dbConfig"))
        Publisher.subscribe(self.onSetScriptTitle, ("script.title"))
        Publisher.subscribe(self.onClose, ("onClose"))

        ############### Ribbon ###################
        self._ribbon = mnuRibbon.mnuRibbon(parent=self, id=wx.ID_ANY, name='ribbon')

        ################ Docking Tools##############
        self.pnlDocking = wx.Panel(id=wxID_ODMTOOLSPANEL1, name='pnlDocking',
                                   parent=self, size=wx.Size(605, 458),
                                   style=wx.TAB_TRAVERSAL)

        ################ Series Selection Panel ##################
        self.pnlSelector = pnlSeriesSelector.pnlSeriesSelector(id=wxID_PNLSELECTOR, name=u'pnlSelector',
                                                               parent=self.pnlDocking, size=wx.Size(770, 388),
                                                               style=wx.TAB_TRAVERSAL, dbservice=self.sc)

        ####################grid Table View##################
        self.dataTable = pnlDataTable.pnlDataTable(id=wxID_ODMTOOLSGRID1, name='dataTable',
                                                   parent=self.pnlDocking, size=wx.Size(376, 280),
                                                   style=0)

        ############# Graph ###############
        self.pnlPlot = pnlPlot.pnlPlot(id=wxID_ODMTOOLSPANEL1, name='pnlPlot',
                                       parent=self.pnlDocking, size=wx.Size(605, 458),
                                       style=wx.TAB_TRAVERSAL)

        ############# Script & Console ###############
        self.txtPythonConsole = wx.py.crust.CrustFrame(id=wxID_TXTPYTHONCONSOLE, size=wx.Size(200, 200), style=0)

        #self.txtPythonConsole = wx.py.shell.Shell(id=wxID_TXTPYTHONCONSOLE, size=wx.Size(500, 800), parent=self.pnlDocking)

        # Console tools object for usability
        self.console_tools = ConsoleTools(self._ribbon)
        self.txtPythonConsole.shell.run("Tools = app.TopWindow.console_tools", prompt=False, verbose=False)

        print "Stuff: ", dir(self.txtPythonConsole)

        self.txtPythonScript = pnlScript(id=wxID_TXTPYTHONSCRIPT,
                                         name=u'txtPython', parent=self,
                                         size=wx.Size(200, 200))

        ############ Docking ###################
        self._mgr = aui.AuiManager()
        self._mgr.SetManagedWindow(self.pnlDocking)

        self._mgr.AddPane(self.pnlPlot, aui.AuiPaneInfo().CenterPane()
                          .Name("Plot").Caption("Plot").MaximizeButton(True))

        self._mgr.AddPane(self.dataTable, aui.AuiPaneInfo().Right()
                          .Name("Table").Show(show=False).Caption('Table View').MinSize(wx.Size(200, 200))
                          .Position(1).MinimizeButton(True))

        self._mgr.AddPane(self.pnlSelector, aui.AuiPaneInfo().Caption('Series Selector').
                          Name("Selector").Show(show=True).MinSize(wx.Size(200, 200)).MinimizeButton(True)
                          .Position(1).CloseButton(False).Bottom())  #, target=self._mgr.GetPane("Script"))

        self._mgr.AddPane(self.txtPythonScript, aui.AuiPaneInfo().Bottom().Caption('Script').
                          Name("Script").Show(show=True).Right().MinimizeButton(True)
                          .Position(1).CloseButton(False))#, target=self._mgr.GetPane("Selector"))

        self._mgr.AddPane(self.txtPythonConsole, aui.AuiPaneInfo().Caption('Python Console').
                          Name("Console").Show(show=True).MinSize(wx.Size(200, 200)).Right().MinimizeButton(True)
                          .Layer(1).Position(1).CloseButton(False), target=self._mgr.GetPane("Script"))

        #print "panes: ", self._mgr.GetAllPanes()

        #self._mgr.AddPane(self.pnlSelector, aui.AuiPaneInfo().Bottom().Name("Selector")
        #       .Caption('Series Selector').Show(show=True).MinSize(wx.Size(100, 200)).Position(2).MinimizeButton(True))

        #self._mgr.AddPane(self.txtPythonScript, aui.AuiPaneInfo().Caption('Script').
        #                  Name("Script").Show(show=False).MinSize(wx.Size(200, 200)).Position(2).MinimizeButton(True),
        #                  target=self._mgr.GetPane("Selector"))

        #self._mgr.AddPane(self.txtPythonConsole, aui.AuiPaneInfo().Caption('Python Console').
        #              Name("Console").Show(show=False).MinimizeButton(True).Position(2),
        #              target=self._mgr.GetPane("Selector"))

        #panedet = self._mgr.GetPane(self.pnlSelector)#.SetSelection(0)
        #print dir(panedet)

        self.loadDockingSettings()

        self._mgr.Update()

        self.Bind(wx.EVT_CLOSE, self.onClose)

        self._init_sizers()
        self._ribbon.Realize()

    def onDocking(self, value):
        panedet = self._mgr.GetPane(self.pnlPlot)

        if value == "Table":
            panedet = self._mgr.GetPane(self.dataTable)
        elif value == "Selector":
            panedet = self._mgr.GetPane(self.pnlSelector)
        elif value == "Script":
            panedet = self._mgr.GetPane(self.txtPythonScript)
        elif value == "Console":
            panedet = self._mgr.GetPane(self.txtPythonConsole)

        if panedet.IsShown():
            panedet.Show(show=False)
        else:
            panedet.Show(show=True)

        self._mgr.Update()

    def onPlotSelection(self, value):
        self.pnlPlot.selectPlot(value)

    def addPlot(self, memDB, seriesID):
        #Memory Database
        self.pnlPlot.addPlot(memDB, seriesID)
        Publisher.sendMessage("EnablePlotButton", plot=self.pnlPlot.getActivePlotID(), isActive=True)
        #self._ribbon.enableButtons(self.pnlPlot.getActivePlotID)

    def onSetScriptTitle(self, title):
        scriptPane = self._mgr.GetPane(self.txtPythonScript)
        scriptPane.Caption(title)
        if scriptPane.IsFloating():
            scriptPane.Restore()
        self._mgr.Update()

    def addEdit(self, seriesID, memDB):

        self.record_service = self.service_manager.get_record_service(self.txtPythonScript, seriesID,
                                                                      connection=memDB.conn)
        self.pnlPlot.addEditPlot(memDB, seriesID, self.record_service)
        self.dataTable.init(memDB, self.record_service)
        self._ribbon.toggleEditButtons(True)

        # set record service for console
        self.console_tools.set_record_service(self.record_service)

    def stopEdit(self):
        self.pnlPlot.stopEdit()
        self.dataTable.stopEdit()
        self.pnlSelector.stopEdit()
        self.record_service = None
        self._ribbon.toggleEditButtons(False)

    def getRecordService(self):
        return self.record_service

    def onChangeDBConn(self, event):
        db_config = frmDBConfiguration.frmDBConfig(None, self.service_manager, False)
        value = db_config.ShowModal()

        print "Value: ", value
        #print "wxID_FRMDBCONFIGBTNSAVE: ", db_config._init_ctrls[2] #wxID_FRMDBCONFIGBTNSAVE
        #print "wxID_FRMDBCONFIGBTNCANCEL: ", db_config._init_ctrls[3] #wxID_FRMDBCONFIGBTNCANCEL
        #print "wxID_FRMDBCONFIGBTNTEST: ", db_config._init_ctrls[4] #wxID_FRMDBCONFIGBTNTEST

        if value == wx.ID_OK:
            #reset Series Selector
            self.createService()
            self.pnlSelector.resetDB(self.sc)
            #clear all plots
            self.pnlPlot.clear()
            #clear table
            self.dataTable.clear()

    def createService(self):
        self.sc = self.service_manager.get_series_service()

    def getDBService(self):
        return self.service_manager

    def toggleConsoleTools(self):
        self.txtPythonConsole.ToggleTools()

    def onExecuteScript(self, value):
        for i in ('red', 'blue', 'green', 'magenta', 'gold', 'cyan', 'brown', 'lime', 'purple', 'navy'):
            self.txtPythonScript('This is a test\n', i)

    def loadDockingSettings(self):
        #test if there is a perspective to load
        try:
            f = open(util.resource_path('ODMTools.config'), 'r')
        except:
            # Create the file if it doesn't exist
            open(util.resource_path('ODMTools.config'), 'w').close()
            f = open(util.resource_path('ODMTools.config'), 'r')

        self._mgr.LoadPerspective(f.read(), True)

    def onClose(self, event):
        # deinitialize the frame manager
        self.pnlPlot.Close()
        try:
            f = open(util.resource_path('ODMTools.config'), 'w')
            f.write(self._mgr.SavePerspective())
        except:
            print "error saving docking data"
        self._mgr.UnInit()
        # delete the frame
        self.Destroy()

if __name__ == '__main__':
    app = wx.App(False)
    frame = create(None)
    frame.Show()

    app.MainLoop()<|endoftext|>


<commit_before># coding=utf-8
import io
import sys
from operator import itemgetter

def collate(iterables, *, reverse=False):
    """
    Accepts multiple iterables and yields items from them in a roughly sorted
    order.

    Each item from each iterator will appear once. If a unique item x is
    produced by an iterator before a unique item y from that same iterator, x
    will ALWAYS come before y in the resulting stream.

    Identical values are taken first from the earliest ordered iterator.

    >>> list(collate([[1, 2, 11, 12], [4, 5, 6]]))
    [1, 2, 4, 5, 6, 11, 12]
    >>> list(collate([
    ...     [5, 6, 7, 4, 5, 6],
    ...     [6, 1, 2, 3],
    ...     [8, 10, 12],
    ...     [8, 9, 11, 13]
    ... ]))
    [5, 6, 6, 1, 2, 3, 7, 4, 5, 6, 8, 8, 9, 10, 11, 12, 13]
    """
    first = max if reverse else min
    opened = []
    current = []
    # get the first items if they exist
    for feed in iterables:
        it = iter(feed)
        try:
            current.append(next(it))
        except StopIteration:
            continue
        else:
            opened.append(it)

    while opened:
        i, result = first(enumerate(current), key=itemgetter(1))
        # advance only the feed we just took the item from
        try:
            current[i] = next(opened[i])
        except StopIteration:
            # feed is finished, remove its entry in the lists
            del current[i]
            del opened[i]

        yield result

# this inspector is kinda dumb
# noinspection PyTypeChecker
def merge(iterables, *, reverse=False):
    """
    Merges multiple possibly incomplete iterators that may include selections
    or stretches of identical content into a single feed that includes every
    unique value at least once.

    Example usage: A sorted list of unique and comparable items is spread out
    among several smaller lists, which is each created by iterating over the
    original list and only taking certain items by an arbitrary standard. Many
    of the lists are very incomplete, but each item in the original list
    appears one or more times among the smaller lists. Passing all these lists
    into merge() will produce an iterator in the exact order and items of the
    original list, with none repeated.

    Likewise: merge(sorted_iterables) is equivalent to
    iter(sorted(set(chain.from_iterable(sorted_iterables))))

    >>> from random import sample, randint
    >>> original = list(range(1000))
    >>> small_lists = [[] for _ in range(10)]
    >>> for item in original:
    ...     for put_in in sample(range(10), randint(1, 5)):
    ...         small_lists[put_in].append(item)
    >>> original == list(merge(small_lists))
    True

    """
    first = max if reverse else min
    opened = []
    current = []
    for feed in iterables:
        it = iter(feed)
        try:
            current.append(next(it))
        except StopIteration:
            continue
        else:
            opened.append(it)

    while opened:
        result = first(current)
        # advance every feed that is going to provide an equal value
        # in reversed order so we can delete in-place while iterating
        for i in reversed(range(len(opened))):
            if current[i] == result:
                try:
                    current[i] = next(opened[i])
                except StopIteration:
                    del current[i]
                    del opened[i]

        yield result

class IteratorFile(io.TextIOBase):
    """ given an iterator which yields strings,
    return a file like object for reading those strings.
    From github/jsheedy """

    def __init__(self, it):
        super().__init__()
        self._it = it
        self._f = io.StringIO()

    def read(self, length=sys.maxsize):

        try:
            while self._f.tell() < length:
                self._f.write(next(self._it))

        finally:
            self._f.seek(0)
            data = self._f.read(length)

            # save the remainder for next read
            remainder = self._f.read()
            self._f.seek(0)
            self._f.truncate(0)
            self._f.write(remainder)
            return data

    def readline(self):
        return next(self._it)
<commit_msg>get iterator from iterable on init<commit_after># coding=utf-8
import io
import sys
from operator import itemgetter

def collate(iterables, *, reverse=False):
    """
    Accepts multiple iterables and yields items from them in a roughly sorted
    order.

    Each item from each iterator will appear once. If a unique item x is
    produced by an iterator before a unique item y from that same iterator, x
    will ALWAYS come before y in the resulting stream.

    Identical values are taken first from the earliest ordered iterator.

    >>> list(collate([[1, 2, 11, 12], [4, 5, 6]]))
    [1, 2, 4, 5, 6, 11, 12]
    >>> list(collate([
    ...     [5, 6, 7, 4, 5, 6],
    ...     [6, 1, 2, 3],
    ...     [8, 10, 12],
    ...     [8, 9, 11, 13]
    ... ]))
    [5, 6, 6, 1, 2, 3, 7, 4, 5, 6, 8, 8, 9, 10, 11, 12, 13]
    """
    first = max if reverse else min
    opened = []
    current = []
    # get the first items if they exist
    for feed in iterables:
        it = iter(feed)
        try:
            current.append(next(it))
        except StopIteration:
            continue
        else:
            opened.append(it)

    while opened:
        i, result = first(enumerate(current), key=itemgetter(1))
        # advance only the feed we just took the item from
        try:
            current[i] = next(opened[i])
        except StopIteration:
            # feed is finished, remove its entry in the lists
            del current[i]
            del opened[i]

        yield result

# this inspector is kinda dumb
# noinspection PyTypeChecker
def merge(iterables, *, reverse=False):
    """
    Merges multiple possibly incomplete iterators that may include selections
    or stretches of identical content into a single feed that includes every
    unique value at least once.

    Example usage: A sorted list of unique and comparable items is spread out
    among several smaller lists, which is each created by iterating over the
    original list and only taking certain items by an arbitrary standard. Many
    of the lists are very incomplete, but each item in the original list
    appears one or more times among the smaller lists. Passing all these lists
    into merge() will produce an iterator in the exact order and items of the
    original list, with none repeated.

    Likewise: merge(sorted_iterables) is equivalent to
    iter(sorted(set(chain.from_iterable(sorted_iterables))))

    >>> from random import sample, randint
    >>> original = list(range(1000))
    >>> small_lists = [[] for _ in range(10)]
    >>> for item in original:
    ...     for put_in in sample(range(10), randint(1, 5)):
    ...         small_lists[put_in].append(item)
    >>> original == list(merge(small_lists))
    True

    """
    first = max if reverse else min
    opened = []
    current = []
    for feed in iterables:
        it = iter(feed)
        try:
            current.append(next(it))
        except StopIteration:
            continue
        else:
            opened.append(it)

    while opened:
        result = first(current)
        # advance every feed that is going to provide an equal value
        # in reversed order so we can delete in-place while iterating
        for i in reversed(range(len(opened))):
            if current[i] == result:
                try:
                    current[i] = next(opened[i])
                except StopIteration:
                    del current[i]
                    del opened[i]

        yield result

class IteratorFile(io.TextIOBase):
    """ given an iterator which yields strings,
    return a file like object for reading those strings.
    From github/jsheedy """

    def __init__(self, it):
        super().__init__()
        self._it = iter(it)
        self._f = io.StringIO()

    def read(self, length=sys.maxsize):

        try:
            while self._f.tell() < length:
                self._f.write(next(self._it))

        finally:
            self._f.seek(0)
            data = self._f.read(length)

            # save the remainder for next read
            remainder = self._f.read()
            self._f.seek(0)
            self._f.truncate(0)
            self._f.write(remainder)
            return data

    def readline(self):
        return next(self._it)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import traceback
import numpy as np
import matplotlib.pyplot as plt

from PyQt4.QtGui import *
from PyQt4.QtCore import *

sys.path.append('..')
import qtutil

from util import fileloader, fileconverter

class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class RawConverterDialog(QDialog):
  def __init__(self, project, filename, parent=None):
    super(RawConverterDialog, self).__init__(parent)
    self.project = project
    self.filename = filename
    self.setup_ui()

  def setup_ui(self):
    self.setWindowTitle('Import Raw File')

    vbox = QVBoxLayout()
    vbox.addWidget(QLabel('Importing \'{}\''.format(
      os.path.basename(self.filename))))
    vbox.addSpacing(10)

    grid = QGridLayout()

    grid.addWidget(QLabel('Width:'), 0, 0)
    self.sb_width = QSpinBox()
    self.sb_width.setMinimum(1)
    self.sb_width.setMaximum(1024)
    self.sb_width.setValue(256)
    grid.addWidget(self.sb_width, 0, 1)

    grid.addWidget(QLabel('Height:'), 1, 0)
    self.sb_height = QSpinBox()
    self.sb_height.setMinimum(1)
    self.sb_height.setMaximum(1024)
    self.sb_height.setValue(256)
    grid.addWidget(self.sb_height, 1, 1)

    grid.addWidget(QLabel('Channel:'), 2, 0)
    self.sb_channel = QSpinBox()
    self.sb_channel.setMinimum(1)
    self.sb_channel.setMaximum(3)
    self.sb_channel.setValue(3)
    grid.addWidget(self.sb_channel, 2, 1)

    grid.addWidget(QLabel('dtype:'), 3, 0)
    self.cb_dtype = QComboBox()
    for t in 'uint8', 'float32', 'float64':
      self.cb_dtype.addItem(t)
    grid.addWidget(self.cb_dtype, 3, 1)

    vbox.addLayout(grid)
    vbox.addStretch()

    pb = QPushButton('&Convert')
    pb.clicked.connect(self.convert_clicked)
    vbox.addWidget(pb)

    self.setLayout(vbox)
    self.resize(400, 220)

  def convert_clicked(self):
    dtype = str(self.cb_dtype.currentText())
    width = int(self.sb_width.value())
    height = int(self.sb_height.value())
    channels = int(self.sb_channel.value())
    path = os.path.splitext(os.path.basename(self.filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting raw to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.raw2npy(self.filename, path, dtype, width, height,
        channels, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      self.ret_filename = path
      self.close()

class Widget(QWidget):
  def __init__(self, project, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
      return

    self.project = project
    self.setup_ui()

    self.listview.setModel(QStandardItemModel())
    for f in self.project.files:
      if f['type'] == 'video':
        self.listview.model().appendRow(QStandardItem(f['path']))

  def setup_ui(self):
    vbox = QVBoxLayout()

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    hbox = QVBoxLayout()
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    hbox.addWidget(pb)

    vbox.addLayout(hbox)
    vbox.addStretch()
    self.setLayout(vbox)

  def convert_raw(self, filename):
    dialog = RawConverterDialog(self.project, filename, self)
    dialog.ret_filename = None
    dialog.exec_()
    return dialog.ret_filename

  def convert_tif(self, filename):
    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.tif2npy(filename, path, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      ret_filename = path
    return ret_filename

  def to_npy(self, filename):
    if filename.endswith('.raw'):
      filename = self.convert_raw(filename)
    elif filename.endswith('.tif'):
      filename = self.convert_tif(filename)
    else:
      raise fileloader.UnknownFileFormatError()
    return filename

  def import_file(self, filename):
    if not filename.endswith('.npy'):
      new_filename = self.to_npy(filename)

    if not new_filename:
      raise NotConvertedError()

    filename = new_filename

    if filename in [f['path'] for f in self.project.files]:
      raise FileAlreadyInProjectError(filename)

    self.project.files.append({
      'path': filename,
      'type': 'video',
      'manipulations': []
    })
    self.project.save()
    return filename

  def import_files(self, filenames):
    for filename in filenames:
      if filename in [f['path'] for f in self.project.files]:
        continue
      try:
        filename = self.import_file(filename)
      except NotConvertedError:
        qtutil.warning('Skipping file \'{}\' since not converted.'.format(filename))
      except FileAlreadyInProjectError as e:
        qtutil.warning('Skipping file \'{}\' since already in project.'.format(e.filename))
      except:
        qtutil.critical('Import of \'{}\' failed:\n'.format(filename) +\
          traceback.format_exc())
      else:
        self.listview.model().appendRow(QStandardItem(filename))

  def new_video(self):
    filenames = QFileDialog.getOpenFileNames(
      self, 'Load images', QSettings().value('last_load_data_path').toString(),
      'Video files (*.npy *.tif *.raw)')
    filenames = map(str, filenames)
    if not filenames:
      return
    QSettings().setValue('last_load_data_path', os.path.dirname(filenames[0]))
    self.import_files(filenames)

class MyPlugin:
  def __init__(self, project):
    self.name = 'Import video files'
    self.widget = Widget(project)

  def run(self):
    pass

if __name__ == '__main__':
  app = QApplication(sys.argv)
  app.aboutToQuit.connect(app.deleteLater)
  w = QMainWindow()
  w.setCentralWidget(Widget())
  w.show()
  app.exec_()
  sys.exit()
<commit_msg>Fix annoying import error<commit_after>#!/usr/bin/env python

import os
import sys
import traceback
import numpy as np
import matplotlib.pyplot as plt

from PyQt4.QtGui import *
from PyQt4.QtCore import *

sys.path.append('..')
import qtutil

from util import fileloader, fileconverter

class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class RawConverterDialog(QDialog):
  def __init__(self, project, filename, parent=None):
    super(RawConverterDialog, self).__init__(parent)
    self.project = project
    self.filename = filename
    self.setup_ui()

  def setup_ui(self):
    self.setWindowTitle('Import Raw File')

    vbox = QVBoxLayout()
    vbox.addWidget(QLabel('Importing \'{}\''.format(
      os.path.basename(self.filename))))
    vbox.addSpacing(10)

    grid = QGridLayout()

    grid.addWidget(QLabel('Width:'), 0, 0)
    self.sb_width = QSpinBox()
    self.sb_width.setMinimum(1)
    self.sb_width.setMaximum(1024)
    self.sb_width.setValue(256)
    grid.addWidget(self.sb_width, 0, 1)

    grid.addWidget(QLabel('Height:'), 1, 0)
    self.sb_height = QSpinBox()
    self.sb_height.setMinimum(1)
    self.sb_height.setMaximum(1024)
    self.sb_height.setValue(256)
    grid.addWidget(self.sb_height, 1, 1)

    grid.addWidget(QLabel('Channel:'), 2, 0)
    self.sb_channel = QSpinBox()
    self.sb_channel.setMinimum(1)
    self.sb_channel.setMaximum(3)
    self.sb_channel.setValue(3)
    grid.addWidget(self.sb_channel, 2, 1)

    grid.addWidget(QLabel('dtype:'), 3, 0)
    self.cb_dtype = QComboBox()
    for t in 'uint8', 'float32', 'float64':
      self.cb_dtype.addItem(t)
    grid.addWidget(self.cb_dtype, 3, 1)

    vbox.addLayout(grid)
    vbox.addStretch()

    pb = QPushButton('&Convert')
    pb.clicked.connect(self.convert_clicked)
    vbox.addWidget(pb)

    self.setLayout(vbox)
    self.resize(400, 220)

  def convert_clicked(self):
    dtype = str(self.cb_dtype.currentText())
    width = int(self.sb_width.value())
    height = int(self.sb_height.value())
    channels = int(self.sb_channel.value())
    path = os.path.splitext(os.path.basename(self.filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting raw to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.raw2npy(self.filename, path, dtype, width, height,
        channels, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      self.ret_filename = path
      self.close()

class Widget(QWidget):
  def __init__(self, project, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
      return

    self.project = project
    self.setup_ui()

    self.listview.setModel(QStandardItemModel())
    for f in self.project.files:
      if f['type'] == 'video':
        self.listview.model().appendRow(QStandardItem(f['path']))

  def setup_ui(self):
    vbox = QVBoxLayout()

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    hbox = QVBoxLayout()
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    hbox.addWidget(pb)

    vbox.addLayout(hbox)
    vbox.addStretch()
    self.setLayout(vbox)

  def convert_raw(self, filename):
    dialog = RawConverterDialog(self.project, filename, self)
    dialog.ret_filename = None
    dialog.exec_()
    return dialog.ret_filename

  def convert_tif(self, filename):
    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.tif2npy(filename, path, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      ret_filename = path
    return ret_filename

  def to_npy(self, filename):
    if filename.endswith('.raw'):
      filename = self.convert_raw(filename)
    elif filename.endswith('.tif'):
      filename = self.convert_tif(filename)
    else:
      raise fileloader.UnknownFileFormatError()
    return filename

  def import_file(self, filename):
    if not filename.endswith('.npy'):
      new_filename = self.to_npy(filename)
      if not new_filename:
        raise NotConvertedError()
      else:
        filename = new_filename

    if filename in [f['path'] for f in self.project.files]:
      raise FileAlreadyInProjectError(filename)

    self.project.files.append({
      'path': filename,
      'type': 'video',
      'manipulations': []
    })
    self.project.save()
    return filename

  def import_files(self, filenames):
    for filename in filenames:
      if filename in [f['path'] for f in self.project.files]:
        continue
      try:
        filename = self.import_file(filename)
      except NotConvertedError:
        qtutil.warning('Skipping file \'{}\' since not converted.'.format(filename))
      except FileAlreadyInProjectError as e:
        qtutil.warning('Skipping file \'{}\' since already in project.'.format(e.filename))
      except:
        qtutil.critical('Import of \'{}\' failed:\n'.format(filename) +\
          traceback.format_exc())
      else:
        self.listview.model().appendRow(QStandardItem(filename))

  def new_video(self):
    filenames = QFileDialog.getOpenFileNames(
      self, 'Load images', QSettings().value('last_load_data_path').toString(),
      'Video files (*.npy *.tif *.raw)')
    filenames = map(str, filenames)
    if not filenames:
      return
    QSettings().setValue('last_load_data_path', os.path.dirname(filenames[0]))
    self.import_files(filenames)

class MyPlugin:
  def __init__(self, project):
    self.name = 'Import video files'
    self.widget = Widget(project)

  def run(self):
    pass

if __name__ == '__main__':
  app = QApplication(sys.argv)
  app.aboutToQuit.connect(app.deleteLater)
  w = QMainWindow()
  w.setCentralWidget(Widget())
  w.show()
  app.exec_()
  sys.exit()
<|endoftext|>


<commit_before>#!/usr/bin/env python
__author__ = 'scott hendrickson'

import multiprocessing.queues
import multiprocessing
import time
import gzip
import os
import logging

write_lock = multiprocessing.RLock()

class SaveThread(object):
    def __init__(self, _upstream, _pool_size, _feedname, _savepath):
        self.queue = _upstream
        self.pool_size = _pool_size
        self.logr = logging.getLogger("SaveThread")
        self.savepath = _savepath
        self.feedName = _feedname
        self.timeStart = time.gmtime(time.time())
        self._stopped = multiprocessing.Event()
        self.run_process = multiprocessing.Process(target=self._run)

    def run(self):
        self.run_process.start()

    def _run(self):
        self.logr.debug("started")
        file_name = self.setup_out_file()
        self.run_loop(file_name)

    def run_loop(self, file_name):
        while not self._stopped.is_set():
            chunk = self.next_message()
            if None != chunk:
                print(str(write_lock))
                with write_lock:
                    self.write(file_name, str(chunk))

    def write(self, file_name, string):
        try:
            fp = gzip.open(file_name, "a")
            fp.write(string)
            fp.close()
            self.logr.info("saved file %s" % file_name)
        except Exception, e:
            self.logr.error("write failed: %s" % e)
            raise e

    def next_message(self):
        ret_val = None
        if self.queue.qsize() > 0:
            try:
                ret_val = self.queue.get(block=False)
            except multiprocessing.queues.Empty:
                self.logr.error("Queue was empty when trying to get next message")
        return ret_val

    def stop(self):
        self._stopped.set()

    def stopped(self):
        return not self._stopped.is_set() and self.queue.qsize() > 0

    def running(self):
        self.run_process.is_alive() and not self._stopped.is_set()

    def setup_out_file(self):
        file_path = "/".join([
            self.savepath,
            "%d" % self.timeStart.tm_year,
            "%02d" % self.timeStart.tm_mon,
            "%02d" % self.timeStart.tm_mday,
            "%02d" % self.timeStart.tm_hour])
        try:
            os.makedirs(file_path)
            self.logr.info("directory created (%s)" % file_path)
        except OSError:
            self.logr.info("directory exists (%s)" % file_path)
        name = self.feedName + "_"
        name += "-".join([
            "%d" % self.timeStart.tm_year,
            "%02d" % self.timeStart.tm_mon,
            "%02d" % self.timeStart.tm_mday])
        name += "_%02d%02d" % (self.timeStart.tm_hour, self.timeStart.tm_min)
        name += ".gz"
        file_name = file_path + "/" + name
        return file_name
<commit_msg>edit stopped logic<commit_after>#!/usr/bin/env python
__author__ = 'scott hendrickson'

import multiprocessing.queues
import multiprocessing
import time
import gzip
import os
import logging

write_lock = multiprocessing.RLock()

class SaveThread(object):
    def __init__(self, _upstream, _pool_size, _feedname, _savepath):
        self.queue = _upstream
        self.pool_size = _pool_size
        self.logr = logging.getLogger("SaveThread")
        self.savepath = _savepath
        self.feedName = _feedname
        self.timeStart = time.gmtime(time.time())
        self._stopped = multiprocessing.Event()
        self.run_process = multiprocessing.Process(target=self._run)

    def run(self):
        self.run_process.start()

    def _run(self):
        self.logr.debug("started")
        file_name = self.setup_out_file()
        self.run_loop(file_name)

    def run_loop(self, file_name):
        while not self._stopped.is_set():
            chunk = self.next_message()
            if None != chunk:
                with write_lock:
                    self.write(file_name, str(chunk))

    def write(self, file_name, string):
        try:
            fp = gzip.open(file_name, "a")
            fp.write(string)
            fp.close()
            self.logr.info("saved file %s" % file_name)
        except Exception, e:
            self.logr.error("write failed: %s" % e)
            raise e

    def next_message(self):
        ret_val = None
        if self.queue.qsize() > 0:
            try:
                ret_val = self.queue.get(block=False)
            except multiprocessing.queues.Empty:
                self.logr.error("Queue was empty when trying to get next message")
        return ret_val

    def stop(self):
        self._stopped.set()

    def stopped(self):
        return self._stopped.is_set() and self.queue.qsize() == 0

    def running(self):
        self.run_process.is_alive() and not self._stopped.is_set()

    def setup_out_file(self):
        file_path = "/".join([
            self.savepath,
            "%d" % self.timeStart.tm_year,
            "%02d" % self.timeStart.tm_mon,
            "%02d" % self.timeStart.tm_mday,
            "%02d" % self.timeStart.tm_hour])
        try:
            os.makedirs(file_path)
            self.logr.info("directory created (%s)" % file_path)
        except OSError:
            self.logr.info("directory exists (%s)" % file_path)
        name = self.feedName + "_"
        name += "-".join([
            "%d" % self.timeStart.tm_year,
            "%02d" % self.timeStart.tm_mon,
            "%02d" % self.timeStart.tm_mday])
        name += "_%02d%02d" % (self.timeStart.tm_hour, self.timeStart.tm_min)
        name += ".gz"
        file_name = file_path + "/" + name
        return file_name
<|endoftext|>


<commit_before># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio, concurrent
import itertools
import h5py
import numpy as np
import os.path

from pycontrol.stream import DataStreamDescriptor
from pycontrol.logging import logger
from pycontrol.filters.filter import Filter, InputConnector, OutputConnector
from tqdm import tqdm, tqdm_notebook

class WriteToHDF5(Filter):
    """Writes data to file."""

    data = InputConnector()
    def __init__(self, filename, compress=True, **kwargs):
        super(WriteToHDF5, self).__init__(**kwargs)
        self.compress = compress
        self.filename = filename
        self.points_taken = 0
        self.file = None

    def final_init(self):
        self.file = self.new_file()

    def new_filename(self):
        # Increment the filename until we find one we want.
        i = 0
        ext = self.filename.find('.h5')
        if ext > -1:
            filename = self.filename[:ext]
        while os.path.exists("{}-{:04d}.h5".format(filename,i)):
            i += 1
        return "{}-{:04d}.h5".format(filename,i)

    def new_file(self):
        """ Open a new data file to write """
        # Close the current file, if any
        if self.file is not None:
            try:
                self.file.close()
            except Exception as e:
                logger.error("Encounter exception: {}".format(e))
                logger.error("Cannot close file '{}'. File may be damaged.".format(self.file.filename))
        # Get new file name
        filename = self.new_filename()
        head = os.path.dirname(filename)
        head = os.path.normpath(head)
        dirs = head.split(os.sep)
        # Check if path exists. If not, create new one(s).
        fulldir = ''
        for d in dirs:
            fulldir = os.path.join(fulldir, d)
            if not os.path.exists(fulldir):
                logger.debug("Create new directory: {}.".format(fulldir))
                os.mkdir(fulldir)
        logger.debug("Create new data file: %s." %filename)
        return h5py.File(filename, 'w')

    async def run(self):
        stream     = self.data.input_streams[0]
        desc       = stream.descriptor
        axes       = stream.descriptor.axes
        params     = stream.descriptor.params
        axis_names = desc.axis_names()

        params['exp_src'] = stream.descriptor.exp_src
        num_axes   = len(axes)

        # All of the combinations for the present values of the sweep parameters only
        tuples     = np.array(stream.descriptor.tuples())

        # Create a 2D dataset with a 1D data column
        dtype = [(a, 'f') for a in axis_names]
        logger.debug("Data type for HDF5: %s", dtype)
        dtype.append((stream.start_connector.name, 'f'))
        if self.compress:
            data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,),
                                        compression='gzip')
        else:
            data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,))

        # Write params into attrs
        for k,v in params.items():
            if k not in axis_names:
                data.attrs[k] = v

        # Write pointer
        w_idx = 0

        while True:
            logger.debug("HDF5 awaiting data")
            done, pending = await asyncio.wait((stream.finished(), stream.queue.get()), return_when=concurrent.futures.FIRST_COMPLETED)
            new_data = list(done)[0].result()
            if type(new_data)==bool:
                if new_data:
                    break
                else:
                    logger.debug("Printer %s awaiting data", self.name)
                    new_data = np.array(await stream.queue.get()).flatten()

            logger.debug("HDF5 stream has %d points", stream.points_taken)
            logger.debug("HDF5: %s got data %s of length %d", stream.name, new_data, new_data.size)

            # import ipdb; ipdb.set_trace()

            # Resize if necessary, also get the new set of sweep tuples since the axes must have changed
            if w_idx + new_data.size >= data.len():
                logger.debug("HDF5 stream was resized to %d points", w_idx + new_data.size)
                data.resize((w_idx + new_data.size,))
                tuples = np.append(tuples, np.array(stream.descriptor.tuples()), axis=0)

            # Write to table
            for i, axis_name in enumerate(axis_names):
                logger.debug("Setting %s to %s", axis_name, tuples[w_idx:w_idx+new_data.size, i])
                data[axis_name, w_idx:w_idx+new_data.size] = tuples[w_idx:w_idx+new_data.size, i]

            data[stream.start_connector.name, w_idx:w_idx+new_data.size] = new_data

            self.file.flush()
            w_idx += new_data.size
            self.points_taken = w_idx

            logger.debug("HDF5: Write index at %d", w_idx)
            logger.debug("HDF5: %s has written %d points", stream.name, w_idx)

        try:
            self.file.close()
        except:
            # This doesn't seem to happen when we don't used named columns
            logger.debug("Ignoring 'dictionary changed sized during iteration' error.")

class ProgressBar(Filter):
    """ Display progress bar(s) on the terminal/notebook.

    num: number of progress bars to be display, \
    corresponding to the number of axes (counting from outer most)

        For running in Jupyter Notebook:
    Needs to open '_tqdm_notebook.py',\
    search for 'n = int(s[:npos])'\
    then replace it with 'n = float(s[:npos])'
    """
    data = InputConnector()
    def __init__(self, num=0, notebook=False):
        super(ProgressBar,self).__init__()
        self.num    = num
        self.notebook = notebook
        self.bars   = []
        self.w_id   = 0

    async def run(self):
        self.stream = self.data.input_streams[0]
        axes = self.stream.descriptor.axes
        num_axes = len(axes)
        totals = [self.stream.descriptor.num_points_through_axis(axis) for axis in range(num_axes)]
        chunk_sizes = [max(1,self.stream.descriptor.num_points_through_axis(axis+1)) for axis in range(num_axes)]
        self.num = min(self.num, num_axes)

        self.bars   = []
        for i in range(self.num):
            if self.notebook:
                self.bars.append(tqdm_notebook(total=totals[i]/chunk_sizes[i]))
            else:
                self.bars.append(tqdm(total=totals[i]/chunk_sizes[i]))
        self.w_id   = 0
        while True:
            if self.stream.done() and self.w_id==self.stream.num_points():
                break

            new_data = np.array(await self.stream.queue.get()).flatten()
            while self.stream.queue.qsize() > 0:
                new_data = np.append(new_data, np.array(self.stream.queue.get_nowait()).flatten())
            self.w_id += new_data.size
            num_data = self.stream.points_taken
            for i in range(self.num):
                if num_data == 0:
                    if self.notebook:
                        self.bars[i].sp(close=True)
                        # Reset the progress bar with a new one
                        self.bars[i] = tqdm_notebook(total=totals[i]/chunk_sizes[i])
                    else:
                        # Reset the progress bar with a new one
                        self.bars[i].close()
                        self.bars[i] = tqdm(total=totals[i]/chunk_sizes[i])
                pos = int(10*num_data / chunk_sizes[i])/10.0 # One decimal is good enough
                if pos > self.bars[i].n:
                    self.bars[i].update(pos - self.bars[i].n)
                num_data = num_data % chunk_sizes[i]
<commit_msg>HDF5 writer now working with structured streams.<commit_after># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio, concurrent
import itertools
import h5py
import numpy as np
import os.path

from pycontrol.stream import DataStreamDescriptor
from pycontrol.logging import logger
from pycontrol.filters.filter import Filter, InputConnector, OutputConnector
from tqdm import tqdm, tqdm_notebook

class WriteToHDF5(Filter):
    """Writes data to file."""

    data = InputConnector()
    def __init__(self, filename, compress=True, **kwargs):
        super(WriteToHDF5, self).__init__(**kwargs)
        self.compress = compress
        self.filename = filename
        self.points_taken = 0
        self.file = None

    def final_init(self):
        self.file = self.new_file()

    def new_filename(self):
        # Increment the filename until we find one we want.
        i = 0
        ext = self.filename.find('.h5')
        if ext > -1:
            filename = self.filename[:ext]
        while os.path.exists("{}-{:04d}.h5".format(filename,i)):
            i += 1
        return "{}-{:04d}.h5".format(filename,i)

    def new_file(self):
        """ Open a new data file to write """
        # Close the current file, if any
        if self.file is not None:
            try:
                self.file.close()
            except Exception as e:
                logger.error("Encounter exception: {}".format(e))
                logger.error("Cannot close file '{}'. File may be damaged.".format(self.file.filename))
        # Get new file name
        filename = self.new_filename()
        head = os.path.dirname(filename)
        head = os.path.normpath(head)
        dirs = head.split(os.sep)
        # Check if path exists. If not, create new one(s).
        fulldir = ''
        for d in dirs:
            fulldir = os.path.join(fulldir, d)
            if not os.path.exists(fulldir):
                logger.debug("Create new directory: {}.".format(fulldir))
                os.mkdir(fulldir)
        logger.debug("Create new data file: %s." %filename)
        return h5py.File(filename, 'w')

    async def run(self):
        stream     = self.data.input_streams[0]
        desc       = stream.descriptor
        axes       = stream.descriptor.axes
        params     = stream.descriptor.params
        axis_names = desc.axis_names()

        params['exp_src'] = stream.descriptor.exp_src
        num_axes   = len(axes)

        # All of the combinations for the present values of the sweep parameters only
        tuples     = np.array(stream.descriptor.tuples())

        # Create a 2D dataset with a 1D data column
        dtype = [(a, 'f') for a in axis_names]
        logger.debug("Data type for HDF5: %s", dtype)
        dtype.append((stream.start_connector.name, 'f'))
        if self.compress:
            data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,),
                                        compression='gzip')
        else:
            data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,))

        # Write params into attrs
        for k,v in params.items():
            if k not in axis_names:
                data.attrs[k] = v

        # Write pointer
        w_idx = 0

        while True:

            message = await stream.queue.get()
            message_type = message['type']
            message_data = message['data']
            message_comp = message['compression']

            if message_comp == 'zlib':
                message_data = pickle.loads(zlib.decompress(message_data))
            # If we receive a message
            if message['type'] == 'event':
                logger.debug('%s "%s" received event "%s"', self.__class__.__name__, self.name, message_data)
                if message['data'] == 'done':
                    break
            elif message['type'] == 'data':
                logger.debug('%s "%s" received %d points.', self.__class__.__name__, self.name, message_data.size)
                logger.debug("Now has %d of %d points.", stream.points_taken, stream.num_points())

                # Resize if necessary, also get the new set of sweep tuples since the axes must have changed
                if w_idx + message_data.size >= data.len():
                    logger.debug("HDF5 stream was resized to %d points", w_idx + message_data.size)
                    data.resize((w_idx + message_data.size,))
                    tuples = np.append(tuples, np.array(stream.descriptor.tuples()), axis=0)

                # Write to table
                for i, axis_name in enumerate(axis_names):
                    logger.debug("Setting %s to %s", axis_name, tuples[w_idx:w_idx+message_data.size, i])
                    data[axis_name, w_idx:w_idx+message_data.size] = tuples[w_idx:w_idx+message_data.size, i]

                data[stream.start_connector.name, w_idx:w_idx+message_data.size] = message_data

                self.file.flush()
                w_idx += message_data.size
                self.points_taken = w_idx

                logger.debug("HDF5: Write index at %d", w_idx)
                logger.debug("HDF5: %s has written %d points", stream.name, w_idx)

        try:
            self.file.close()
        except:
            # This doesn't seem to happen when we don't used named columns
            logger.debug("Ignoring 'dictionary changed sized during iteration' error.")

class ProgressBar(Filter):
    """ Display progress bar(s) on the terminal/notebook.

    num: number of progress bars to be display, \
    corresponding to the number of axes (counting from outer most)

        For running in Jupyter Notebook:
    Needs to open '_tqdm_notebook.py',\
    search for 'n = int(s[:npos])'\
    then replace it with 'n = float(s[:npos])'
    """
    data = InputConnector()
    def __init__(self, num=0, notebook=False):
        super(ProgressBar,self).__init__()
        self.num    = num
        self.notebook = notebook
        self.bars   = []
        self.w_id   = 0

    async def run(self):
        self.stream = self.data.input_streams[0]
        axes = self.stream.descriptor.axes
        num_axes = len(axes)
        totals = [self.stream.descriptor.num_points_through_axis(axis) for axis in range(num_axes)]
        chunk_sizes = [max(1,self.stream.descriptor.num_points_through_axis(axis+1)) for axis in range(num_axes)]
        self.num = min(self.num, num_axes)

        self.bars   = []
        for i in range(self.num):
            if self.notebook:
                self.bars.append(tqdm_notebook(total=totals[i]/chunk_sizes[i]))
            else:
                self.bars.append(tqdm(total=totals[i]/chunk_sizes[i]))
        self.w_id   = 0
        while True:
            if self.stream.done() and self.w_id==self.stream.num_points():
                break

            new_data = np.array(await self.stream.queue.get()).flatten()
            while self.stream.queue.qsize() > 0:
                new_data = np.append(new_data, np.array(self.stream.queue.get_nowait()).flatten())
            self.w_id += new_data.size
            num_data = self.stream.points_taken
            for i in range(self.num):
                if num_data == 0:
                    if self.notebook:
                        self.bars[i].sp(close=True)
                        # Reset the progress bar with a new one
                        self.bars[i] = tqdm_notebook(total=totals[i]/chunk_sizes[i])
                    else:
                        # Reset the progress bar with a new one
                        self.bars[i].close()
                        self.bars[i] = tqdm(total=totals[i]/chunk_sizes[i])
                pos = int(10*num_data / chunk_sizes[i])/10.0 # One decimal is good enough
                if pos > self.bars[i].n:
                    self.bars[i].update(pos - self.bars[i].n)
                num_data = num_data % chunk_sizes[i]
<|endoftext|>


<commit_before>"""
Parse to and from a Excel format for isotherms.

The _parser_version variable documents any changes to the format,
and is used to check for any deprecations.

"""

import ast
import warnings

import pandas
import xlrd
import xlwt

from pygaps.core.baseisotherm import BaseIsotherm
from pygaps.core.modelisotherm import ModelIsotherm
from pygaps.core.pointisotherm import PointIsotherm
from pygaps.modelling import model_from_dict
from pygaps.utilities.exceptions import ParsingError

_parser_version = "3.0"

_META_DICT = {
    'material': {
        'text': ('Material name', ),
        'name': 'material',
        'row': 0,
        'column': 0,
    },
    'temperature': {
        'text': ('Experiment temperature (K)', ),
        'name': 'temperature',
        'row': 1,
        'column': 0,
    },
    'adsorbate': {
        'text': ('Adsorbate used', ),
        'name': 'adsorbate',
        'row': 2,
        'column': 0,
    },
    'pressure_mode': {
        'text': ('Pressure mode', ),
        'name': 'pressure_mode',
        'row': 3,
        'column': 0,
    },
    'pressure_unit': {
        'text': ('Pressure unit', ),
        'name': 'pressure_unit',
        'row': 4,
        'column': 0,
    },
    'loading_basis': {
        'text': ('Loading basis', ),
        'name': 'loading_basis',
        'row': 5,
        'column': 0,
    },
    'loading_unit': {
        'text': ('Loading unit', ),
        'name': 'loading_unit',
        'row': 6,
        'column': 0,
    },
    'material_basis': {
        'text': ('Material basis', ),
        'name': 'material_basis',
        'row': 7,
        'column': 0,
    },
    'material_unit': {
        'text': ('Material unit', ),
        'name': 'material_unit',
        'row': 8,
        'column': 0,
    },
    'isotherm_data': {
        'text': ('Isotherm type', ),
        'name': 'isotherm_data',
        'row': 9,
        'column': 0,
    },
}

def isotherm_to_xl(isotherm, path):
    """
    Save an isotherm to an excel file.

    Parameters
    ----------
    isotherm : Isotherm
        Isotherm to be written to excel.
    path : str
        Path to the file to be written.
    """
    # create a new workbook and select first sheet
    wb = xlwt.Workbook()
    sht = wb.add_sheet('data')

    # get the required dictionaries
    iso_dict = isotherm.to_dict()
    iso_dict['file_version'] = _parser_version  # version

    # Parse material
    material = iso_dict['material']
    if isinstance(material, dict):
        iso_dict['material'] = material.pop('name')
        iso_dict.update({f"_material_{key}": val for key, val in material.items()})

    # Add the required named properties
    prop_style = xlwt.easyxf('align: horiz left; pattern: pattern solid, fore_colour grey25;')
    for field in _META_DICT.values():
        val = iso_dict.pop(field['name'], None)
        sht.write(field['row'], field['column'], field['text'][0], prop_style)
        if val:
            sht.write(field['row'], field['column'] + 1, val, prop_style)

    # Get the isotherm type header
    type_row = _META_DICT['isotherm_data']['row']
    type_col = _META_DICT['isotherm_data']['column']

    col_width = 256 * 25  # 25 characters wide (-ish)
    sht.col(type_col).width = col_width
    sht.col(type_col + 1).width = col_width

    if isinstance(isotherm, BaseIsotherm):
        # Write the type
        sht.write(type_row, type_col + 1, 'metadata', prop_style)

    if isinstance(isotherm, PointIsotherm):

        # Write the type
        sht.write(type_row, type_col + 1, 'data', prop_style)

        # Get the data row
        data_row = type_row + 1

        # We get data and replace adsorption terminology
        data = isotherm.data_raw.copy()
        data['branch'] = data['branch'].replace(0, 'ads').replace(1, 'des')

        # Write all data
        for col_index, heading in enumerate(data.columns):
            sht.write(data_row, col_index, heading)
            for row_index, datapoint in enumerate(data[heading]):
                sht.write(data_row + row_index + 1, col_index, datapoint)

    if isinstance(isotherm, ModelIsotherm):

        # Write the type
        sht.write(type_row, type_col + 1, 'model', prop_style)

        # Get the model row
        model_row = type_row

        # Generate the headings (i know it's a lame way to do it)
        sht.write(model_row + 1, 0, 'Model name')
        sht.write(model_row + 1, 1, isotherm.model.name)
        sht.write(model_row + 2, 0, 'RMSE')
        sht.write(model_row + 2, 1, isotherm.model.rmse)
        sht.write(model_row + 3, 0, 'Pressure range')
        sht.write(model_row + 3, 1, str(isotherm.model.pressure_range))
        sht.write(model_row + 4, 0, 'Loading range')
        sht.write(model_row + 4, 1, str(isotherm.model.loading_range))
        sht.write(model_row + 5, 0, 'Model parameters')
        model_row = model_row + 5
        for row_index, param in enumerate(isotherm.model.params):
            sht.write(model_row + row_index + 1, 0, param)
            sht.write(model_row + row_index + 1, 1, isotherm.model.params[param])

    # Now add the other keys
    sht = wb.add_sheet('otherdata')
    row = 0
    col = 0
    for prop in iso_dict:
        sht.write(row, col, prop)
        sht.write(row, col + 1, iso_dict[prop])
        row += 1

    wb.save(path)

def isotherm_from_xl(path, *isotherm_parameters):
    """
    Load an isotherm from a pyGAPS Excel file.

    Parameters
    ----------
    path : str
        Path to the file to be read.
    isotherm_parameters :
        Any other options to be overridden in the isotherm creation.

    Returns
    -------
    Isotherm
        The isotherm contained in the excel file.

    """

    # isotherm type (point/model)
    isotype = 0

    raw_dict = {}

    # Get excel workbook and sheet
    wb = xlrd.open_workbook(path)
    if 'data' in wb.sheet_names():
        sht = wb.sheet_by_name('data')
    else:
        sht = wb.sheet_by_index(0)

    # read the main isotherm parameters
    for field in _META_DICT.values():
        raw_dict[field['name']] = sht.cell(field['row'], field['column'] + 1).value

    # find data/model limits
    type_row = _META_DICT['isotherm_data']['row']

    if sht.cell(type_row, 1).value.lower().startswith('data'):

        # Store isotherm type
        isotype = 1

        header_row = type_row + 1
        start_row = header_row + 1
        final_row = start_row

        while final_row < sht.nrows:
            point = sht.cell(final_row, 0).value
            if point == '':
                break
            final_row += 1

        # read the data in
        header_col = 0
        headers = []
        experiment_data = {}
        while header_col < sht.ncols:
            header = sht.cell(header_row, header_col).value
            if header == '':
                break
            headers.append(header)
            experiment_data[header] = [
                sht.cell(i, header_col).value for i in range(start_row, final_row)
            ]
            header_col += 1
        data = pandas.DataFrame(experiment_data)

        raw_dict['loading_key'] = headers[0]
        raw_dict['pressure_key'] = headers[1]

        # process isotherm branches if they exist
        if 'branch' in data.columns:
            data['branch'] = data['branch'].apply(lambda x: 0 if x == 'ads' else 1)
        else:
            raw_dict['branch'] = 'guess'

    if sht.cell(type_row, 1).value.lower().startswith('model'):

        # Store isotherm type
        isotype = 2
        model = {
            "name": sht.cell(type_row + 1, 1).value,
            "rmse": sht.cell(type_row + 2, 1).value,
            "pressure_range": ast.literal_eval(sht.cell(type_row + 3, 1).value),
            "loading_range": ast.literal_eval(sht.cell(type_row + 4, 1).value),
            "parameters": {},
        }

        final_row = type_row + 6

        while final_row < sht.nrows:
            point = sht.cell(final_row, 0).value
            if point == '':
                break
            model["parameters"][point] = sht.cell(final_row, 1).value
            final_row += 1

    # read the secondary isotherm parameters
    if 'otherdata' in wb.sheet_names():
        sht = wb.sheet_by_name('otherdata')
        row_index = 0
        while row_index < sht.nrows:
            namec = sht.cell(row_index, 0)
            valc = sht.cell(row_index, 1)
            if namec.ctype == xlrd.XL_CELL_EMPTY:
                break
            if valc.ctype == xlrd.XL_CELL_BOOLEAN:
                val = bool(valc.value)
            elif valc.ctype == xlrd.XL_CELL_EMPTY:
                val = None
            else:
                val = valc.value
            raw_dict[namec.value] = val
            row_index += 1

    # Put data in order
    # version check
    version = raw_dict.pop("file_version", None)
    if not version or float(version) < float(_parser_version):
        warnings.warn(
            f"The file version is {version} while the parser uses version {_parser_version}. "
            "Strange things might happen, so double check your data."
        )
    raw_dict.pop('isotherm_data')  # remove useless field
    raw_dict.pop('iso_id', None)  # make sure id is not passed

    # check if material needs parsing
    material = {}
    for key, val in raw_dict.items():
        if key.startswith("_material_"):
            material[key.replace("_material_", "")] = val
    if material:
        for key in material.keys():
            raw_dict.pop("_material_" + key)
        material['name'] = raw_dict['material']
        raw_dict['material'] = material

    # Update dictionary with any user parameters
    raw_dict.update(isotherm_parameters)

    if isotype == 1:
        return PointIsotherm(isotherm_data=data, **raw_dict)

    if isotype == 2:
        return ModelIsotherm(model=model_from_dict(model), **raw_dict)

    return BaseIsotherm(**raw_dict)
<commit_msg>fix: overwrite cell error<commit_after>"""
Parse to and from a Excel format for isotherms.

The _parser_version variable documents any changes to the format,
and is used to check for any deprecations.

"""

import ast
import warnings

import pandas
import xlrd
import xlwt

from pygaps.core.baseisotherm import BaseIsotherm
from pygaps.core.modelisotherm import ModelIsotherm
from pygaps.core.pointisotherm import PointIsotherm
from pygaps.modelling import model_from_dict
from pygaps.utilities.exceptions import ParsingError

_parser_version = "3.0"

_META_DICT = {
    'material': {
        'text': ('Material name', ),
        'name': 'material',
        'row': 0,
        'column': 0,
    },
    'temperature': {
        'text': ('Experiment temperature (K)', ),
        'name': 'temperature',
        'row': 1,
        'column': 0,
    },
    'adsorbate': {
        'text': ('Adsorbate used', ),
        'name': 'adsorbate',
        'row': 2,
        'column': 0,
    },
    'pressure_mode': {
        'text': ('Pressure mode', ),
        'name': 'pressure_mode',
        'row': 3,
        'column': 0,
    },
    'pressure_unit': {
        'text': ('Pressure unit', ),
        'name': 'pressure_unit',
        'row': 4,
        'column': 0,
    },
    'loading_basis': {
        'text': ('Loading basis', ),
        'name': 'loading_basis',
        'row': 5,
        'column': 0,
    },
    'loading_unit': {
        'text': ('Loading unit', ),
        'name': 'loading_unit',
        'row': 6,
        'column': 0,
    },
    'material_basis': {
        'text': ('Material basis', ),
        'name': 'material_basis',
        'row': 7,
        'column': 0,
    },
    'material_unit': {
        'text': ('Material unit', ),
        'name': 'material_unit',
        'row': 8,
        'column': 0,
    },
    'isotherm_data': {
        'text': ('Isotherm type', ),
        'name': 'isotherm_data',
        'row': 9,
        'column': 0,
    },
}

def isotherm_to_xl(isotherm, path):
    """
    Save an isotherm to an excel file.

    Parameters
    ----------
    isotherm : Isotherm
        Isotherm to be written to excel.
    path : str
        Path to the file to be written.
    """
    # create a new workbook and select first sheet
    wb = xlwt.Workbook()
    sht = wb.add_sheet('data')

    # get the required dictionaries
    iso_dict = isotherm.to_dict()
    iso_dict['file_version'] = _parser_version  # version

    # Parse material
    material = iso_dict['material']
    if isinstance(material, dict):
        iso_dict['material'] = material.pop('name')
        iso_dict.update({f"_material_{key}": val for key, val in material.items()})

    # Add the required named properties
    prop_style = xlwt.easyxf('align: horiz left; pattern: pattern solid, fore_colour grey25;')
    for field in _META_DICT.values():
        val = iso_dict.pop(field['name'], None)
        sht.write(field['row'], field['column'], field['text'][0], prop_style)
        if val:
            sht.write(field['row'], field['column'] + 1, val, prop_style)

    # Get the isotherm type header
    type_row = _META_DICT['isotherm_data']['row']
    type_col = _META_DICT['isotherm_data']['column']

    col_width = 256 * 25  # 25 characters wide (-ish)
    sht.col(type_col).width = col_width
    sht.col(type_col + 1).width = col_width

    if isinstance(isotherm, PointIsotherm):

        # Write the type
        sht.write(type_row, type_col + 1, 'data', prop_style)

        # Get the data row
        data_row = type_row + 1

        # We get data and replace adsorption terminology
        data = isotherm.data_raw.copy()
        data['branch'] = data['branch'].replace(0, 'ads').replace(1, 'des')

        # Write all data
        for col_index, heading in enumerate(data.columns):
            sht.write(data_row, col_index, heading)
            for row_index, datapoint in enumerate(data[heading]):
                sht.write(data_row + row_index + 1, col_index, datapoint)

    elif isinstance(isotherm, ModelIsotherm):

        # Write the type
        sht.write(type_row, type_col + 1, 'model', prop_style)

        # Get the model row
        model_row = type_row

        # Generate the headings (i know it's a lame way to do it)
        sht.write(model_row + 1, 0, 'Model name')
        sht.write(model_row + 1, 1, isotherm.model.name)
        sht.write(model_row + 2, 0, 'RMSE')
        sht.write(model_row + 2, 1, isotherm.model.rmse)
        sht.write(model_row + 3, 0, 'Pressure range')
        sht.write(model_row + 3, 1, str(isotherm.model.pressure_range))
        sht.write(model_row + 4, 0, 'Loading range')
        sht.write(model_row + 4, 1, str(isotherm.model.loading_range))
        sht.write(model_row + 5, 0, 'Model parameters')
        model_row = model_row + 5
        for row_index, param in enumerate(isotherm.model.params):
            sht.write(model_row + row_index + 1, 0, param)
            sht.write(model_row + row_index + 1, 1, isotherm.model.params[param])

    elif isinstance(isotherm, BaseIsotherm):
        # Write the type
        sht.write(type_row, type_col + 1, 'metadata', prop_style)

    # Now add the other keys
    sht = wb.add_sheet('otherdata')
    row = 0
    col = 0
    for prop in iso_dict:
        sht.write(row, col, prop)
        sht.write(row, col + 1, iso_dict[prop])
        row += 1

    wb.save(path)

def isotherm_from_xl(path, *isotherm_parameters):
    """
    Load an isotherm from a pyGAPS Excel file.

    Parameters
    ----------
    path : str
        Path to the file to be read.
    isotherm_parameters :
        Any other options to be overridden in the isotherm creation.

    Returns
    -------
    Isotherm
        The isotherm contained in the excel file.

    """

    # isotherm type (point/model)
    isotype = 0

    raw_dict = {}

    # Get excel workbook and sheet
    wb = xlrd.open_workbook(path)
    if 'data' in wb.sheet_names():
        sht = wb.sheet_by_name('data')
    else:
        sht = wb.sheet_by_index(0)

    # read the main isotherm parameters
    for field in _META_DICT.values():
        raw_dict[field['name']] = sht.cell(field['row'], field['column'] + 1).value

    # find data/model limits
    type_row = _META_DICT['isotherm_data']['row']

    if sht.cell(type_row, 1).value.lower().startswith('data'):

        # Store isotherm type
        isotype = 1

        header_row = type_row + 1
        start_row = header_row + 1
        final_row = start_row

        while final_row < sht.nrows:
            point = sht.cell(final_row, 0).value
            if point == '':
                break
            final_row += 1

        # read the data in
        header_col = 0
        headers = []
        experiment_data = {}
        while header_col < sht.ncols:
            header = sht.cell(header_row, header_col).value
            if header == '':
                break
            headers.append(header)
            experiment_data[header] = [
                sht.cell(i, header_col).value for i in range(start_row, final_row)
            ]
            header_col += 1
        data = pandas.DataFrame(experiment_data)

        raw_dict['loading_key'] = headers[0]
        raw_dict['pressure_key'] = headers[1]

        # process isotherm branches if they exist
        if 'branch' in data.columns:
            data['branch'] = data['branch'].apply(lambda x: 0 if x == 'ads' else 1)
        else:
            raw_dict['branch'] = 'guess'

    if sht.cell(type_row, 1).value.lower().startswith('model'):

        # Store isotherm type
        isotype = 2
        model = {
            "name": sht.cell(type_row + 1, 1).value,
            "rmse": sht.cell(type_row + 2, 1).value,
            "pressure_range": ast.literal_eval(sht.cell(type_row + 3, 1).value),
            "loading_range": ast.literal_eval(sht.cell(type_row + 4, 1).value),
            "parameters": {},
        }

        final_row = type_row + 6

        while final_row < sht.nrows:
            point = sht.cell(final_row, 0).value
            if point == '':
                break
            model["parameters"][point] = sht.cell(final_row, 1).value
            final_row += 1

    # read the secondary isotherm parameters
    if 'otherdata' in wb.sheet_names():
        sht = wb.sheet_by_name('otherdata')
        row_index = 0
        while row_index < sht.nrows:
            namec = sht.cell(row_index, 0)
            valc = sht.cell(row_index, 1)
            if namec.ctype == xlrd.XL_CELL_EMPTY:
                break
            if valc.ctype == xlrd.XL_CELL_BOOLEAN:
                val = bool(valc.value)
            elif valc.ctype == xlrd.XL_CELL_EMPTY:
                val = None
            else:
                val = valc.value
            raw_dict[namec.value] = val
            row_index += 1

    # Put data in order
    # version check
    version = raw_dict.pop("file_version", None)
    if not version or float(version) < float(_parser_version):
        warnings.warn(
            f"The file version is {version} while the parser uses version {_parser_version}. "
            "Strange things might happen, so double check your data."
        )
    raw_dict.pop('isotherm_data')  # remove useless field
    raw_dict.pop('iso_id', None)  # make sure id is not passed

    # check if material needs parsing
    material = {}
    for key, val in raw_dict.items():
        if key.startswith("_material_"):
            material[key.replace("_material_", "")] = val
    if material:
        for key in material.keys():
            raw_dict.pop("_material_" + key)
        material['name'] = raw_dict['material']
        raw_dict['material'] = material

    # Update dictionary with any user parameters
    raw_dict.update(isotherm_parameters)

    if isotype == 1:
        return PointIsotherm(isotherm_data=data, **raw_dict)

    if isotype == 2:
        return ModelIsotherm(model=model_from_dict(model), **raw_dict)

    return BaseIsotherm(**raw_dict)
<|endoftext|>