From baaa3b2445562020b8087c6b469e3a528d596ff2 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Wed, 2 Apr 2025 12:39:45 +0200 Subject: [PATCH 1/6] LBN: Do labnotebook upgrade on most common MIES entry points When loading experiments, possibly old experiments the data structures might be present in an old version. These need to be upgraded before used by the current code. The points where the LBN wave is upgraded are: - opening the Data Browser - Locking a device in the DAEphys panel - NWB Export At these locations the LBN upgrade for values and keys waves were added through the common function UpgradeLabNotebook. --- Packages/MIES/MIES_DAEphys.ipf | 6 +----- Packages/MIES/MIES_DataBrowser.ipf | 4 +--- Packages/MIES/MIES_NeuroDataWithoutBorders.ipf | 2 ++ Packages/MIES/MIES_WaveDataFolderGetters.ipf | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Packages/MIES/MIES_DAEphys.ipf b/Packages/MIES/MIES_DAEphys.ipf index a6d4697cf7..d596cee293 100644 --- a/Packages/MIES/MIES_DAEphys.ipf +++ b/Packages/MIES/MIES_DAEphys.ipf @@ -4540,11 +4540,7 @@ Function DAP_LockDevice(string win) headstage = GetSliderPositionIndex(deviceLocked, "slider_DataAcq_ActiveHeadstage") P_SaveUserSelectedHeadstage(deviceLocked, headstage) - // upgrade all four labnotebook waves in wanna-be atomic way - GetLBNumericalKeys(deviceLocked) - GetLBNumericalValues(deviceLocked) - GetLBTextualKeys(deviceLocked) - GetLBTextualValues(deviceLocked) + UpgradeLabNotebook(deviceLocked) NVAR sessionStartTime = $GetSessionStartTime() sessionStartTime = DateTimeInUTC() diff --git a/Packages/MIES/MIES_DataBrowser.ipf b/Packages/MIES/MIES_DataBrowser.ipf index ada6fc7f69..d2bbe5414c 100644 --- a/Packages/MIES/MIES_DataBrowser.ipf +++ b/Packages/MIES/MIES_DataBrowser.ipf @@ -421,9 +421,7 @@ Function DB_UpdateSweepPlot(string win) return NaN endif - // fetch keys waves to trigger a potential labnotebook upgrade - WAVE numericalKeys = DB_GetLBNWave(win, LBN_NUMERICAL_KEYS) - WAVE textualKeys = DB_GetLBNWave(win, LBN_TEXTUAL_KEYS) + UpgradeLabNotebook(device) WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES) WAVE textualValues = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES) diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf index 1720b4804c..9f1ed258e2 100644 --- a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf +++ b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf @@ -679,6 +679,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF for(device : devicesWithContent) + UpgradeLabNotebook(device) + if(ParamIsDefault(overrideFullFilePath) && ParamIsDefault(overrideFileTemplate)) [locationID, createdNewNWBFile] = NWB_GetFileForExport(nwbVersion, device) elseif(!ParamIsDefault(overrideFullFilePath)) diff --git a/Packages/MIES/MIES_WaveDataFolderGetters.ipf b/Packages/MIES/MIES_WaveDataFolderGetters.ipf index 3ae72cf33d..ec2d2f6805 100644 --- a/Packages/MIES/MIES_WaveDataFolderGetters.ipf +++ b/Packages/MIES/MIES_WaveDataFolderGetters.ipf @@ -1385,7 +1385,7 @@ End /// - Making dimension labels valid liberal object names /// - Extending the row dimension to 6 for the key waves /// - Fixing empty column dimension labels in key waves -static Function UpgradeLabNotebook(string device) +Function UpgradeLabNotebook(string device) variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow string list, key From c53ed013b5b401e8a7aec4068476e9965e319c6f Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Thu, 10 Apr 2025 14:26:13 +0200 Subject: [PATCH 2/6] DB: Optimize LBN Upgrade location The LBN Upgrade location was moved in the DB code from DB_UpdateSweepPlot to the device locking in DB_LockToDevice. This reduces overhead because the upgrade is not attempted on each plot update. --- Packages/MIES/MIES_DataBrowser.ipf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Packages/MIES/MIES_DataBrowser.ipf b/Packages/MIES/MIES_DataBrowser.ipf index d2bbe5414c..ba5acff9e0 100644 --- a/Packages/MIES/MIES_DataBrowser.ipf +++ b/Packages/MIES/MIES_DataBrowser.ipf @@ -310,6 +310,7 @@ static Function/S DB_LockToDevice(string win, string device) BSP_UnsetDynamicStartupSettings(win) else newWindow = "DB_" + device + UpgradeLabNotebook(device) endif DB_SetUserData(win, device) @@ -421,8 +422,6 @@ Function DB_UpdateSweepPlot(string win) return NaN endif - UpgradeLabNotebook(device) - WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES) WAVE textualValues = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES) From ccecdf6bdc18bb400f207d549aee1d531e961102 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Wed, 2 Apr 2025 15:50:19 +0200 Subject: [PATCH 3/6] ED: Add feature to ED_AddEntriesToLabnotebook for value insertion ED_AddEntriesToLabnotebook and further down functions were extended to add the feature that a given value is inserted into the LNB at the end of the sweepNumber/entrySourceType block. This row is also marked as postprocessed data. The row and index cache is invalidated when a row was inserted. Added tests for insertion in numerical and textual LNB. Raise LABNOTEBOOK_VERSION to 82 --- Packages/MIES/MIES_Constants.ipf | 3 +- .../MIES/MIES_ExperimentDocumentation.ipf | 106 +++++++++++-- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 9 +- .../labnotebook_numerical_description.itx | 3 +- .../MIES/labnotebook_textual_description.itx | 3 +- Packages/tests/Basic/UTF_Labnotebook.ipf | 147 ++++++++++++++++++ Packages/tests/UTF_DataGenerators.ipf | 14 ++ 7 files changed, 270 insertions(+), 15 deletions(-) diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index 6e57ac4e4a..4065540243 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,7 +39,7 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 81 +Constant LABNOTEBOOK_VERSION = 82 Constant RESULTS_VERSION = 3 ///@} @@ -996,6 +996,7 @@ StrConstant STIMSET_SIZE_KEY = "Stimset Size" StrConstant STIMSET_ERROR_KEY = "Wavebuilder Error" StrConstant AUTOBIAS_PERC_KEY = "Autobias %" StrConstant SWEEP_EPOCH_VERSION_ENTRY_KEY = "Epochs Version" +StrConstant POSTPROCESSED_ENTRY_KEY = "PostProcessed" Constant WAVEBUILDER_STATUS_ERROR = 1 diff --git a/Packages/MIES/MIES_ExperimentDocumentation.ipf b/Packages/MIES/MIES_ExperimentDocumentation.ipf index 7e6b243eae..f731b97c0e 100644 --- a/Packages/MIES/MIES_ExperimentDocumentation.ipf +++ b/Packages/MIES/MIES_ExperimentDocumentation.ipf @@ -62,14 +62,16 @@ /// It is recommended to gather all entries to be written in keys/values and call ED_AddEntriesToLabnotebook then once. /// /// @see ED_createTextNotes, ED_createWaveNote -Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType) +Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType, [variable insertAsPostProc]) + + insertAsPostProc = ParamIsDefault(insertAsPostProc) ? 0 : !!insertAsPostProc ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) else - ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) endif End @@ -79,9 +81,9 @@ Function ED_AddEntriesToResults(WAVE vals, WAVE/T keys, variable entrySourceType ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) else - ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) endif End @@ -109,6 +111,38 @@ static Function ED_CheckValuesAndKeys(WAVE vals, WAVE keys) endif End +static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row) + + variable col, unused + + ASSERT(DimSize(values, ROWS) >= row, "Row does not exist in LBN values wave") + + WAVE keys = GetLogbookKeysFromValues(values) + + if(IsTextWave(values)) + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesT + WAVE incomingValues = incomingValuesT + else + Make/FREE/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesNum + WAVE incomingValues = incomingValuesNum + endif + + Make/FREE/T/N=(3, 1) incomingKeys + incomingKeys[0] = POSTPROCESSED_ENTRY_KEY + incomingKeys[1] = LABNOTEBOOK_BINARY_UNIT + incomingKeys[2] = LABNOTEBOOK_NO_TOLERANCE + [WAVE indizes, unused] = ED_FindIndizesAndRedimension(incomingKeys, incomingValues, keys, values, LBT_LABNOTEBOOK) + ASSERT(WaveExists(indizes), "Missing indizes") + + col = indizes[0] + if(IsTextWave(values)) + WAVE/T valuesT = values + valuesT[row][col][] = "1" + else + values[row][col][] = 1 + endif +End + static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState) variable timestamp @@ -142,6 +176,29 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, endif End +/// @brief Inserts a row after the sweep block and returns the inserted row number +static Function ED_InsertRowAfterSweepBlock(WAVE values, variable sweepNo, variable entrySourceType) + + variable sweepCol, firstRow, lastRow, rowIndex, size + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweepNo, entrySourceType, firstRow, lastRow) + ASSERT(!IsNaN(firstRow) && !IsNaN(lastRow), "FindRange could not determine start and/or end of sweep block") + rowIndex = lastRow + 1 + InsertPoints/M=(ROWS) rowIndex, 1, values + + if(!IsTextWave(values)) + values[rowIndex][][] = NaN + endif + + size = GetNumberFromWaveNote(values, NOTE_INDEX) + SetNumberInWaveNote(values, NOTE_INDEX, size + 1) + + InvalidateLBIndexAndRowCache(values) + + return rowIndex +End + /// @brief Add textual entries to the logbook /// /// The text documentation wave will use layers to report the different headstages. @@ -156,9 +213,11 @@ End /// @param device [optional for logbookType LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType type of the logbook, one of @ref LogbookTypes -static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, i, lastValidIncomingLayer, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -173,8 +232,14 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) @@ -196,7 +261,9 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = NormalizeToEOL(incomingTextualValues[0][i][r], "\n") endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End static Function ED_ParseHeadstageContingencyMode(string str) @@ -266,9 +333,11 @@ End /// @param device [optional for logbooktype LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType one of @ref LogbookTypes -static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, lastValidIncomingLayer, i, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -284,8 +353,14 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) @@ -299,7 +374,9 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = incomingNumericalValues[0][i][r] endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End /// @brief Add custom entries to the numerical/textual labnotebook for the very last sweep acquired. @@ -706,6 +783,13 @@ static Function [WAVE colIndizes, variable rowIndex] ED_FindIndizesAndRedimensio LBN_SetDimensionLabels(key, values, start = ((rowIndex == 0) ? 0 : numKeyCols)) endif + // Notes on the LBN row expansion here: + // - Initialisation with NaN, "" takes only place when the wave is expanded for rows/cols. + // - It does not initialise the row returned to the caller. + // Thus, for the unlucky case that a previous LBN write attempt ended up to be partial (not seen happening so far) + // then the row returned might be partially filled. + // - This part is also called if a row is inserted in the LBN from postprocessing. The LBN wave size is increased if required. + // The new row at the end is not used by the insertion. It is still consistent as the NOTE_INDEX is not updated here. if(IsNumericWave(values)) EnsureLargeEnoughWave(values, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) if(numAdditions) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 5b722be6d1..90ef26dabd 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1278,7 +1278,7 @@ End /// @param[in] entrySourceType type of the labnotebook entry, one of @ref DataAcqModes /// @param[out] first point index of the beginning of the range /// @param[out] last point index of the end of the range -threadsafe static Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) +threadsafe Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) variable numRows, i, j, sourceTypeCol, firstRow, lastRow, isNumeric, index, startRow, endRow @@ -1879,6 +1879,13 @@ Function GetTotalOnsetDelayFromDevice(string device) return DAG_GetNumericalValue(device, "setvar_DataAcq_OnsetDelayUser") + TPSettingsCalculated[%totalLengthMS] End +/// @brief Invalidates the row and index cache for the given LBN +Function InvalidateLBIndexAndRowCache(WAVE values) + + CA_DeleteCacheEntry(CA_CreateLBIndexCacheKey(values)) + CA_DeleteCacheEntry(CA_CreateLBRowCacheKey(values)) +End + /// @brief Retrieve the analysis function that was run for a given sweep / channelNumber / channelType /// /// @param numericalValues numerical labnotebook diff --git a/Packages/MIES/labnotebook_numerical_description.itx b/Packages/MIES/labnotebook_numerical_description.itx index ce017fdca3..650e388ea8 100644 --- a/Packages/MIES/labnotebook_numerical_description.itx +++ b/Packages/MIES/labnotebook_numerical_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(196,6) labnotebook_numerical_description +WAVES/T/N=(197,6) labnotebook_numerical_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "SweepNum" "" "-" "Sweep number: Non-repeating non-negative numeric identifier for sweep time series. Increments in the order of acquisition. Starts at zero." "ALL" "" @@ -197,6 +197,7 @@ BEGIN "Require amplifier" "On/Off" "-" "ON if data acquisition was done with an amplifier, OFF if not." "INDEP" "IC;VC;I=0" "Skip Ahead" "" "1" "Determines how many sweeps are skipped from the stimset whan starting data acquisition." "INDEP" "IC;VC;I=0" "Skip Sweeps source" "" "0.1" "Stores who is responsible for sweep skipping. Current values are 0x1 for the user and 0x2 for automatic/internal reasons." "INDEP" "IC;VC;I=0" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_numerical_description; SetScale/P y 0,1,"", labnotebook_numerical_description; SetScale d 0,0,"", labnotebook_numerical_description X Note labnotebook_numerical_description, "WAVE_LAYOUT_VERSION:2;" diff --git a/Packages/MIES/labnotebook_textual_description.itx b/Packages/MIES/labnotebook_textual_description.itx index 2f5b5eb0e2..030dba1270 100644 --- a/Packages/MIES/labnotebook_textual_description.itx +++ b/Packages/MIES/labnotebook_textual_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(78,6) labnotebook_textual_description +WAVES/T/N=(79,6) labnotebook_textual_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "TimeStamp" "s" "-" "Time Stamp: Seconds since Igor epoch (1/1/1904) in local time zone with millisecond precision. Written at time of labnotebook entry." "ALL" "" @@ -79,6 +79,7 @@ BEGIN "TTL Epochs Channel 5" "" "-" "Epochs of TTL Channel 5" "INDEP" "" "TTL Epochs Channel 6" "" "-" "Epochs of TTL Channel 6" "INDEP" "" "TTL Epochs Channel 7" "" "-" "Epochs of TTL Channel 7" "INDEP" "" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_textual_description; SetScale/P y 0,1,"", labnotebook_textual_description; SetScale d 0,0,"", labnotebook_textual_description X Note labnotebook_textual_description, "WAVE_LAYOUT_VERSION:1;" diff --git a/Packages/tests/Basic/UTF_Labnotebook.ipf b/Packages/tests/Basic/UTF_Labnotebook.ipf index 2337da1095..42e049673c 100644 --- a/Packages/tests/Basic/UTF_Labnotebook.ipf +++ b/Packages/tests/Basic/UTF_Labnotebook.ipf @@ -1440,3 +1440,150 @@ Function EmptyLabnotebookWorks() WAVE/Z entries = GetLastSetting(textualValues, 0, "Sweep Number", UNKNOWN_MODE) CHECK_WAVE(entries, NULL_WAVE) End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForProstProcessingSweepIndexerText +static Function InsertRowForProstProcessingTextual([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col + string str + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + sizeBefore = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, EPOCHS_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[testHS], keyItem) + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[INDEP_HEADSTAGE], "1") + + FindValue/TEXT=keyItem/TXOP=4 textualValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + str = textualValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_STR(str, "1") +End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForProstProcessingSweepIndexerNum +static Function InsertRowForProstProcessingNumerical([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col, val + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, "DAC", DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[testHS], keyValue) + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[INDEP_HEADSTAGE], 1) + + FindValue/V=(keyValue) numericalValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + val = numericalValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_VAR(val, 1) +End + +static Function InsertRowForProstProcessingTextualUnknownSweep() + + variable sweepNo = 1337 + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + try + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + FAIL() + catch + PASS() + endtry +End + +static Function InsertRowForProstProcessingNumericalMultiple() + + variable sizeBefore, sizeAfter, sweepNo, index, beforeVal + + variable startSweep = 4 + variable endSweep = 15 + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + beforeVal = setting[index] + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + for(sweepNo = startSweep; sweepNo <= endSweep; sweepNo += 1) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + endfor + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1 + endSweep - startSweep, sizeAfter) + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(beforeVal, setting[index]) + +End diff --git a/Packages/tests/UTF_DataGenerators.ipf b/Packages/tests/UTF_DataGenerators.ipf index f65dd95df4..4a755a1b7f 100644 --- a/Packages/tests/UTF_DataGenerators.ipf +++ b/Packages/tests/UTF_DataGenerators.ipf @@ -1965,3 +1965,17 @@ static Function/WAVE DG_SourceLocationsContent() return wv End + +static Function/WAVE InsertRowForProstProcessingSweepIndexerText() + + Make/FREE wv = {0, 1} + + return wv +End + +static Function/WAVE InsertRowForProstProcessingSweepIndexerNum() + + Make/FREE wv = {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + + return wv +End From c6345aa2f4dece3f5c2d4d75bb67f673467d8606 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Mon, 14 Apr 2025 23:38:27 +0200 Subject: [PATCH 4/6] LBN: Store LBN capabilities on LBN Upgrade - Added capability SupportsEntrySourceType - Added function to determine that support - Added function to get a LBN capability This should improve performance because capabilities stay constant through the lifetime of the LBN. Raise LBN version to 83 Raise Results wave version to 4 The information about the EntrySourceType support is required for LBN row insertion. The insertion is done for a specific EntrySourceType at the end of the block defined by sweepNumber/EntrySourceType in the LBN. However, old LBNs do not feature the EntrySourceType column. In the LBN wave upgrade process this column is created but keeps the initialization values of NaN for UNKNOWN_MODE. Now if a row is inserted, we have to insert it with the same sweepNumber/EntrySourceType combination as the block for which FindRange determined the start/end row index. FindRange has a fallback for the old LBN properties. If we would blindly write the given EntrySourceType (which is typically not UNKNOWN_MODE) on insertion in such an old LBN, effectively a different sweepNumber/EntrySourceType combination is created that is then an own block. This new block makes the fallback of FindRange fail to determine the "old" LBN entries boundaries, making them non-retrievable. As the determination of EntrySourceType support is expensive and a static property of the LBN, it can be done in the LBN upgrade process and saved in the wavenote (of the LBN key wave). --- Packages/MIES/MIES_Constants.ipf | 10 ++++- .../MIES/MIES_ExperimentDocumentation.ipf | 14 +++--- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 44 +++++++++++++++++++ Packages/MIES/MIES_WaveDataFolderGetters.ipf | 31 ++++++++++++- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index 4065540243..4d016dd74e 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,8 +39,8 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 82 -Constant RESULTS_VERSION = 3 +Constant LABNOTEBOOK_VERSION = 83 +Constant RESULTS_VERSION = 4 ///@} /// @name Analysis function versions @@ -2558,3 +2558,9 @@ Constant SF_STEP_PARSER = 1 Constant SF_STEP_EXECUTOR = 2 Constant SF_STEP_OUTSIDE = 3 ///@} + +/// Labnotebook capabilities are stored in the key wave note +/// @anchor LabnotebookCapabilityKeys +///@{ +StrConstant LBN_CAP_SUPPORTS_ENTRYSOURCETYPE = "SupportsEntrySourceType" // since 81 +///@} diff --git a/Packages/MIES/MIES_ExperimentDocumentation.ipf b/Packages/MIES/MIES_ExperimentDocumentation.ipf index f731b97c0e..796f1f58b5 100644 --- a/Packages/MIES/MIES_ExperimentDocumentation.ipf +++ b/Packages/MIES/MIES_ExperimentDocumentation.ipf @@ -143,7 +143,7 @@ static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row) endif End -static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState) +static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState, variable insertAsPostProc) variable timestamp string timestampStr @@ -151,7 +151,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, if(IsTextWave(values)) WAVE/T valuesT = values valuesT[rowIndex][0][] = num2istr(sweepNo) - valuesT[rowIndex][3][] = num2istr(entrySourceType) + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + valuesT[rowIndex][3][] = num2istr(entrySourceType) + endif valuesT[rowIndex][4][] = num2istr(acqState) @@ -163,7 +165,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, valuesT[rowIndex][2][] = timestampStr else values[rowIndex][0][] = sweepNo - values[rowIndex][3][] = entrySourceType + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + values[rowIndex][3][] = entrySourceType + endif values[rowIndex][4][] = acqState @@ -241,7 +245,7 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming ASSERT(WaveExists(indizes), "Missing indizes") rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS) @@ -362,7 +366,7 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming ASSERT(WaveExists(indizes), "Missing indizes") rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 90ef26dabd..d15be1210b 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1704,6 +1704,50 @@ threadsafe Function ReverseEntrySourceTypeMapper(variable mapped) return ((mapped == 0) ? NaN : --mapped) End +/// @brief Tests if the LBN value wave supports the EntrySourceType column +/// Only used in UpgradeLabNotebook +/// returns 0 if not, 1 is yes +Function HasLBNEntrySourceTypeCapability(WAVE values) + + variable entryCol + + entryCol = FindDimLabel(values, COLS, "EntrySourceType") + if(entryCol == -2) + return 0 + endif + + if(!DimSize(values, ROWS)) + return 1 + endif + + if(IsNumericWave(values)) + WaveStats/Q/M=1/RMD=[][entryCol][] values + return !IsNaN(V_max) + endif + + Duplicate/FREE/RMD=[][entryCol][] values, entrySourceTypeValues + return HasOneValidEntry(entrySourceTypeValues) +End + +/// @brief Returns a labnotebook capability +/// +/// @param values LBN values wave +/// @param capabilityKey Capabilities key, one of @ref LabnotebookCapabilityKeys +/// +/// @returns capability value +threadsafe Function GetLBNCapability(WAVE values, string capabilityKey) + + variable cap + + WAVE/Z keys = GetLogbookKeysFromValues(values) + ASSERT_TS(WaveExists(keys), "Can not resolve LBN keys wave") + cap = GetNumberFromWaveNote(keys, capabilityKey) + // Triggers if LBN was not correctly upgraded in UpgradeLabNotebook + ASSERT_TS(!IsNaN(cap), "Requested LBN capability not found: " + capabilityKey) + + return cap +End + /// @brief Return labnotebook keys for patch seq analysis functions /// /// @param type One of @ref SpecialAnalysisFunctionTypes diff --git a/Packages/MIES/MIES_WaveDataFolderGetters.ipf b/Packages/MIES/MIES_WaveDataFolderGetters.ipf index ec2d2f6805..5abdd50b75 100644 --- a/Packages/MIES/MIES_WaveDataFolderGetters.ipf +++ b/Packages/MIES/MIES_WaveDataFolderGetters.ipf @@ -1385,9 +1385,11 @@ End /// - Making dimension labels valid liberal object names /// - Extending the row dimension to 6 for the key waves /// - Fixing empty column dimension labels in key waves +/// - add capabilities to wavenote Function UpgradeLabNotebook(string device) variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow + variable hasCapability string list, key // we only have to check the new place and name as we are called @@ -1658,6 +1660,18 @@ Function UpgradeLabNotebook(string device) endif endif // END add note index + + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalKeys, 83)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalValues) + SetNumberInWaveNote(numericalKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualKeys, 83)) + hasCapability = HasLBNEntrySourceTypeCapability(textualValues) + SetNumberInWaveNote(textualKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType End static Function/S FixInvalidLabnotebookKey(string name) @@ -1731,6 +1745,7 @@ Function/WAVE GetLBTextualKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1785,6 +1800,7 @@ Function/WAVE GetLBNumericalKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1918,7 +1934,7 @@ End /// - Fixing empty column dimension labels in key waves static Function UpgradeResultsNotebook() - variable i, numCols + variable i, numCols, hasCapability DFREF dfr = GetResultsFolder() WAVE/Z/SDFR=dfr numericalResultValues = $LBN_NUMERICALRESULT_VALUES_NAME @@ -1960,6 +1976,17 @@ static Function UpgradeResultsNotebook() endfor endif // END fix missing column dimension labels in keyWaves + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalResultKeys, 4)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalResultValues) + SetNumberInWaveNote(numericalResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualResultKeys, 4)) + hasCapability = HasLBNEntrySourceTypeCapability(textualResultValues) + SetNumberInWaveNote(textualResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType End /// @brief Return a wave reference to the numeric labnotebook keys @@ -2029,6 +2056,7 @@ Function/WAVE GetNumericalResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -2124,6 +2152,7 @@ Function/WAVE GetTextualResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" From 70d5785196e526f80a68393ae4bf130c0221ec4f Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Tue, 15 Apr 2025 13:45:56 +0200 Subject: [PATCH 5/6] NWB: Recreate Epochs on export and insert this info into LBN - Epochs for DA channels get recreated. - Epoch recreation is an optional argument for NWB_ExportAllData and is by default off. - The location where the LNB is written to the NWB file through NWB_WriteLabnotebooksAndComments was moved to a spot after the LNB was adapted. Added tests for postProcessed epoch addition for vintage files. --- Packages/MIES/MIES_Epochs.ipf | 2 +- Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 66 +++++++++ .../MIES/MIES_NeuroDataWithoutBorders.ipf | 17 ++- .../UTF_AttemptNWB2ExportOnOldData.ipf | 6 +- Packages/tests/UTF_HelperFunctions.ipf | 135 ++++++++++++++++++ 5 files changed, 221 insertions(+), 5 deletions(-) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index f319d954e2..edb493467a 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1778,7 +1778,7 @@ End /// @param sweepDFR single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) /// @param sweepNo sweep number /// @returns recreated 4D epoch wave -static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) +Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) STRUCT DataConfigurationResult s variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index d15be1210b..a947da629f 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -2035,3 +2035,69 @@ Function/S StringifyLogbookMode(variable mode) break endswitch End + +Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues, string device, variable sweepNo) + + string epochList + variable channelNumber, channelType, headstage, numChannelTypes, colCount, allocatedCols + variable assocCol = NaN + + DFREF deviceDFR = GetDeviceDataPath(device) + DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) + WAVE/Z recEpochs = EP_RecreateEpochsFromLoadedData(numericalValues, textualValues, sweepDFR, sweepNo) + if(!WaveExists(recEpochs)) + print "Could not recreate Epochs." + return NaN + endif + + Make/FREE channelTypes = {XOP_CHANNEL_TYPE_DAC, XOP_CHANNEL_TYPE_TTL} + + numChannelTypes = DimSize(channelTypes, ROWS) + allocatedCols = numChannelTypes * NUM_DA_TTL_CHANNELS + Make/FREE/T/N=(1, allocatedCols) keys + Make/FREE/T/N=(1, allocatedCols, LABNOTEBOOK_LAYER_COUNT) values + + for(channelType : channelTypes) + for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1) + // Currently only implemented for DAC channel type + if(channelType != XOP_CHANNEL_TYPE_DAC) + continue + endif + + epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType) + if(IsEmpty(epochList)) + continue + endif + + headstage = GetHeadstageForChannel(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE) + if(IsAssociatedChannel(headstage)) + if(IsNaN(assocCol)) + assocCol = colCount + colCount += 1 + keys[0][assocCol] = EPOCHS_ENTRY_KEY + endif + values[0][assocCol][headstage] = epochList + continue + endif + + values[0][colCount][INDEP_HEADSTAGE] = epochList + keys[0][colCount] = CreateLBNUnassocKey(EPOCHS_ENTRY_KEY, channelNumber, channelType) + colCount += 1 + endfor + endfor + if(!colCount) + // No Epochs could be recreated for any channel + return NaN + endif + + Redimension/N=(-1, colCount) keys + Redimension/N=(-1, colCount, -1) values + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + + Redimension/N=(-1, 1) keys + keys[0][0] = SWEEP_EPOCH_VERSION_ENTRY_KEY + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) valuesNum + FastOp valuesNum = (NaN) + valuesNum[0][0][INDEP_HEADSTAGE] = SWEEP_EPOCH_VERSION + ED_AddEntriesToLabnotebook(valuesNum, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) +End diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf index 9f1ed258e2..782437f1ab 100644 --- a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf +++ b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf @@ -479,7 +479,6 @@ static Function NWB_AddDeviceSpecificData(STRUCT NWBAsyncParameters &s, variable AddModificationTimeEntry(s.locationID, s.nwbVersion) NWB_AddDevice(s) - NWB_WriteLabnotebooksAndComments(s) NWB_WriteTestpulseData(s, writeStoredTestPulses) End @@ -588,12 +587,13 @@ End /// @param overwrite [optional, defaults to false] overwrite any existing NWB file with the same name, only /// used when overrideFilePath is passed /// @param verbose [optional, defaults to true] get diagnostic output to the command line +/// @param recreateEpochs [optional, defaults to false] when set to true, epoch information is recreated if the version of the data in the experiment is older than the latest version /// /// @return 0 on success, non-zero on failure -Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose]) +Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose, variable recreateEpochs]) string list, name, fileName - variable locationID, sweep, createdNewNWBFile, argCheck + variable locationID, sweep, createdNewNWBFile, argCheck, epochVersion string stimsetList = "" if(ParamIsDefault(keepFileOpen)) @@ -630,6 +630,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF verbose = !!verbose endif + recreateEpochs = ParamIsDefault(recreateEpochs) ? 0 : !!recreateEpochs + argCheck = ParamIsDefault(overrideFullFilePath) + ParamIsDefault(overrideFileTemplate) ASSERT(argCheck >= 1 && argCheck <= 2, "Either arg overrideFullFilePath or arg overrideFileTemplate must be given or none (auto-gen)") @@ -736,6 +738,13 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF WAVE s.DAQDataWave = TextSweepToWaveRef(sweepWave) WAVE s.DAQConfigWave = configWave + if(recreateEpochs) + epochVersion = GetLastSettingIndep(s.numericalValues, s.sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + if(IsNaN(epochVersion) || epochVersion < SWEEP_EPOCH_VERSION) + InsertRecreatedEpochsIntoLBN(s.numericalValues, s.textualValues, s.device, s.sweep) + endif + endif + NWB_AppendSweepLowLevel(s) stimsetList += AB_GetStimsetFromPanel(device, sweep) @@ -744,6 +753,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF endfor LOG_AddEntry(PACKAGE_MIES, "export", keys = {"size [MiB]"}, values = {num2str(NWB_GetExportedFileSize(device))}) + NWB_WriteLabnotebooksAndComments(s) + NWB_AppendStimset(nwbVersion, s.locationID, stimsetList, compressionMode) if(writeIgorHistory) diff --git a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf index 55aa458cea..67ba23525c 100644 --- a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf +++ b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf @@ -11,10 +11,14 @@ static Function TestExportingDataToNWB([string str]) LoadMIESFolderFromPXP("input:" + str) + WAVE devEpochVersionPre = GatherEpochVersions() + PathInfo home templateName = S_path + GetBaseName(str) // attempt export - NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1) + NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1, recreateEpochs = 1) + + CheckIfPostProcessedEpochsAreAdded(devEpochVersionPre) CHECK_NO_RTE() diff --git a/Packages/tests/UTF_HelperFunctions.ipf b/Packages/tests/UTF_HelperFunctions.ipf index 8129c16331..a60783e144 100644 --- a/Packages/tests/UTF_HelperFunctions.ipf +++ b/Packages/tests/UTF_HelperFunctions.ipf @@ -2063,3 +2063,138 @@ Function RunWithOpts([string testcase, string testsuite, variable allowdebug, va RunTest(testsuite, name = name, enableJU = enableJU, enableRegExp = enableRegExp, debugMode = debugMode, testcase = testcase, traceOptions = traceOptions, traceWinList = traceWinList, keepDataFolder = keepDataFolder, waveTrackingMode = waveTrackingMode, retry = IUTF_RETRY_FAILED_UNTIL_PASS) endif End + +Function/WAVE GatherEpochVersions() + + variable i + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + Make/FREE/WAVE/N=(DimSize(devicesWithContent, ROWS)) devEpochVersion + for(device : devicesWithContent) + WAVE numericalValues = GetLBNumericalValues(device) + DFREF dfr = GetDeviceDataPath(device) + + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + Make/FREE/N=(DimSize(sweepWaveNames, ROWS)) sweepNums + sweepNums[] = ExtractSweepNumber(sweepWaveNames[p]) + Make/FREE/D/N=(WaveMax(sweepNums) + 1) epochVersion + for(sweep : sweepNums) + epochVersion[sweep] = GetLastSettingIndep(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + endfor + devEpochVersion[i] = epochVersion + i += 1 + endfor + + return devEpochVersion +End + +/// @brief A simple function to find the last row of a sweep number/DATA_ACQUISITION_MODE block for a specific headstage in the LBN +/// Depending on the flag findAny there are two search modes +/// findAny 0 (default): a value val or string str must have been specified, the search finds the last entry with that value +/// (in case of text the comparison is case-insensitive) +/// findAny 1: The first non-empty/non-NaN entry is found +/// The search is backwards and returns the LBN row of the first match or NaN if nothing was found +/// +/// @param values LBN values wave +/// @param sweep sweep number +/// @param key key of the LBN column to search +/// @param headstage headstage (LBN layer) +/// @param val [optional] value to search for, use if LBN values wave is numeric +/// @param str [optional] string to search for, use if LBN values wave is text +/// @param findAny [optional, default 0] For default, search explicitly for a value (val/str), when set find the last row with any value +/// +/// @returns last LBN row found or NaN if not found +Function FindLastLBNRow(WAVE values, variable sweep, string key, variable headstage, [variable val, string str, variable findAny]) + + variable sweepCol, i, keyCol + variable firstValue, lastValue + + findAny = ParamIsDefault(findAny) ? 0 : !!findAny + if(!findAny) + REQUIRE_EQUAL_VAR(ParamIsDefault(val) + ParamIsDefault(str), 1) + endif + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweep, DATA_ACQUISITION_MODE, firstValue, lastValue) + REQUIRE_NEQ_VAR(firstValue, NaN) + REQUIRE_NEQ_VAR(lastValue, NaN) + REQUIRE_LE_VAR(firstValue, lastValue) + keyCol = FindDimLabel(values, COLS, key) + CHECK_NEQ_VAR(keyCol, -2) + if(IsTextWave(values)) + WAVE/T valuesT = values + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsEmpty(valuesT[i][keyCol][headstage])) + return i + endif + else + if(!CmpStr(valuesT[i][keyCol][headstage], str)) + return i + endif + endif + endfor + else + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsNaN(values[i][keyCol][headstage])) + return i + endif + else + if(values[i][keyCol][headstage] == val) + return i + endif + endif + endfor + endif + + return NaN +End + +Function CheckIfPostProcessedEpochsAreAdded(WAVE/WAVE devEpochVersionPre) + + variable devCnt, sweep, keyCol, row, i + + WAVE/WAVE devEpochVersionAfter = GatherEpochVersions() + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + for(device : devicesWithContent) + // check for postProcessed epoch info + WAVE numericalValues = GetLBNumericalValues(device) + WAVE/T textualValues = GetLBTextualValues(device) + + DFREF dfr = GetDeviceDataPath(device) + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + + WAVE epochVersionsPre = devEpochVersionPre[devCnt] + WAVE epochVersionsAfter = devEpochVersionAfter[devCnt] + + for(name : sweepWaveNames) + sweep = ExtractSweepNumber(name) + if(epochVersionsPre[sweep] < SWEEP_EPOCH_VERSION) + CHECK_EQUAL_VAR(SWEEP_EPOCH_VERSION, epochVersionsAfter[sweep]) + + row = FindLastLBNRow(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, INDEP_HEADSTAGE, val = SWEEP_EPOCH_VERSION) + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_VAR(numericalValues[row][keyCol][INDEP_HEADSTAGE], 1) + + for(i = 0; i < NUM_HEADSTAGES; i += 1) + row = FindLastLBNRow(textualValues, sweep, EPOCHS_ENTRY_KEY, i, findAny = 1) + if(!IsNaN(row)) + break + endif + endfor + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_STR(textualValues[row][keyCol][INDEP_HEADSTAGE], "1") + endif + + endfor + devCnt += 1 + endfor +End From 456d73447648cb6368c8090f91e59df59ec427d1 Mon Sep 17 00:00:00 2001 From: Michael Huth Date: Thu, 10 Apr 2025 14:50:18 +0200 Subject: [PATCH 6/6] LBN PostProc: Also add TTL epochs to postproc LBN insertion on NWB export TTL epochs are now also inserted into the LBN. --- Packages/MIES/MIES_Epochs.ipf | 1 + Packages/MIES/MIES_MiesUtilities_Logbook.ipf | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index edb493467a..5d7b06f8d8 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1794,6 +1794,7 @@ Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textu WAVE/T recEpochWave = GetEpochsWaveAsFree() EP_CollectEpochInfoDA(recEpochWave, s) + EP_CollectEpochInfoTTL(recEpochWave, s) EP_AddRecreatedUserEpochs(numericalValues, textualValues, sweepDFR, sweepNo, s, recEpochWave) WAVE/Z channelDA = GetDAQDataSingleColumnWaveNG(numericalValues, textualValues, sweepNo, sweepDFR, XOP_CHANNEL_TYPE_DAC, s.DACList[0]) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index a947da629f..79bb0c2241 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -2059,16 +2059,19 @@ Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues for(channelType : channelTypes) for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1) - // Currently only implemented for DAC channel type - if(channelType != XOP_CHANNEL_TYPE_DAC) - continue - endif epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType) if(IsEmpty(epochList)) continue endif + if(channelType == XOP_CHANNEL_TYPE_TTL) + keys[0][colCount] = CreateTTLChannelLBNKey(EPOCHS_ENTRY_KEY, channelNumber) + values[0][colCount][INDEP_HEADSTAGE] = epochList + colCount += 1 + continue + endif + headstage = GetHeadstageForChannel(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE) if(IsAssociatedChannel(headstage)) if(IsNaN(assocCol))