diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index 6e57ac4e4a..4d016dd74e 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,8 +39,8 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 81 -Constant RESULTS_VERSION = 3 +Constant LABNOTEBOOK_VERSION = 83 +Constant RESULTS_VERSION = 4 ///@} /// @name Analysis function versions @@ -996,6 +996,7 @@ StrConstant STIMSET_SIZE_KEY = "Stimset Size" StrConstant STIMSET_ERROR_KEY = "Wavebuilder Error" StrConstant AUTOBIAS_PERC_KEY = "Autobias %" StrConstant SWEEP_EPOCH_VERSION_ENTRY_KEY = "Epochs Version" +StrConstant POSTPROCESSED_ENTRY_KEY = "PostProcessed" Constant WAVEBUILDER_STATUS_ERROR = 1 @@ -2557,3 +2558,9 @@ Constant SF_STEP_PARSER = 1 Constant SF_STEP_EXECUTOR = 2 Constant SF_STEP_OUTSIDE = 3 ///@} + +/// Labnotebook capabilities are stored in the key wave note +/// @anchor LabnotebookCapabilityKeys +///@{ +StrConstant LBN_CAP_SUPPORTS_ENTRYSOURCETYPE = "SupportsEntrySourceType" // since 81 +///@} diff --git a/Packages/MIES/MIES_DAEphys.ipf b/Packages/MIES/MIES_DAEphys.ipf index a6d4697cf7..d596cee293 100644 --- a/Packages/MIES/MIES_DAEphys.ipf +++ b/Packages/MIES/MIES_DAEphys.ipf @@ -4540,11 +4540,7 @@ Function DAP_LockDevice(string win) headstage = GetSliderPositionIndex(deviceLocked, "slider_DataAcq_ActiveHeadstage") P_SaveUserSelectedHeadstage(deviceLocked, headstage) - // upgrade all four labnotebook waves in wanna-be atomic way - GetLBNumericalKeys(deviceLocked) - GetLBNumericalValues(deviceLocked) - GetLBTextualKeys(deviceLocked) - GetLBTextualValues(deviceLocked) + UpgradeLabNotebook(deviceLocked) NVAR sessionStartTime = $GetSessionStartTime() sessionStartTime = DateTimeInUTC() diff --git a/Packages/MIES/MIES_DataBrowser.ipf b/Packages/MIES/MIES_DataBrowser.ipf index ada6fc7f69..ba5acff9e0 100644 --- a/Packages/MIES/MIES_DataBrowser.ipf +++ b/Packages/MIES/MIES_DataBrowser.ipf @@ -310,6 +310,7 @@ static Function/S DB_LockToDevice(string win, string device) BSP_UnsetDynamicStartupSettings(win) else newWindow = "DB_" + device + UpgradeLabNotebook(device) endif DB_SetUserData(win, device) @@ -421,10 +422,6 @@ Function DB_UpdateSweepPlot(string win) return NaN endif - // fetch keys waves to trigger a potential labnotebook upgrade - WAVE numericalKeys = DB_GetLBNWave(win, LBN_NUMERICAL_KEYS) - WAVE textualKeys = DB_GetLBNWave(win, LBN_TEXTUAL_KEYS) - WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES) WAVE textualValues = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES) diff --git a/Packages/MIES/MIES_Epochs.ipf b/Packages/MIES/MIES_Epochs.ipf index f319d954e2..5d7b06f8d8 100644 --- a/Packages/MIES/MIES_Epochs.ipf +++ b/Packages/MIES/MIES_Epochs.ipf @@ -1778,7 +1778,7 @@ End /// @param sweepDFR single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) /// @param sweepNo sweep number /// @returns recreated 4D epoch wave -static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) +Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo) STRUCT DataConfigurationResult s variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex @@ -1794,6 +1794,7 @@ static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/ WAVE/T recEpochWave = GetEpochsWaveAsFree() EP_CollectEpochInfoDA(recEpochWave, s) + EP_CollectEpochInfoTTL(recEpochWave, s) EP_AddRecreatedUserEpochs(numericalValues, textualValues, sweepDFR, sweepNo, s, recEpochWave) WAVE/Z channelDA = GetDAQDataSingleColumnWaveNG(numericalValues, textualValues, sweepNo, sweepDFR, XOP_CHANNEL_TYPE_DAC, s.DACList[0]) diff --git a/Packages/MIES/MIES_ExperimentDocumentation.ipf b/Packages/MIES/MIES_ExperimentDocumentation.ipf index 7e6b243eae..796f1f58b5 100644 --- a/Packages/MIES/MIES_ExperimentDocumentation.ipf +++ b/Packages/MIES/MIES_ExperimentDocumentation.ipf @@ -62,14 +62,16 @@ /// It is recommended to gather all entries to be written in keys/values and call ED_AddEntriesToLabnotebook then once. /// /// @see ED_createTextNotes, ED_createWaveNote -Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType) +Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType, [variable insertAsPostProc]) + + insertAsPostProc = ParamIsDefault(insertAsPostProc) ? 0 : !!insertAsPostProc ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) else - ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device) + ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device) endif End @@ -79,9 +81,9 @@ Function ED_AddEntriesToResults(WAVE vals, WAVE/T keys, variable entrySourceType ED_CheckValuesAndKeys(vals, keys) if(IsTextWave(vals)) - ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) else - ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS) + ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0) endif End @@ -109,7 +111,39 @@ static Function ED_CheckValuesAndKeys(WAVE vals, WAVE keys) endif End -static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState) +static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row) + + variable col, unused + + ASSERT(DimSize(values, ROWS) >= row, "Row does not exist in LBN values wave") + + WAVE keys = GetLogbookKeysFromValues(values) + + if(IsTextWave(values)) + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesT + WAVE incomingValues = incomingValuesT + else + Make/FREE/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesNum + WAVE incomingValues = incomingValuesNum + endif + + Make/FREE/T/N=(3, 1) incomingKeys + incomingKeys[0] = POSTPROCESSED_ENTRY_KEY + incomingKeys[1] = LABNOTEBOOK_BINARY_UNIT + incomingKeys[2] = LABNOTEBOOK_NO_TOLERANCE + [WAVE indizes, unused] = ED_FindIndizesAndRedimension(incomingKeys, incomingValues, keys, values, LBT_LABNOTEBOOK) + ASSERT(WaveExists(indizes), "Missing indizes") + + col = indizes[0] + if(IsTextWave(values)) + WAVE/T valuesT = values + valuesT[row][col][] = "1" + else + values[row][col][] = 1 + endif +End + +static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState, variable insertAsPostProc) variable timestamp string timestampStr @@ -117,7 +151,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, if(IsTextWave(values)) WAVE/T valuesT = values valuesT[rowIndex][0][] = num2istr(sweepNo) - valuesT[rowIndex][3][] = num2istr(entrySourceType) + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + valuesT[rowIndex][3][] = num2istr(entrySourceType) + endif valuesT[rowIndex][4][] = num2istr(acqState) @@ -129,7 +165,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, valuesT[rowIndex][2][] = timestampStr else values[rowIndex][0][] = sweepNo - values[rowIndex][3][] = entrySourceType + if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE)) + values[rowIndex][3][] = entrySourceType + endif values[rowIndex][4][] = acqState @@ -142,6 +180,29 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, endif End +/// @brief Inserts a row after the sweep block and returns the inserted row number +static Function ED_InsertRowAfterSweepBlock(WAVE values, variable sweepNo, variable entrySourceType) + + variable sweepCol, firstRow, lastRow, rowIndex, size + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweepNo, entrySourceType, firstRow, lastRow) + ASSERT(!IsNaN(firstRow) && !IsNaN(lastRow), "FindRange could not determine start and/or end of sweep block") + rowIndex = lastRow + 1 + InsertPoints/M=(ROWS) rowIndex, 1, values + + if(!IsTextWave(values)) + values[rowIndex][][] = NaN + endif + + size = GetNumberFromWaveNote(values, NOTE_INDEX) + SetNumberInWaveNote(values, NOTE_INDEX, size + 1) + + InvalidateLBIndexAndRowCache(values) + + return rowIndex +End + /// @brief Add textual entries to the logbook /// /// The text documentation wave will use layers to report the different headstages. @@ -156,9 +217,11 @@ End /// @param device [optional for logbookType LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType type of the logbook, one of @ref LogbookTypes -static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, i, lastValidIncomingLayer, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -173,10 +236,16 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS) @@ -196,7 +265,9 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = NormalizeToEOL(incomingTextualValues[0][i][r], "\n") endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End static Function ED_ParseHeadstageContingencyMode(string str) @@ -266,9 +337,11 @@ End /// @param device [optional for logbooktype LBT_RESULTS only] device /// @param entrySourceType type of reporting subsystem, one of @ref DataAcqModes /// @param logbookType one of @ref LogbookTypes -static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device]) +/// @param insertAsPostProc when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data +static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device]) variable rowIndex, numCols, lastValidIncomingLayer, i, state + variable addIndex, insertIndex if(ParamIsDefault(device)) ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type") @@ -284,10 +357,16 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming state = ROVar(GetAcquisitionState(device)) endif - [WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) + if(insertAsPostProc) + insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType) + ED_SetLabnotebookRowToPostProcessed(values, insertIndex) + endif + + [WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType) ASSERT(WaveExists(indizes), "Missing indizes") + rowIndex = insertAsPostProc ? insertIndex : addIndex - ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state) + ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc) WAVE valuesDat = ExtractLogbookSliceTimeStamp(values) EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) @@ -299,7 +378,9 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = incomingNumericalValues[0][i][r] endfor - SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + if(!insertAsPostProc) + SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1) + endif End /// @brief Add custom entries to the numerical/textual labnotebook for the very last sweep acquired. @@ -706,6 +787,13 @@ static Function [WAVE colIndizes, variable rowIndex] ED_FindIndizesAndRedimensio LBN_SetDimensionLabels(key, values, start = ((rowIndex == 0) ? 0 : numKeyCols)) endif + // Notes on the LBN row expansion here: + // - Initialisation with NaN, "" takes only place when the wave is expanded for rows/cols. + // - It does not initialise the row returned to the caller. + // Thus, for the unlucky case that a previous LBN write attempt ended up to be partial (not seen happening so far) + // then the row returned might be partially filled. + // - This part is also called if a row is inserted in the LBN from postprocessing. The LBN wave size is increased if required. + // The new row at the end is not used by the insertion. It is still consistent as the NOTE_INDEX is not updated here. if(IsNumericWave(values)) EnsureLargeEnoughWave(values, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN) if(numAdditions) diff --git a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf index 5b722be6d1..79bb0c2241 100644 --- a/Packages/MIES/MIES_MiesUtilities_Logbook.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Logbook.ipf @@ -1278,7 +1278,7 @@ End /// @param[in] entrySourceType type of the labnotebook entry, one of @ref DataAcqModes /// @param[out] first point index of the beginning of the range /// @param[out] last point index of the end of the range -threadsafe static Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) +threadsafe Function FindRange(WAVE wv, variable col, variable val, variable entrySourceType, variable &first, variable &last) variable numRows, i, j, sourceTypeCol, firstRow, lastRow, isNumeric, index, startRow, endRow @@ -1704,6 +1704,50 @@ threadsafe Function ReverseEntrySourceTypeMapper(variable mapped) return ((mapped == 0) ? NaN : --mapped) End +/// @brief Tests if the LBN value wave supports the EntrySourceType column +/// Only used in UpgradeLabNotebook +/// returns 0 if not, 1 is yes +Function HasLBNEntrySourceTypeCapability(WAVE values) + + variable entryCol + + entryCol = FindDimLabel(values, COLS, "EntrySourceType") + if(entryCol == -2) + return 0 + endif + + if(!DimSize(values, ROWS)) + return 1 + endif + + if(IsNumericWave(values)) + WaveStats/Q/M=1/RMD=[][entryCol][] values + return !IsNaN(V_max) + endif + + Duplicate/FREE/RMD=[][entryCol][] values, entrySourceTypeValues + return HasOneValidEntry(entrySourceTypeValues) +End + +/// @brief Returns a labnotebook capability +/// +/// @param values LBN values wave +/// @param capabilityKey Capabilities key, one of @ref LabnotebookCapabilityKeys +/// +/// @returns capability value +threadsafe Function GetLBNCapability(WAVE values, string capabilityKey) + + variable cap + + WAVE/Z keys = GetLogbookKeysFromValues(values) + ASSERT_TS(WaveExists(keys), "Can not resolve LBN keys wave") + cap = GetNumberFromWaveNote(keys, capabilityKey) + // Triggers if LBN was not correctly upgraded in UpgradeLabNotebook + ASSERT_TS(!IsNaN(cap), "Requested LBN capability not found: " + capabilityKey) + + return cap +End + /// @brief Return labnotebook keys for patch seq analysis functions /// /// @param type One of @ref SpecialAnalysisFunctionTypes @@ -1879,6 +1923,13 @@ Function GetTotalOnsetDelayFromDevice(string device) return DAG_GetNumericalValue(device, "setvar_DataAcq_OnsetDelayUser") + TPSettingsCalculated[%totalLengthMS] End +/// @brief Invalidates the row and index cache for the given LBN +Function InvalidateLBIndexAndRowCache(WAVE values) + + CA_DeleteCacheEntry(CA_CreateLBIndexCacheKey(values)) + CA_DeleteCacheEntry(CA_CreateLBRowCacheKey(values)) +End + /// @brief Retrieve the analysis function that was run for a given sweep / channelNumber / channelType /// /// @param numericalValues numerical labnotebook @@ -1984,3 +2035,72 @@ Function/S StringifyLogbookMode(variable mode) break endswitch End + +Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues, string device, variable sweepNo) + + string epochList + variable channelNumber, channelType, headstage, numChannelTypes, colCount, allocatedCols + variable assocCol = NaN + + DFREF deviceDFR = GetDeviceDataPath(device) + DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo) + WAVE/Z recEpochs = EP_RecreateEpochsFromLoadedData(numericalValues, textualValues, sweepDFR, sweepNo) + if(!WaveExists(recEpochs)) + print "Could not recreate Epochs." + return NaN + endif + + Make/FREE channelTypes = {XOP_CHANNEL_TYPE_DAC, XOP_CHANNEL_TYPE_TTL} + + numChannelTypes = DimSize(channelTypes, ROWS) + allocatedCols = numChannelTypes * NUM_DA_TTL_CHANNELS + Make/FREE/T/N=(1, allocatedCols) keys + Make/FREE/T/N=(1, allocatedCols, LABNOTEBOOK_LAYER_COUNT) values + + for(channelType : channelTypes) + for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1) + + epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType) + if(IsEmpty(epochList)) + continue + endif + + if(channelType == XOP_CHANNEL_TYPE_TTL) + keys[0][colCount] = CreateTTLChannelLBNKey(EPOCHS_ENTRY_KEY, channelNumber) + values[0][colCount][INDEP_HEADSTAGE] = epochList + colCount += 1 + continue + endif + + headstage = GetHeadstageForChannel(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE) + if(IsAssociatedChannel(headstage)) + if(IsNaN(assocCol)) + assocCol = colCount + colCount += 1 + keys[0][assocCol] = EPOCHS_ENTRY_KEY + endif + values[0][assocCol][headstage] = epochList + continue + endif + + values[0][colCount][INDEP_HEADSTAGE] = epochList + keys[0][colCount] = CreateLBNUnassocKey(EPOCHS_ENTRY_KEY, channelNumber, channelType) + colCount += 1 + endfor + endfor + if(!colCount) + // No Epochs could be recreated for any channel + return NaN + endif + + Redimension/N=(-1, colCount) keys + Redimension/N=(-1, colCount, -1) values + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + + Redimension/N=(-1, 1) keys + keys[0][0] = SWEEP_EPOCH_VERSION_ENTRY_KEY + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) valuesNum + FastOp valuesNum = (NaN) + valuesNum[0][0][INDEP_HEADSTAGE] = SWEEP_EPOCH_VERSION + ED_AddEntriesToLabnotebook(valuesNum, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) +End diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf index 1720b4804c..782437f1ab 100644 --- a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf +++ b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf @@ -479,7 +479,6 @@ static Function NWB_AddDeviceSpecificData(STRUCT NWBAsyncParameters &s, variable AddModificationTimeEntry(s.locationID, s.nwbVersion) NWB_AddDevice(s) - NWB_WriteLabnotebooksAndComments(s) NWB_WriteTestpulseData(s, writeStoredTestPulses) End @@ -588,12 +587,13 @@ End /// @param overwrite [optional, defaults to false] overwrite any existing NWB file with the same name, only /// used when overrideFilePath is passed /// @param verbose [optional, defaults to true] get diagnostic output to the command line +/// @param recreateEpochs [optional, defaults to false] when set to true, epoch information is recreated if the version of the data in the experiment is older than the latest version /// /// @return 0 on success, non-zero on failure -Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose]) +Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose, variable recreateEpochs]) string list, name, fileName - variable locationID, sweep, createdNewNWBFile, argCheck + variable locationID, sweep, createdNewNWBFile, argCheck, epochVersion string stimsetList = "" if(ParamIsDefault(keepFileOpen)) @@ -630,6 +630,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF verbose = !!verbose endif + recreateEpochs = ParamIsDefault(recreateEpochs) ? 0 : !!recreateEpochs + argCheck = ParamIsDefault(overrideFullFilePath) + ParamIsDefault(overrideFileTemplate) ASSERT(argCheck >= 1 && argCheck <= 2, "Either arg overrideFullFilePath or arg overrideFileTemplate must be given or none (auto-gen)") @@ -679,6 +681,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF for(device : devicesWithContent) + UpgradeLabNotebook(device) + if(ParamIsDefault(overrideFullFilePath) && ParamIsDefault(overrideFileTemplate)) [locationID, createdNewNWBFile] = NWB_GetFileForExport(nwbVersion, device) elseif(!ParamIsDefault(overrideFullFilePath)) @@ -734,6 +738,13 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF WAVE s.DAQDataWave = TextSweepToWaveRef(sweepWave) WAVE s.DAQConfigWave = configWave + if(recreateEpochs) + epochVersion = GetLastSettingIndep(s.numericalValues, s.sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + if(IsNaN(epochVersion) || epochVersion < SWEEP_EPOCH_VERSION) + InsertRecreatedEpochsIntoLBN(s.numericalValues, s.textualValues, s.device, s.sweep) + endif + endif + NWB_AppendSweepLowLevel(s) stimsetList += AB_GetStimsetFromPanel(device, sweep) @@ -742,6 +753,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF endfor LOG_AddEntry(PACKAGE_MIES, "export", keys = {"size [MiB]"}, values = {num2str(NWB_GetExportedFileSize(device))}) + NWB_WriteLabnotebooksAndComments(s) + NWB_AppendStimset(nwbVersion, s.locationID, stimsetList, compressionMode) if(writeIgorHistory) diff --git a/Packages/MIES/MIES_WaveDataFolderGetters.ipf b/Packages/MIES/MIES_WaveDataFolderGetters.ipf index 3ae72cf33d..5abdd50b75 100644 --- a/Packages/MIES/MIES_WaveDataFolderGetters.ipf +++ b/Packages/MIES/MIES_WaveDataFolderGetters.ipf @@ -1385,9 +1385,11 @@ End /// - Making dimension labels valid liberal object names /// - Extending the row dimension to 6 for the key waves /// - Fixing empty column dimension labels in key waves -static Function UpgradeLabNotebook(string device) +/// - add capabilities to wavenote +Function UpgradeLabNotebook(string device) variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow + variable hasCapability string list, key // we only have to check the new place and name as we are called @@ -1658,6 +1660,18 @@ static Function UpgradeLabNotebook(string device) endif endif // END add note index + + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalKeys, 83)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalValues) + SetNumberInWaveNote(numericalKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualKeys, 83)) + hasCapability = HasLBNEntrySourceTypeCapability(textualValues) + SetNumberInWaveNote(textualKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType End static Function/S FixInvalidLabnotebookKey(string name) @@ -1731,6 +1745,7 @@ Function/WAVE GetLBTextualKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1785,6 +1800,7 @@ Function/WAVE GetLBNumericalKeys(string device) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -1918,7 +1934,7 @@ End /// - Fixing empty column dimension labels in key waves static Function UpgradeResultsNotebook() - variable i, numCols + variable i, numCols, hasCapability DFREF dfr = GetResultsFolder() WAVE/Z/SDFR=dfr numericalResultValues = $LBN_NUMERICALRESULT_VALUES_NAME @@ -1960,6 +1976,17 @@ static Function UpgradeResultsNotebook() endfor endif // END fix missing column dimension labels in keyWaves + // BEGIN add capability note for EntrySourceType + if(WaveVersionIsSmaller(numericalResultKeys, 4)) + hasCapability = HasLBNEntrySourceTypeCapability(numericalResultValues) + SetNumberInWaveNote(numericalResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + + if(WaveVersionIsSmaller(textualResultKeys, 4)) + hasCapability = HasLBNEntrySourceTypeCapability(textualResultValues) + SetNumberInWaveNote(textualResultKeys, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, hasCapability) + endif + // END add capability note for EntrySourceType End /// @brief Return a wave reference to the numeric labnotebook keys @@ -2029,6 +2056,7 @@ Function/WAVE GetNumericalResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" @@ -2124,6 +2152,7 @@ Function/WAVE GetTextualResultsKeys() return wv else Make/T/N=(3, INITIAL_KEY_WAVE_COL_COUNT) dfr:$name/WAVE=wv + SetNumberInWaveNote(wv, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE, 1) endif wv = "" diff --git a/Packages/MIES/labnotebook_numerical_description.itx b/Packages/MIES/labnotebook_numerical_description.itx index ce017fdca3..650e388ea8 100644 --- a/Packages/MIES/labnotebook_numerical_description.itx +++ b/Packages/MIES/labnotebook_numerical_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(196,6) labnotebook_numerical_description +WAVES/T/N=(197,6) labnotebook_numerical_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "SweepNum" "" "-" "Sweep number: Non-repeating non-negative numeric identifier for sweep time series. Increments in the order of acquisition. Starts at zero." "ALL" "" @@ -197,6 +197,7 @@ BEGIN "Require amplifier" "On/Off" "-" "ON if data acquisition was done with an amplifier, OFF if not." "INDEP" "IC;VC;I=0" "Skip Ahead" "" "1" "Determines how many sweeps are skipped from the stimset whan starting data acquisition." "INDEP" "IC;VC;I=0" "Skip Sweeps source" "" "0.1" "Stores who is responsible for sweep skipping. Current values are 0x1 for the user and 0x2 for automatic/internal reasons." "INDEP" "IC;VC;I=0" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_numerical_description; SetScale/P y 0,1,"", labnotebook_numerical_description; SetScale d 0,0,"", labnotebook_numerical_description X Note labnotebook_numerical_description, "WAVE_LAYOUT_VERSION:2;" diff --git a/Packages/MIES/labnotebook_textual_description.itx b/Packages/MIES/labnotebook_textual_description.itx index 2f5b5eb0e2..030dba1270 100644 --- a/Packages/MIES/labnotebook_textual_description.itx +++ b/Packages/MIES/labnotebook_textual_description.itx @@ -1,5 +1,5 @@ IGOR -WAVES/T/N=(78,6) labnotebook_textual_description +WAVES/T/N=(79,6) labnotebook_textual_description BEGIN "Name" "Unit" "Tolerance" "Description" "Headstage Contingency" "ClampMode" "TimeStamp" "s" "-" "Time Stamp: Seconds since Igor epoch (1/1/1904) in local time zone with millisecond precision. Written at time of labnotebook entry." "ALL" "" @@ -79,6 +79,7 @@ BEGIN "TTL Epochs Channel 5" "" "-" "Epochs of TTL Channel 5" "INDEP" "" "TTL Epochs Channel 6" "" "-" "Epochs of TTL Channel 6" "INDEP" "" "TTL Epochs Channel 7" "" "-" "Epochs of TTL Channel 7" "INDEP" "" + "PostProcessed" "On/Off" "-" "Information added at a later time through a post processing step. (e.g. recreated epoch information)" "ALL" "" END X SetScale/P x 0,1,"", labnotebook_textual_description; SetScale/P y 0,1,"", labnotebook_textual_description; SetScale d 0,0,"", labnotebook_textual_description X Note labnotebook_textual_description, "WAVE_LAYOUT_VERSION:1;" diff --git a/Packages/tests/Basic/UTF_Labnotebook.ipf b/Packages/tests/Basic/UTF_Labnotebook.ipf index 2337da1095..42e049673c 100644 --- a/Packages/tests/Basic/UTF_Labnotebook.ipf +++ b/Packages/tests/Basic/UTF_Labnotebook.ipf @@ -1440,3 +1440,150 @@ Function EmptyLabnotebookWorks() WAVE/Z entries = GetLastSetting(textualValues, 0, "Sweep Number", UNKNOWN_MODE) CHECK_WAVE(entries, NULL_WAVE) End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForProstProcessingSweepIndexerText +static Function InsertRowForProstProcessingTextual([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col + string str + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + sizeBefore = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, EPOCHS_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[testHS], keyItem) + WAVE/Z/T settings = GetLastSetting(textualValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_STR(settings[INDEP_HEADSTAGE], "1") + + FindValue/TEXT=keyItem/TXOP=4 textualValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + str = textualValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_STR(str, "1") +End + +// IUTF_TD_GENERATOR DataGenerators#InsertRowForProstProcessingSweepIndexerNum +static Function InsertRowForProstProcessingNumerical([variable sweepNo]) + + variable sizeBefore, sizeAfter, row, col, val + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1, sizeAfter) + + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, "DAC", DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[testHS], keyValue) + WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, POSTPROCESSED_ENTRY_KEY, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(settings[INDEP_HEADSTAGE], 1) + + FindValue/V=(keyValue) numericalValues + CHECK_NEQ_VAR(V_value, -1) + row = V_row + col = FindDimlabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(col, -2) + val = numericalValues[row][col][INDEP_HEADSTAGE] + CHECK_EQUAL_VAR(val, 1) +End + +static Function InsertRowForProstProcessingTextualUnknownSweep() + + variable sweepNo = 1337 + + variable testHS = 2 + string device = "dummyDevice" + string keyItem = "Cintamani Stone" + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr textualValuesSrc = textualValues + WAVE textualValuesTest = PrepareLBNTextualValues(textualValuesSrc) + WAVE/T textualValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device) + Duplicate/O/T textualValuesTest, textualValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = EPOCHS_ENTRY_KEY + values[0][0][testHS] = keyItem + + try + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + FAIL() + catch + PASS() + endtry +End + +static Function InsertRowForProstProcessingNumericalMultiple() + + variable sizeBefore, sizeAfter, sweepNo, index, beforeVal + + variable startSweep = 4 + variable endSweep = 15 + + variable testHS = 2 + string device = "dummyDevice" + variable keyValue = 3292385893 + + DFREF dfr = root:Labnotebook_misc: + WAVE/SDFR=dfr numericalValuesSrc = numericalValues + WAVE numericalValuesTest = PrepareLBNNumericalValues(numericalValuesSrc) + WAVE numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device) + Duplicate/O numericalValuesTest, numericalValues + + Make/FREE/T/N=(1, 1) keys + Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) values + + keys[0][0] = "DAC" + values[0][0][testHS] = keyValue + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + beforeVal = setting[index] + + sizeBefore = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + for(sweepNo = startSweep; sweepNo <= endSweep; sweepNo += 1) + ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1) + endfor + sizeAfter = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(sizeBefore + 1 + endSweep - startSweep, sizeAfter) + + [WAVE setting, index] = GetLastSettingChannel(numericalValues, $"", endSweep, "DA Gain", 0, XOP_CHANNEL_TYPE_DAC, DATA_ACQUISITION_MODE) + CHECK_EQUAL_VAR(beforeVal, setting[index]) + +End diff --git a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf index 55aa458cea..67ba23525c 100644 --- a/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf +++ b/Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf @@ -11,10 +11,14 @@ static Function TestExportingDataToNWB([string str]) LoadMIESFolderFromPXP("input:" + str) + WAVE devEpochVersionPre = GatherEpochVersions() + PathInfo home templateName = S_path + GetBaseName(str) // attempt export - NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1) + NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1, recreateEpochs = 1) + + CheckIfPostProcessedEpochsAreAdded(devEpochVersionPre) CHECK_NO_RTE() diff --git a/Packages/tests/UTF_DataGenerators.ipf b/Packages/tests/UTF_DataGenerators.ipf index f65dd95df4..4a755a1b7f 100644 --- a/Packages/tests/UTF_DataGenerators.ipf +++ b/Packages/tests/UTF_DataGenerators.ipf @@ -1965,3 +1965,17 @@ static Function/WAVE DG_SourceLocationsContent() return wv End + +static Function/WAVE InsertRowForProstProcessingSweepIndexerText() + + Make/FREE wv = {0, 1} + + return wv +End + +static Function/WAVE InsertRowForProstProcessingSweepIndexerNum() + + Make/FREE wv = {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + + return wv +End diff --git a/Packages/tests/UTF_HelperFunctions.ipf b/Packages/tests/UTF_HelperFunctions.ipf index 8129c16331..a60783e144 100644 --- a/Packages/tests/UTF_HelperFunctions.ipf +++ b/Packages/tests/UTF_HelperFunctions.ipf @@ -2063,3 +2063,138 @@ Function RunWithOpts([string testcase, string testsuite, variable allowdebug, va RunTest(testsuite, name = name, enableJU = enableJU, enableRegExp = enableRegExp, debugMode = debugMode, testcase = testcase, traceOptions = traceOptions, traceWinList = traceWinList, keepDataFolder = keepDataFolder, waveTrackingMode = waveTrackingMode, retry = IUTF_RETRY_FAILED_UNTIL_PASS) endif End + +Function/WAVE GatherEpochVersions() + + variable i + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + Make/FREE/WAVE/N=(DimSize(devicesWithContent, ROWS)) devEpochVersion + for(device : devicesWithContent) + WAVE numericalValues = GetLBNumericalValues(device) + DFREF dfr = GetDeviceDataPath(device) + + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + Make/FREE/N=(DimSize(sweepWaveNames, ROWS)) sweepNums + sweepNums[] = ExtractSweepNumber(sweepWaveNames[p]) + Make/FREE/D/N=(WaveMax(sweepNums) + 1) epochVersion + for(sweep : sweepNums) + epochVersion[sweep] = GetLastSettingIndep(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN) + endfor + devEpochVersion[i] = epochVersion + i += 1 + endfor + + return devEpochVersion +End + +/// @brief A simple function to find the last row of a sweep number/DATA_ACQUISITION_MODE block for a specific headstage in the LBN +/// Depending on the flag findAny there are two search modes +/// findAny 0 (default): a value val or string str must have been specified, the search finds the last entry with that value +/// (in case of text the comparison is case-insensitive) +/// findAny 1: The first non-empty/non-NaN entry is found +/// The search is backwards and returns the LBN row of the first match or NaN if nothing was found +/// +/// @param values LBN values wave +/// @param sweep sweep number +/// @param key key of the LBN column to search +/// @param headstage headstage (LBN layer) +/// @param val [optional] value to search for, use if LBN values wave is numeric +/// @param str [optional] string to search for, use if LBN values wave is text +/// @param findAny [optional, default 0] For default, search explicitly for a value (val/str), when set find the last row with any value +/// +/// @returns last LBN row found or NaN if not found +Function FindLastLBNRow(WAVE values, variable sweep, string key, variable headstage, [variable val, string str, variable findAny]) + + variable sweepCol, i, keyCol + variable firstValue, lastValue + + findAny = ParamIsDefault(findAny) ? 0 : !!findAny + if(!findAny) + REQUIRE_EQUAL_VAR(ParamIsDefault(val) + ParamIsDefault(str), 1) + endif + + sweepCol = GetSweepColumn(values) + FindRange(values, sweepCol, sweep, DATA_ACQUISITION_MODE, firstValue, lastValue) + REQUIRE_NEQ_VAR(firstValue, NaN) + REQUIRE_NEQ_VAR(lastValue, NaN) + REQUIRE_LE_VAR(firstValue, lastValue) + keyCol = FindDimLabel(values, COLS, key) + CHECK_NEQ_VAR(keyCol, -2) + if(IsTextWave(values)) + WAVE/T valuesT = values + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsEmpty(valuesT[i][keyCol][headstage])) + return i + endif + else + if(!CmpStr(valuesT[i][keyCol][headstage], str)) + return i + endif + endif + endfor + else + for(i = lastValue; i >= firstValue; i -= 1) + if(findAny) + if(!IsNaN(values[i][keyCol][headstage])) + return i + endif + else + if(values[i][keyCol][headstage] == val) + return i + endif + endif + endfor + endif + + return NaN +End + +Function CheckIfPostProcessedEpochsAreAdded(WAVE/WAVE devEpochVersionPre) + + variable devCnt, sweep, keyCol, row, i + + WAVE/WAVE devEpochVersionAfter = GatherEpochVersions() + + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + for(device : devicesWithContent) + // check for postProcessed epoch info + WAVE numericalValues = GetLBNumericalValues(device) + WAVE/T textualValues = GetLBTextualValues(device) + + DFREF dfr = GetDeviceDataPath(device) + WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";") + + WAVE epochVersionsPre = devEpochVersionPre[devCnt] + WAVE epochVersionsAfter = devEpochVersionAfter[devCnt] + + for(name : sweepWaveNames) + sweep = ExtractSweepNumber(name) + if(epochVersionsPre[sweep] < SWEEP_EPOCH_VERSION) + CHECK_EQUAL_VAR(SWEEP_EPOCH_VERSION, epochVersionsAfter[sweep]) + + row = FindLastLBNRow(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, INDEP_HEADSTAGE, val = SWEEP_EPOCH_VERSION) + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_VAR(numericalValues[row][keyCol][INDEP_HEADSTAGE], 1) + + for(i = 0; i < NUM_HEADSTAGES; i += 1) + row = FindLastLBNRow(textualValues, sweep, EPOCHS_ENTRY_KEY, i, findAny = 1) + if(!IsNaN(row)) + break + endif + endfor + CHECK_NEQ_VAR(row, NaN) + + keyCol = FindDimLabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY) + CHECK_NEQ_VAR(keyCol, -2) + CHECK_EQUAL_STR(textualValues[row][keyCol][INDEP_HEADSTAGE], "1") + endif + + endfor + devCnt += 1 + endfor +End