Insert recreated epoch info on massexport

Packages/MIES/MIES_Constants.ipf

@@ -39,8 +39,8 @@ Constant SWEEP_EPOCH_VERSION = 9
 /// - New/Changed layers of entries
 ///
 ///@{
-Constant LABNOTEBOOK_VERSION = 81
-Constant RESULTS_VERSION     = 3
+Constant LABNOTEBOOK_VERSION = 83
+Constant RESULTS_VERSION     = 4
 ///@}
 
 /// @name Analysis function versions
@@ -996,6 +996,7 @@ StrConstant STIMSET_SIZE_KEY              = "Stimset Size"
 StrConstant STIMSET_ERROR_KEY             = "Wavebuilder Error"
 StrConstant AUTOBIAS_PERC_KEY             = "Autobias %"
 StrConstant SWEEP_EPOCH_VERSION_ENTRY_KEY = "Epochs Version"
+StrConstant POSTPROCESSED_ENTRY_KEY       = "PostProcessed"
 
 Constant WAVEBUILDER_STATUS_ERROR = 1
 
@@ -2557,3 +2558,9 @@ Constant SF_STEP_PARSER   = 1
 Constant SF_STEP_EXECUTOR = 2
 Constant SF_STEP_OUTSIDE  = 3
 ///@}
+
+/// Labnotebook capabilities are stored in the key wave note
+/// @anchor LabnotebookCapabilityKeys
+///@{
+StrConstant LBN_CAP_SUPPORTS_ENTRYSOURCETYPE = "SupportsEntrySourceType" // since 81
+///@}

Packages/MIES/MIES_DAEphys.ipf

@@ -4540,11 +4540,7 @@ Function DAP_LockDevice(string win)
 	headstage = GetSliderPositionIndex(deviceLocked, "slider_DataAcq_ActiveHeadstage")
 	P_SaveUserSelectedHeadstage(deviceLocked, headstage)
 
-	// upgrade all four labnotebook waves in wanna-be atomic way
-	GetLBNumericalKeys(deviceLocked)
-	GetLBNumericalValues(deviceLocked)
-	GetLBTextualKeys(deviceLocked)
-	GetLBTextualValues(deviceLocked)
+	UpgradeLabNotebook(deviceLocked)
 
 	NVAR sessionStartTime = $GetSessionStartTime()
 	sessionStartTime = DateTimeInUTC()

Packages/MIES/MIES_DataBrowser.ipf

@@ -310,6 +310,7 @@ static Function/S DB_LockToDevice(string win, string device)
 		BSP_UnsetDynamicStartupSettings(win)
 	else
 		newWindow = "DB_" + device
+		UpgradeLabNotebook(device)
 	endif
 
 	DB_SetUserData(win, device)
@@ -421,10 +422,6 @@ Function DB_UpdateSweepPlot(string win)
 		return NaN
 	endif
 
-	// fetch keys waves to trigger a potential labnotebook upgrade
-	WAVE numericalKeys = DB_GetLBNWave(win, LBN_NUMERICAL_KEYS)
-	WAVE textualKeys   = DB_GetLBNWave(win, LBN_TEXTUAL_KEYS)
-
 	WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES)
 	WAVE textualValues   = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES)
 

Packages/MIES/MIES_Epochs.ipf

@@ -1778,7 +1778,7 @@ End
 /// @param sweepDFR        single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
 /// @param sweepNo         sweep number
 /// @returns recreated 4D epoch wave
-static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
+Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
 
 	STRUCT DataConfigurationResult s
 	variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex
@@ -1794,6 +1794,7 @@ static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/
 
 	WAVE/T recEpochWave = GetEpochsWaveAsFree()
 	EP_CollectEpochInfoDA(recEpochWave, s)
+	EP_CollectEpochInfoTTL(recEpochWave, s)
 	EP_AddRecreatedUserEpochs(numericalValues, textualValues, sweepDFR, sweepNo, s, recEpochWave)
 
 	WAVE/Z channelDA = GetDAQDataSingleColumnWaveNG(numericalValues, textualValues, sweepNo, sweepDFR, XOP_CHANNEL_TYPE_DAC, s.DACList[0])

Packages/MIES/MIES_ExperimentDocumentation.ipf

@@ -62,14 +62,16 @@
 /// It is recommended to gather all entries to be written in keys/values and call ED_AddEntriesToLabnotebook then once.
 ///
 /// @see ED_createTextNotes, ED_createWaveNote
-Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType)
+Function ED_AddEntriesToLabnotebook(WAVE vals, WAVE/T keys, variable sweepNo, string device, variable entrySourceType, [variable insertAsPostProc])
+
+	insertAsPostProc = ParamIsDefault(insertAsPostProc) ? 0 : !!insertAsPostProc
 
 	ED_CheckValuesAndKeys(vals, keys)
 
 	if(IsTextWave(vals))
-		ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device)
+		ED_createTextNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device)
 	else
-		ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, device = device)
+		ED_createWaveNotes(vals, keys, sweepNo, entrySourceType, LBT_LABNOTEBOOK, insertAsPostProc, device = device)
 	endif
 End
 
@@ -79,9 +81,9 @@ Function ED_AddEntriesToResults(WAVE vals, WAVE/T keys, variable entrySourceType
 	ED_CheckValuesAndKeys(vals, keys)
 
 	if(IsTextWave(vals))
-		ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS)
+		ED_createTextNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0)
 	else
-		ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS)
+		ED_createWaveNotes(vals, keys, NaN, entrySourceType, LBT_RESULTS, 0)
 	endif
 End
 
@@ -109,15 +111,49 @@ static Function ED_CheckValuesAndKeys(WAVE vals, WAVE keys)
 	endif
 End
 
-static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState)
+static Function ED_SetLabnotebookRowToPostProcessed(WAVE values, variable row)
+
+	variable col, unused
+
+	ASSERT(DimSize(values, ROWS) >= row, "Row does not exist in LBN values wave")
+
+	WAVE keys = GetLogbookKeysFromValues(values)
+
+	if(IsTextWave(values))
+		Make/FREE/T/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesT
+		WAVE incomingValues = incomingValuesT
+	else
+		Make/FREE/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) incomingValuesNum
+		WAVE incomingValues = incomingValuesNum
+	endif
+
+	Make/FREE/T/N=(3, 1) incomingKeys
+	incomingKeys[0]        = POSTPROCESSED_ENTRY_KEY
+	incomingKeys[1]        = LABNOTEBOOK_BINARY_UNIT
+	incomingKeys[2]        = LABNOTEBOOK_NO_TOLERANCE
+	[WAVE indizes, unused] = ED_FindIndizesAndRedimension(incomingKeys, incomingValues, keys, values, LBT_LABNOTEBOOK)
+	ASSERT(WaveExists(indizes), "Missing indizes")
+
+	col = indizes[0]
+	if(IsTextWave(values))
+		WAVE/T valuesT = values
+		valuesT[row][col][] = "1"
+	else
+		values[row][col][] = 1
+	endif
+End
+
+static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo, variable entrySourceType, variable acqState, variable insertAsPostProc)
 
 	variable timestamp
 	string   timestampStr
 
 	if(IsTextWave(values))
 		WAVE/T valuesT = values
 		valuesT[rowIndex][0][] = num2istr(sweepNo)
-		valuesT[rowIndex][3][] = num2istr(entrySourceType)
+		if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE))
+			valuesT[rowIndex][3][] = num2istr(entrySourceType)
+		endif
 
 		valuesT[rowIndex][4][] = num2istr(acqState)
 
@@ -129,7 +165,9 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo,
 		valuesT[rowIndex][2][] = timestampStr
 	else
 		values[rowIndex][0][] = sweepNo
-		values[rowIndex][3][] = entrySourceType
+		if(GetLBNCapability(values, LBN_CAP_SUPPORTS_ENTRYSOURCETYPE))
+			values[rowIndex][3][] = entrySourceType
+		endif
 
 		values[rowIndex][4][] = acqState
 
@@ -142,6 +180,29 @@ static Function ED_InitNewRow(WAVE values, variable rowIndex, variable sweepNo,
 	endif
 End
 
+/// @brief Inserts a row after the sweep block and returns the inserted row number
+static Function ED_InsertRowAfterSweepBlock(WAVE values, variable sweepNo, variable entrySourceType)
+
+	variable sweepCol, firstRow, lastRow, rowIndex, size
+
+	sweepCol = GetSweepColumn(values)
+	FindRange(values, sweepCol, sweepNo, entrySourceType, firstRow, lastRow)
+	ASSERT(!IsNaN(firstRow) && !IsNaN(lastRow), "FindRange could not determine start and/or end of sweep block")
+	rowIndex = lastRow + 1
+	InsertPoints/M=(ROWS) rowIndex, 1, values
+
+	if(!IsTextWave(values))
+		values[rowIndex][][] = NaN
+	endif
+
+	size = GetNumberFromWaveNote(values, NOTE_INDEX)
+	SetNumberInWaveNote(values, NOTE_INDEX, size + 1)
+
+	InvalidateLBIndexAndRowCache(values)
+
+	return rowIndex
+End
+
 /// @brief Add textual entries to the logbook
 ///
 /// The text documentation wave will use layers to report the different headstages.
@@ -156,9 +217,11 @@ End
 /// @param device                [optional for logbookType LBT_RESULTS only] device
 /// @param entrySourceType       type of reporting subsystem, one of @ref DataAcqModes
 /// @param logbookType           type of the logbook, one of @ref LogbookTypes
-static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device])
+/// @param insertAsPostProc      when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data
+static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incomingTextualKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device])
 
 	variable rowIndex, numCols, i, lastValidIncomingLayer, state
+	variable addIndex, insertIndex
 
 	if(ParamIsDefault(device))
 		ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type")
@@ -173,10 +236,16 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming
 		state = ROVar(GetAcquisitionState(device))
 	endif
 
-	[WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType)
+	if(insertAsPostProc)
+		insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType)
+		ED_SetLabnotebookRowToPostProcessed(values, insertIndex)
+	endif
+
+	[WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingTextualKeys, incomingTextualValues, keys, values, logbookType)
 	ASSERT(WaveExists(indizes), "Missing indizes")
+	rowIndex = insertAsPostProc ? insertIndex : addIndex
 
-	ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state)
+	ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc)
 
 	WAVE valuesDat = ExtractLogbookSliceTimeStamp(values)
 	EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS)
@@ -196,7 +265,9 @@ static Function ED_createTextNotes(WAVE/T incomingTextualValues, WAVE/T incoming
 		values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = NormalizeToEOL(incomingTextualValues[0][i][r], "\n")
 	endfor
 
-	SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
+	if(!insertAsPostProc)
+		SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
+	endif
 End
 
 static Function ED_ParseHeadstageContingencyMode(string str)
@@ -266,9 +337,11 @@ End
 /// @param device                  [optional for logbooktype LBT_RESULTS only] device
 /// @param entrySourceType         type of reporting subsystem, one of @ref DataAcqModes
 /// @param logbookType             one of @ref LogbookTypes
-static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, [string device])
+/// @param insertAsPostProc        when set to 1 then the values are inserted at the end of the sweep block flagged as postprocessed data
+static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incomingNumericalKeys, variable sweepNo, variable entrySourceType, variable logbookType, variable insertAsPostProc, [string device])
 
 	variable rowIndex, numCols, lastValidIncomingLayer, i, state
+	variable addIndex, insertIndex
 
 	if(ParamIsDefault(device))
 		ASSERT(logbookType == LBT_RESULTS, "Invalid logbook type")
@@ -284,10 +357,16 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming
 		state = ROVar(GetAcquisitionState(device))
 	endif
 
-	[WAVE indizes, rowIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType)
+	if(insertAsPostProc)
+		insertIndex = ED_InsertRowAfterSweepBlock(values, sweepNo, entrySourceType)
+		ED_SetLabnotebookRowToPostProcessed(values, insertIndex)
+	endif
+
+	[WAVE indizes, addIndex] = ED_FindIndizesAndRedimension(incomingNumericalKeys, incomingNumericalValues, keys, values, logbookType)
 	ASSERT(WaveExists(indizes), "Missing indizes")
+	rowIndex = insertAsPostProc ? insertIndex : addIndex
 
-	ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state)
+	ED_InitNewRow(values, rowIndex, sweepNo, entrySourceType, state, insertAsPostProc)
 
 	WAVE valuesDat = ExtractLogbookSliceTimeStamp(values)
 	EnsureLargeEnoughWave(valuesDat, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN)
@@ -299,7 +378,9 @@ static Function ED_createWaveNotes(WAVE incomingNumericalValues, WAVE/T incoming
 		values[rowIndex][indizes[i]][0, lastValidIncomingLayer] = incomingNumericalValues[0][i][r]
 	endfor
 
-	SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
+	if(!insertAsPostProc)
+		SetNumberInWaveNote(values, NOTE_INDEX, rowIndex + 1)
+	endif
 End
 
 /// @brief Add custom entries to the numerical/textual labnotebook for the very last sweep acquired.
@@ -706,6 +787,13 @@ static Function [WAVE colIndizes, variable rowIndex] ED_FindIndizesAndRedimensio
 		LBN_SetDimensionLabels(key, values, start = ((rowIndex == 0) ? 0 : numKeyCols))
 	endif
 
+	// Notes on the LBN row expansion here:
+	// - Initialisation with NaN, "" takes only place when the wave is expanded for rows/cols.
+	// - It does not initialise the row returned to the caller.
+	//   Thus, for the unlucky case that a previous LBN write attempt ended up to be partial (not seen happening so far)
+	//   then the row returned might be partially filled.
+	// - This part is also called if a row is inserted in the LBN from postprocessing. The LBN wave size is increased if required.
+	//   The new row at the end is not used by the insertion. It is still consistent as the NOTE_INDEX is not updated here.
 	if(IsNumericWave(values))
 		EnsureLargeEnoughWave(values, indexShouldExist = rowIndex, dimension = ROWS, initialValue = NaN)
 		if(numAdditions)