Conversion/MIES_MassExperimentProcessing.ipf: Fix it

Packages/Conversion/MIES_MassExperimentProcessing.ipf

@@ -16,6 +16,9 @@
 /// - Ensure that only MIES is installed and no other Igor Pro packages
 /// - In the MIES installation folder (All Users: `C:\Program Files\MIES`, User: `C:\Users\$User\Documents\MIES`)
 ///   create an empty file named `UserConfig.txt`.
+/// - Execute CreateEmptyFiles() to create required empty files in `User Procedures`
+/// - If the files are on a Windows/SMB network share, create a drive letter
+///   via `net use Z: \\$Server\$Share` as UNC paths don't work everywhere in IP.
 ///
 /// Running:
 /// - Start Igor Pro
@@ -30,8 +33,8 @@
 
 #ifdef MEP_DEBUGGING
 
-static StrConstant INPUT_FOLDER  = "C:tim-data:pxp_examples_for_nwb_2:"
-static StrConstant OUTPUT_FOLDER = "C:tim-data:output:"
+static StrConstant INPUT_FOLDER  = "E:tim-data:pxp_examples_for_nwb_2:"
+static StrConstant OUTPUT_FOLDER = "E:tim-data:output:"
 
 #else
 
@@ -85,8 +88,8 @@ End
 
 static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &prefs)
 
-	variable jsonID, index
-	string outputFilePath, inputFile, outputFolder
+	variable jsonID, index, ref, error
+	string outputFileTemplate, inputFile, outputFolder, history, path, message, file, regex
 
 	jsonID = GetJSON(prefs)
 
@@ -97,28 +100,44 @@ static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &pre
 		PathInfo home
 		inputFile = S_path + GetExperimentName() + ".pxp"
 
-		outputFilePath = outputFolder + S_path + GetExperimentName() + ".nwb"
+		outputFileTemplate = outputFolder + S_path + GetExperimentName()
 
-		index = JSON_GetVariable(jsonID, "/index")
-		JSON_AddString(jsonID, "/log/" + num2str(index) + "/from", inputFile)
-		JSON_AddString(jsonID, "/log/" + num2str(index) + "/to", outputFilePath)
+		path = "/log/" + num2str(JSON_GetVariable(jsonID, "/index"))
+		JSON_AddString(jsonID, path + "/from", inputFile)
+		JSON_AddString(jsonID, path + "/to", outputFileTemplate)
 
-		DoWindow/K HistoryCarbonCopy
-		NewNotebook/V=0/F=0/N=HistoryCarbonCopy
+		ref = CaptureHistoryStart()
 
+		AssertOnAndClearRTError()
 		try
-			PerformMiesTasks(outputFilePath); AbortOnRTE
+			PerformMiesTasks(outputFileTemplate); AbortOnRTE
 		catch
-			ClearRTError()
-			print "Caught an RTE"
-			JSON_AddBoolean(jsonID, "/log/" + num2str(index) + "/error", 1)
+			message = GetRTErrMessage()
+			error   = ClearRTError()
+
+			if(error >= 0)
+				printf "Encountered lingering RTE of %d (message: %s) after executing PerformMiesTasks.\r", error, message
+			else
+				printf "Encountered Abort with V_AbortCode: %d\r", V_AbortCode
+			endif
+
+			JSON_AddBoolean(jsonID, path + "/error", 1)
 			JSON_SetVariable(jsonID, "/errors", JSON_GetVariable(jsonID, "/errors") + 1)
 			HDF5CloseFile/A/Z 0
-			DeleteFile/Z outputFilePath
+
+			regex = "\\Q" + outputFileTemplate + "\\E" + ".*\.nwb$"
+			WAVE/Z/T files = GetAllFilesRecursivelyFromPath("home", regex = regex)
+
+			if(WaveExists(files))
+				for(file : files)
+					DeleteFile/Z file
+				endfor
+			endif
 		endtry
 
-		Notebook HistoryCarbonCopy, getData=1
-		JSON_AddString(jsonID, "/log/" + num2str(index) + "/output", trimstring(S_Value))
+		history = CaptureHistory(ref, 1)
+
+		JSON_AddString(jsonID, path + "/output", trimstring(history))
 
 		JSON_SetVariable(jsonID, "/processed", JSON_GetVariable(jsonID, "/processed") + 1)
 	else
@@ -130,28 +149,20 @@ static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &pre
 	StoreJSON(prefs, jsonID)
 End
 
-static Function PerformMiesTasks(string outputFilePath)
+static Function PerformMiesTasks(string outputFileTemplate)
 
-	string folder, message
-	variable nwbVersion, error
+	string folder
 
 	printf "Free Memory: %g GB\r", GetFreeMemory()
 
-	folder = GetFolder(outputFilePath)
+	folder = GetFolder(outputFileTemplate)
 
 	if(!FolderExists(folder))
 		CreateFolderOnDisk(folder)
 	endif
 
-	ClearRTError()
-
-	nwbVersion = 2
-	NWB_ExportAllData(nwbVersion, overrideFullFilePath = outputFilePath)
+	NWB_ExportAllData(NWB_VERSION_LATEST, overrideFileTemplate = outputFileTemplate)
 	HDF5CloseFile/A/Z 0
-
-	message = GetRTErrMessage()
-	error   = GetRTError(1)
-	ASSERT(error == 0, "Encountered lingering RTE of " + num2str(error) + "(message: " + message + ") after executing NWB_ExportAllData.")
 End
 
 static Function IsAppropriateExperiment()
@@ -232,23 +243,32 @@ static Function AfterFileOpenHook(variable refNum, string file, string pathName,
 
 	ProcessCurrentExperiment(prefs)
 
-	// See if there are more experiments to process.
-	string nextExperimentFullPath = FindNextExperiment(prefs)
-	if(strlen(nextExperimentFullPath) == 0)
-		// Process is finished
-		prefs.processRunning = 0 // Flag process is finished.
-		Execute/P "NEWEXPERIMENT " // Post command to close this experiment.
-		print "Multi-experiment process is finished."
-	else
-		// Load the next experiment in the designated folder, if any.
-		PostLoadNextExperiment(nextExperimentFullPath) // Post operation queue commands to load next experiment
-	endif
-
-	SavePackagePrefs(prefs)
+	NextFile(prefs)
 
 	return 0 // Tell Igor to handle file in default fashion.
 End
 
+Function CreateEmptyFiles()
+
+	string file
+	string path = SpecialDirPath("Igor Pro User Files", 0, 0, 0) + "User Procedures:"
+
+	Make/T/FREE files = {"MIES_Include.ipf",                \
+	                     "TJ_MIES_AnalysisBrowser.ipf",     \
+	                     "TJ_MIES_Include.ipf",             \
+	                     "UTF_HardwareHelperFunctions.ipf", \
+	                     "UTF_HardwareMain.ipf",            \
+	                     "UserAnalysisFunctions.ipf",       \
+	                     "tango_Panel.ipf",                 \
+	                     "tango_loader.ipf",                \
+	                     "unit-testing.ipf",                \
+	                     "UserConfig.txt"}
+
+	for(file : files)
+		SaveTextFile("", path + file)
+	endfor
+End
+
 // This function enables our special Igor hooks which skip saving the experiment
 Function StartMultiExperimentProcess()
 
@@ -260,7 +280,7 @@ End
 // Allow user to choose the folder containing the experiment files and start the process.
 Function StartMultiExperimentProcessWrapper()
 
-	string message, settingsFile, inputFolder, outputFolder, files
+	string message, settingsFile, inputFolder, outputFolder
 	variable jsonID
 
 	STRUCT MultiExperimentProcessPrefs prefs
@@ -296,7 +316,7 @@ Function StartMultiExperimentProcessWrapper()
 	outputFolder = S_Path
 	ASSERT(V_flag, "Invalid path")
 
-	WAVE/Z files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", regex = "(?i)\.pxp$")
+	WAVE/Z/T files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", regex = "(?i)\.pxp$")
 
 	if(WaveExists(files))
 		Sort/A=2 files, files
@@ -312,10 +332,10 @@ Function StartMultiExperimentProcessWrapper()
 	JSON_AddVariable(jsonID, "/processed", 0)
 	JSON_AddVariable(jsonID, "/errors", 0)
 	JSON_AddVariable(jsonID, "/skipped", 0)
-	JSON_AddVariable(jsonID, "/total", DimSize(inputPXPs, ROWS))
+	JSON_AddVariable(jsonID, "/total", DimSize(files, ROWS))
 
 	JSON_AddTreeArray(jsonID, "/log")
-	JSON_AddObjects(jsonID, "/log", objCount = DimSize(inputPXPs, ROWS))
+	JSON_AddObjects(jsonID, "/log", objCount = DimSize(files, ROWS))
 
 	prefs.settingsFile = outputFolder + "conversion.json"
 	StoreJSON(prefs, jsonID)
@@ -331,6 +351,23 @@ Function StartMultiExperimentProcessWrapper()
 	return 0
 End
 
+Function NextFile(STRUCT MultiExperimentProcessPrefs &prefs)
+
+	// See if there are more experiments to process.
+	string nextExperimentFullPath = FindNextExperiment(prefs)
+	if(strlen(nextExperimentFullPath) == 0)
+		// Process is finished
+		prefs.processRunning = 0 // Flag process is finished.
+		Execute/P "NEWEXPERIMENT " // Post command to close this experiment.
+		print "Multi-experiment process is finished."
+	else
+		// Load the next experiment in the designated folder, if any.
+		PostLoadNextExperiment(nextExperimentFullPath) // Post operation queue commands to load next experiment
+	endif
+
+	SavePackagePrefs(prefs)
+End
+
 #ifdef MEP_DEBUGGING
 
 Function TestMe()
@@ -339,6 +376,7 @@ Function TestMe()
 
 	LoadPackagePrefs(prefs)
 	ProcessCurrentExperiment(prefs)
+	NextFile(prefs)
 End
 
 #endif // MEP_DEBUGGING

Packages/MIES/MIES_AnalysisFunctionHelpers.ipf

@@ -469,19 +469,19 @@ End
 ///        parameters, possibly including the type, as specified by the function
 ///        `$func_GetParams`
 ///
-/// @param func Analysis function `V3` which must be valid and existing
-/// @param mode Bit mask values from @ref GetListOfParamsModeFlags
-Function/S AFH_GetListOfAnalysisParams(string func, variable mode)
+/// @param funcname Analysis function `V3` which must be valid and existing
+/// @param mode     Bit mask values from @ref GetListOfParamsModeFlags
+Function/S AFH_GetListOfAnalysisParams(string funcname, variable mode)
 
 	string params, re
 
-	FUNCREF AFP_PARAM_GETTER_V3 f = $(func + "_GetParams")
+	FUNCREF AFP_PARAM_GETTER_V3 func = $(funcname + "_GetParams")
 
-	if(!FuncRefIsAssigned(FuncRefInfo(f))) // no such getter functions
+	if(!FuncRefIsAssigned(FuncRefInfo(func))) // no such getter functions
 		return ""
 	endif
 
-	params = f()
+	params = func()
 
 	ASSERT(strsearch(params, ";", 0) == -1, "Entries must be separated with ,")
 
@@ -501,19 +501,19 @@ End
 
 /// @brief Get help string from optional `$func_GetHelp`
 ///
-/// @param func Analysis function `V3`
-/// @param name Parameter name
-Function/S AFH_GetHelpForAnalysisParameter(string func, string name)
+/// @param funcname Analysis function `V3`
+/// @param name     Parameter name
+Function/S AFH_GetHelpForAnalysisParameter(string funcname, string name)
 
-	FUNCREF AFP_PARAM_HELP_GETTER_V3 f = $(func + "_GetHelp")
+	FUNCREF AFP_PARAM_HELP_GETTER_V3 func = $(funcname + "_GetHelp")
 
-	if(!FuncRefIsAssigned(FuncRefInfo(f)))
+	if(!FuncRefIsAssigned(FuncRefInfo(func)))
 		return ""
 	endif
 
 	AssertOnAndClearRTError()
 	try
-		return f(name); AbortOnRTE
+		return func(name); AbortOnRTE
 	catch
 		ClearRTError()
 		// ignoring errors here

Packages/MIES/MIES_Async.ipf

@@ -124,7 +124,7 @@ threadsafe static Function/DF ASYNC_Run_Worker(DFREF dfr)
 	DFREF dfrOut, dfrTemp
 
 	SVAR                 WFunc = dfr:$ASYNC_WORKERFUNC_STR
-	FUNCREF ASYNC_Worker f     = $WFunc
+	FUNCREF ASYNC_Worker func  = $WFunc
 
 	DFREF dfrInp   = dfr:input
 	DFREF dfrAsync = dfr:async
@@ -139,7 +139,7 @@ threadsafe static Function/DF ASYNC_Run_Worker(DFREF dfr)
 	dfrOut = $""
 	AssertOnAndClearRTError()
 	try
-		dfrOut = f(dfrInp); AbortOnRTE
+		dfrOut = func(dfrInp); AbortOnRTE
 	catch
 		rtErrMsg = GetRTErrMessage()
 		rtErr    = ClearRTError()
@@ -243,7 +243,7 @@ Function ASYNC_ThreadReadOut()
 		track[%$workloadClass][%OUTPUTCOUNT] += 1
 
 		SVAR                  RFunc     = dfr:$ASYNC_READOUTFUNC_STR
-		FUNCREF ASYNC_ReadOut f         = $RFunc
+		FUNCREF ASYNC_ReadOut func      = $RFunc
 		NVAR                  rtErr     = dfr:$ASYNC_RTERROR_STR
 		SVAR                  rtErrMsg  = dfr:$ASYNC_RTERRORMSG_STR
 		NVAR                  abortCode = dfr:$ASYNC_ABORTCODE_STR
@@ -256,7 +256,7 @@ Function ASYNC_ThreadReadOut()
 		statCnt += 1
 		AssertOnAndClearRTError()
 		try
-			f(ar); AbortOnRTE
+			func(ar); AbortOnRTE
 		catch
 			msg = GetRTErrMessage()
 			ASSERT(!ClearRTError(), "ReadOut function " + RFunc + " encountered an RTE: " + msg)

Packages/MIES/MIES_DAEphys.ipf

@@ -4645,11 +4645,7 @@ Function DAP_LockDevice(string win)
 	headstage = GetSliderPositionIndex(deviceLocked, "slider_DataAcq_ActiveHeadstage")
 	P_SaveUserSelectedHeadstage(deviceLocked, headstage)
 
-	// upgrade all four labnotebook waves in wanna-be atomic way
-	GetLBNumericalKeys(deviceLocked)
-	GetLBNumericalValues(deviceLocked)
-	GetLBTextualKeys(deviceLocked)
-	GetLBTextualValues(deviceLocked)
+	UpgradeLabNotebook(deviceLocked)
 
 	NVAR sessionStartTime = $GetSessionStartTime()
 	sessionStartTime = DateTimeInUTC()

Packages/MIES/MIES_DataBrowser.ipf

@@ -326,6 +326,8 @@ static Function/S DB_LockToDevice(string win, string device)
 		for(i = first; i <= last; i += 1)
 			SplitAndUpgradeSweepGlobal(device, i)
 		endfor
+
+		UpgradeLabNotebook(device)
 	endif
 
 	UpdateSweepPlot(win)
@@ -421,10 +423,6 @@ Function DB_UpdateSweepPlot(string win)
 		return NaN
 	endif
 
-	// fetch keys waves to trigger a potential labnotebook upgrade
-	WAVE numericalKeys = DB_GetLBNWave(win, LBN_NUMERICAL_KEYS)
-	WAVE textualKeys   = DB_GetLBNWave(win, LBN_TEXTUAL_KEYS)
-
 	WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES)
 	WAVE textualValues   = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES)
 

Packages/MIES/MIES_MiesUtilities_Device.ipf

@@ -14,12 +14,13 @@
 Function/S GetAllDevices()
 
 	variable i, j, numEntries, numDevices
-	string folder, number, device, folders, subFolders, subFolder
+	string folder, number, device, folders, subFolders, subFolder, deviceType, deviceNumber
 	string path
 	string list = ""
 
-	string devicesFolderPath = GetDAQDevicesFolderAsString()
+	// ensure that the folder location upgrade in GetDAQDevicesFolder is done first
 	DFREF  devicesFolder     = GetDAQDevicesFolder()
+	string devicesFolderPath = GetDAQDevicesFolderAsString()
 
 	folders    = GetListOfObjects(devicesFolder, ".*", typeFlag = COUNTOBJECTS_DATAFOLDER)
 	numEntries = ItemsInList(folders)
@@ -50,7 +51,12 @@ Function/S GetAllDevices()
 		else
 			// other hardware has no subfolder
 			device = folder
-			path   = GetDevicePathAsString(device)
+
+			if(ParseDeviceString(device, deviceType, deviceNumber) == 0)
+				continue
+			endif
+
+			path = GetDevicePathAsString(device)
 
 			if(DataFolderExists(path))
 				DFREF dfr = $path
@@ -73,7 +79,7 @@ static Function DeviceHasUserComments(string device)
 	userComment = ROStr(GetUserComment(device))
 
 	if(WindowExists(device))
-		userCommentDraft = DAG_GetTextualValue(device, "SetVar_DataAcq_Comment")
+		userCommentDraft = GetSetVariableString(device, "SetVar_DataAcq_Comment")
 
 		commentNotebook = DAP_GetCommentNotebook(device)
 		if(WindowExists(commentNotebook))