Ewm3645 mask workspace diffcal

.readthedocs.yaml

@@ -21,7 +21,7 @@ build:
       # Install the package in editable mode using pixi
       - $HOME/.pixi/bin/pixi run --environment docs pip install -e .
       # Install docs dependencies in the RTD Python environment too
-      - pip install erdantic versioningit sphinx_rtd_theme sphinxcontrib-mermaid types-pyyaml h5py numpy matplotlib ruamel.yaml
+      - pip install erdantic versioningit sphinx_rtd_theme sphinxcontrib-mermaid types-pyyaml h5py numpy matplotlib ruamel.yaml==0.18.16
 
 sphinx:
   builder: html

src/snapred/backend/dao/request/CalibrationAssessmentRequest.py

@@ -37,6 +37,7 @@ class CalibrationAssessmentRequest(BaseModel):
         default_factory=lambda: Pair.model_validate(Config["calibration.parameters.default.FWHMMultiplier"])
     )
     maxChiSq: float = Field(default_factory=lambda: Config["constants.GroupDiffractionCalibration.MaxChiSq"])
+    combinedPixelMask: WorkspaceName | None = None
 
     @field_validator("fwhmMultipliers", mode="before")
     @classmethod

src/snapred/backend/dao/request/DiffractionCalibrationRequest.py

@@ -43,6 +43,7 @@ class DiffractionCalibrationRequest(BaseModel):
     maxChiSq: float = Field(default_factory=lambda: Config["constants.GroupDiffractionCalibration.MaxChiSq"])
     removeBackground: bool = False
     pixelMasks: List[WorkspaceName] = []
+    combinedPixelMask: WorkspaceName | None = None
 
     continueFlags: Optional[ContinueWarning.Type] = ContinueWarning.Type.UNSET
 

src/snapred/backend/dao/request/FocusSpectraRequest.py

@@ -1,8 +1,7 @@
-from typing import Optional
-
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from snapred.backend.dao.state.FocusGroup import FocusGroup
+from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
 
 
 class FocusSpectraRequest(BaseModel):
@@ -13,4 +12,7 @@ class FocusSpectraRequest(BaseModel):
 
     inputWorkspace: str
     groupingWorkspace: str
-    outputWorkspace: Optional[str] = None
+    maskWorkspace: WorkspaceName | None = None
+    outputWorkspace: WorkspaceName | None = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)

src/snapred/backend/data/GroceryService.py

@@ -1515,6 +1515,36 @@ def fetchGroceryList(self, groceryList: Iterable[GroceryListItem]) -> List[Works
 
         return groceries
 
+    def combinePixelMasks(self, outputMaskWsName: WorkspaceName, masks2Combine: List[WorkspaceName]):
+        if not masks2Combine:
+            raise ValueError("Internal Error: Lists of masks to combine is empty")
+
+        if not self.mantidSnapper.mtd.doesExist(outputMaskWsName):
+            raise ValueError(
+                (
+                    "Internal Error: outputMaskWs should exist before attempting to combine masks with it. "
+                    "Consider using fetchCompatiblePixelMask to generate it."
+                )
+            )
+
+        for maskWsName in masks2Combine:
+            if maskWsName == outputMaskWsName:
+                continue
+            if not self.mantidSnapper.mtd.doesExist(maskWsName):
+                raise ValueError(
+                    f"Mask {maskWsName} of mask set {masks2Combine} does not exist, cannot combine into pixel mask."
+                )
+            self.mantidSnapper.BinaryOperateMasks(
+                f"combine from pixel mask: '{maskWsName}'...",
+                InputWorkspace1=outputMaskWsName,
+                InputWorkspace2=maskWsName,
+                OperationType="OR",
+                OutputWorkspace=outputMaskWsName,
+            )
+
+        self.mantidSnapper.executeQueue()
+        return outputMaskWsName
+
     def fetchGroceryDict(self, groceryDict: Dict[str, GroceryListItem], **kwargs) -> Dict[str, WorkspaceName]:
         """
         This is the primary method you should use for fetching groceries, in almost all cases.

src/snapred/backend/recipe/CalculateDiffCalResidualRecipe.py

@@ -66,7 +66,6 @@ def queueAlgos(self):
         # Step 2: Check for overlapping spectra and manage them
         fitPeaksWorkspace = self.mantidSnapper.mtd[self.fitPeaksDiagnosticWorkspaceName]
         numHistograms = fitPeaksWorkspace.getNumberHistograms()
-        processedSpectra = []
         spectrumDict = {}
 
         for i in range(numHistograms):
@@ -102,9 +101,7 @@ def queueAlgos(self):
 
                 spectrumDict[spectrumId] = singleSpectrumName
 
-            # Append the processed spectrum to the list
-            processedSpectra.append(singleSpectrumName)
-
+        processedSpectra = list(spectrumDict.values())
         # Step 3: Combine all processed spectra into a single workspace
         combinedWorkspace = processedSpectra[0]
         for spectrum in processedSpectra[1:]:

src/snapred/backend/recipe/GroupDiffCalRecipe.py

@@ -77,7 +77,7 @@ def validateInputs(self, ingredients: Ingredients, groceries: Dict[str, Workspac
             )
 
         diffocWS = self.mantidSnapper.mtd[groceries["groupingWorkspace"]]
-        if groupIDs != diffocWS.getGroupIDs().tolist():
+        if not set(groupIDs).issubset(set(diffocWS.getGroupIDs().tolist())):
             raise RuntimeError(
                 f"Group IDs do not match between peak list and focus WS: '{groupIDs}' vs. '{diffocWS.getGroupIDs()}'"
             )

src/snapred/backend/recipe/PixelDiffCalRecipe.py

@@ -1,15 +1,16 @@
 from typing import Any, Dict, List, Set
 
 import numpy as np
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from snapred.backend.dao.ingredients import DiffractionCalibrationIngredients as Ingredients
 from snapred.backend.log.logger import snapredLogger
 from snapred.backend.profiling.ProgressRecorder import ComputationalOrder, WallClockTime
 from snapred.backend.recipe.algorithm.Utensils import Utensils
-from snapred.backend.recipe.Recipe import Recipe, WorkspaceName
+from snapred.backend.recipe.Recipe import Recipe
 from snapred.meta.Config import Config
 from snapred.meta.decorators.classproperty import classproperty
+from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceName
 from snapred.meta.mantid.WorkspaceNameGenerator import WorkspaceNameGenerator as wng
 
 logger = snapredLogger.getLogger(__name__)
@@ -18,10 +19,12 @@
 class PixelDiffCalServing(BaseModel):
     result: bool
     medianOffsets: List[float]
-    maskWorkspace: str
+    maskWorkspace: WorkspaceName | None = None
     calibrationTable: str
     outputWorkspace: str
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
 
 # Decorating with the `WallClockTime` profiler here somewhat duplicates the objective of the decoration
 #   at `CalibrationService.pixelCalibration`.  However, by default, there will be no logging output from this instance.
@@ -101,11 +104,20 @@ def unbagGroceries(self, groceries: Dict[str, WorkspaceName]) -> None:  # noqa A
         """
         self.wsTOF = groceries["inputWorkspace"]
         self.groupingWS = groceries["groupingWorkspace"]
-        self.maskWS = groceries["maskWorkspace"]
+        self.maskWS = groceries.get("maskWorkspace")
         # the name of the output calibration table
         self.DIFCpixel = groceries["calibrationTable"]
         self.DIFCprev = groceries.get("previousCalibration", "")
         # the input data converted to d-spacing
+
+        if self.maskWS and self.mantidSnapper.mtd.doesExist(self.maskWS):
+            # if user supplied mask exists, apply it
+            # A user mask may not exist, but the maskws name is used to store
+            # the mask generated as part of PixelDiffCalRecipe
+            self.mantidSnapper.MaskDetectors(
+                "applying user generated mask", Workspace=self.wsTOF, MaskedWorkspace=self.maskWS
+            )
+
         self.wsDSP = wng.diffCalInputDSP().runNumber(self.runNumber).build()
         self.convertUnitsAndRebin(self.wsTOF, self.wsDSP)
         self.mantidSnapper.CloneWorkspace(
@@ -215,11 +227,14 @@ def queueAlgos(self):
         wscc: str = f"__{self.runNumber}_tmp_group_CC_{self._counts}"
 
         for i, (groupID, workspaceIndices) in enumerate(self.groupWorkspaceIndices.items()):
+            if groupID not in self.maxDSpaceShifts:
+                # group has been fully masked.
+                continue
             workspaceIndices = list(workspaceIndices)
             refID: int = self.getRefID(workspaceIndices)
 
             self.mantidSnapper.CrossCorrelate(
-                f"Cross-Correlating spectra for {wscc}",
+                f"Cross-Correlating spectra for {wscc}, subgroup {groupID}",
                 InputWorkspace=self.wsDSP,
                 OutputWorkspace=wscc + f"_group{groupID}",
                 ReferenceSpectra=refID,
@@ -230,7 +245,7 @@ def queueAlgos(self):
             )
 
             self.mantidSnapper.GetDetectorOffsets(
-                f"Calculate offset workspace {wsoff}",
+                f"Calculate offset workspace {wsoff}, group {groupID}",
                 InputWorkspace=wscc + f"_group{groupID}",
                 OutputWorkspace=wsoff,
                 MaskWorkspace=self.maskWS,

src/snapred/backend/recipe/algorithm/ConjoinDiagnosticWorkspaces.py

@@ -15,8 +15,11 @@
     mtd,
 )
 
+from snapred.backend.log.logger import snapredLogger
 from snapred.meta.mantid.FitPeaksOutput import FIT_PEAK_DIAG_SUFFIX, FitOutputEnum
 
+logger = snapredLogger.getLogger(__name__)
+
 
 class ConjoinDiagnosticWorkspaces(PythonAlgorithm):
     """
@@ -127,12 +130,24 @@ def conjoinMatrixWorkspaces(self, inws, outws, index):
                 InputWorkspace=inws,
                 OutputWorkspace=tmpws,
             )
+        logger.debug(f"{outws} already has spectrum numbers of {mtd[outws].getSpectrumNumbers()}")
+        logger.debug(f"Conjoining {tmpws} with {outws}, adding spectrum numbers of {mtd[tmpws].getSpectrumNumbers()}")
+        specNumbers = list(mtd[outws].getSpectrumNumbers())
+        specNumbers.extend(list(mtd[tmpws].getSpectrumNumbers()))
         ConjoinWorkspaces(
             InputWorkspace1=outws,
             InputWorkspace2=tmpws,
             CheckOverlapping=False,
             CheckMatchingBins=False,  # Not available in 6.11.0.3rc2
         )
+
+        # TODO: Remove when Defect 14460 is resolved.
+        #       There is a defect in ConjoinWorkspaces that incorrectly determines
+        #       if spectrum numbers need to be remapped.
+        for i, specNum in enumerate(specNumbers):
+            mtd[outws].getSpectrum(i).setSpectrumNo(specNum)
+
+        logger.debug(f"resulting spectrum numbers: {mtd[outws].getSpectrumNumbers()}")
         if self.autoDelete and inws in mtd:
             DeleteWorkspace(inws)
         assert outws in mtd

src/snapred/backend/recipe/algorithm/FitMultiplePeaksAlgorithm.py

@@ -79,7 +79,20 @@ def PyExec(self):
         self.chopIngredients(reducedPeakList)
         self.unbagGroceries()
 
+        numHisto = self.mantidSnapper.mtd[self.inputWorkspaceName].getNumberHistograms()
+        if numHisto < len(self.groupIDs):
+            raise ValueError(
+                f"Number of histograms and number of GroupPeakLists do not match: {numHisto} vs {len(self.groupIDs)}"
+            )
+
         for index, groupID in enumerate(self.groupIDs):
+            spectrumInfo = self.mantidSnapper.mtd[self.inputWorkspaceName].spectrumInfo()
+            numHisto = self.mantidSnapper.mtd[self.inputWorkspaceName].getNumberHistograms()
+            if spectrumInfo.detectorCount() == 0:
+                raise ValueError(
+                    f"Spectrum with NO DETECTORS encountered on {self.inputWorkspaceName} at index {index}"
+                )
+
             tmpSpecName = mtd.unique_name(prefix=f"__tmp_fitspec_{index}_")
             outputNameTmp = mtd.unique_name(prefix=f"__tmp_fitdiag_{index}_")
             outputNamesTmp = {x: f"{outputNameTmp}{self.outputSuffix[x]}_{index}" for x in FitOutputEnum}
@@ -120,7 +133,7 @@ def PyExec(self):
                 OutputWorkspace=outputNameTmp,
             )
             self.mantidSnapper.ConjoinDiagnosticWorkspaces(
-                "Conjoin the diagnostic group workspaces",
+                f"Conjoin the diagnostic group workspaces for subgroup {groupID}",
                 DiagnosticWorkspace=outputNameTmp,
                 TotalDiagnosticWorkspace=self.outputWorkspaceName,
                 AddAtIndex=index,

src/snapred/backend/service/CalibrationService.py

@@ -129,6 +129,7 @@ def _calibration_N_ref(self, request: DiffractionCalibrationRequest) -> float |
     def prepDiffractionCalibrationIngredients(
         self, request: DiffractionCalibrationRequest
     ) -> DiffractionCalibrationIngredients:
+        self.validateRequest(request)
         # fetch the ingredients needed to focus and plot the peaks
         cifPath = self.dataFactoryService.getCifFilePath(Path(request.calibrantSamplePath).stem)
         state, _ = self.dataFactoryService.constructStateId(request.runNumber)
@@ -147,15 +148,15 @@ def prepDiffractionCalibrationIngredients(
             maxChiSq=request.maxChiSq,
             state=state,
         )
-        ingredients = self.sousChef.prepDiffractionCalibrationIngredients(farmFresh)
+        ingredients = self.sousChef.prepDiffractionCalibrationIngredients(farmFresh, request.combinedPixelMask)
         ingredients.removeBackground = request.removeBackground
         return ingredients
 
     @FromString
     @Register("groceries")
     def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRequest) -> Dict[str, str]:
         # groceries
-
+        self.validateRequest(request)
         # TODO:  It would be nice for groceryclerk to be smart enough to flatten versions
         # However I will save that scope for another time
         if request.startingTableVersion == VersionState.DEFAULT:
@@ -193,23 +194,21 @@ def fetchDiffractionCalibrationGroceries(self, request: DiffractionCalibrationRe
             maskWorkspace=combinedMask,
         )
 
-        if self.groceryService.workspaceDoesExist(calibrationMaskName):
-            self.groceryService.renameWorkspace(calibrationMaskName, combinedMask)
-        elif request.pixelMasks:
-            self.groceryService.getCloneOfWorkspace(request.pixelMasks[0], combinedMask)
+        allMasks = []
 
+        if self.groceryService.workspaceDoesExist(calibrationMaskName):
+            allMasks.append(calibrationMaskName)
         if request.pixelMasks:
-            for mask in request.pixelMasks:
-                if not self.groceryService.workspaceDoesExist(mask):
-                    raise RuntimeError([f"Pixel mask workspace '{mask}' does not exist"])
-                self.mantidSnapper.BinaryOperateMasks(
-                    f"Combine pixel mask workspace {mask} with existing masks",
-                    InputWorkspace1=combinedMask,
-                    InputWorkspace2=mask,
-                    OutputWorkspace=combinedMask,
-                    OperationType="OR",
-                )
-                self.mantidSnapper.executeQueue()
+            allMasks.extend(request.pixelMasks)
+
+        allMasks.append(
+            self.groceryService.fetchCompatiblePixelMask(combinedMask, request.runNumber, request.useLiteMode)
+        )
+
+        if len(allMasks) > 0:
+            combinedMask = self.groceryService.combinePixelMasks(combinedMask, allMasks)
+        else:
+            combinedMask = ""
 
         return groceryDict
 
@@ -222,11 +221,13 @@ def diffractionCalibration(self, request: DiffractionCalibrationRequest) -> Dict
         # Profiling note: none of the following should be marked as sub-steps:
         #   their service methods are decorated separately.
 
+        groceries = self.fetchDiffractionCalibrationGroceries(request)
+        if not request.combinedPixelMask:
+            request.combinedPixelMask = groceries.get("maskWorkspace")
         payload = SimpleDiffCalRequest(
             ingredients=self.prepDiffractionCalibrationIngredients(request),
-            groceries=self.fetchDiffractionCalibrationGroceries(request),
+            groceries=groceries,
         )
-
         pixelRes = self.pixelCalibration(payload)
         if not pixelRes.result:
             raise RuntimeError("Pixel Calibration failed")
@@ -275,9 +276,14 @@ def _calibration_substep_N_ref(self, request: SimpleDiffCalRequest) -> float | N
     @Register("pixel")
     def pixelCalibration(self, request: SimpleDiffCalRequest) -> PixelDiffCalServing:
         # cook recipe
+        userMaskWs = request.groceries.get("maskWorkspace")
+        userMaskWs = self.groceryService.getWorkspaceForName(userMaskWs)
+        numMaskedBeforePixelCal = 0
+        if userMaskWs:
+            numMaskedBeforePixelCal = userMaskWs.getNumberMasked()
         res = PixelDiffCalRecipe().cook(request.ingredients, request.groceries)
         maskWS = self.groceryService.getWorkspaceForName(res.maskWorkspace)
-        percentMasked = maskWS.getNumberMasked() / maskWS.getNumberHistograms()
+        percentMasked = (maskWS.getNumberMasked() - numMaskedBeforePixelCal) / maskWS.getNumberHistograms()
         threshold = Config["constants.maskedPixelThreshold"]
         if percentMasked > threshold:
             res.result = False
@@ -311,6 +317,7 @@ def validateRequest(self, request: DiffractionCalibrationRequest):
             runNumber=request.runNumber, continueFlags=request.continueFlags
         )
         self.validateWritePermissions(permissionsRequest)
+        self.sousChef.verifyCalibrationExists(request.runNumber, request.useLiteMode)
 
     @Register("validateWritePermissions")
     def validateWritePermissions(self, request: CalibrationWritePermissionsRequest):
@@ -348,7 +355,7 @@ def focusSpectra(self, request: FocusSpectraRequest):
         farmFresh = FarmFreshIngredients(
             runNumber=request.runNumber, useLiteMode=request.useLiteMode, focusGroups=[request.focusGroup], state=state
         )
-        pixelGroup = self.sousChef.prepPixelGroup(farmFresh)
+        pixelGroup = self.sousChef.prepPixelGroup(farmFresh, pixelMask=request.maskWorkspace)
         # fetch the grouping workspace
         self.groceryClerk.grouping(request.focusGroup.name).fromRun(request.runNumber).useLiteMode(request.useLiteMode)
         groupingWorkspace = self.groceryService.fetchGroupingDefinition(self.groceryClerk.build())["workspace"]
@@ -681,8 +688,8 @@ def assessQuality(self, request: CalibrationAssessmentRequest):
             maxChiSq=request.maxChiSq,
             state=state,
         )
-        pixelGroup = self.sousChef.prepPixelGroup(farmFresh)
-        detectorPeaks = self.sousChef.prepDetectorPeaks(farmFresh)
+        pixelGroup = self.sousChef.prepPixelGroup(farmFresh, pixelMask=request.combinedPixelMask)
+        detectorPeaks = self.sousChef.prepDetectorPeaks(farmFresh, pixelMask=request.combinedPixelMask)
 
         # TODO: We Need to Fit the Data
         fitResults = FitMultiplePeaksRecipe().executeRecipe(