Merge pull request #703 from intersystems/v0.10.x-feat-init-cpf-merge

Feat: CPF Merge

Author: isc-shuliu

.github/workflows/main.yml

@@ -190,7 +190,7 @@ jobs:
           CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
           docker cp tests/migration/v0.7-to-v0.9/. $CONTAINER:/tmp/test-package/
           sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
-          docker exec -i $CONTAINER iris session iris -UUSER << EOF 
+          docker exec -i $CONTAINER iris session iris -UUSER << 'EOF' 
             s version="0.7.4" s r=##class(%Net.HttpRequest).%New(),r.Server="pm.community.intersystems.com",r.SSLConfiguration="ISC.FeatureTracker.SSL.Config" d r.Get("/packages/zpm/"_version_"/installer"),$system.OBJ.LoadStream(r.HttpResponse.Data,"c")
             zpm "list":1
             zpm "install dsw":1

CHANGELOG.md

@@ -15,10 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - #609 Added support for `-export-deps` when running the "Package" phase of lifecycle
 - #541 Added support for ORAS repository
 - #704 Added support for passing in env files via `-env /path/to/env1.json;/path/to/env2.json` syntax
+- #702 Added a new lifecycle phase `Initialize` which is used for preload
+- #702 Added a `<CPF/>` resource, which can be used for CPF merge before/after a specified lifecycle phase or in a custom lifecycle phase.
 - #716 Added support to publish under external name by passing `-use-external-name` or `-use-ext`. 
 
 ### Changed
-- 
+- #702 Preload now happens as part of the new `Initialize` lifecycle phase. `zpm "<module> reload -only"` will no longer auto compile resources in `/preload` directory.
 
 ### Fixed
 - #474: When loading a .tgz/.tar.gz package, automatically locate the top-most module.xml in case there is nested directory structure (e.g., GitHub releases)

src/cls/IPM/DataType/CustomPhaseName.cls

@@ -0,0 +1,7 @@
+Class %IPM.DataType.CustomPhaseName Extends %Library.String [ ClassType = datatype ]
+{
+
+/// The maximum number of characters the string can contain.
+Parameter MAXLEN As INTEGER = 255;
+
+}

src/cls/IPM/DataType/PhaseName.cls

@@ -0,0 +1,14 @@
+Class %IPM.DataType.PhaseName Extends %Library.String [ ClassType = datatype ]
+{
+
+/// The maximum number of characters the string can contain.
+Parameter MAXLEN As INTEGER = 50;
+
+/// Used for enumerated (multiple-choice) attributes.
+/// <var>VALUELIST</var> is either a null string ("") or a delimiter
+/// separated list (where the delimiter is the first character) of logical values.
+/// If a non-null value is present, then the attribute is restricted to values
+/// in the list, and the validation code simply checks to see if the value is in the list.
+Parameter VALUELIST = ",Clean,Initialize,Reload,*,Validate,ExportData,Compile,Activate,Document,MakeDeployed,Test,Package,Verify,Publish,Configure,Unconfigure";
+
+}

src/cls/IPM/DataType/PhaseWhen.cls

@@ -0,0 +1,14 @@
+Class %IPM.DataType.PhaseWhen Extends %Library.String [ ClassType = datatype ]
+{
+
+/// The maximum number of characters the string can contain.
+Parameter MAXLEN As INTEGER = 50;
+
+/// Used for enumerated (multiple-choice) attributes.
+/// <var>VALUELIST</var> is either a null string ("") or a delimiter
+/// separated list (where the delimiter is the first character) of logical values.
+/// If a non-null value is present, then the attribute is restricted to values
+/// in the list, and the validation code simply checks to see if the value is in the list.
+Parameter VALUELIST = ",Before,After";
+
+}

src/cls/IPM/DataType/ResourceDirectory.cls

@@ -0,0 +1,24 @@
+Class %IPM.DataType.ResourceDirectory Extends %Library.String [ ClassType = datatype ]
+{
+
+Parameter MAXLEN = 255;
+
+/// Tests if the logical value <var>%val</var>, which is a string, is valid.
+/// The validation is based on the class parameter settings used for the class attribute this data type is associated with.
+/// In this case, <a href="#MINLEN">MINLEN</a>, <a href="#MAXLEN">MAXLEN</a>, <a href="#VALUELIST">VALUELIST</a>, and <a href="#PATTERN">PATTERN</a>.
+ClassMethod IsValid(%val As %RawString) As %Status [ ServerOnly = 0 ]
+{
+    If $Extract(%val) = "/" {
+        Return $$$ERROR($$$GeneralError, "Resource directory cannot start with a slash.")
+    }
+    Set segments = $ListFromString(%val, "/")
+    Set ptr = 0
+    While $ListNext(segments, ptr, seg) {
+        If seg = ".." {
+            Return $$$ERROR($$$GeneralError, "For security reasons, resource directory cannot contain '..'.")
+        }
+    }
+    Return $$$OK
+}
+
+}

src/cls/IPM/Lifecycle/Base.cls

@@ -9,7 +9,7 @@ Property PhaseList As %List;
 
 /// $ListBuild list of phases in this lifecycle. <br />
 /// For each phase name, an instance method named "%<phase name>" must be defined in the class with a return type of %Status.
-Parameter PHASES = {$ListBuild("Clean","Reload","*","Validate","ExportData","Compile","Activate","Document","MakeDeployed","Test","Package","Verify", "Publish", "Configure","Unconfigure")};
+Parameter PHASES = {$ListBuild("Clean","Initialize", "Reload","*","Validate","ExportData","Compile","Activate","Document","MakeDeployed","Test","Package","Verify", "Publish", "Configure","Unconfigure")};
 
 Property Payload As %Stream.Object [ Private ];
 
@@ -121,7 +121,7 @@ ClassMethod GetDefaultResourceProcessorProc(pLifecycleClass As %Dictionary.Class
 ClassMethod GetCompletePhases(pPhases As %List) As %List
 {
 	// If there is only phase, and the phase is not found in standard phases, it is a custom phase. Return as-is.
-	If ($ListLength(pPhases) = 1) && ('$ListFind(..#PHASES, $List(pPhases,1))) {
+	If ($ListLength(pPhases) = 1) && ('$ListFind(..#PHASES, ..MatchSinglePhase($List(pPhases,1)))) {
 		Return pPhases
 	}
 	For i=1:1:$ListLength(pPhases) {
@@ -142,18 +142,19 @@ ClassMethod GetCompletePhasesForOne(pOnePhase As %String) As %List
 
 	Quit $Case(pOnePhase,
 		"clean":		$ListBuild("Clean"),
-		"reload":		$ListBuild("Reload","*"),
-		"validate":		$ListBuild("Reload","*","Validate"),
+		"initialize":   $ListBuild("Initialize"),
+		"reload":		$ListBuild("Initialize","Reload","*"),
+		"validate":		$ListBuild("Initialize","Reload","*","Validate"),
 		"exportdata":	$ListBuild("ExportData"),
-		"compile":		$ListBuild("Reload","*","Validate","Compile"),
-		"activate":		$ListBuild("Reload","*","Validate","Compile","Activate"),
+		"compile":		$ListBuild("Initialize","Reload","*","Validate","Compile"),
+		"activate":		$ListBuild("Initialize","Reload","*","Validate","Compile","Activate"),
 		"document":		$ListBuild("Document"),
 		"makedeployed":	$ListBuild("MakeDeployed"),
-		"test":			$ListBuild("Reload","*","Validate","Compile","Activate","Test"),
-		"package":		$ListBuild("Reload","*","Validate","Compile","Activate","Package"),
-		"verify":		$ListBuild("Reload","*","Validate","Compile","Activate","Package","Verify"),
-		"register":		$ListBuild("Reload","*","Validate","Compile","Activate","Package","Register"),
-		"publish":		$ListBuild("Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
+		"test":			$ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Test"),
+		"package":		$ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package"),
+		"verify":		$ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Verify"),
+		"register":		$ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Register"),
+		"publish":		$ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
 		"configure":	$ListBuild("Configure"),
 		"unconfigure":	$ListBuild("Unconfigure"),
 		:				""
@@ -166,6 +167,7 @@ ClassMethod MatchSinglePhase(pOnePhase As %String) As %String
 	Set phase = $ZCONVERT(pOnePhase, "L")
 	Quit $Case(phase,
 		"clean":		"Clean",
+		"initialize":	"Initialize",
 		"reload":		"Reload",
 		"validate":		"Validate",
 		"exportdata":	"ExportData",
@@ -505,6 +507,35 @@ Method %Unconfigure(ByRef pParams) As %Status
 	Quit tSC
 }
 
+Method %Initialize(ByRef pParams) As %Status
+{
+	Set status = $$$OK
+	Try {
+		Set key = ""
+		For {
+			Set resource = ..Module.Resources.GetNext(.key)
+			Quit:key=""
+			If $IsObject(resource.Processor) {
+				Set status = $$$ADDSC(status,resource.Processor.OnPhase("Initialize",.pParams))
+			}
+		}
+
+		Set preloadRoot = $Get(pParams("RootDirectory"))_"preload"
+		Set verbose = $Get(pParams("Verbose"))
+		If ##class(%File).DirectoryExists(preloadRoot) {
+			Set tSC = $System.OBJ.ImportDir(preloadRoot, "*", $Select(verbose:"d",1:"-d")_$Select($Tlevel:"/multicompile=0", 1: ""), , 1, .tImported)
+			If $$$ISERR(tSC) { Quit }
+			Set tSC = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.tImported)
+			If $$$ISERR(tSC) { Quit }
+		} ElseIf verbose {
+			Write !,"Skipping preload - directory does not exist."
+		}
+	} Catch ex {
+		Set status = ex.AsStatus()
+	}
+	Quit status
+}
+
 Method %Reload(ByRef pParams) As %Status
 {
 	Set tSC = $$$OK
@@ -581,16 +612,6 @@ Method %Reload(ByRef pParams) As %Status
 
     $$$ThrowOnError(..InstallPythonRequirements(tRoot, .pParams))
 
-		Set tPreloadRoot = tRoot_"preload"
-		If ##class(%File).DirectoryExists(tPreloadRoot) {
-			Set tSC = $System.OBJ.ImportDir(tPreloadRoot, "*", $Select(tVerbose:"d",1:"-d")_$Select($Tlevel:"/multicompile=0", 1: ""), , 1, .tImported)
-			If $$$ISERR(tSC) { Quit }
-			Set tSC = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.tImported)
-			If $$$ISERR(tSC) { Quit }
-		} ElseIf tVerbose {
-			Write !,"Skipping preload - directory does not exist."
-		}
-		
 		// Reload the module definition
 		Set tSC = ..Module.%Reload()
 		If $$$ISERR(tSC) {

src/cls/IPM/ResourceProcessor/Abstract.cls

@@ -231,7 +231,6 @@ Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.Res
 	Quit $$$OK
 }
 
-
 /// Evaluates an expression in a provided string. <br />
 /// These special expressions are case-insensitive. <br />
 /// Current valid expressions:

src/cls/IPM/ResourceProcessor/CPF.cls

@@ -0,0 +1,93 @@
+Include %IPM.Formatting
+
+Class %IPM.ResourceProcessor.CPF Extends (%IPM.ResourceProcessor.Abstract, %IPM.ResourceProcessor.CustomPhaseMixin)
+{
+
+/// Comma-separated list of resource attribute names that this processor uses
+Parameter ATTRIBUTES As STRING = "Name,Directory,Phase,CustomPhase,When";
+
+/// Description of resource processor class (shown in UI)
+Parameter DESCRIPTION As STRING = "Merges the specified CPF file in the specified lifecycle phase (""Initialize"" by default).";
+
+/// Directory containing the CPF file to merge
+Property Directory As %IPM.DataType.ResourceDirectory [ InitialExpression = "cpf" ];
+
+/// FileN ame of the CPF merge file
+Property Name As %IPM.DataType.ResourceName [ Required ];
+
+/// The phase before/after which the CPF file should be merged
+Property Phase As %IPM.DataType.PhaseName [ InitialExpression = "Initialize" ];
+
+/// When to merge the CPF file: Before or After the specified phase. This only applies to the standard phases.
+Property When As %IPM.DataType.PhaseWhen [ InitialExpression = "Before" ];
+
+Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
+{
+    If (..When = "Before") && (..Phase = pPhase) && (..CustomPhase = "") {
+        Quit ..DoMerge(.pParams)
+    }
+    Quit $$$OK
+}
+
+Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
+{
+    If (..When = "After") && (..Phase = pPhase) && (..CustomPhase = "") {
+        Quit ..DoMerge(.pParams)
+    }
+    Quit $$$OK
+}
+
+Method OnCustomPhase(pCustomPhase As %String, ByRef pParams) As %Status
+{
+    If (..CustomPhase = pCustomPhase) {
+        Quit ..DoMerge(.pParams)
+    }
+    Quit $$$OK
+}
+
+Method DoMerge(ByRef pParams) As %Status
+{
+    Try {
+        Set verbose = $GET(pParams("Verbose"))
+        Set root = ..ResourceReference.Module.Root
+        Set sourcesRoot = ..ResourceReference.Module.SourcesRoot
+        // Use Construct first, rather than NormalizeFilename, so we don't have to deal with leading/trailing slashes
+        Set dir = $SELECT($$$isWINDOWS: $REPLACE(..Directory, "/", "\"), 1: ..Directory)
+        Set dir = ##class(%File).Construct(root, sourcesRoot, dir)
+        Set filename = ##class(%File).NormalizeFilename(..Name, dir)
+        If (filename = "") || ('##class(%File).Exists(filename)) {
+            $$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("CPF file '%1' not found in directory '%2'", ..Name, dir)))
+        }
+
+        Set stream = ##class(%Stream.FileCharacter).%New()  
+        $$$ThrowOnError(stream.LinkToFile(filename))
+        If verbose {
+            Write !, "Merging CPF file: ", filename, !
+            Do stream.OutputToDevice()
+        }
+        Do ..MergeCPF(filename)
+    } Catch ex {
+        Return ex.AsStatus()
+    }
+    Return $$$OK
+}
+
+ClassMethod MergeCPF(file As %String)
+{
+    // TODO The $zf(-100) callout is much slower than ##class(Config.CPF).Merge() 
+    //      Figure out why ##class(Config.CPF).Merge() doesn't work
+    //      c.f. https://github.com/intersystems/ipm/pull/703#discussion_r1917290136
+
+    Set args($INCREMENT(args)) = "merge"
+    Set args($INCREMENT(args)) = ##class(%SYS.System).GetInstanceName()
+    Set args($INCREMENT(args)) = file
+
+    // Somehow, if the STDOUT is not set, the merge will silently fail
+    Set flags = "/SHELL/LOGCMD/STDOUT=""zf100stdout""/STDERR=""zf100stderr"""
+    Set status = $ZF(-100, flags, "iris", .args)
+    If status '= 0 {
+        $$$ThrowStatus($$$ERROR($$$GeneralError, "Error merging CPF file. $zf(-100) exited with "_status))
+    }
+}
+
+}

src/cls/IPM/ResourceProcessor/CustomPhaseMixin.cls

@@ -0,0 +1,14 @@
+/// Some processors may need to do something custom when a custom phase is executed.
+/// Such as the CPF processor, which optionally merges a CPF file during a custom phase.
+/// This  mixin provides the CustomPhase property and the OnCustomPhase method.
+Class %IPM.ResourceProcessor.CustomPhaseMixin
+{
+
+Property CustomPhase As %IPM.DataType.CustomPhaseName;
+
+Method OnCustomPhase(pCustomPhase As %String, ByRef pParams) As %Status
+{
+	Quit $$$OK
+}
+
+}

src/cls/IPM/Storage/InvokeReference.cls

@@ -9,12 +9,12 @@ Property Class As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ Required
 
 Property Method As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ Required ];
 
-Property Phase As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "Configure" ];
+Property Phase As %IPM.DataType.PhaseName(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "Configure" ];
 
 /// If provided, the Phase property will be ignored. This CustomPhase will be used and no corresponding lifecycle is required.
-Property CustomPhase As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE");
+Property CustomPhase As %IPM.DataType.CustomPhaseName(XMLPROJECTION = "ATTRIBUTE");
 
-Property When As %String(MAXLEN = 255, VALUELIST = ",Before,After", XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "After", SqlFieldName = _WHEN ];
+Property When As %IPM.DataType.PhaseWhen(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "After", SqlFieldName = _WHEN ];
 
 Property CheckStatus As %Boolean(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = 0 ];
 

src/cls/IPM/Storage/Module.cls

@@ -177,11 +177,27 @@ Method GetCustomPhases(Output pPhases)
     Set key = ""
     For {
         Set tInvoke = ..Invokes.GetNext(.key)
-        Quit:(key = "")
+		If key = "" {
+			Quit
+		}
         If (tInvoke.CustomPhase '= "") {
             Set pPhases($$$lcase(tInvoke.CustomPhase)) = tInvoke.CustomPhase
         }
     }
+	For {
+		Set tResource = ..Resources.GetNext(.key)
+		If key = "" {
+			Quit
+		}
+		Set tProcessor = tResource.Processor
+		// %IsA() only returns true if the class is the "primary" superclass, while %Extends() works for other superclasses (such as mixins).
+		If $IsObject(tProcessor) && tProcessor.%Extends("%IPM.ResourceProcessor.CustomPhaseMixin") {
+			Set cp = tProcessor.CustomPhase
+			If cp '= "" {
+				Set pPhases($$$lcase(cp)) = cp
+			}
+		}
+	}
 }
 
 /// Execute multiple lifecycle phases in sequence. Execution is terminated if one fails.
@@ -350,7 +366,20 @@ ClassMethod ExecutePhases(pModuleName As %String, pPhases As %List, pIsComplete
         }
         Quit:$$$ISERR(tSC)
 
-        If 'tIsCustomPhase {
+        If tIsCustomPhase {
+            Set tKey = ""
+            For {
+                Set tResource = tModule.Resources.GetNext(.tKey)
+                If tKey = "" {
+                    Quit
+                }
+                Set tProcessor = tResource.Processor
+                // %IsA() only returns true if the class is the "primary" superclass, while %Extends() works for other superclasses (such as mixins).
+                If $IsObject(tProcessor) && tProcessor.%Extends("%IPM.ResourceProcessor.CustomPhaseMixin") {
+                    $$$ThrowOnError(tProcessor.OnCustomPhase(tOnePhase, .pParams))
+                }
+            }
+        } Else {
             // Lifecycle before / (phase) / after
             $$$ThrowOnError(tLifecycle.OnBeforePhase(tOnePhase,.pParams))
             $$$ThrowOnError($Method(tLifecycle,"%"_tOnePhase,.pParams))
@@ -1612,4 +1641,3 @@ Storage Default
 }
 
 }
-