From 037abadebf37b7db2ad3ccbf46689309abd4b5cf Mon Sep 17 00:00:00 2001 From: Fuqing Wang Date: Wed, 21 Jan 2026 13:38:11 -0800 Subject: [PATCH 1/3] local version bump script --- .../V3/README_LocalVersion.md | 51 ++ .../V3/createSolutionV3_LocalVersion.ps1 | 463 ++++++++++++++++++ .../common/commonFunctions.ps1 | 31 +- 3 files changed, 539 insertions(+), 6 deletions(-) create mode 100644 Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md create mode 100644 Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md b/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md new file mode 100644 index 00000000000..83dbb8bc0c0 --- /dev/null +++ b/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md @@ -0,0 +1,51 @@ +# Azure Sentinel Solution Creator V3 - Local Version + +## Overview + +This is a modified version of the `createSolutionV3.ps1` script that uses the version number from the current solution's data file instead of fetching it from the Microsoft catalog API. + +## Files + +- **`createSolutionV3_LocalVersion.ps1`** - Modified script that uses local version +- **`createSolutionV3.ps1`** - Original script that uses Microsoft catalog API +- **`README_LocalVersion.md`** - This documentation file + +## Key Differences + +### Original Script (`createSolutionV3.ps1`) +- Calls `GetCatalogDetails()` to fetch solution details from Microsoft catalog API +- Uses `GetPackageVersion()` to determine version based on catalog data +- May increment version automatically based on published versions +- Requires internet connectivity to Microsoft catalog API + +### Modified Script (`createSolutionV3_LocalVersion.ps1`) +- Uses `GetLocalPackageVersion()` function instead of catalog API calls +- Reads version directly from the solution's data file (`Version` field) +- No internet connectivity required for version determination +- No automatic version incrementation based on published versions + +## Usage + +### Command Line +```powershell +# Run with interactive prompt for path +.\createSolutionV3_LocalVersion.ps1 + +# Run with specified solution data folder path +.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\path\to\solution\Data" + +# Run with specific version bump type +.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\path\to\solution\Data" -VersionBump "minor" +``` + +### Examples +```powershell +# Patch version bump (default): 3.1.8 -> 3.1.9 +.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" + +# Minor version bump: 3.1.8 -> 3.2.0 +.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" -VersionBump "minor" + +# Major version bump: 3.1.8 -> 4.0.0 +.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" -VersionBump "major" +``` \ No newline at end of file diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 new file mode 100644 index 00000000000..55d7b884344 --- /dev/null +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 @@ -0,0 +1,463 @@ +param( + [string]$SolutionDataFolderPath = $null, + [ValidateSet("patch", "minor", "major")] + [string]$VersionBump = "patch" +) + +Write-Host '=======Starting Package Creation using V3 tool with Local Version=========' +if ($null -eq $SolutionDataFolderPath -or $SolutionDataFolderPath -eq '') { + $path = Read-Host "Enter solution data folder path " +} else { + $path = $SolutionDataFolderPath + Write-Host "Solution Data folder path specified is : $path" +} + +$defaultPackageVersion = "3.0.0" # for templateSpec this will be 2.0.0 +Write-Host "Path $path, DefaultPackageVersion is $defaultPackageVersion" + +if ($path.length -eq 0) +{ + # path is not provided so check first file from input folder + $path = "$PSScriptRoot\input" + + $inputFile = $(Get-ChildItem $path) + if ($inputFile.Count -gt 0) + { + $inputFile = $inputFile[0] + $inputJsonPath = Join-Path -Path $path -ChildPath "$($inputFile.Name)" + + $contentToImport = Get-Content -Raw $inputJsonPath | Out-String | ConvertFrom-Json + # BELOW LINE MAKES USE OF BASEPATH FROM DATA FILE AND ADDS DATA FOLDER NAME. + $path = $contentToImport.BasePath + "/Data" + } + else { + Write-Host "Path is not specified and also input folder doesnt have input file. Please make sure to have path specified or add file in side of V3/input folder!" + } +} + +$path = $path.Replace('\', '/') +$indexOfSolutions = $path.IndexOf('Solutions') + +if ($indexOfSolutions -le 0) { + Write-Host "Please provide data folder path from Solutions folder!" + exit 1 +} +else { + $hasDataFolder = $path -like '*/data' + if ($hasDataFolder) { + # DATA FOLDER PRESENT + $dataFolderIndex = $path.LastIndexOf("/data", [StringComparison]"CurrentCultureIgnoreCase") + + if ($dataFolderIndex -le 0) { + Write-Host "Given path is not from Solutions data folders. Please provide data file path from Solution" + exit 1 + } + else { + $dataFolderName = $path.Substring($dataFolderIndex + 1) + $solutionName = $path.Substring($indexOfSolutions + 10, $dataFolderIndex - ($indexOfSolutions + 10)) + $solutionFolderBasePath = $path.Substring(0, $dataFolderIndex) + + # GET DATA FOLDER FILE NAME + $excluded = @("parameters.json", "parameter.json", "system_generated_metadata.json", "testParameters.json") + $dataFileName = Get-ChildItem -Path "$solutionFolderBasePath\$dataFolderName\" -recurse -exclude $excluded | ForEach-Object -Process { [System.IO.Path]::GetFileName($_) } + + if ($dataFileName.Length -le 0) { + Write-Host "Data File not present in given folder path!" + exit 1 + } + } + } + else { + Write-Host "Data File not present in given folder path!" + exit 1 + } +} + +$solutionBasePath = $path.Substring(0, $indexOfSolutions + 10) +$repositoryBasePath = $path.Substring(0, $indexOfSolutions) +Write-Host "SolutionBasePath is $solutionBasePath, Solution Name $solutionName" + +$isPipelineRun = $false + +$commonFunctionsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1" +$getccpDetailsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1" + +. $commonFunctionsFilePath # load common functions +. $getccpDetailsFilePath # load ccp functions + +# Function to increment version based on bump type +function GetIncrementedVersion($version, $bumpType = "patch") +{ + if ([string]::IsNullOrWhiteSpace($version)) { + Write-Host "Warning: Version is null or empty, using default 1.0.0" + $version = "1.0.0" + } + + $versionParts = $version.split(".") + + # Ensure we have at least 3 parts (major.minor.patch) + if ($versionParts.Length -lt 3) { + Write-Host "Warning: Version format invalid, padding with zeros" + while ($versionParts.Length -lt 3) { + $versionParts += "0" + } + } + + $major = $versionParts[0] + $minor = $versionParts[1] + $patch = $versionParts[2] + + # Convert to integers with error handling + try { + [int]$majorInt = [int]$major + [int]$minorInt = [int]$minor + [int]$patchInt = [int]$patch + } + catch { + Write-Host "Error: Cannot convert version parts to integers. Version: $version" + Write-Host "Using fallback version 1.0.0" + $majorInt = 1 + $minorInt = 0 + $patchInt = 0 + } + + switch ($bumpType.ToLower()) { + "major" { + $majorInt += 1 + $minorInt = 0 + $patchInt = 0 + Write-Host "Incrementing MAJOR version: $version -> $majorInt.$minorInt.$patchInt" + } + "minor" { + $minorInt += 1 + $patchInt = 0 + Write-Host "Incrementing MINOR version: $version -> $majorInt.$minorInt.$patchInt" + } + "patch" { + $patchInt += 1 + Write-Host "Incrementing PATCH version: $version -> $majorInt.$minorInt.$patchInt" + } + default { + $patchInt += 1 + Write-Host "Incrementing PATCH version (default): $version -> $majorInt.$minorInt.$patchInt" + } + } + + return "$majorInt.$minorInt.$patchInt" +} + +# Function to get package version from local solution data file instead of Microsoft catalog +function GetLocalPackageVersion($defaultPackageVersion, $userInputPackageVersion, $packageVersionAttribute, $versionBumpType, $dataFilePath) +{ + Write-Host "Using local version from solution data file instead of Microsoft catalog (Bump type: $versionBumpType)" + + if ($packageVersionAttribute -and $null -ne $userInputPackageVersion -and $userInputPackageVersion -ne '') + { + Write-Host "Current version from data file: $userInputPackageVersion" + + # Validate version format before processing + if ([string]::IsNullOrWhiteSpace($userInputPackageVersion)) { + Write-Host "Warning: User input version is null or empty, using default" + return $defaultPackageVersion + } + + $versionParts = $userInputPackageVersion.split(".") + if ($versionParts.Length -lt 3) { + Write-Host "Warning: Invalid version format in data file: $userInputPackageVersion. Using default: $defaultPackageVersion" + return $defaultPackageVersion + } + + try { + $userInputMajor = [int]$versionParts[0] + $userInputMinor = [int]$versionParts[1] + $userInputBuild = [int]$versionParts[2] + } + catch { + Write-Host "Error: Cannot parse version numbers from: $userInputPackageVersion. Using default: $defaultPackageVersion" + return $defaultPackageVersion + } + + # Always increment the version based on the specified bump type, regardless of major version + $incrementedVersion = GetIncrementedVersion $userInputPackageVersion $versionBumpType + Write-Host "Package version incremented to $incrementedVersion (from local solution data: $userInputPackageVersion)" + + # Update the version in the original data file + if ($null -ne $dataFilePath -and $dataFilePath -ne '') { + try { + Write-Host "Updating version in data file: $dataFilePath" + $dataFileContent = Get-Content -Raw $dataFilePath | Out-String | ConvertFrom-Json + $dataFileContent.Version = $incrementedVersion + $updatedJson = $dataFileContent | ConvertTo-Json -Depth 100 + Set-Content -Path $dataFilePath -Value $updatedJson -Encoding UTF8 + Write-Host "Successfully updated version to $incrementedVersion in data file: $dataFilePath" + } + catch { + Write-Host "Warning: Could not update version in data file: $dataFilePath. Error: $_" -ForegroundColor Yellow + } + } + + return $incrementedVersion + } + else { + # No version specified in data file, use default + Write-Host "Package version set to $defaultPackageVersion (default - no version in data file)" + return $defaultPackageVersion + } +} + +try { + $ccpDict = @(); + $ccpTablesFilePaths = @(); + $ccpTablesCounter = 1; + $isCCPConnector = $false; + foreach ($inputFile in $(Get-ChildItem -Path "$solutionFolderBasePath\$dataFolderName\$dataFileName")) { + #$inputJsonPath = Join-Path -Path $path -ChildPath "$($inputFile.Name)" + $contentToImport = Get-Content -Raw $inputFile | Out-String | ConvertFrom-Json + + $has1PConnectorProperty = [bool]($contentToImport.PSobject.Properties.Name -match "Is1PConnector") + if ($has1PConnectorProperty) { + $Is1PConnectorPropertyValue = [bool]($contentToImport.Is1PConnector) + if ($Is1PConnectorPropertyValue) { + # when true we terminate package creation. + Write-Host "ERROR: Is1PConnector property is deprecated. Please use StaticDataConnector property. Refer link https://github.com/Azure/Azure-Sentinel/blob/master/Tools/Create-Azure-Sentinel-Solution/V3/README.md for more details!" + exit 1; + } + } + + $basePath = $(if ($solutionBasePath) { $solutionBasePath } else { "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/" }) + $metadataAuthor = $contentToImport.Author.Split(" - "); + if ($null -ne $metadataAuthor[1]) { + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "email" -NotePropertyValue $($metadataAuthor[1]) + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_email" -NotePropertyValue "[variables('email')]" + } + + $solutionName = $contentToImport.Name + #$metadataPath = "$PSScriptRoot/../../../Solutions/$($contentToImport.Name)/$($contentToImport.Metadata)" + $metadataPath = $solutionBasePath + "$($contentToImport.Name)/$($contentToImport.Metadata)" + + $baseMetadata = Get-Content -Raw $metadataPath | Out-String | ConvertFrom-Json + if ($null -eq $baseMetadata) { + Write-Host "Please verify if the given path is correct and/or Solution folder name and Data file Name attribute value is correct!" + exit 1 + } + + #================START: IDENTIFY PACKAGE VERSION FROM LOCAL SOLUTION============= + $userInputPackageVersion = $contentToImport.version + $packageVersionAttribute = [bool]($contentToImport.PSobject.Properties.Name -match "version") + $packageVersion = GetLocalPackageVersion $defaultPackageVersion $userInputPackageVersion $packageVersionAttribute $VersionBump $inputFile.FullName + + if ($packageVersion -ne $contentToImport.version) { + $contentToImport.PSObject.Properties.Remove('version') + $contentToImport | Add-Member -MemberType NoteProperty -Name 'version' -Value $packageVersion + Write-Host "Package version updated to $packageVersion" + } + + $TemplateSpecAttribute = [bool]($contentToImport.PSobject.Properties.Name -match "TemplateSpec") + if (!$TemplateSpecAttribute) { + $contentToImport | Add-Member -MemberType NoteProperty -Name 'TemplateSpec' -Value $true + } + + $major = $contentToImport.version.split(".")[0] + if ($TemplateSpecAttribute -and $contentToImport.TemplateSpec -eq $false -and $major -gt 1) { + $contentToImport.PSObject.Properties.Remove('TemplateSpec') + $contentToImport | Add-Member -MemberType NoteProperty -Name 'TemplateSpec' -Value $true + } + #================END: IDENTIFY PACKAGE VERSION FROM LOCAL SOLUTION============= + + Write-Host "Package version identified is $packageVersion" + + if ($major -ge 3) { + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionName" -NotePropertyValue $solutionName + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionVersion" -NotePropertyValue $contentToImport.version + } + + $metadataAuthor = $contentToImport.Author.Split(" - "); + + $global:solutionId = $baseMetadata.publisherId + "." + $baseMetadata.offerId + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "solutionId" -NotePropertyValue "$global:solutionId" + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionId" -NotePropertyValue "[variables('solutionId')]" + + # VERIFY IF IT IS A CONTENTSPEC OR CONTENTPACKAGE RESOURCE TYPE BY VERIFYING VERSION FROM DATA FILE + $contentResourceDetails = returnContentResources($contentToImport.Version) + if ($null -eq $contentResourceDetails) { + Write-Host "Not able to identify content resource details based on Version. Please verify if Version in data input file is correct!" + exit 1; + } + + $dcWithoutSpace = ($baseFolderPath + $solutionName + "/DataConnectors/").Replace("//", "/") + $hasDCWithoutSpace = Test-Path -Path $dcWithoutSpace + + if ($hasDCWithoutSpace) { + $DCFolderName = "DataConnectors" + } + + if ($isCCPConnector -eq $false) { + $DCFolderName = "Data Connectors" + + $ccpDict = Get-CCP-Dict -dataFileMetadata $contentToImport -baseFolderPath $solutionBasePath -solutionName $solutionName -DCFolderName $DCFolderName + + if ($null -ne $ccpDict -and $ccpDict.count -gt 0) { + $isCCPConnector = $true + [array]$ccpTablesFilePaths = GetCCPTableFilePaths -existingCCPDict $ccpDict -baseFolderPath $solutionBasePath -solutionName $solutionName -DCFolderName $DCFolderName + } + } + Write-Host "isCCPConnector $isCCPConnector" + $ccpConnectorCodeExecutionCounter = 1; + foreach ($objectProperties in $contentToImport.PsObject.Properties) { + if ($objectProperties.Value -is [System.Array] -and $objectProperties.Name.ToLower() -ne 'dependentdomainsolutionids' -and $objectProperties.Name.ToLower() -ne 'staticdataconnectorids') { + foreach ($file in $objectProperties.Value) { + $file = $file.Replace("$basePath/", "").Replace("Solutions/", "").Replace("$solutionName/", "") + $finalPath = ($basePath + $solutionName + "/" + $file).Replace("//", "/") + $rawData = $null + try { + Write-Host "Downloading $finalPath" + $rawData = (New-Object System.Net.WebClient).DownloadString($finalPath) + } + catch { + Write-Host "Failed to download $finalPath -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red + break; + } + + try { + $json = ConvertFrom-Json $rawData -ErrorAction Stop; # Determine whether content is JSON or YAML + $validJson = $true; + } + catch { + $validJson = $false; + } + + if ($validJson) { + # If valid JSON, must be Workbook or Playbook + $objectKeyLowercase = $objectProperties.Name.ToLower() + if ($objectKeyLowercase -eq "workbooks") { + GetWorkbookDataMetadata -file $file -isPipelineRun $isPipelineRun -contentResourceDetails $contentResourceDetails -baseFolderPath $repositoryBasePath -contentToImport $contentToImport + } + elseif ($objectKeyLowercase -eq "playbooks") { + GetPlaybookDataMetadata -file $file -contentToImport $contentToImport -contentResourceDetails $contentResourceDetails -json $json -isPipelineRun $isPipelineRun + } + elseif ($objectKeyLowercase -eq "data connectors" -or $objectKeyLowercase -eq "dataconnectors") { + if ($ccpDict.Count -gt 0) { + $isCCPConnectorFile = $false; + foreach($item in $ccpDict) { + if ($item.DCDefinitionFullPath -eq $finalPath) { + $isCCPConnectorFile = $true + break; + } + } + + if ($isCCPConnectorFile -and $ccpConnectorCodeExecutionCounter -eq 1) { + # current file is a ccp connector + GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $ccpDict -solutionBasePath $basePath -solutionName $solutionName -ccpTables $ccpTablesFilePaths -ccpTablesCounter $ccpTablesCounter + + $ccpConnectorCodeExecutionCounter += 1 + } + elseif ($isCCPConnectorFile -and $ccpConnectorCodeExecutionCounter -gt 1) { + continue; + } + else { + # current file is a normal connector + GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $null -solutionBasePath $basePath -solutionName $solutionName -ccpTables $null -ccpTablesCounter $ccpTablesCounter + } + } + else { + # current file is a normal connector + GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $null -solutionBasePath $basePath -solutionName $solutionName -ccpTables $null -ccpTablesCounter $ccpTablesCounter + } + } + elseif ($objectKeyLowercase -eq "savedsearches") { + GenerateSavedSearches -json $json -contentResourceDetails $contentResourceDetails + } + elseif ($objectKeyLowercase -eq "watchlists") { + $watchListFileName = Get-ChildItem $finalPath + + GenerateWatchList -json $json -isPipelineRun $isPipelineRun -watchListFileName $watchListFileName.BaseName + } + } + else { + if ($file -match "(\.yaml)$" -and $objectProperties.Name.ToLower() -ne "parsers") { + $objectKeyLowercase = $objectProperties.Name.ToLower() + if ($objectKeyLowercase -eq "hunting queries") { + GetHuntingDataMetadata -file $file -rawData $rawData -contentResourceDetails $contentResourceDetails + } elseif ($objectKeyLowercase -eq "summary rules" -or $objectKeyLowercase -eq "summaryrules") { + $summaryRuleFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/summaryRules.ps1" + . $summaryRuleFilePath + GenerateSummaryRules -solutionName $solutionName -file $file -rawData $rawData -contentResourceDetails $contentResourceDetails + } + else { + GenerateAlertRule -file $file -contentResourceDetails $contentResourceDetails + } + } + else { + GenerateParsersList -file $file -contentToImport $contentToImport -contentResourceDetails $contentResourceDetails + } + } + } + } + elseif ($objectProperties.Name.ToLower() -eq "metadata") { + try { + $finalPath = $metadataPath + $rawData = $null + try { + Write-Host "Downloading $finalPath" + $rawData = (New-Object System.Net.WebClient).DownloadString($finalPath) + } + catch { + Write-Host "Failed to download $finalPath -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red + break; + } + + try { + $json = ConvertFrom-Json $rawData -ErrorAction Stop; # Determine whether content is JSON or YAML + $validJson = $true; + } + catch { + $validJson = $false; + } + + if ($validJson -and $json) { + PrepareSolutionMetadata -solutionMetadataRawContent $json -contentResourceDetails $contentResourceDetails -defaultPackageVersion $defaultPackageVersion + } + else { + Write-Host "Failed to load Metadata file $file -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red + } + } + catch { + Write-Host "Failed to load Metadata file $file -- Please ensure that the SolutionMetadata file exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red + break; + } + } + } + + $global:analyticRuleCounter -= 1 +$global:workbookCounter -= 1 +$global:playbookCounter -= 1 +$global:connectorCounter -= 1 +$global:parserCounter -= 1 +$global:huntingQueryCounter -= 1 +$global:watchlistCounter -= 1 + $global:summaryRuleCounter -= 1 + +updateDescriptionCount $global:connectorCounter "**Data Connectors:** " "{{DataConnectorCount}}" $(checkResourceCounts $global:parserCounter, $global:analyticRuleCounter, $global:workbookCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) +updateDescriptionCount $global:parserCounter "**Parsers:** " "{{ParserCount}}" $(checkResourceCounts $global:analyticRuleCounter, $global:workbookCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) +updateDescriptionCount $global:workbookCounter "**Workbooks:** " "{{WorkbookCount}}" $(checkResourceCounts $global:analyticRuleCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) +updateDescriptionCount $global:analyticRuleCounter "**Analytic Rules:** " "{{AnalyticRuleCount}}" $(checkResourceCounts $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) +updateDescriptionCount $global:huntingQueryCounter "**Hunting Queries:** " "{{HuntingQueryCount}}" $(checkResourceCounts $global:playbookCounter, $global:watchlistCounter, $global:summaryRuleCounter) + +updateDescriptionCount $global:watchlistCounter "**Watchlists:** " "{{WatchlistCount}}" $(checkResourceCounts $global:playbookCounter, $global:summaryRuleCounter) + + updateDescriptionCount $global:summaryRuleCounter "**Summary Rules:** " "{{SummaryRuleCount}}" $(checkResourceCounts @($global:playbookCounter)) + +updateDescriptionCount $global:customConnectorsList.Count "**Custom Azure Logic Apps Connectors:** " "{{LogicAppCustomConnectorCount}}" $(checkResourceCounts @($global:playbookCounter)) +updateDescriptionCount $global:functionAppList.Count "**Function Apps:** " "{{FunctionAppsCount}}" $(checkResourceCounts @($global:playbookCounter)) + updateDescriptionCount ($global:playbookCounter - $global:customConnectorsList.Count - $global:functionAppList.Count) "**Playbooks:** " "{{PlaybookCount}}" $false + + GeneratePackage -solutionName $solutionName -contentToImport $contentToImport -calculatedBuildPipelinePackageVersion $contentToImport.Version; + RunArmTtkOnPackage -solutionName $solutionName -isPipelineRun $false; + + # check if mainTemplate and createUiDefinition json files are valid or not + CheckJsonIsValid($solutionFolderBasePath) + } +} +catch { + Write-Host "Error occurred in catch of createSolutionV3_LocalVersion file Error details are $_" -ForegroundColor Red +} diff --git a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 index 2ebec4af90b..7a28bc47abb 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1 @@ -336,9 +336,23 @@ function getParserDetails($solutionName,$yaml,$isyaml) $variableExpressionRegex = "\[\s?variables\(\'_([\w\W]+)\'\)\s?\]" $parserDisplayDetails = New-Object PSObject - $functionAlias = ($isyaml -eq $true) ? $yaml.FunctionName : $(getFileNameFromPath $file) - $displayName = ($isyaml -eq $true) ? "$($yaml.Function.Title)" : "$($fileName)" - $name = ($isyaml -eq $true) ? "$($yaml.FunctionName)" : "$($fileName)" + if ($isyaml -eq $true) { + $functionAlias = $yaml.FunctionName + } else { + $functionAlias = $(getFileNameFromPath $file) + } + + if ($isyaml -eq $true) { + $displayName = "$($yaml.Function.Title)" + } else { + $displayName = "$($fileName)" + } + + if ($isyaml -eq $true) { + $name = "$($yaml.FunctionName)" + } else { + $name = "$($fileName)" + } $parserDisplayDetails | Add-Member -NotePropertyName "functionAlias" -NotePropertyValue $functionAlias $parserDisplayDetails | Add-Member -NotePropertyName "displayName" -NotePropertyValue $displayName $parserDisplayDetails | Add-Member -NotePropertyName "name" -NotePropertyValue $name @@ -558,7 +572,12 @@ function PrepareSolutionMetadata($solutionMetadataRawContent, $contentResourceDe $newMetadata.Properties | Add-Member -Name 'descriptionHtml' -Type NoteProperty -Value $contentToImport.Description; $newMetadata.Properties | Add-Member -Name 'contentKind' -Type NoteProperty -Value "Solution"; - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutioncontentProductId" -NotePropertyValue "[concat(take(variables('_solutionId'),50),'-','$($ContentKindDict.ContainsKey("Solution") ? $ContentKindDict["Solution"] : '')','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]" + if ($ContentKindDict.ContainsKey("Solution")) { + $solutionContentKind = $ContentKindDict["Solution"] + } else { + $solutionContentKind = '' + } + $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutioncontentProductId" -NotePropertyValue "[concat(take(variables('_solutionId'),50),'-','$solutionContentKind','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]" $newMetadata.Properties | Add-Member -Name 'contentProductId' -Type NoteProperty -Value "[variables('_solutioncontentProductId')]" $newMetadata.Properties | Add-Member -Name 'id' -Type NoteProperty -Value "[variables('_solutioncontentProductId')]" $newMetadata.Properties | Add-Member -Name 'icon' -Type NoteProperty -Value $contentToImport.Logo; @@ -3318,7 +3337,7 @@ function PrepareSolutionMetadata($solutionMetadataRawContent, $contentResourceDe { $dict = $null; $version = constructVersionNumber($item) - if($version.Major -eq 3) + if($version.Major -ge 3) { $dict = @{ #'resourcetype' = "Microsoft.OperationInsights/workspaces/providers/contentPackages" @@ -3796,4 +3815,4 @@ function generateParserContent($file, $contentToImport, $contentResourceDetails) } $global:parserCounter += 1 -} \ No newline at end of file +} From 87acf4e698e459618fb2e20ce9c0d9abee6c4e14 Mon Sep 17 00:00:00 2001 From: Fuqing Wang Date: Fri, 23 Jan 2026 12:23:28 -0800 Subject: [PATCH 2/3] unified scripts --- .../V3/README.md | 82 +++- .../V3/README_LocalVersion.md | 51 -- .../V3/createSolutionV3.ps1 | 182 ++++++- .../V3/createSolutionV3_LocalVersion.ps1 | 463 ------------------ 4 files changed, 251 insertions(+), 527 deletions(-) delete mode 100644 Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md delete mode 100644 Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README.md b/Tools/Create-Azure-Sentinel-Solution/V3/README.md index c7450358c5a..36bf08e09a7 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/README.md +++ b/Tools/Create-Azure-Sentinel-Solution/V3/README.md @@ -249,13 +249,85 @@ Create a file and place it in the base path of solution `https://raw.githubuser NOTE: It is now recommended to use 'createSolutionV3.ps1' file instead of 'createSolutionV2.ps1'. 'createSolutionV2.ps1' is not recommended going forward. 'createSolutionV4.ps1' file is used for GitHub pipeline and is not used for local use. `'createSolutionV3.ps1' requires 'commonFunctions.ps1' file which is placed under 'Tools\Create-Azure-Sentinel-Solution\common' path and this file 'commonFunctions.ps1' has all core logic to create package.` -To generate the solution package, run the `createSolutionV3.ps1` script in the automation folder, `Tools/Create-Azure-Sentinel-Solution/V3`. -> Ex. From repository root, run: `./Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1` +The `createSolutionV3.ps1` script supports two version management modes: -Executing above command with ask you to enter the data file path in the solution as 'Enter solution data file path'. Just specify the data folder path from Solutions. No need to specify data file path. This will generate and compress the solution package, and name the package using the version provided in the input file. -eg: Enter solution data file path : C:\Github\Azure-Sentinel\Solutions\Agari\data +#### **Catalog Mode (Default)** +Uses Microsoft Catalog API for version management. This is the original behavior and is recommended for production deployments. -In above example we have provided path of data folder only without file name. Also there is NO need to copy paste data input file to Tools/input folder. +#### **Local Mode** +Uses local version bumping with semantic versioning (major.minor.patch). This mode works offline and is ideal for development and testing. + +### Usage Examples + +#### **Basic Usage (Catalog Mode)** +```powershell +# Navigate to the script directory +cd Tools/Create-Azure-Sentinel-Solution/V3 + +# Run with catalog mode (default behavior) +./createSolutionV3.ps1 + +# Or specify the data folder path directly +./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" + +# Explicitly specify catalog mode +./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "catalog" +``` + +#### **Local Version Bumping Mode** +```powershell +# Patch version bump (1.0.0 -> 1.0.1) +./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "local" -VersionBump "patch" + +# Minor version bump (1.0.0 -> 1.1.0) +./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "local" -VersionBump "minor" + +# Major version bump (1.0.0 -> 2.0.0) +./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "local" -VersionBump "major" +``` + +#### **Interactive Mode** +If you don't specify the `-SolutionDataFolderPath` parameter, the script will prompt you for the data folder path: +```powershell +./createSolutionV3.ps1 -VersionMode "local" -VersionBump "patch" +# You will be prompted: "Enter solution data folder path" +# Just specify the data folder path from Solutions, e.g.: C:\Github\Azure-Sentinel\Solutions\Agari\Data +``` + +### Version Management Modes Comparison + +| Feature | Catalog Mode | Local Mode | +|---------|--------------|------------| +| **Version Source** | Microsoft Catalog API | Local solution data file | +| **Version Logic** | Increments based on catalog version | Semantic versioning (major.minor.patch) | +| **Internet Required** | Yes (for API calls) | No | +| **Use Case** | Production deployments | Development/testing | +| **File Updates** | No automatic updates | Automatically updates data and metadata files | + +### What Gets Updated in Local Mode + +When using local mode, the script automatically updates: +1. **Solution data file** (`Solution_[Name].json`) - Version property updated +2. **Solution metadata file** (referenced in data file) - Version property updated +3. **Generated package files** - Only version numbers change, no other content modifications + +### Parameters + +- **`SolutionDataFolderPath`** (Optional): Path to the solution data folder. If not provided, you'll be prompted to enter it. +- **`VersionMode`** (Optional): Choose between `"catalog"` (default) or `"local"` version management. +- **`VersionBump`** (Optional): For local mode, specify `"patch"`, `"minor"`, or `"major"` (default: `"patch"`). + +### Legacy Usage + +The original usage still works for backward compatibility: +```powershell +# From repository root, run: +./Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 +# You will be prompted: "Enter solution data folder path" +# Example input: C:\Github\Azure-Sentinel\Solutions\Agari\Data +``` + +In the above examples, provide the path to the data folder only (without the file name). There is NO need to copy the data input file to the `Tools/input` folder. The package consists of the following files: diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md b/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md deleted file mode 100644 index 83dbb8bc0c0..00000000000 --- a/Tools/Create-Azure-Sentinel-Solution/V3/README_LocalVersion.md +++ /dev/null @@ -1,51 +0,0 @@ -# Azure Sentinel Solution Creator V3 - Local Version - -## Overview - -This is a modified version of the `createSolutionV3.ps1` script that uses the version number from the current solution's data file instead of fetching it from the Microsoft catalog API. - -## Files - -- **`createSolutionV3_LocalVersion.ps1`** - Modified script that uses local version -- **`createSolutionV3.ps1`** - Original script that uses Microsoft catalog API -- **`README_LocalVersion.md`** - This documentation file - -## Key Differences - -### Original Script (`createSolutionV3.ps1`) -- Calls `GetCatalogDetails()` to fetch solution details from Microsoft catalog API -- Uses `GetPackageVersion()` to determine version based on catalog data -- May increment version automatically based on published versions -- Requires internet connectivity to Microsoft catalog API - -### Modified Script (`createSolutionV3_LocalVersion.ps1`) -- Uses `GetLocalPackageVersion()` function instead of catalog API calls -- Reads version directly from the solution's data file (`Version` field) -- No internet connectivity required for version determination -- No automatic version incrementation based on published versions - -## Usage - -### Command Line -```powershell -# Run with interactive prompt for path -.\createSolutionV3_LocalVersion.ps1 - -# Run with specified solution data folder path -.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\path\to\solution\Data" - -# Run with specific version bump type -.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\path\to\solution\Data" -VersionBump "minor" -``` - -### Examples -```powershell -# Patch version bump (default): 3.1.8 -> 3.1.9 -.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" - -# Minor version bump: 3.1.8 -> 3.2.0 -.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" -VersionBump "minor" - -# Major version bump: 3.1.8 -> 4.0.0 -.\createSolutionV3_LocalVersion.ps1 -SolutionDataFolderPath "C:\Users\fuqingwang\repos\Azure-Sentinel\Solutions\CrowdStrike Falcon Endpoint Protection\Data" -VersionBump "major" -``` \ No newline at end of file diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 index 70886d04d94..5e48414a053 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 @@ -1,9 +1,18 @@ param( - [string]$SolutionDataFolderPath = $null + [string]$SolutionDataFolderPath = $null, + [ValidateSet("catalog", "local")] + [string]$VersionMode = "catalog", + [ValidateSet("patch", "minor", "major")] + [string]$VersionBump = "patch" ) Write-Host '=======Starting Package Creation using V3 tool=========' +Write-Host "Version Mode: $VersionMode" +if ($VersionMode -eq "local") { + Write-Host "Version Bump Type: $VersionBump" +} + if ($null -eq $SolutionDataFolderPath -or $SolutionDataFolderPath -eq '') { $path = Read-Host "Enter solution data folder path " } else { @@ -79,13 +88,161 @@ Write-Host "SolutionBasePath is $solutionBasePath, Solution Name $solutionName" $isPipelineRun = $false $commonFunctionsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1" -$catalogAPIFilePath = $repositoryBasePath + ".script/package-automation/catalogAPI.ps1" $getccpDetailsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1" . $commonFunctionsFilePath # load common functions -. $catalogAPIFilePath # load catalog api functions . $getccpDetailsFilePath # load ccp functions +# Load catalog API functions only if using catalog mode +if ($VersionMode -eq "catalog") { + $catalogAPIFilePath = $repositoryBasePath + ".script/package-automation/catalogAPI.ps1" + . $catalogAPIFilePath # load catalog api functions +} + +# Function to increment version based on bump type (for local mode) +function GetIncrementedVersion($version, $bumpType = "patch") +{ + if ([string]::IsNullOrWhiteSpace($version)) { + Write-Host "Warning: Version is null or empty, using default 1.0.0" + $version = "1.0.0" + } + + $versionParts = $version.split(".") + + # Ensure we have at least 3 parts (major.minor.patch) + if ($versionParts.Length -lt 3) { + Write-Host "Warning: Version format invalid, padding with zeros" + while ($versionParts.Length -lt 3) { + $versionParts += "0" + } + } + + $major = $versionParts[0].Trim() + $minor = $versionParts[1].Trim() + $patch = $versionParts[2].Trim() + + # Convert to integers with error handling + try { + [int]$majorInt = [Convert]::ToInt32($major) + [int]$minorInt = [Convert]::ToInt32($minor) + [int]$patchInt = [Convert]::ToInt32($patch) + } + catch { + Write-Host "Error: Cannot convert version parts to integers. Version: $version, Parts: $($versionParts -join ', ')" + Write-Host "Using fallback version 1.0.0" + $majorInt = 1 + $minorInt = 0 + $patchInt = 0 + } + + $originalMajor = $majorInt + $originalMinor = $minorInt + $originalPatch = $patchInt + + switch ($bumpType.ToLower().Trim()) { + "major" { + $majorInt += 1 + $minorInt = 0 + $patchInt = 0 + Write-Host "Incrementing MAJOR version: $originalMajor.$originalMinor.$originalPatch -> $majorInt.$minorInt.$patchInt" + } + "minor" { + $minorInt += 1 + $patchInt = 0 + Write-Host "Incrementing MINOR version: $originalMajor.$originalMinor.$originalPatch -> $majorInt.$minorInt.$patchInt" + } + "patch" { + $patchInt += 1 + Write-Host "Incrementing PATCH version: $originalMajor.$originalMinor.$originalPatch -> $majorInt.$minorInt.$patchInt" + } + default { + $patchInt += 1 + Write-Host "Incrementing PATCH version (default): $originalMajor.$originalMinor.$originalPatch -> $majorInt.$minorInt.$patchInt" + } + } + + return "$majorInt.$minorInt.$patchInt" +} + +# Function to get package version from local solution data file instead of Microsoft catalog +function GetLocalPackageVersion($defaultPackageVersion, $userInputPackageVersion, $packageVersionAttribute, $versionBumpType, $dataFilePath) +{ + Write-Host "Using local version from solution data file instead of Microsoft catalog (Bump type: $versionBumpType)" + + if ($packageVersionAttribute -and $null -ne $userInputPackageVersion -and $userInputPackageVersion -ne '') + { + Write-Host "Current version from data file: $userInputPackageVersion" + + # Validate version format before processing + if ([string]::IsNullOrWhiteSpace($userInputPackageVersion)) { + Write-Host "Warning: User input version is null or empty, using default" + return $defaultPackageVersion + } + + $versionParts = $userInputPackageVersion.split(".") + if ($versionParts.Length -lt 3) { + Write-Host "Warning: Invalid version format in data file: $userInputPackageVersion. Using default: $defaultPackageVersion" + return $defaultPackageVersion + } + + try { + $userInputMajor = [int]$versionParts[0] + $userInputMinor = [int]$versionParts[1] + $userInputBuild = [int]$versionParts[2] + } + catch { + Write-Host "Error: Cannot parse version numbers from: $userInputPackageVersion. Using default: $defaultPackageVersion" + return $defaultPackageVersion + } + + # Always increment the version based on the specified bump type, regardless of major version + $incrementedVersion = GetIncrementedVersion $userInputPackageVersion $versionBumpType + Write-Host "Package version incremented to $incrementedVersion (from local solution data: $userInputPackageVersion)" + + # Update the version in the original data file + if ($null -ne $dataFilePath -and $dataFilePath -ne '') { + try { + Write-Host "Updating version in data file: $dataFilePath" + $dataFileContent = Get-Content -Raw $dataFilePath | Out-String | ConvertFrom-Json + $dataFileContent.Version = $incrementedVersion + $updatedJson = $dataFileContent | ConvertTo-Json -Depth 100 + Set-Content -Path $dataFilePath -Value $updatedJson -Encoding UTF8 + Write-Host "Successfully updated version to $incrementedVersion in data file: $dataFilePath" + + # Also update the solution metadata file version if it exists + $solutionDir = Split-Path $dataFilePath -Parent | Split-Path -Parent + $metadataFile = $dataFileContent.Metadata + if ($metadataFile) { + $metadataPath = Join-Path $solutionDir $metadataFile + if (Test-Path $metadataPath) { + try { + Write-Host "Updating version in solution metadata file: $metadataPath" + $metadataContent = Get-Content -Raw $metadataPath | Out-String | ConvertFrom-Json + $metadataContent.version = $incrementedVersion + $updatedMetadataJson = $metadataContent | ConvertTo-Json -Depth 100 + Set-Content -Path $metadataPath -Value $updatedMetadataJson -Encoding UTF8 + Write-Host "Successfully updated version to $incrementedVersion in metadata file: $metadataPath" + } + catch { + Write-Host "Warning: Could not update version in metadata file: $metadataPath. Error: $_" -ForegroundColor Yellow + } + } + } + } + catch { + Write-Host "Warning: Could not update version in data file: $dataFilePath. Error: $_" -ForegroundColor Yellow + } + } + + return $incrementedVersion + } + else { + # No version specified in data file, use default + Write-Host "Package version set to $defaultPackageVersion (default - no version in data file)" + return $defaultPackageVersion + } +} + try { $ccpDict = @(); $ccpTablesFilePaths = @(); @@ -123,11 +280,20 @@ try { } #================START: IDENTIFY PACKAGE VERSION============= - $solutionOfferId = $baseMetadata.offerId - $offerId = "$solutionOfferId" - $offerDetails = GetCatalogDetails $offerId $userInputPackageVersion = $contentToImport.version - $packageVersion = GetPackageVersion $defaultPackageVersion $offerId $offerDetails $true $userInputPackageVersion + $packageVersionAttribute = [bool]($contentToImport.PSobject.Properties.Name -match "version") + + if ($VersionMode -eq "local") { + # Use local version bumping + $packageVersion = GetLocalPackageVersion $defaultPackageVersion $userInputPackageVersion $packageVersionAttribute $VersionBump $inputFile.FullName + } else { + # Use catalog API for version management + $solutionOfferId = $baseMetadata.offerId + $offerId = "$solutionOfferId" + $offerDetails = GetCatalogDetails $offerId + $packageVersion = GetPackageVersion $defaultPackageVersion $offerId $offerDetails $packageVersionAttribute $userInputPackageVersion + } + if ($packageVersion -ne $contentToImport.version) { $contentToImport.PSObject.Properties.Remove('version') $contentToImport | Add-Member -MemberType NoteProperty -Name 'version' -Value $packageVersion @@ -144,7 +310,7 @@ try { $contentToImport.PSObject.Properties.Remove('TemplateSpec') $contentToImport | Add-Member -MemberType NoteProperty -Name 'TemplateSpec' -Value $true } - #================START: IDENTIFY PACKAGE VERSION============= + #================END: IDENTIFY PACKAGE VERSION============= Write-Host "Package version identified is $packageVersion" diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 deleted file mode 100644 index 55d7b884344..00000000000 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_LocalVersion.ps1 +++ /dev/null @@ -1,463 +0,0 @@ -param( - [string]$SolutionDataFolderPath = $null, - [ValidateSet("patch", "minor", "major")] - [string]$VersionBump = "patch" -) - -Write-Host '=======Starting Package Creation using V3 tool with Local Version=========' -if ($null -eq $SolutionDataFolderPath -or $SolutionDataFolderPath -eq '') { - $path = Read-Host "Enter solution data folder path " -} else { - $path = $SolutionDataFolderPath - Write-Host "Solution Data folder path specified is : $path" -} - -$defaultPackageVersion = "3.0.0" # for templateSpec this will be 2.0.0 -Write-Host "Path $path, DefaultPackageVersion is $defaultPackageVersion" - -if ($path.length -eq 0) -{ - # path is not provided so check first file from input folder - $path = "$PSScriptRoot\input" - - $inputFile = $(Get-ChildItem $path) - if ($inputFile.Count -gt 0) - { - $inputFile = $inputFile[0] - $inputJsonPath = Join-Path -Path $path -ChildPath "$($inputFile.Name)" - - $contentToImport = Get-Content -Raw $inputJsonPath | Out-String | ConvertFrom-Json - # BELOW LINE MAKES USE OF BASEPATH FROM DATA FILE AND ADDS DATA FOLDER NAME. - $path = $contentToImport.BasePath + "/Data" - } - else { - Write-Host "Path is not specified and also input folder doesnt have input file. Please make sure to have path specified or add file in side of V3/input folder!" - } -} - -$path = $path.Replace('\', '/') -$indexOfSolutions = $path.IndexOf('Solutions') - -if ($indexOfSolutions -le 0) { - Write-Host "Please provide data folder path from Solutions folder!" - exit 1 -} -else { - $hasDataFolder = $path -like '*/data' - if ($hasDataFolder) { - # DATA FOLDER PRESENT - $dataFolderIndex = $path.LastIndexOf("/data", [StringComparison]"CurrentCultureIgnoreCase") - - if ($dataFolderIndex -le 0) { - Write-Host "Given path is not from Solutions data folders. Please provide data file path from Solution" - exit 1 - } - else { - $dataFolderName = $path.Substring($dataFolderIndex + 1) - $solutionName = $path.Substring($indexOfSolutions + 10, $dataFolderIndex - ($indexOfSolutions + 10)) - $solutionFolderBasePath = $path.Substring(0, $dataFolderIndex) - - # GET DATA FOLDER FILE NAME - $excluded = @("parameters.json", "parameter.json", "system_generated_metadata.json", "testParameters.json") - $dataFileName = Get-ChildItem -Path "$solutionFolderBasePath\$dataFolderName\" -recurse -exclude $excluded | ForEach-Object -Process { [System.IO.Path]::GetFileName($_) } - - if ($dataFileName.Length -le 0) { - Write-Host "Data File not present in given folder path!" - exit 1 - } - } - } - else { - Write-Host "Data File not present in given folder path!" - exit 1 - } -} - -$solutionBasePath = $path.Substring(0, $indexOfSolutions + 10) -$repositoryBasePath = $path.Substring(0, $indexOfSolutions) -Write-Host "SolutionBasePath is $solutionBasePath, Solution Name $solutionName" - -$isPipelineRun = $false - -$commonFunctionsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/commonFunctions.ps1" -$getccpDetailsFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1" - -. $commonFunctionsFilePath # load common functions -. $getccpDetailsFilePath # load ccp functions - -# Function to increment version based on bump type -function GetIncrementedVersion($version, $bumpType = "patch") -{ - if ([string]::IsNullOrWhiteSpace($version)) { - Write-Host "Warning: Version is null or empty, using default 1.0.0" - $version = "1.0.0" - } - - $versionParts = $version.split(".") - - # Ensure we have at least 3 parts (major.minor.patch) - if ($versionParts.Length -lt 3) { - Write-Host "Warning: Version format invalid, padding with zeros" - while ($versionParts.Length -lt 3) { - $versionParts += "0" - } - } - - $major = $versionParts[0] - $minor = $versionParts[1] - $patch = $versionParts[2] - - # Convert to integers with error handling - try { - [int]$majorInt = [int]$major - [int]$minorInt = [int]$minor - [int]$patchInt = [int]$patch - } - catch { - Write-Host "Error: Cannot convert version parts to integers. Version: $version" - Write-Host "Using fallback version 1.0.0" - $majorInt = 1 - $minorInt = 0 - $patchInt = 0 - } - - switch ($bumpType.ToLower()) { - "major" { - $majorInt += 1 - $minorInt = 0 - $patchInt = 0 - Write-Host "Incrementing MAJOR version: $version -> $majorInt.$minorInt.$patchInt" - } - "minor" { - $minorInt += 1 - $patchInt = 0 - Write-Host "Incrementing MINOR version: $version -> $majorInt.$minorInt.$patchInt" - } - "patch" { - $patchInt += 1 - Write-Host "Incrementing PATCH version: $version -> $majorInt.$minorInt.$patchInt" - } - default { - $patchInt += 1 - Write-Host "Incrementing PATCH version (default): $version -> $majorInt.$minorInt.$patchInt" - } - } - - return "$majorInt.$minorInt.$patchInt" -} - -# Function to get package version from local solution data file instead of Microsoft catalog -function GetLocalPackageVersion($defaultPackageVersion, $userInputPackageVersion, $packageVersionAttribute, $versionBumpType, $dataFilePath) -{ - Write-Host "Using local version from solution data file instead of Microsoft catalog (Bump type: $versionBumpType)" - - if ($packageVersionAttribute -and $null -ne $userInputPackageVersion -and $userInputPackageVersion -ne '') - { - Write-Host "Current version from data file: $userInputPackageVersion" - - # Validate version format before processing - if ([string]::IsNullOrWhiteSpace($userInputPackageVersion)) { - Write-Host "Warning: User input version is null or empty, using default" - return $defaultPackageVersion - } - - $versionParts = $userInputPackageVersion.split(".") - if ($versionParts.Length -lt 3) { - Write-Host "Warning: Invalid version format in data file: $userInputPackageVersion. Using default: $defaultPackageVersion" - return $defaultPackageVersion - } - - try { - $userInputMajor = [int]$versionParts[0] - $userInputMinor = [int]$versionParts[1] - $userInputBuild = [int]$versionParts[2] - } - catch { - Write-Host "Error: Cannot parse version numbers from: $userInputPackageVersion. Using default: $defaultPackageVersion" - return $defaultPackageVersion - } - - # Always increment the version based on the specified bump type, regardless of major version - $incrementedVersion = GetIncrementedVersion $userInputPackageVersion $versionBumpType - Write-Host "Package version incremented to $incrementedVersion (from local solution data: $userInputPackageVersion)" - - # Update the version in the original data file - if ($null -ne $dataFilePath -and $dataFilePath -ne '') { - try { - Write-Host "Updating version in data file: $dataFilePath" - $dataFileContent = Get-Content -Raw $dataFilePath | Out-String | ConvertFrom-Json - $dataFileContent.Version = $incrementedVersion - $updatedJson = $dataFileContent | ConvertTo-Json -Depth 100 - Set-Content -Path $dataFilePath -Value $updatedJson -Encoding UTF8 - Write-Host "Successfully updated version to $incrementedVersion in data file: $dataFilePath" - } - catch { - Write-Host "Warning: Could not update version in data file: $dataFilePath. Error: $_" -ForegroundColor Yellow - } - } - - return $incrementedVersion - } - else { - # No version specified in data file, use default - Write-Host "Package version set to $defaultPackageVersion (default - no version in data file)" - return $defaultPackageVersion - } -} - -try { - $ccpDict = @(); - $ccpTablesFilePaths = @(); - $ccpTablesCounter = 1; - $isCCPConnector = $false; - foreach ($inputFile in $(Get-ChildItem -Path "$solutionFolderBasePath\$dataFolderName\$dataFileName")) { - #$inputJsonPath = Join-Path -Path $path -ChildPath "$($inputFile.Name)" - $contentToImport = Get-Content -Raw $inputFile | Out-String | ConvertFrom-Json - - $has1PConnectorProperty = [bool]($contentToImport.PSobject.Properties.Name -match "Is1PConnector") - if ($has1PConnectorProperty) { - $Is1PConnectorPropertyValue = [bool]($contentToImport.Is1PConnector) - if ($Is1PConnectorPropertyValue) { - # when true we terminate package creation. - Write-Host "ERROR: Is1PConnector property is deprecated. Please use StaticDataConnector property. Refer link https://github.com/Azure/Azure-Sentinel/blob/master/Tools/Create-Azure-Sentinel-Solution/V3/README.md for more details!" - exit 1; - } - } - - $basePath = $(if ($solutionBasePath) { $solutionBasePath } else { "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/" }) - $metadataAuthor = $contentToImport.Author.Split(" - "); - if ($null -ne $metadataAuthor[1]) { - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "email" -NotePropertyValue $($metadataAuthor[1]) - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_email" -NotePropertyValue "[variables('email')]" - } - - $solutionName = $contentToImport.Name - #$metadataPath = "$PSScriptRoot/../../../Solutions/$($contentToImport.Name)/$($contentToImport.Metadata)" - $metadataPath = $solutionBasePath + "$($contentToImport.Name)/$($contentToImport.Metadata)" - - $baseMetadata = Get-Content -Raw $metadataPath | Out-String | ConvertFrom-Json - if ($null -eq $baseMetadata) { - Write-Host "Please verify if the given path is correct and/or Solution folder name and Data file Name attribute value is correct!" - exit 1 - } - - #================START: IDENTIFY PACKAGE VERSION FROM LOCAL SOLUTION============= - $userInputPackageVersion = $contentToImport.version - $packageVersionAttribute = [bool]($contentToImport.PSobject.Properties.Name -match "version") - $packageVersion = GetLocalPackageVersion $defaultPackageVersion $userInputPackageVersion $packageVersionAttribute $VersionBump $inputFile.FullName - - if ($packageVersion -ne $contentToImport.version) { - $contentToImport.PSObject.Properties.Remove('version') - $contentToImport | Add-Member -MemberType NoteProperty -Name 'version' -Value $packageVersion - Write-Host "Package version updated to $packageVersion" - } - - $TemplateSpecAttribute = [bool]($contentToImport.PSobject.Properties.Name -match "TemplateSpec") - if (!$TemplateSpecAttribute) { - $contentToImport | Add-Member -MemberType NoteProperty -Name 'TemplateSpec' -Value $true - } - - $major = $contentToImport.version.split(".")[0] - if ($TemplateSpecAttribute -and $contentToImport.TemplateSpec -eq $false -and $major -gt 1) { - $contentToImport.PSObject.Properties.Remove('TemplateSpec') - $contentToImport | Add-Member -MemberType NoteProperty -Name 'TemplateSpec' -Value $true - } - #================END: IDENTIFY PACKAGE VERSION FROM LOCAL SOLUTION============= - - Write-Host "Package version identified is $packageVersion" - - if ($major -ge 3) { - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionName" -NotePropertyValue $solutionName - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionVersion" -NotePropertyValue $contentToImport.version - } - - $metadataAuthor = $contentToImport.Author.Split(" - "); - - $global:solutionId = $baseMetadata.publisherId + "." + $baseMetadata.offerId - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "solutionId" -NotePropertyValue "$global:solutionId" - $global:baseMainTemplate.variables | Add-Member -NotePropertyName "_solutionId" -NotePropertyValue "[variables('solutionId')]" - - # VERIFY IF IT IS A CONTENTSPEC OR CONTENTPACKAGE RESOURCE TYPE BY VERIFYING VERSION FROM DATA FILE - $contentResourceDetails = returnContentResources($contentToImport.Version) - if ($null -eq $contentResourceDetails) { - Write-Host "Not able to identify content resource details based on Version. Please verify if Version in data input file is correct!" - exit 1; - } - - $dcWithoutSpace = ($baseFolderPath + $solutionName + "/DataConnectors/").Replace("//", "/") - $hasDCWithoutSpace = Test-Path -Path $dcWithoutSpace - - if ($hasDCWithoutSpace) { - $DCFolderName = "DataConnectors" - } - - if ($isCCPConnector -eq $false) { - $DCFolderName = "Data Connectors" - - $ccpDict = Get-CCP-Dict -dataFileMetadata $contentToImport -baseFolderPath $solutionBasePath -solutionName $solutionName -DCFolderName $DCFolderName - - if ($null -ne $ccpDict -and $ccpDict.count -gt 0) { - $isCCPConnector = $true - [array]$ccpTablesFilePaths = GetCCPTableFilePaths -existingCCPDict $ccpDict -baseFolderPath $solutionBasePath -solutionName $solutionName -DCFolderName $DCFolderName - } - } - Write-Host "isCCPConnector $isCCPConnector" - $ccpConnectorCodeExecutionCounter = 1; - foreach ($objectProperties in $contentToImport.PsObject.Properties) { - if ($objectProperties.Value -is [System.Array] -and $objectProperties.Name.ToLower() -ne 'dependentdomainsolutionids' -and $objectProperties.Name.ToLower() -ne 'staticdataconnectorids') { - foreach ($file in $objectProperties.Value) { - $file = $file.Replace("$basePath/", "").Replace("Solutions/", "").Replace("$solutionName/", "") - $finalPath = ($basePath + $solutionName + "/" + $file).Replace("//", "/") - $rawData = $null - try { - Write-Host "Downloading $finalPath" - $rawData = (New-Object System.Net.WebClient).DownloadString($finalPath) - } - catch { - Write-Host "Failed to download $finalPath -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red - break; - } - - try { - $json = ConvertFrom-Json $rawData -ErrorAction Stop; # Determine whether content is JSON or YAML - $validJson = $true; - } - catch { - $validJson = $false; - } - - if ($validJson) { - # If valid JSON, must be Workbook or Playbook - $objectKeyLowercase = $objectProperties.Name.ToLower() - if ($objectKeyLowercase -eq "workbooks") { - GetWorkbookDataMetadata -file $file -isPipelineRun $isPipelineRun -contentResourceDetails $contentResourceDetails -baseFolderPath $repositoryBasePath -contentToImport $contentToImport - } - elseif ($objectKeyLowercase -eq "playbooks") { - GetPlaybookDataMetadata -file $file -contentToImport $contentToImport -contentResourceDetails $contentResourceDetails -json $json -isPipelineRun $isPipelineRun - } - elseif ($objectKeyLowercase -eq "data connectors" -or $objectKeyLowercase -eq "dataconnectors") { - if ($ccpDict.Count -gt 0) { - $isCCPConnectorFile = $false; - foreach($item in $ccpDict) { - if ($item.DCDefinitionFullPath -eq $finalPath) { - $isCCPConnectorFile = $true - break; - } - } - - if ($isCCPConnectorFile -and $ccpConnectorCodeExecutionCounter -eq 1) { - # current file is a ccp connector - GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $ccpDict -solutionBasePath $basePath -solutionName $solutionName -ccpTables $ccpTablesFilePaths -ccpTablesCounter $ccpTablesCounter - - $ccpConnectorCodeExecutionCounter += 1 - } - elseif ($isCCPConnectorFile -and $ccpConnectorCodeExecutionCounter -gt 1) { - continue; - } - else { - # current file is a normal connector - GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $null -solutionBasePath $basePath -solutionName $solutionName -ccpTables $null -ccpTablesCounter $ccpTablesCounter - } - } - else { - # current file is a normal connector - GetDataConnectorMetadata -file $file -contentResourceDetails $contentResourceDetails -dataFileMetadata $contentToImport -solutionFileMetadata $baseMetadata -dcFolderName $DCFolderName -ccpDict $null -solutionBasePath $basePath -solutionName $solutionName -ccpTables $null -ccpTablesCounter $ccpTablesCounter - } - } - elseif ($objectKeyLowercase -eq "savedsearches") { - GenerateSavedSearches -json $json -contentResourceDetails $contentResourceDetails - } - elseif ($objectKeyLowercase -eq "watchlists") { - $watchListFileName = Get-ChildItem $finalPath - - GenerateWatchList -json $json -isPipelineRun $isPipelineRun -watchListFileName $watchListFileName.BaseName - } - } - else { - if ($file -match "(\.yaml)$" -and $objectProperties.Name.ToLower() -ne "parsers") { - $objectKeyLowercase = $objectProperties.Name.ToLower() - if ($objectKeyLowercase -eq "hunting queries") { - GetHuntingDataMetadata -file $file -rawData $rawData -contentResourceDetails $contentResourceDetails - } elseif ($objectKeyLowercase -eq "summary rules" -or $objectKeyLowercase -eq "summaryrules") { - $summaryRuleFilePath = $repositoryBasePath + "Tools/Create-Azure-Sentinel-Solution/common/summaryRules.ps1" - . $summaryRuleFilePath - GenerateSummaryRules -solutionName $solutionName -file $file -rawData $rawData -contentResourceDetails $contentResourceDetails - } - else { - GenerateAlertRule -file $file -contentResourceDetails $contentResourceDetails - } - } - else { - GenerateParsersList -file $file -contentToImport $contentToImport -contentResourceDetails $contentResourceDetails - } - } - } - } - elseif ($objectProperties.Name.ToLower() -eq "metadata") { - try { - $finalPath = $metadataPath - $rawData = $null - try { - Write-Host "Downloading $finalPath" - $rawData = (New-Object System.Net.WebClient).DownloadString($finalPath) - } - catch { - Write-Host "Failed to download $finalPath -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red - break; - } - - try { - $json = ConvertFrom-Json $rawData -ErrorAction Stop; # Determine whether content is JSON or YAML - $validJson = $true; - } - catch { - $validJson = $false; - } - - if ($validJson -and $json) { - PrepareSolutionMetadata -solutionMetadataRawContent $json -contentResourceDetails $contentResourceDetails -defaultPackageVersion $defaultPackageVersion - } - else { - Write-Host "Failed to load Metadata file $file -- Please ensure that it exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red - } - } - catch { - Write-Host "Failed to load Metadata file $file -- Please ensure that the SolutionMetadata file exists in $([System.Uri]::EscapeUriString($basePath))" -ForegroundColor Red - break; - } - } - } - - $global:analyticRuleCounter -= 1 -$global:workbookCounter -= 1 -$global:playbookCounter -= 1 -$global:connectorCounter -= 1 -$global:parserCounter -= 1 -$global:huntingQueryCounter -= 1 -$global:watchlistCounter -= 1 - $global:summaryRuleCounter -= 1 - -updateDescriptionCount $global:connectorCounter "**Data Connectors:** " "{{DataConnectorCount}}" $(checkResourceCounts $global:parserCounter, $global:analyticRuleCounter, $global:workbookCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) -updateDescriptionCount $global:parserCounter "**Parsers:** " "{{ParserCount}}" $(checkResourceCounts $global:analyticRuleCounter, $global:workbookCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) -updateDescriptionCount $global:workbookCounter "**Workbooks:** " "{{WorkbookCount}}" $(checkResourceCounts $global:analyticRuleCounter, $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) -updateDescriptionCount $global:analyticRuleCounter "**Analytic Rules:** " "{{AnalyticRuleCount}}" $(checkResourceCounts $global:playbookCounter, $global:huntingQueryCounter, $global:watchlistCounter, $global:summaryRuleCounter) -updateDescriptionCount $global:huntingQueryCounter "**Hunting Queries:** " "{{HuntingQueryCount}}" $(checkResourceCounts $global:playbookCounter, $global:watchlistCounter, $global:summaryRuleCounter) - -updateDescriptionCount $global:watchlistCounter "**Watchlists:** " "{{WatchlistCount}}" $(checkResourceCounts $global:playbookCounter, $global:summaryRuleCounter) - - updateDescriptionCount $global:summaryRuleCounter "**Summary Rules:** " "{{SummaryRuleCount}}" $(checkResourceCounts @($global:playbookCounter)) - -updateDescriptionCount $global:customConnectorsList.Count "**Custom Azure Logic Apps Connectors:** " "{{LogicAppCustomConnectorCount}}" $(checkResourceCounts @($global:playbookCounter)) -updateDescriptionCount $global:functionAppList.Count "**Function Apps:** " "{{FunctionAppsCount}}" $(checkResourceCounts @($global:playbookCounter)) - updateDescriptionCount ($global:playbookCounter - $global:customConnectorsList.Count - $global:functionAppList.Count) "**Playbooks:** " "{{PlaybookCount}}" $false - - GeneratePackage -solutionName $solutionName -contentToImport $contentToImport -calculatedBuildPipelinePackageVersion $contentToImport.Version; - RunArmTtkOnPackage -solutionName $solutionName -isPipelineRun $false; - - # check if mainTemplate and createUiDefinition json files are valid or not - CheckJsonIsValid($solutionFolderBasePath) - } -} -catch { - Write-Host "Error occurred in catch of createSolutionV3_LocalVersion file Error details are $_" -ForegroundColor Red -} From 4f6ea682fdc1deb41108f87cbc856aea0767e22d Mon Sep 17 00:00:00 2001 From: Fuqing Wang Date: Fri, 23 Jan 2026 13:02:52 -0800 Subject: [PATCH 3/3] update readme --- .../V3/README.md | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/README.md b/Tools/Create-Azure-Sentinel-Solution/V3/README.md index 36bf08e09a7..db82548a707 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/README.md +++ b/Tools/Create-Azure-Sentinel-Solution/V3/README.md @@ -269,9 +269,6 @@ cd Tools/Create-Azure-Sentinel-Solution/V3 # Or specify the data folder path directly ./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" - -# Explicitly specify catalog mode -./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "catalog" ``` #### **Local Version Bumping Mode** @@ -286,49 +283,6 @@ cd Tools/Create-Azure-Sentinel-Solution/V3 ./createSolutionV3.ps1 -SolutionDataFolderPath "C:\Github\Azure-Sentinel\Solutions\YourSolution\Data" -VersionMode "local" -VersionBump "major" ``` -#### **Interactive Mode** -If you don't specify the `-SolutionDataFolderPath` parameter, the script will prompt you for the data folder path: -```powershell -./createSolutionV3.ps1 -VersionMode "local" -VersionBump "patch" -# You will be prompted: "Enter solution data folder path" -# Just specify the data folder path from Solutions, e.g.: C:\Github\Azure-Sentinel\Solutions\Agari\Data -``` - -### Version Management Modes Comparison - -| Feature | Catalog Mode | Local Mode | -|---------|--------------|------------| -| **Version Source** | Microsoft Catalog API | Local solution data file | -| **Version Logic** | Increments based on catalog version | Semantic versioning (major.minor.patch) | -| **Internet Required** | Yes (for API calls) | No | -| **Use Case** | Production deployments | Development/testing | -| **File Updates** | No automatic updates | Automatically updates data and metadata files | - -### What Gets Updated in Local Mode - -When using local mode, the script automatically updates: -1. **Solution data file** (`Solution_[Name].json`) - Version property updated -2. **Solution metadata file** (referenced in data file) - Version property updated -3. **Generated package files** - Only version numbers change, no other content modifications - -### Parameters - -- **`SolutionDataFolderPath`** (Optional): Path to the solution data folder. If not provided, you'll be prompted to enter it. -- **`VersionMode`** (Optional): Choose between `"catalog"` (default) or `"local"` version management. -- **`VersionBump`** (Optional): For local mode, specify `"patch"`, `"minor"`, or `"major"` (default: `"patch"`). - -### Legacy Usage - -The original usage still works for backward compatibility: -```powershell -# From repository root, run: -./Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3.ps1 -# You will be prompted: "Enter solution data folder path" -# Example input: C:\Github\Azure-Sentinel\Solutions\Agari\Data -``` - -In the above examples, provide the path to the data folder only (without the file name). There is NO need to copy the data input file to the `Tools/input` folder. - The package consists of the following files: * `createUIDefinition.json`: Template containing the definition for the Deployment Creation UI