From 134a60c64d47d28dabaf12ef25d58af28c5c75d6 Mon Sep 17 00:00:00 2001 From: swagger-automation Date: Thu, 21 Sep 2023 08:21:22 +0000 Subject: [PATCH 1/6] [AutoSync] 5a9813b211 Add lakehouse swagger --- .../2018-06-01/entityTypes/Dataset.json | 36 ++++++ .../2018-06-01/entityTypes/LinkedService.json | 63 ++++++++++ .../2018-06-01/entityTypes/Pipeline.json | 115 ++++++++++++++++++ 3 files changed, 214 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index d50df09e48e3..a33539d119c8 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -290,6 +290,15 @@ } ] }, + "LakehouseLocation": { + "description": "The location of Microsoft Fabric Lakehouse Files dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, "DatasetDataElement": { "description": "Columns that define the structure of the dataset.", "type": "object", @@ -3578,6 +3587,33 @@ "description": "The database name of delta table. Type: string (or Expression with resultType string)." } } + }, + "LakehouseTableDataset": { + "x-ms-discriminator-value": "LakehouseTable", + "description": "Microsoft Fabric Lakehouse Table.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "Microsoft Fabric Lakehouse Table dataset properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/AzureBlobFSDatasetTypeProperties" + } + } + }, + "LakehouseTableDatasetTypeProperties": { + "description": "Microsoft Fabric Lakehouse Table dataset properties.", + "properties": { + "table": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The name of Microsoft Fabric Lakehouse Table. Type: string (or Expression with resultType string)." + } + } } } } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index df295b051853..8518dd3f1a9a 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -7464,6 +7464,69 @@ "required": [ "endpoint" ] + }, + "LakehouseLinkedService": { + "x-ms-discriminator-value": "Lakehouse", + "description": "Microsoft Fabric Lakehouse linked service.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/LinkedService" + } + ], + "properties": { + "typeProperties": { + "description": "Microsoft Fabric Lakehouse linked service properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/LakehouseLinkedServiceTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "LakehouseLinkedServiceTypeProperties": { + "description": "Microsoft Fabric Lakehouse linked service properties.", + "type": "object", + "properties": { + "workspaceId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of Microsoft Fabric workspace. Type: string (or Expression with resultType string)." + }, + "artifactId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of Microsoft Fabric Lakehouse artifact. Type: string (or Expression with resultType string)." + }, + "servicePrincipalId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of the application used to authenticate against Microsoft Fabric Lakehouse. Type: string (or Expression with resultType string)." + }, + "servicePrincipalKey": { + "description": "The Key of the application used to authenticate against Microsoft Fabric Lakehouse.", + "$ref": "../datafactory.json#/definitions/SecretBase" + }, + "tenant": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The name or ID of the tenant to which the service principal belongs. Type: string (or Expression with resultType string)." + }, + "encryptedCredential": { + "type": "string", + "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string." + }, + "servicePrincipalCredentialType": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The service principal credential type to use in Server-To-Server authentication. 'ServicePrincipalKey' for key/secret, 'ServicePrincipalCert' for certificate. Type: string (or Expression with resultType string)." + }, + "servicePrincipalCredential": { + "description": "The credential of the service principal object in Azure Active Directory. If servicePrincipalCredentialType is 'ServicePrincipalKey', servicePrincipalCredential can be SecureString or AzureKeyVaultSecretReference. If servicePrincipalCredentialType is 'ServicePrincipalCert', servicePrincipalCredential can only be AzureKeyVaultSecretReference.", + "$ref": "../datafactory.json#/definitions/SecretBase" + } + } } } } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 9f358df1122c..3cec737e9d35 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1092,6 +1092,62 @@ } } }, + "LakehouseReadSettings": { + "description": "Microsoft Fabric Lakehouse Files read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/StoreReadSettings" + } + ], + "properties": { + "recursive": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Microsoft Fabric Lakehouse Files wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Microsoft Fabric Lakehouse Files wildcardFileName. Type: string (or Expression with resultType string)." + }, + "fileListPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Point to a text file that lists each file (relative path to the path configured in the dataset) that you want to copy. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "Indicates whether to enable partition discovery. Type: boolean (or Expression with resultType boolean)." + }, + "partitionRootPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Specify the root path where partition discovery starts from. Type: string (or Expression with resultType string)." + }, + "deleteFilesAfterCompletion": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "Indicates whether the source files need to be deleted after copy completion. Default is false. Type: boolean (or Expression with resultType boolean)." + }, + "modifiedDatetimeStart": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, "StoreWriteSettings": { "description": "Connector write settings.", "discriminator": "type", @@ -1189,6 +1245,15 @@ } ] }, + "LakehouseWriteSettings": { + "description": "Microsoft Fabric Lakehouse Files write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/StoreWriteSettings" + } + ] + }, "FormatReadSettings": { "description": "Format read settings.", "discriminator": "type", @@ -4118,6 +4183,31 @@ } } }, + "LakehouseTableSource": { + "description": "A copy activity source for Microsoft Fabric Lakehouse Table.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "timestampAsOf": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Query an older snapshot by timestamp. Type: string (or Expression with resultType string)." + }, + "versionAsOf": { + "type": "object", + "x-ms-format": "dfe-int", + "description": "Query an older snapshot by version. Type: integer (or Expression with resultType integer)." + }, + "additionalColumns": { + "type": "object", + "description": "Specifies the additional columns to be added to source data. Type: array of objects(AdditionalColumns) (or Expression with resultType array of objects)." + } + } + }, "RedshiftUnloadSettings": { "description": "The Amazon S3 settings needed for the interim Amazon S3 when copying from Amazon Redshift with unload. With this, data from Amazon Redshift source will be unloaded into S3 first and then copied into the targeted sink from the interim S3.", "type": "object", @@ -5599,6 +5689,31 @@ } } }, + "LakehouseTableSink": { + "description": "A copy activity for Microsoft Fabric Lakehouse Table sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": { + "tableActionOption": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The type of table action for Lakehouse Table sink. Possible values include: \"None\", \"Append\", \"Overwrite\"." + }, + "partitionOption": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Create partitions in folder structure based on one or multiple columns. Each distinct column value (pair) will be a new partition. Possible values include: \"None\", \"PartitionByKey\"." + }, + "partitionNameList": { + "type": "object", + "description": "Specify the partition column names from sink columns. Type: array of objects (or Expression with resultType array of objects)." + } + } + }, "CopyTranslator": { "discriminator": "type", "description": "A copy activity translator.", From d5976b635bea67fc914ce2889a0d1d115dee7b01 Mon Sep 17 00:00:00 2001 From: swagger-automation Date: Thu, 21 Sep 2023 08:55:38 +0000 Subject: [PATCH 2/6] [AutoSync] 1cc7912bdd Fix spelling --- .../2018-06-01/entityTypes/Dataset.json | 18 +++++++-------- .../2018-06-01/entityTypes/LinkedService.json | 20 ++++++++--------- .../2018-06-01/entityTypes/Pipeline.json | 22 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index a33539d119c8..a771b83f0608 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -290,8 +290,8 @@ } ] }, - "LakehouseLocation": { - "description": "The location of Microsoft Fabric Lakehouse Files dataset.", + "LakeHouseLocation": { + "description": "The location of Microsoft Fabric LakeHouse Files dataset.", "type": "object", "allOf": [ { @@ -3588,9 +3588,9 @@ } } }, - "LakehouseTableDataset": { - "x-ms-discriminator-value": "LakehouseTable", - "description": "Microsoft Fabric Lakehouse Table.", + "LakeHouseTableDataset": { + "x-ms-discriminator-value": "LakeHouseTable", + "description": "Microsoft Fabric LakeHouse Table.", "type": "object", "allOf": [ { @@ -3599,19 +3599,19 @@ ], "properties": { "typeProperties": { - "description": "Microsoft Fabric Lakehouse Table dataset properties.", + "description": "Microsoft Fabric LakeHouse Table dataset properties.", "x-ms-client-flatten": true, "$ref": "#/definitions/AzureBlobFSDatasetTypeProperties" } } }, - "LakehouseTableDatasetTypeProperties": { - "description": "Microsoft Fabric Lakehouse Table dataset properties.", + "LakeHouseTableDatasetTypeProperties": { + "description": "Microsoft Fabric LakeHouse Table dataset properties.", "properties": { "table": { "type": "object", "x-ms-format": "dfe-string", - "description": "The name of Microsoft Fabric Lakehouse Table. Type: string (or Expression with resultType string)." + "description": "The name of Microsoft Fabric LakeHouse Table. Type: string (or Expression with resultType string)." } } } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index 8518dd3f1a9a..53cb598af54f 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -7465,9 +7465,9 @@ "endpoint" ] }, - "LakehouseLinkedService": { - "x-ms-discriminator-value": "Lakehouse", - "description": "Microsoft Fabric Lakehouse linked service.", + "LakeHouseLinkedService": { + "x-ms-discriminator-value": "LakeHouse", + "description": "Microsoft Fabric LakeHouse linked service.", "type": "object", "allOf": [ { @@ -7476,17 +7476,17 @@ ], "properties": { "typeProperties": { - "description": "Microsoft Fabric Lakehouse linked service properties.", + "description": "Microsoft Fabric LakeHouse linked service properties.", "x-ms-client-flatten": true, - "$ref": "#/definitions/LakehouseLinkedServiceTypeProperties" + "$ref": "#/definitions/LakeHouseLinkedServiceTypeProperties" } }, "required": [ "typeProperties" ] }, - "LakehouseLinkedServiceTypeProperties": { - "description": "Microsoft Fabric Lakehouse linked service properties.", + "LakeHouseLinkedServiceTypeProperties": { + "description": "Microsoft Fabric LakeHouse linked service properties.", "type": "object", "properties": { "workspaceId": { @@ -7497,15 +7497,15 @@ "artifactId": { "type": "object", "x-ms-format": "dfe-string", - "description": "The ID of Microsoft Fabric Lakehouse artifact. Type: string (or Expression with resultType string)." + "description": "The ID of Microsoft Fabric LakeHouse artifact. Type: string (or Expression with resultType string)." }, "servicePrincipalId": { "type": "object", "x-ms-format": "dfe-string", - "description": "The ID of the application used to authenticate against Microsoft Fabric Lakehouse. Type: string (or Expression with resultType string)." + "description": "The ID of the application used to authenticate against Microsoft Fabric LakeHouse. Type: string (or Expression with resultType string)." }, "servicePrincipalKey": { - "description": "The Key of the application used to authenticate against Microsoft Fabric Lakehouse.", + "description": "The Key of the application used to authenticate against Microsoft Fabric LakeHouse.", "$ref": "../datafactory.json#/definitions/SecretBase" }, "tenant": { diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 3cec737e9d35..d110fbe1e9cc 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1092,8 +1092,8 @@ } } }, - "LakehouseReadSettings": { - "description": "Microsoft Fabric Lakehouse Files read settings.", + "LakeHouseReadSettings": { + "description": "Microsoft Fabric LakeHouse Files read settings.", "type": "object", "allOf": [ { @@ -1109,12 +1109,12 @@ "wildcardFolderPath": { "type": "object", "x-ms-format": "dfe-string", - "description": "Microsoft Fabric Lakehouse Files wildcardFolderPath. Type: string (or Expression with resultType string)." + "description": "Microsoft Fabric LakeHouse Files wildcardFolderPath. Type: string (or Expression with resultType string)." }, "wildcardFileName": { "type": "object", "x-ms-format": "dfe-string", - "description": "Microsoft Fabric Lakehouse Files wildcardFileName. Type: string (or Expression with resultType string)." + "description": "Microsoft Fabric LakeHouse Files wildcardFileName. Type: string (or Expression with resultType string)." }, "fileListPath": { "type": "object", @@ -1245,8 +1245,8 @@ } ] }, - "LakehouseWriteSettings": { - "description": "Microsoft Fabric Lakehouse Files write settings.", + "LakeHouseWriteSettings": { + "description": "Microsoft Fabric LakeHouse Files write settings.", "type": "object", "allOf": [ { @@ -4183,8 +4183,8 @@ } } }, - "LakehouseTableSource": { - "description": "A copy activity source for Microsoft Fabric Lakehouse Table.", + "LakeHouseTableSource": { + "description": "A copy activity source for Microsoft Fabric LakeHouse Table.", "type": "object", "allOf": [ { @@ -5689,8 +5689,8 @@ } } }, - "LakehouseTableSink": { - "description": "A copy activity for Microsoft Fabric Lakehouse Table sink.", + "LakeHouseTableSink": { + "description": "A copy activity for Microsoft Fabric LakeHouse Table sink.", "type": "object", "allOf": [ { @@ -5701,7 +5701,7 @@ "tableActionOption": { "type": "object", "x-ms-format": "dfe-string", - "description": "The type of table action for Lakehouse Table sink. Possible values include: \"None\", \"Append\", \"Overwrite\"." + "description": "The type of table action for LakeHouse Table sink. Possible values include: \"None\", \"Append\", \"Overwrite\"." }, "partitionOption": { "type": "object", From 449f59f551c1be9a285d469be2198bdd0a33b9bf Mon Sep 17 00:00:00 2001 From: swagger-automation Date: Tue, 31 Oct 2023 03:17:56 +0000 Subject: [PATCH 3/6] [AutoSync] bc53c0c6db Add type --- .../stable/2018-06-01/entityTypes/Dataset.json | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index a771b83f0608..dd22ea07b506 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -3607,6 +3607,7 @@ }, "LakeHouseTableDatasetTypeProperties": { "description": "Microsoft Fabric LakeHouse Table dataset properties.", + "type": "object", "properties": { "table": { "type": "object", From c058548baf8c963f7c9c43cc6ad03029e615a45c Mon Sep 17 00:00:00 2001 From: Jaylen Zhang Date: Mon, 6 Nov 2023 11:27:14 +0800 Subject: [PATCH 4/6] Add swagger change for Synapse --- .../2020-12-01/entityTypes/Dataset.json | 37 ++++++ .../2020-12-01/entityTypes/LinkedService.json | 63 ++++++++++ .../2020-12-01/entityTypes/Pipeline.json | 115 ++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json index d4cd76efa674..b69046feee28 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json @@ -223,6 +223,15 @@ } ] }, + "LakeHouseLocation": { + "description": "The location of Microsoft Fabric LakeHouse Files dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/DatasetLocation" + } + ] + }, "DatasetDataElement": { "description": "Columns that define the structure of the dataset.", "type": "object", @@ -3423,6 +3432,34 @@ "description": "The database name of delta table. Type: string (or Expression with resultType string)." } } + }, + "LakeHouseTableDataset": { + "x-ms-discriminator-value": "LakeHouseTable", + "description": "Microsoft Fabric LakeHouse Table.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "Microsoft Fabric LakeHouse Table dataset properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/AzureBlobFSDatasetTypeProperties" + } + } + }, + "LakeHouseTableDatasetTypeProperties": { + "description": "Microsoft Fabric LakeHouse Table dataset properties.", + "type": "object", + "properties": { + "table": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The name of Microsoft Fabric LakeHouse Table. Type: string (or Expression with resultType string)." + } + } } } } diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json index bbca978ede88..717477291f00 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json @@ -6864,6 +6864,69 @@ "endpoint" ] }, + "LakeHouseLinkedService": { + "x-ms-discriminator-value": "LakeHouse", + "description": "Microsoft Fabric LakeHouse linked service.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/LinkedService" + } + ], + "properties": { + "typeProperties": { + "description": "Microsoft Fabric LakeHouse linked service properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/LakeHouseLinkedServiceTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "LakeHouseLinkedServiceTypeProperties": { + "description": "Microsoft Fabric LakeHouse linked service properties.", + "type": "object", + "properties": { + "workspaceId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of Microsoft Fabric workspace. Type: string (or Expression with resultType string)." + }, + "artifactId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of Microsoft Fabric LakeHouse artifact. Type: string (or Expression with resultType string)." + }, + "servicePrincipalId": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The ID of the application used to authenticate against Microsoft Fabric LakeHouse. Type: string (or Expression with resultType string)." + }, + "servicePrincipalKey": { + "description": "The Key of the application used to authenticate against Microsoft Fabric LakeHouse.", + "$ref": "../datafactory.json#/definitions/SecretBase" + }, + "tenant": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The name or ID of the tenant to which the service principal belongs. Type: string (or Expression with resultType string)." + }, + "encryptedCredential": { + "type": "string", + "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string." + }, + "servicePrincipalCredentialType": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The service principal credential type to use in Server-To-Server authentication. 'ServicePrincipalKey' for key/secret, 'ServicePrincipalCert' for certificate. Type: string (or Expression with resultType string)." + }, + "servicePrincipalCredential": { + "description": "The credential of the service principal object in Azure Active Directory. If servicePrincipalCredentialType is 'ServicePrincipalKey', servicePrincipalCredential can be SecureString or AzureKeyVaultSecretReference. If servicePrincipalCredentialType is 'ServicePrincipalCert', servicePrincipalCredential can only be AzureKeyVaultSecretReference.", + "$ref": "../datafactory.json#/definitions/SecretBase" + } + } + }, "PowerBIWorkspaceLinkedService": { "x-ms-discriminator-value": "PowerBIWorkspace", "description": "Power BI Workspace linked service.", diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json index 94797ac53a01..49b03edc89d3 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json @@ -825,6 +825,62 @@ } } }, + "LakeHouseReadSettings": { + "description": "Microsoft Fabric LakeHouse Files read settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/StoreReadSettings" + } + ], + "properties": { + "recursive": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." + }, + "wildcardFolderPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Microsoft Fabric LakeHouse Files wildcardFolderPath. Type: string (or Expression with resultType string)." + }, + "wildcardFileName": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Microsoft Fabric LakeHouse Files wildcardFileName. Type: string (or Expression with resultType string)." + }, + "fileListPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Point to a text file that lists each file (relative path to the path configured in the dataset) that you want to copy. Type: string (or Expression with resultType string)." + }, + "enablePartitionDiscovery": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "Indicates whether to enable partition discovery. Type: boolean (or Expression with resultType boolean)." + }, + "partitionRootPath": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Specify the root path where partition discovery starts from. Type: string (or Expression with resultType string)." + }, + "deleteFilesAfterCompletion": { + "type": "object", + "x-ms-format": "dfe-bool", + "description": "Indicates whether the source files need to be deleted after copy completion. Default is false. Type: boolean (or Expression with resultType boolean)." + }, + "modifiedDatetimeStart": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." + }, + "modifiedDatetimeEnd": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." + } + } + }, "StoreWriteSettings": { "description": "Connector write settings.", "discriminator": "type", @@ -913,6 +969,15 @@ } ] }, + "LakeHouseWriteSettings": { + "description": "Microsoft Fabric LakeHouse Files write settings.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/StoreWriteSettings" + } + ] + }, "FormatReadSettings": { "description": "Format read settings.", "discriminator": "type", @@ -3669,6 +3734,31 @@ } } }, + "LakeHouseTableSource": { + "description": "A copy activity source for Microsoft Fabric LakeHouse Table.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "timestampAsOf": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Query an older snapshot by timestamp. Type: string (or Expression with resultType string)." + }, + "versionAsOf": { + "type": "object", + "x-ms-format": "dfe-int", + "description": "Query an older snapshot by version. Type: integer (or Expression with resultType integer)." + }, + "additionalColumns": { + "type": "object", + "description": "Specifies the additional columns to be added to source data. Type: array of objects(AdditionalColumns) (or Expression with resultType array of objects)." + } + } + }, "RedshiftUnloadSettings": { "description": "The Amazon S3 settings needed for the interim Amazon S3 when copying from Amazon Redshift with unload. With this, data from Amazon Redshift source will be unloaded into S3 first and then copied into the targeted sink from the interim S3.", "type": "object", @@ -4908,6 +4998,31 @@ } } }, + "LakeHouseTableSink": { + "description": "A copy activity for Microsoft Fabric LakeHouse Table sink.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySink" + } + ], + "properties": { + "tableActionOption": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "The type of table action for LakeHouse Table sink. Possible values include: \"None\", \"Append\", \"Overwrite\"." + }, + "partitionOption": { + "type": "object", + "x-ms-format": "dfe-string", + "description": "Create partitions in folder structure based on one or multiple columns. Each distinct column value (pair) will be a new partition. Possible values include: \"None\", \"PartitionByKey\"." + }, + "partitionNameList": { + "type": "object", + "description": "Specify the partition column names from sink columns. Type: array of objects (or Expression with resultType array of objects)." + } + } + }, "CopyTranslator": { "discriminator": "type", "description": "A copy activity translator.", From dfae4ca3320a8d63be9ab84054fba5c5fe8a28e8 Mon Sep 17 00:00:00 2001 From: Jaylen Zhang Date: Mon, 6 Nov 2023 13:27:24 +0800 Subject: [PATCH 5/6] Fix Synapse --- .../stable/2020-12-01/entityTypes/Dataset.json | 1 - .../2020-12-01/entityTypes/LinkedService.json | 9 ++------- .../stable/2020-12-01/entityTypes/Pipeline.json | 13 ------------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json index b69046feee28..93f888863cf4 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json @@ -3456,7 +3456,6 @@ "properties": { "table": { "type": "object", - "x-ms-format": "dfe-string", "description": "The name of Microsoft Fabric LakeHouse Table. Type: string (or Expression with resultType string)." } } diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json index 717477291f00..b0cd320db02c 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/LinkedService.json @@ -6890,26 +6890,22 @@ "properties": { "workspaceId": { "type": "object", - "x-ms-format": "dfe-string", "description": "The ID of Microsoft Fabric workspace. Type: string (or Expression with resultType string)." }, "artifactId": { "type": "object", - "x-ms-format": "dfe-string", "description": "The ID of Microsoft Fabric LakeHouse artifact. Type: string (or Expression with resultType string)." }, "servicePrincipalId": { "type": "object", - "x-ms-format": "dfe-string", "description": "The ID of the application used to authenticate against Microsoft Fabric LakeHouse. Type: string (or Expression with resultType string)." }, "servicePrincipalKey": { "description": "The Key of the application used to authenticate against Microsoft Fabric LakeHouse.", - "$ref": "../datafactory.json#/definitions/SecretBase" + "$ref": "../artifacts.json#/definitions/SecretBase" }, "tenant": { "type": "object", - "x-ms-format": "dfe-string", "description": "The name or ID of the tenant to which the service principal belongs. Type: string (or Expression with resultType string)." }, "encryptedCredential": { @@ -6918,12 +6914,11 @@ }, "servicePrincipalCredentialType": { "type": "object", - "x-ms-format": "dfe-string", "description": "The service principal credential type to use in Server-To-Server authentication. 'ServicePrincipalKey' for key/secret, 'ServicePrincipalCert' for certificate. Type: string (or Expression with resultType string)." }, "servicePrincipalCredential": { "description": "The credential of the service principal object in Azure Active Directory. If servicePrincipalCredentialType is 'ServicePrincipalKey', servicePrincipalCredential can be SecureString or AzureKeyVaultSecretReference. If servicePrincipalCredentialType is 'ServicePrincipalCert', servicePrincipalCredential can only be AzureKeyVaultSecretReference.", - "$ref": "../datafactory.json#/definitions/SecretBase" + "$ref": "../artifacts.json#/definitions/SecretBase" } } }, diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json index 49b03edc89d3..95fbd77db21c 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Pipeline.json @@ -836,47 +836,38 @@ "properties": { "recursive": { "type": "object", - "x-ms-format": "dfe-bool", "description": "If true, files under the folder path will be read recursively. Default is true. Type: boolean (or Expression with resultType boolean)." }, "wildcardFolderPath": { "type": "object", - "x-ms-format": "dfe-string", "description": "Microsoft Fabric LakeHouse Files wildcardFolderPath. Type: string (or Expression with resultType string)." }, "wildcardFileName": { "type": "object", - "x-ms-format": "dfe-string", "description": "Microsoft Fabric LakeHouse Files wildcardFileName. Type: string (or Expression with resultType string)." }, "fileListPath": { "type": "object", - "x-ms-format": "dfe-string", "description": "Point to a text file that lists each file (relative path to the path configured in the dataset) that you want to copy. Type: string (or Expression with resultType string)." }, "enablePartitionDiscovery": { "type": "object", - "x-ms-format": "dfe-bool", "description": "Indicates whether to enable partition discovery. Type: boolean (or Expression with resultType boolean)." }, "partitionRootPath": { "type": "object", - "x-ms-format": "dfe-string", "description": "Specify the root path where partition discovery starts from. Type: string (or Expression with resultType string)." }, "deleteFilesAfterCompletion": { "type": "object", - "x-ms-format": "dfe-bool", "description": "Indicates whether the source files need to be deleted after copy completion. Default is false. Type: boolean (or Expression with resultType boolean)." }, "modifiedDatetimeStart": { "type": "object", - "x-ms-format": "dfe-string", "description": "The start of file's modified datetime. Type: string (or Expression with resultType string)." }, "modifiedDatetimeEnd": { "type": "object", - "x-ms-format": "dfe-string", "description": "The end of file's modified datetime. Type: string (or Expression with resultType string)." } } @@ -3745,12 +3736,10 @@ "properties": { "timestampAsOf": { "type": "object", - "x-ms-format": "dfe-string", "description": "Query an older snapshot by timestamp. Type: string (or Expression with resultType string)." }, "versionAsOf": { "type": "object", - "x-ms-format": "dfe-int", "description": "Query an older snapshot by version. Type: integer (or Expression with resultType integer)." }, "additionalColumns": { @@ -5009,12 +4998,10 @@ "properties": { "tableActionOption": { "type": "object", - "x-ms-format": "dfe-string", "description": "The type of table action for LakeHouse Table sink. Possible values include: \"None\", \"Append\", \"Overwrite\"." }, "partitionOption": { "type": "object", - "x-ms-format": "dfe-string", "description": "Create partitions in folder structure based on one or multiple columns. Each distinct column value (pair) will be a new partition. Possible values include: \"None\", \"PartitionByKey\"." }, "partitionNameList": { From ec49e5c5b828a2a10e201d6705b2ffa38f022883 Mon Sep 17 00:00:00 2001 From: Jaylen Zhang Date: Mon, 6 Nov 2023 13:33:14 +0800 Subject: [PATCH 6/6] Fix reference --- .../stable/2018-06-01/entityTypes/Dataset.json | 2 +- .../stable/2020-12-01/entityTypes/Dataset.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index dd22ea07b506..0db9c0206342 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -3601,7 +3601,7 @@ "typeProperties": { "description": "Microsoft Fabric LakeHouse Table dataset properties.", "x-ms-client-flatten": true, - "$ref": "#/definitions/AzureBlobFSDatasetTypeProperties" + "$ref": "#/definitions/LakeHouseTableDatasetTypeProperties" } } }, diff --git a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json index 93f888863cf4..3a69786afcf8 100644 --- a/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json +++ b/specification/synapse/data-plane/Microsoft.Synapse/stable/2020-12-01/entityTypes/Dataset.json @@ -3446,7 +3446,7 @@ "typeProperties": { "description": "Microsoft Fabric LakeHouse Table dataset properties.", "x-ms-client-flatten": true, - "$ref": "#/definitions/AzureBlobFSDatasetTypeProperties" + "$ref": "#/definitions/LakeHouseTableDatasetTypeProperties" } } },