Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
94 commits
Select commit Hold shift + click to select a range
884a340
tests: add coverlet.msbuild dependency for coverage instrumentation
adamnova Aug 20, 2025
ee84e4a
tests(stream-processor): add encryptor tests (primitives, containers,…
adamnova Aug 20, 2025
cca71da
tests(stream-processor): add comprehensive decryptor tests incl. erro…
adamnova Aug 20, 2025
5b6d22c
tests: add StreamProcessor decrypt coverage for TypeMarker.Null and _…
adamnova Aug 21, 2025
3006f4d
update contracts and bump coverlet to 6.0.0
adamnova Aug 21, 2025
6120ba8
coverlet collector
adamnova Aug 21, 2025
659bea2
remove solution file from tracking to avoid local dev environment cha…
adamnova Aug 21, 2025
afb4dc6
Added new tests
adamnova Aug 22, 2025
99e8340
Added tests
adamnova Aug 22, 2025
ad7bfeb
Replaced parser with Utf8Parser
adamnova Aug 22, 2025
87c3f0b
Added new tests
adamnova Aug 22, 2025
96263aa
Cleanup
adamnova Aug 22, 2025
6d82c06
Update StreamProcessor.Decryptor.cs
adamnova Aug 22, 2025
33cbc55
Update StreamProcessor.Decryptor.cs
adamnova Aug 22, 2025
4a1a553
Merge branch 'master' into feature/stream-processor-tests-baseline
adamnova Aug 22, 2025
782e194
chore: restore Microsoft.Azure.Cosmos.sln from origin/master
adamnova Aug 22, 2025
5868509
Arrange Act Assert
adamnova Aug 22, 2025
7abd591
StreamProcessor: fix end-of-stream buffer growth; adopt safe fallback…
adamnova Aug 26, 2025
1092ffe
StreamProcessor: rename MdeEncryptor field to MdeEngine for clarity; …
adamnova Aug 26, 2025
56beaa4
ArrayPool: add Return() and early-return temp buffers in stream proce…
adamnova Aug 26, 2025
48538a6
Diagnostics: add basic metrics bag to CosmosDiagnosticsContext and in…
adamnova Aug 26, 2025
2b92b99
StreamProcessor: smarter buffer growth with 8MB cap and min increment…
adamnova Aug 26, 2025
baa05f3
Merge branch 'master' into feature/stream-processor-improvements
adamnova Aug 27, 2025
bc00584
Encryption.Custom: StreamProcessor.Decryptor – return old buffer to p…
adamnova Aug 27, 2025
4bcc056
Encryption.Custom: StreamProcessor.Decryptor – robust cross-buffer sk…
adamnova Aug 27, 2025
6dd9795
Encryption.Custom: StreamProcessor.Decryptor – support multi-segment …
adamnova Aug 27, 2025
1f8e775
Encryption.Custom: StreamProcessor.Decryptor – use stackalloc for sma…
adamnova Aug 27, 2025
7f88cf0
Encryption.Custom.Tests: add coverage for decrypt stream changes – cr…
adamnova Aug 27, 2025
3d576bb
tests(encryption-custom): temporarily ignore ContractChanges API base…
adamnova Aug 28, 2025
ad329c8
Encryption.Custom: avoid per-hit /name concat; reduce path lookups; z…
adamnova Aug 28, 2025
caf4c5c
Perf: add StreamProcessor path-matching benchmarks (Encrypt/Decrypt E…
adamnova Aug 28, 2025
b9d4dbc
Optimize buffer management in encryption/decryption
adamnova Aug 29, 2025
b4829a3
Improve encrypted payload handling in StreamProcessor
adamnova Aug 29, 2025
2db5959
Refactor buffer ownership and improve decryption handling
adamnova Aug 29, 2025
71c8c87
Encryptor: reduce allocations for fragmented string/number tokens by …
adamnova Sep 1, 2025
4f6c9c3
Encryptor: pre-size collections and manually write _ei metadata with …
adamnova Sep 1, 2025
77a3240
Encryptor: avoid Substring by precomputing UTF-8 property names and u…
adamnova Sep 1, 2025
838a1a7
Encryptor: revert manual _ei serialization in favor of EncryptionProp…
adamnova Sep 1, 2025
6975520
Optimize top-level encrypted path matching
adamnova Sep 1, 2025
65e2829
Optimize CandidatePaths for allocation-free matching
adamnova Sep 1, 2025
f8a3eea
Encryption.Custom: centralize _ei writer via WriteEncryptionInfo usin…
adamnova Sep 1, 2025
b47a5e3
Encryption.Custom: cut allocations on decrypt path by decoding base64…
adamnova Sep 2, 2025
f400d61
Add encryptedData to encryption info serialization
adamnova Sep 2, 2025
b385a13
Decryptor: add non-seekable small payload single-pass fast path + test
adamnova Sep 4, 2025
fa0e4c9
Decryptor: reduce initial buffer to 2KB + adaptive first rent & small…
adamnova Sep 4, 2025
bfb90bb
Decryptor: stackalloc small tokens & reuse single cipher decode buffe…
adamnova Sep 4, 2025
bab56d1
Encryptor: introduce DecryptInto span-based API (placeholder implemen…
adamnova Sep 4, 2025
cf2f60a
Decryptor: direct decrypt into shared plaintext buffer for base encry…
adamnova Sep 4, 2025
bf5db2f
Optimizations and improvements
adamnova Sep 4, 2025
c6061aa
Optimized Json buffer
adamnova Sep 4, 2025
09beeae
StreamProcessor: bucket EnsureCapacity to next power-of-two for scrat…
adamnova Sep 5, 2025
2f7acde
StreamProcessor: extract and cache CandidatePaths with UTF8 pre-encod…
adamnova Sep 5, 2025
bffb3d0
Optimizations
adamnova Sep 5, 2025
6803c12
Refactor
adamnova Sep 5, 2025
184bab8
Merge branch 'master' into feature/stream-processor-improvements
adamnova Sep 5, 2025
239e7d0
Delete tasks.json
adamnova Sep 5, 2025
8d11dee
Added diagnostics context
adamnova Sep 5, 2025
81671ae
Removed Brotli test
adamnova Sep 5, 2025
dc630bc
Added tests
adamnova Sep 5, 2025
8c5a582
Simplifications
adamnova Sep 5, 2025
165728d
Update StreamProcessor.Decryptor.cs
adamnova Sep 5, 2025
15f3a74
Encryption.Custom: replace magic format version numbers with Encrypti…
adamnova Sep 8, 2025
ce0cb0c
StreamProcessor: add test hook for max buffer size and overflow test …
adamnova Sep 8, 2025
93521b0
StreamProcessor tests: add forged Double/Boolean type marker invalid …
adamnova Sep 8, 2025
a42fb33
Cleanup
adamnova Sep 8, 2025
636f41d
StreamProcessor tests: add negative decompressed size guard test
adamnova Sep 8, 2025
b7e41cb
Encryption.Custom: Simplify CandidatePaths (remove cache & length mas…
adamnova Sep 8, 2025
d8d3dab
StreamProcessor: early finalize on root close for non-seekable stream…
adamnova Sep 8, 2025
30a9f15
Encryptor tests: add container buffering, compression threshold, and …
adamnova Sep 8, 2025
b37fb96
Emulator tests: enable ENCRYPTION_CUSTOM_PREVIEW and add initial Stre…
adamnova Sep 8, 2025
be0adbc
Added emulator tests
adamnova Sep 8, 2025
ace48ab
Update StreamProcessorEmulatorTests.cs
adamnova Sep 8, 2025
d9ce6bf
Encryption: make EncryptionProperties.EncryptedPaths IReadOnlyCollect…
adamnova Sep 10, 2025
8a27b76
Encryption.Custom: Remove non-seekable probe logic and related tests;…
adamnova Sep 10, 2025
d66de4a
Remove payload compression support from encryption
adamnova Sep 11, 2025
a7ef854
Pass Utf8JsonWriter to EncryptionPipelineState constructor
adamnova Sep 11, 2025
0656584
Remove manual bucketing from StreamProcessor
adamnova Sep 11, 2025
818ca62
Remove TryProcessSmallSeekableAsync micro optimization
adamnova Sep 11, 2025
fcccb07
Refactor Base64 decoding in Decryptor
adamnova Sep 11, 2025
45ac643
test(stream-processor): add strong encrypted plaintext absence verifi…
adamnova Sep 11, 2025
2cad2c0
test(stream-processor): unify encryption verification via base64 + ty…
adamnova Sep 11, 2025
9a19b66
Refactor EncryptionVerificationTestHelper imports and types
adamnova Sep 11, 2025
72a7560
Delete encustom-.net8.0.info
adamnova Sep 11, 2025
f0ae5b4
Refactor DecodeValidated to static method with more args
adamnova Sep 11, 2025
5835bba
Refactor ValidateRawEncryptedAsync to use property map
adamnova Sep 11, 2025
926dc5d
Refactor plainMap initialization in test helper
adamnova Sep 11, 2025
bc7fae1
Update type check for MdeEncryptor in Decryptor
adamnova Sep 11, 2025
a7c1431
Re-enable ContractChanges test
adamnova Sep 11, 2025
f5f08dc
Merge branch 'master' into feature/stream-processor-improvements
adamnova Oct 2, 2025
ad793ab
Merge branch 'master' into feature/stream-processor-improvements
adamnova Oct 6, 2025
9076ba9
Merge branch 'master' into feature/stream-processor-improvements
adamnova Oct 22, 2025
f449464
Remove ENCRYPTION_CUSTOM_PREVIEW flag and add tests for ValueSequence…
adamnova Oct 27, 2025
f4e3759
Merge branch 'master' into feature/stream-processor-improvements
adamnova Nov 5, 2025
5855757
Add diagnostics context to EncryptAsync interface
adamnova Nov 5, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,19 @@ public static async Task<Stream> EncryptAsync(
throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}.");
}

// Materialize paths to an IReadOnlyCollection to satisfy ctor signature explicitly
IReadOnlyCollection<string> pathsToEncryptCollection = encryptionOptions.PathsToEncrypt as IReadOnlyCollection<string>;
if (pathsToEncryptCollection == null)
{
pathsToEncryptCollection = new List<string>(encryptionOptions.PathsToEncrypt);
}

encryptionProperties = new EncryptionProperties(
encryptionFormatVersion: 2,
encryptionOptions.EncryptionAlgorithm,
encryptionOptions.DataEncryptionKeyId,
encryptedData: cipherText,
encryptionOptions.PathsToEncrypt);
encryptionFormatVersion: EncryptionFormatVersion.AeAes,
encryptionOptions.EncryptionAlgorithm,
encryptionOptions.DataEncryptionKeyId,
encryptedData: cipherText,
pathsToEncryptCollection);

itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,34 @@ public T[] Rent(int minimumLength)
return buffer;
}

public void Return(T[] buffer)
{
if (buffer == null)
{
return;
}

int idx = this.rentedBuffers?.IndexOf(buffer) ?? -1;
if (idx >= 0)
{
this.rentedBuffers.RemoveAt(idx);
ArrayPool<T>.Shared.Return(buffer, clearArray: true);
}
}

#if DEBUG
// Debug-only ownership probe to assert correct pooling contracts without impacting release perf
public bool IsOwned(T[] buffer)
{
if (buffer == null || this.rentedBuffers == null)
{
return false;
}

return this.rentedBuffers.IndexOf(buffer) >= 0;
}
#endif

protected virtual void Dispose(bool disposing)
{
if (!this.disposedValue)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;

/// <summary>
/// Lightweight diagnostics context for Custom Encryption extension.
/// Manages Activity creation for OpenTelemetry integration.
/// Manages Activity creation for OpenTelemetry integration and collects metrics.
/// </summary>
internal class CosmosDiagnosticsContext
{
Expand All @@ -27,6 +28,9 @@ internal class CosmosDiagnosticsContext
/// </summary>
internal const string ScopeDecryptModeSelectionPrefix = "EncryptionProcessor.Decrypt.Mde.";

// Simple metric bag for counters and durations (best-effort; thread-safe and optional)
private readonly ConcurrentDictionary<string, long> metrics = new (StringComparer.Ordinal);

internal CosmosDiagnosticsContext()
{
}
Expand Down Expand Up @@ -59,6 +63,34 @@ public Scope CreateScope(string scope)
return new Scope(activity);
}

// Record or update a metric with an absolute value.
public void SetMetric(string name, long value)
{
if (string.IsNullOrEmpty(name))
{
return;
}

this.metrics[name] = value;
}

// Increment a metric by delta (can be negative).
public void AddMetric(string name, long delta = 1)
{
if (string.IsNullOrEmpty(name))
{
return;
}

this.metrics.AddOrUpdate(name, delta, (_, current) => current + delta);
}

// Snapshot of current metrics (intended for diagnostics/testing only).
public IReadOnlyDictionary<string, long> GetMetricsSnapshot()
{
return new Dictionary<string, long>(this.metrics);
}

/// <summary>
/// Represents a diagnostic scope for Activity tracking.
/// Must be used with the 'using' pattern to ensure proper disposal.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -831,7 +831,12 @@ public override Task<ItemResponse<T>> PatchItemAsync<T>(
PatchItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
return this.PatchItemCoreAsync<T>(
id,
partitionKey,
patchOperations,
requestOptions,
cancellationToken);
}

public override Task<ResponseMessage> PatchItemStreamAsync(
Expand All @@ -841,7 +846,71 @@ public override Task<ResponseMessage> PatchItemStreamAsync(
PatchItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
return this.PatchItemCoreStreamAsync(
id,
partitionKey,
patchOperations,
requestOptions,
cancellationToken);
}

private async Task<ItemResponse<T>> PatchItemCoreAsync<T>(
string id,
PartitionKey partitionKey,
IReadOnlyList<PatchOperation> patchOperations,
PatchItemRequestOptions requestOptions,
CancellationToken cancellationToken)
{
ResponseMessage response = await this.PatchItemCoreStreamAsync(
id,
partitionKey,
patchOperations,
requestOptions,
cancellationToken);

return this.ResponseFactory.CreateItemResponse<T>(response);
}

// Minimal patch implementation: only supports patching non-encrypted paths. If an encrypted path is modified,
// the value won't be automatically encrypted – future enhancement would mirror EncryptPatchOperationsAsync from base encryption container.
private async Task<ResponseMessage> PatchItemCoreStreamAsync(
string id,
PartitionKey partitionKey,
IReadOnlyList<PatchOperation> patchOperations,
PatchItemRequestOptions requestOptions,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentNullException(nameof(id));
}

if (patchOperations == null || patchOperations.Count == 0)
{
throw new ArgumentNullException(nameof(patchOperations));
}

CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateScope("PatchItemStream"))
{
ResponseMessage response = await this.container.PatchItemStreamAsync(
id,
partitionKey,
patchOperations,
requestOptions,
cancellationToken);

if (response.IsSuccessStatusCode && response.Content != null)
{
(response.Content, _) = await EncryptionProcessor.DecryptAsync(
response.Content,
this.Encryptor,
diagnosticsContext,
cancellationToken);
}

return response;
}
}

public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder<T>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ internal class EncryptionProperties

[JsonProperty(PropertyName = Constants.EncryptedPaths)]
[JsonPropertyName(Constants.EncryptedPaths)]
public IEnumerable<string> EncryptedPaths { get; }
public IReadOnlyCollection<string> EncryptedPaths { get; }

public EncryptionProperties(
int encryptionFormatVersion,
string encryptionAlgorithm,
string dataEncryptionKeyId,
byte[] encryptedData,
IEnumerable<string> encryptedPaths)
IReadOnlyCollection<string> encryptedPaths)
{
this.EncryptionFormatVersion = encryptionFormatVersion;
this.EncryptionAlgorithm = encryptionAlgorithm;
Expand Down
45 changes: 45 additions & 0 deletions Microsoft.Azure.Cosmos.Encryption.Custom/src/PooledByteOwner.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------

namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System;
using System.Buffers;

// Small wrapper to make pooled ownership explicit and efficient via Memory/Span
internal sealed class PooledByteOwner : IMemoryOwner<byte>, IDisposable
{
private readonly ArrayPoolManager pool;

internal byte[] Buffer { get; private set; }

internal int Length { get; private set; }

private bool disposed;

public PooledByteOwner(ArrayPoolManager pool, byte[] buffer, int length)
{
this.pool = pool ?? throw new ArgumentNullException(nameof(pool));
this.Buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
this.Length = length;
}

public Memory<byte> Memory => new Memory<byte>(this.Buffer, 0, this.Length);

internal byte[] Array => this.Buffer;

public void Dispose()
{
if (this.disposed)
{
return;
}

this.disposed = true;
this.pool.Return(this.Buffer);
this.Buffer = System.Array.Empty<byte>();
this.Length = 0;
}
}
}
Loading