[Server] Handle ApplicationCertificate updates using events: close channels on update and block new Connections / Requests

Libraries/Opc.Ua.Gds.Client.Common/GlobalDiscoveryServerClient.cs

@@ -376,6 +376,7 @@ await CoreClientUtils.SelectEndpointAsync(
                 }
                 catch (ServiceResultException e) when ((e.StatusCode is
                     StatusCodes.BadServerHalted or
+                    StatusCodes.BadConnectionClosed or
                     StatusCodes.BadSecureChannelClosed or
                     StatusCodes.BadNoCommunication) &&
                     attempt < maxAttempts)
@@ -432,6 +433,7 @@ public async Task ConnectAsync(ConfiguredEndpoint endpoint, CancellationToken ct
                 catch (ServiceResultException e) when ((e.StatusCode is
                     StatusCodes.BadServerHalted or
                     StatusCodes.BadSecureChannelClosed or
+                    StatusCodes.BadConnectionClosed or
                     StatusCodes.BadNoCommunication) &&
                     attempt < maxAttempts)
                 {

Libraries/Opc.Ua.Gds.Client.Common/ServerPushConfigurationClient.cs

@@ -290,6 +290,7 @@ await CoreClientUtils.SelectEndpointAsync(
                 catch (ServiceResultException e) when ((e.StatusCode is
                     StatusCodes.BadServerHalted or
                     StatusCodes.BadSecureChannelClosed or
+                    StatusCodes.BadConnectionClosed or
                     StatusCodes.BadNoCommunication) &&
                     attempt < maxAttempts)
                 {
@@ -345,6 +346,7 @@ public async Task ConnectAsync(ConfiguredEndpoint endpoint, CancellationToken ct
                 catch (ServiceResultException e) when ((e.StatusCode is
                     StatusCodes.BadServerHalted or
                     StatusCodes.BadSecureChannelClosed or
+                    StatusCodes.BadConnectionClosed or
                     StatusCodes.BadNoCommunication) &&
                     attempt < maxAttempts)
                 {

Libraries/Opc.Ua.Server/Server/StandardServer.cs

@@ -3340,6 +3340,7 @@ await subscriptionManager.StartupAsync(cancellationToken)
             }
 
             CertificateValidator.CertificateUpdate += OnCertificateUpdateAsync;
+            CertificateValidator.CertificateUpdateStarted += OnCertificateUpdateStartedAsync;
         }
 
         /// <summary>

Stack/Opc.Ua.Bindings.Https/Stack/Https/HttpsTransportListener.cs

@@ -354,6 +354,9 @@ public void Stop()
         /// </summary>
         public async Task SendAsync(HttpContext context)
         {
+            // wait for certificate update to complete before processing requests.
+            m_quotas.CertificateValidator.CertificateUpdateInProgress.WaitOne();
+
             string message = string.Empty;
             CancellationToken ct = context.RequestAborted;
             try
@@ -608,6 +611,13 @@ private bool ValidateClientCertificate(
             return true;
         }
 
+
+        /// <inheritdoc/>
+        public void CloseAllChannels(string reason)
+        {
+            // nothing to do
+        }
+
         private EndpointDescriptionCollection m_descriptions;
         private ChannelQuotas m_quotas;
         private ITransportListenerCallback m_callback;

Stack/Opc.Ua.Core/Security/Certificates/CertificateValidator.cs

@@ -97,6 +97,15 @@ public event CertificateUpdateEventHandler CertificateUpdate
             remove => m_CertificateUpdate -= value;
         }
 
+        /// <summary>
+        /// Raised before an application certificate update occurs.
+        /// </summary>
+        public event CertificateUpdateEventHandler CertificateUpdateStarted
+        {
+            add => m_CertificateUpdateStarted += value;
+            remove => m_CertificateUpdateStarted -= value;
+        }
+
         /// <summary>
         /// Updates the validator with the current state of the configuration.
         /// </summary>
@@ -284,51 +293,69 @@ public virtual async Task UpdateCertificateAsync(
             string applicationUri = null,
             CancellationToken ct = default)
         {
-            await m_semaphore.WaitAsync(ct).ConfigureAwait(false);
+            m_updateEvent.Reset();
 
             try
             {
-                m_applicationCertificates.Clear();
-                //
-                // crash occurs if the cert is in use still and this has not run yet.
-                // This might be the intended design but this runs on a free task that
-                // might not be scheduled right away.
-                //
-                // TODO: We need a better way to disconnect all sessions when the cert is
-                // updated. (See caller of this method)
-                //
-                // foreach (CertificateIdentifier applicationCertificate in securityConfiguration
-                //     .ApplicationCertificates)
-                // {
-                //     applicationCertificate.DisposeCertificate();
-                // }
-
-                foreach (CertificateIdentifier applicationCertificate in securityConfiguration
-                    .ApplicationCertificates)
-                {
-                    await applicationCertificate
-                        .LoadPrivateKeyExAsync(
-                            securityConfiguration.CertificatePasswordProvider,
-                            applicationUri,
-                            m_telemetry,
-                            ct)
-                        .ConfigureAwait(false);
+                CertificateUpdateEventHandler started_callback = m_CertificateUpdateStarted;
+                if (started_callback != null)
+                {
+                    var args = new CertificateUpdateEventArgs(
+                        securityConfiguration,
+                        GetChannelValidator());
+                    await started_callback(this, args).ConfigureAwait(false);
+                }
+
+                await m_semaphore.WaitAsync(ct).ConfigureAwait(false);
+
+                try
+                {
+                    m_applicationCertificates.Clear();
+                    //
+                    // crash occurs if the cert is in use still and this has not run yet.
+                    // This might be the intended design but this runs on a free task that
+                    // might not be scheduled right away.
+                    //
+                    // TODO: We need a better way to disconnect all sessions when the cert is
+                    // updated. (See caller of this method)
+                    //
+                    // foreach (CertificateIdentifier applicationCertificate in securityConfiguration
+                    //     .ApplicationCertificates)
+                    // {
+                    //     applicationCertificate.DisposeCertificate();
+                    // }
+
+                    foreach (CertificateIdentifier applicationCertificate in securityConfiguration
+                        .ApplicationCertificates)
+                    {
+                        await applicationCertificate
+                            .LoadPrivateKeyExAsync(
+                                securityConfiguration.CertificatePasswordProvider,
+                                applicationUri,
+                                m_telemetry,
+                                ct)
+                            .ConfigureAwait(false);
+                    }
+                }
+                finally
+                {
+                    m_semaphore.Release();
                 }
-            }
-            finally
-            {
-                m_semaphore.Release();
-            }
 
-            await UpdateAsync(securityConfiguration, applicationUri, ct).ConfigureAwait(false);
+                await UpdateAsync(securityConfiguration, applicationUri, ct).ConfigureAwait(false);
 
-            CertificateUpdateEventHandler callback = m_CertificateUpdate;
-            if (callback != null)
+                CertificateUpdateEventHandler callback = m_CertificateUpdate;
+                if (callback != null)
+                {
+                    var args = new CertificateUpdateEventArgs(
+                        securityConfiguration,
+                        GetChannelValidator());
+                    await callback(this, args).ConfigureAwait(false);
+                }
+            }
+            finally
             {
-                var args = new CertificateUpdateEventArgs(
-                    securityConfiguration,
-                    GetChannelValidator());
-                callback(this, args);
+                m_updateEvent.Set();
             }
         }
 
@@ -2234,6 +2261,9 @@ public static bool IsECSecureForProfile(
             throw new NotSupportedException("Unsupported curve type.");
         }
 
+        /// <inheritdoc/>
+        public WaitHandle CertificateUpdateInProgress => m_updateEvent.WaitHandle;
+
         /// <summary>
         /// Flag to protect setting by application
         /// from a modification by a SecurityConfiguration.
@@ -2254,13 +2284,15 @@ private enum ProtectFlags
         private readonly ILogger m_logger;
         private readonly ITelemetryContext m_telemetry;
         private readonly Dictionary<string, X509Certificate2> m_validatedCertificates;
+        private readonly ManualResetEventSlim m_updateEvent = new(true);
         private CertificateStoreIdentifier m_trustedCertificateStore;
         private CertificateIdentifierCollection m_trustedCertificateList;
         private CertificateStoreIdentifier m_issuerCertificateStore;
         private CertificateIdentifierCollection m_issuerCertificateList;
         private CertificateStoreIdentifier m_rejectedCertificateStore;
         private event CertificateValidationEventHandler m_CertificateValidation;
         private event CertificateUpdateEventHandler m_CertificateUpdate;
+        private event CertificateUpdateEventHandler m_CertificateUpdateStarted;
         private readonly List<X509Certificate2> m_applicationCertificates;
         private ProtectFlags m_protectFlags;
         private bool m_autoAcceptUntrustedCertificates;
@@ -2350,7 +2382,7 @@ public CertificateUpdateEventArgs(
     /// <summary>
     /// Used to handle certificate update events.
     /// </summary>
-    public delegate void CertificateUpdateEventHandler(
+    public delegate Task CertificateUpdateEventHandler(
         CertificateValidator sender,
         CertificateUpdateEventArgs e);
 }

Stack/Opc.Ua.Core/Security/Certificates/ICertificateValidator.cs

@@ -38,6 +38,29 @@ namespace Opc.Ua
     /// </summary>
     public interface ICertificateValidator
     {
+        /// <summary>
+        /// Raised when an application certificate update occurs.
+        /// </summary>
+        event CertificateUpdateEventHandler CertificateUpdate;
+
+        /// <summary>
+        /// Raised before an application certificate update occurs.
+        /// </summary>
+        event CertificateUpdateEventHandler CertificateUpdateStarted;
+
+        /// <summary>
+        /// An event that signals that a certificate update is in progress.
+        /// </summary>
+        WaitHandle CertificateUpdateInProgress { get; }
+
+        /// <summary>
+        /// Updates the validator with a new application certificate.
+        /// </summary>
+        Task UpdateCertificateAsync(
+            SecurityConfiguration securityConfiguration,
+            string applicationUri = null,
+            CancellationToken ct = default);
+
         /// <summary>
         /// Validates a certificate.
         /// </summary>

Stack/Opc.Ua.Core/Stack/Server/ServerBase.cs

@@ -771,7 +771,7 @@ protected virtual EndpointBase GetEndpointInstance(ServerBase server)
         /// <summary>
         /// Called after the application certificate update.
         /// </summary>
-        protected virtual async void OnCertificateUpdateAsync(object sender, CertificateUpdateEventArgs e)
+        protected virtual async Task OnCertificateUpdateAsync(object sender, CertificateUpdateEventArgs e)
         {
             try
             {
@@ -810,6 +810,25 @@ await InstanceCertificateTypesProvider.LoadCertificateChainAsync(certificate)
             }
         }
 
+        /// <summary>
+        /// Called before the application certificate update.
+        /// </summary>
+        protected virtual Task OnCertificateUpdateStartedAsync(object sender, CertificateUpdateEventArgs e)
+        {
+            try
+            {
+                foreach (ITransportListener listener in TransportListeners)
+                {
+                    listener.CloseAllChannels("Update of ApplicationCertificate");
+                }
+            }
+            catch (Exception ex)
+            {
+                m_logger.LogError(ex, "Failed to close all channels on certificate update: {EventArgs}", e);
+            }
+            return Task.CompletedTask;
+        }
+
         /// <summary>
         /// Create the transport listener for the service host endpoint.
         /// </summary>

Stack/Opc.Ua.Core/Stack/Tcp/TcpListenerChannel.cs

@@ -189,6 +189,28 @@ public void IdleCleanup()
             }
         }
 
+        /// <summary>
+        /// Force the channel to close immediately, e.g. due to certificate update.
+        /// </summary>
+        public void ForceClose(string reason)
+        {
+            TcpChannelState state;
+
+            lock (DataLock)
+            {
+                state = State;
+                if (state is TcpChannelState.Open or TcpChannelState.Connecting)
+                {
+                    state = State = TcpChannelState.Closing;
+                }
+            }
+
+            if (state is TcpChannelState.Closing or TcpChannelState.Opening or TcpChannelState.Faulted)
+            {
+                OnForceClosed(reason);
+            }
+        }
+
         /// <summary>
         /// The time in milliseconds elapsed since the channel received or sent messages
         /// or received a keep alive.
@@ -349,6 +371,32 @@ private void OnCleanup(object state)
             }
         }
 
+        /// <summary>
+        /// Called when the channel is force closed.
+        /// </summary>
+        private void OnForceClosed(string reason)
+        {
+            lock (DataLock)
+            {
+                // nothing to do if the channel is now open or closed.
+                if (State is TcpChannelState.Closed or TcpChannelState.Open)
+                {
+                    return;
+                }
+
+                m_logger.LogInformation(
+                    "{Channel} Force Close Socket={SocketHandle:X8}, ChannelId={ChannelId}, TokenId={TokenId}, Reason={Reason}",
+                    ChannelName,
+                    (Socket?.Handle) ?? 0,
+                    CurrentToken != null ? CurrentToken.ChannelId : 0,
+                    CurrentToken != null ? CurrentToken.TokenId : 0,
+                    reason);
+
+                // close channel.
+                ChannelClosed();
+            }
+        }
+
         /// <summary>
         /// Closes the channel and releases resources.
         /// Sets state to Closed and notifies monitors.

Stack/Opc.Ua.Core/Stack/Tcp/TcpTransportListener.cs

@@ -401,6 +401,15 @@ public void Close()
             Stop();
         }
 
+        /// <inheritdoc/>
+        public void CloseAllChannels(string reason)
+        {
+            foreach (TcpListenerChannel channel in m_channels.Values.ToList())
+            {
+                channel.ForceClose(reason);
+            }
+        }
+
         /// <inheritdoc/>
         public void UpdateChannelLastActiveTime(string globalChannelId)
         {
@@ -807,6 +816,9 @@ private void OnAccept(object sender, SocketAsyncEventArgs e)
             {
                 bool isBlocked = false;
 
+                // wait for certificate update to complete
+                m_quotas.CertificateValidator.CertificateUpdateInProgress.WaitOne();
+
                 // Track potential problematic client behavior only if Basic128Rsa15 security policy is offered
                 if (m_activeClientTracker != null)
                 {

Stack/Opc.Ua.Core/Stack/Transport/ITransportListener.cs

@@ -74,6 +74,11 @@ void Open(
         /// <exception cref="ServiceResultException">Thrown if any communication error occurs.</exception>
         void Close();
 
+        /// <summary>
+        /// Closes all connections of the listener.
+        /// </summary>
+        void CloseAllChannels(string reason);
+
         /// <summary>
         /// Updates the application certificate for a listener.
         /// </summary>