Add LTS 03/2024 refresh (LTS_03_2024_Ref02) (#2625)

* Port e0c0c2c

* Port 9f3ef4f

* Port 1d98530

* Port fce8040

* Port 614cb35

* Port f449635

* Port 8ebc30e

* Update submodules

* Bump version of LTS_03_2024_Ref02 from 1.12.0 to 1.12.1

* Update LTS Schedule for LTS_03_2024_Ref02

build/.vsts-ci.yml

@@ -525,7 +525,7 @@ jobs:
       gcc --version
       openssl version
       curl --version
-      sudo -E env PATH="$PATH" ./build_all/linux/build.sh --run-unittests --run-e2e-tests --provisioning --use-hsmsymmkey --use-hsmsas --use-hsmx509 --use-edge-modules --use-ipv6-dualstack --config Debug
+      sudo -E env PATH="$PATH" ./build_all/linux/build.sh --run-unittests --run-e2e-tests --no_uploadtoblob --provisioning --use-hsmsymmkey --use-hsmsas --use-hsmx509 --use-edge-modules --use-ipv6-dualstack --config Debug
     displayName: 'Build'
   - script: |
       export OPENSSL_ia32cap=0x00000000

build_all/docs/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "Microsoft Azure IoT Device SDK for C"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.12.0
+PROJECT_NUMBER         = 1.12.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

build_all/windows/build.cmd

@@ -41,6 +41,7 @@ set make=yes
 set build_traceabilitytool=0
 set prov_auth=OFF
 set use_edge_modules=OFF
+set no_uploadtoblob=OFF
 
 :args-loop
 if "%1" equ "" goto args-done
@@ -57,6 +58,7 @@ if "%1" equ "--no-make" goto arg-no-make
 if "%1" equ "--build-traceabilitytool" goto arg-build-traceabilitytool
 if "%1" equ "--provisioning" goto arg-provisioning
 if "%1" equ "--use-edge-modules" goto arg-edge-modules
+if "%1" equ "--no-uploadtoblob" goto arg-uploadtoblob
 call :usage && exit /b 1
 
 :arg-build-clean
@@ -119,6 +121,10 @@ goto args-continue
 set use_edge_modules=ON
 goto args-continue
 
+:arg-uploadtoblob
+set no_uploadtoblob=ON
+goto args-continue
+
 :args-continue
 shift
 goto args-loop
@@ -154,15 +160,15 @@ cmake --version
 
 if %build-platform% == x64 (
     echo ***Running CMAKE for Win64***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Duse_cppunittest:BOOL=OFF -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A x64 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Ddont_use_uploadtoblob:BOOL=%no_uploadtoblob% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Duse_cppunittest:BOOL=OFF -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A x64 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 ) else if %build-platform% == arm (
     echo ***Running CMAKE for ARM***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A ARM -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Ddont_use_uploadtoblob:BOOL=%no_uploadtoblob% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A ARM -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 ) else (
     echo ***Running CMAKE for Win32***
-    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A Win32 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
+    cmake -LAH -Drun_longhaul_tests:BOOL=%CMAKE_run_longhaul_tests% -Drun_e2e_tests:BOOL=%CMAKE_run_e2e_tests% -Ddont_use_uploadtoblob:BOOL=%no_uploadtoblob% -Drun_sfc_tests:BOOL=%CMAKE_run_sfc_tests% -Drun_unittests:BOOL=%CMAKE_run_unittests% %build-root% -Duse_prov_client:BOOL=%prov_auth% -G %VSVERSION% -A Win32 -Duse_edge_modules=%use_edge_modules% -DCMAKE_BUILD_TYPE=%build-config%
     if not !ERRORLEVEL!==0 exit /b !ERRORLEVEL!
 )
 

iothub_client/inc/iothub_client_version.h

@@ -9,7 +9,7 @@
 #define IOTHUB_CLIENT_VERSION_H
 
 /** @brief Macro representing current IoT Hub Client SDK version. */
-#define IOTHUB_SDK_VERSION "1.12.0"
+#define IOTHUB_SDK_VERSION "1.12.1"
 
 #include "umock_c/umock_c_prod.h"
 

iothub_client/samples/iothub_ll_client_shared_sample/iothub_ll_client_shared_sample.c

@@ -303,7 +303,7 @@ int main(void)
                 ThreadAPI_Sleep((unsigned int)sleep_ms);
                 counter += sleep_ms;
 
-                g_continueRunning = !(messages_sent > MESSAGE_COUNT);
+                g_continueRunning &= !(messages_sent > MESSAGE_COUNT);
             } while (g_continueRunning);
 
             (void)printf("client_amqp_shared_sample has received a quit message, call DoWork %d more time to complete final sending...\r\n", DOWORK_LOOP_NUM);

iothub_client/src/iothub_client_core.c

@@ -73,6 +73,7 @@ typedef struct UPLOADTOBLOB_MULTIBLOCK_SAVED_DATA_TAG
 {
     IOTHUB_CLIENT_FILE_UPLOAD_GET_DATA_CALLBACK getDataCallback;
     IOTHUB_CLIENT_FILE_UPLOAD_GET_DATA_CALLBACK_EX getDataCallbackEx;
+    IOTHUB_CLIENT_FILE_UPLOAD_RESULT blobStorageUploadResult;
 }UPLOADTOBLOB_MULTIBLOCK_SAVED_DATA;
 
 typedef struct INVOKE_METHOD_SAVED_DATA_TAG
@@ -2298,6 +2299,7 @@ static HTTPWORKER_THREAD_INFO* allocateUploadToBlob(const char* destinationFileN
         threadInfo->workerThreadType = HTTPWORKER_THREAD_UPLOAD_TO_BLOB;
         threadInfo->iotHubClientHandle = iotHubClientHandle;
         threadInfo->context = context;
+        threadInfo->uploadBlobMultiblockSavedData.blobStorageUploadResult = FILE_UPLOAD_ERROR;
 
         if (mallocAndStrcpy_s(&threadInfo->destinationFileName, destinationFileName) != 0)
         {
@@ -2419,22 +2421,78 @@ IOTHUB_CLIENT_RESULT IoTHubClientCore_UploadToBlobAsync(IOTHUB_CLIENT_CORE_HANDL
     return result;
 }
 
+// This callback wrapper allows iothub_client_core to trap the final result from the upload to blob callback.
+// That is because IoTHubClientCore_LL_UploadMultipleBlocksToBlob
+// will fire the user callback with succes or error only in specific situations (i.e., uploading to Azure Storage).
+// All other stages within IoTHubClientCore_LL_UploadMultipleBlocksToBlob do not cause the callback to be invoked,
+// only signaling the final upload result through the return code. However, the return code is not exposed to the user application
+// by iothub_client_core. So iothub_client_core must invoke the user callback so the user can get a feedback that
+// an upload has succeeded or failed.
+static void uploadToBlobMultiblockCallbackWrapper(IOTHUB_CLIENT_FILE_UPLOAD_RESULT result, unsigned char const ** data, size_t* size, void* context)
+{
+    HTTPWORKER_THREAD_INFO* threadInfo = (HTTPWORKER_THREAD_INFO*)context;
+
+    if (data == NULL || size == NULL)
+    {
+        // Capture the last callback invocation and save result.
+        threadInfo->uploadBlobMultiblockSavedData.blobStorageUploadResult = result;
+    }
+    else
+    {
+        threadInfo->uploadBlobMultiblockSavedData.getDataCallback(result, data, size, threadInfo->context);
+    }
+}
+
+// This callback wrapper allows iothub_client_core to trap the final result from the upload to blob callback.
+// That is because IoTHubClientCore_LL_UploadMultipleBlocksToBlobEx
+// will fire the user callback with succes or error only in specific situations (i.e., uploading to Azure Storage).
+// All other stages within IoTHubClientCore_LL_UploadMultipleBlocksToBlobEx do not cause the callback to be invoked,
+// only signaling the final upload result through the return code. However, the return code is not exposed to the user application
+// by iothub_client_core. So iothub_client_core must invoke the user callback so the user can get a feedback that
+// an upload has succeeded or failed.
+static IOTHUB_CLIENT_FILE_UPLOAD_GET_DATA_RESULT uploadToBlobMultiblockCallbackWrapperEx(IOTHUB_CLIENT_FILE_UPLOAD_RESULT result, unsigned char const ** data, size_t* size, void* context)
+{
+    HTTPWORKER_THREAD_INFO* threadInfo = (HTTPWORKER_THREAD_INFO*)context;
+
+    if (data == NULL || size == NULL)
+    {
+        // Capture the last callback invocation and save result.
+        threadInfo->uploadBlobMultiblockSavedData.blobStorageUploadResult = result;
+        return IOTHUB_CLIENT_FILE_UPLOAD_GET_DATA_OK;
+    }
+    else
+    {
+        return threadInfo->uploadBlobMultiblockSavedData.getDataCallbackEx(result, data, size, threadInfo->context);
+    }
+}
+
 static int uploadMultipleBlock_thread(void* data)
 {
     HTTPWORKER_THREAD_INFO* threadInfo = (HTTPWORKER_THREAD_INFO*)data;
     IOTHUB_CLIENT_CORE_LL_HANDLE llHandle = threadInfo->iotHubClientHandle->IoTHubClientLLHandle;
 
     IOTHUB_CLIENT_RESULT result;
+    IOTHUB_CLIENT_FILE_UPLOAD_RESULT finalUploadToBlobResult;
 
     srand((unsigned int)get_time(NULL));
 
     if (threadInfo->uploadBlobMultiblockSavedData.getDataCallback != NULL)
     {
-        result = IoTHubClientCore_LL_UploadMultipleBlocksToBlob(llHandle, threadInfo->destinationFileName, threadInfo->uploadBlobMultiblockSavedData.getDataCallback, threadInfo->context);
+        result = IoTHubClientCore_LL_UploadMultipleBlocksToBlob(llHandle, threadInfo->destinationFileName, uploadToBlobMultiblockCallbackWrapper, threadInfo);
+
+        // See comments in uploadToBlobMultiblockCallbackWrapper for further details.
+        finalUploadToBlobResult = (result != IOTHUB_CLIENT_OK ? FILE_UPLOAD_ERROR : threadInfo->uploadBlobMultiblockSavedData.blobStorageUploadResult);
+
+        threadInfo->uploadBlobMultiblockSavedData.getDataCallback(finalUploadToBlobResult, NULL, NULL, threadInfo->context);
     }
     else
     {
-        result = IoTHubClientCore_LL_UploadMultipleBlocksToBlobEx(llHandle, threadInfo->destinationFileName, threadInfo->uploadBlobMultiblockSavedData.getDataCallbackEx, threadInfo->context);
+        result = IoTHubClientCore_LL_UploadMultipleBlocksToBlobEx(llHandle, threadInfo->destinationFileName, uploadToBlobMultiblockCallbackWrapperEx, threadInfo);
+
+        // See comments in uploadToBlobMultiblockCallbackWrapperEx for further details.
+        finalUploadToBlobResult = (result != IOTHUB_CLIENT_OK ? FILE_UPLOAD_ERROR : threadInfo->uploadBlobMultiblockSavedData.blobStorageUploadResult);
+
+        (void)threadInfo->uploadBlobMultiblockSavedData.getDataCallbackEx(finalUploadToBlobResult, NULL, NULL, threadInfo->context);
     }
     (void)markThreadReadyToBeGarbageCollected(threadInfo);
 

iothub_client/src/iothubtransport_amqp_messenger.c

@@ -14,6 +14,7 @@
 #include "azure_uamqp_c/messaging.h"
 #include "azure_uamqp_c/message_sender.h"
 #include "azure_uamqp_c/message_receiver.h"
+#include "azure_uamqp_c/async_operation.h"
 #include "internal/message_queue.h"
 #include "internal/iothub_client_retry_control.h"
 #include "internal/iothubtransport_amqp_messenger.h"
@@ -79,17 +80,31 @@ typedef struct MESSAGE_SEND_CONTEXT_TAG
 {
     uint32_t mq_message_id;
     MESSAGE_HANDLE message;
-    bool is_destroyed;
 
     AMQP_MESSENGER_INSTANCE* messenger;
 
     AMQP_MESSENGER_SEND_COMPLETE_CALLBACK on_send_complete_callback;
     void* user_context;
 
     PROCESS_MESSAGE_COMPLETED_CALLBACK on_process_message_completed_callback;
+
+    // Handle to the async operation instance returned
+    // by messagesender_send_async().
+    ASYNC_OPERATION_HANDLE async_operation;
 } MESSAGE_SEND_CONTEXT;
 
+static void destroy_message_send_context(MESSAGE_SEND_CONTEXT* context)
+{
+    if (context->async_operation != NULL)
+    {
+        (void)async_operation_cancel(context->async_operation);
+    }
+
+    free(context);
+}
 
+#define mark_message_send_context_completed(context) \
+    context->async_operation = NULL
 
 static MESSAGE_SEND_CONTEXT* create_message_send_context(void)
 {
@@ -273,11 +288,6 @@ static AMQP_MESSENGER_CONFIG* clone_configuration(const AMQP_MESSENGER_CONFIG* c
     return result;
 }
 
-static void destroy_message_send_context(MESSAGE_SEND_CONTEXT* context)
-{
-    free(context);
-}
-
 static STRING_HANDLE create_link_address(const char* host_fqdn, const char* device_id, const char* module_id, const char* address_suffix)
 {
     STRING_HANDLE link_address;
@@ -851,6 +861,8 @@ static void on_send_complete_callback(void* context, MESSAGE_SEND_RESULT send_re
         MESSAGE_QUEUE_RESULT mq_result;
         MESSAGE_SEND_CONTEXT* msg_ctx = (MESSAGE_SEND_CONTEXT*)context;
 
+        mark_message_send_context_completed(msg_ctx);
+
         if (send_result == MESSAGE_SEND_OK)
         {
             mq_result = MESSAGE_QUEUE_SUCCESS;
@@ -875,15 +887,18 @@ static void on_process_message_callback(MESSAGE_QUEUE_HANDLE message_queue, MQ_M
         MESSAGE_SEND_CONTEXT* message_context = (MESSAGE_SEND_CONTEXT*)context;
         message_context->mq_message_id = message_id;
         message_context->on_process_message_completed_callback = on_process_message_completed_callback;
+        message_context->async_operation = messagesender_send_async(message_context->messenger->message_sender, (MESSAGE_HANDLE)message, on_send_complete_callback, context, 0);
 
-        if (messagesender_send_async(message_context->messenger->message_sender, (MESSAGE_HANDLE)message, on_send_complete_callback, context, 0) == NULL)
+        if (message_context->async_operation == NULL)
         {
             LogError("Failed sending AMQP message");
             on_process_message_completed_callback(message_queue, message_id, MESSAGE_QUEUE_ERROR, NULL);
         }
 
-        message_destroy((MESSAGE_HANDLE)message);
-        message_context->is_destroyed = true;
+        // An optimization was done here before to destroy `message` right away,
+        // freeing up memory as early as possible, but it led to a double-free
+        // situation when on_message_processing_completed_callback is called.
+        // Removing the optimization is the easiest fix.
     }
 }
 
@@ -930,16 +945,12 @@ static void on_message_processing_completed_callback(MQ_MESSAGE_HANDLE message,
             msg_ctx->messenger->send_error_count++;
         }
 
-        if (!msg_ctx->is_destroyed)
+        if (msg_ctx->on_send_complete_callback != NULL)
         {
-            if (msg_ctx->on_send_complete_callback != NULL)
-            {
-                msg_ctx->on_send_complete_callback(messenger_send_result, messenger_send_reason, msg_ctx->user_context);
-            }
-
-            message_destroy((MESSAGE_HANDLE)message);
+            msg_ctx->on_send_complete_callback(messenger_send_result, messenger_send_reason, msg_ctx->user_context);
         }
 
+        message_destroy((MESSAGE_HANDLE)message);
         destroy_message_send_context(msg_ctx);
     }
 }

iothub_client/src/iothubtransport_amqp_telemetry_messenger.c

@@ -1338,6 +1338,7 @@ static void remove_timed_out_events(TELEMETRY_MESSENGER_INSTANCE* instance)
     while (list_item != NULL)
     {
         MESSENGER_SEND_EVENT_TASK* task = (MESSENGER_SEND_EVENT_TASK*)singlylinkedlist_item_get_value(list_item);
+        list_item = singlylinkedlist_get_next_item(list_item);
 
         if (task != NULL && task->is_timed_out == true)
         {
@@ -1346,7 +1347,6 @@ static void remove_timed_out_events(TELEMETRY_MESSENGER_INSTANCE* instance)
             free_task(task);
         }
 
-        list_item = singlylinkedlist_get_next_item(list_item);
     }
 }
 

iothub_client/src/message_queue.c

@@ -258,10 +258,7 @@ static void process_pending_messages(MESSAGE_QUEUE_HANDLE message_queue)
         {
             LogError("failed moving message out of pending list (%p)", mq_item->message);
 
-            if (mq_item->on_message_processing_completed_callback != NULL)
-            {
-                mq_item->on_message_processing_completed_callback(mq_item->message, MESSAGE_QUEUE_ERROR, NULL, mq_item->user_context);
-            }
+            fire_message_callback(mq_item, MESSAGE_QUEUE_ERROR, NULL);
 
             // Not freeing since this would cause a memory A/V on the next call.
 
@@ -271,22 +268,14 @@ static void process_pending_messages(MESSAGE_QUEUE_HANDLE message_queue)
         {
             LogError("failed setting message processing_start_time (%p)", mq_item->message);
 
-            if (mq_item->on_message_processing_completed_callback != NULL)
-            {
-                mq_item->on_message_processing_completed_callback(mq_item->message, MESSAGE_QUEUE_ERROR, NULL, mq_item->user_context);
-            }
-
+            fire_message_callback(mq_item, MESSAGE_QUEUE_ERROR, NULL);
             free(mq_item);
         }
         else if (singlylinkedlist_add(message_queue->in_progress, (const void*)mq_item) == NULL)
         {
             LogError("failed moving message to in-progress list (%p)", mq_item->message);
 
-            if (mq_item->on_message_processing_completed_callback != NULL)
-            {
-                mq_item->on_message_processing_completed_callback(mq_item->message, MESSAGE_QUEUE_ERROR, NULL, mq_item->user_context);
-            }
-
+            fire_message_callback(mq_item, MESSAGE_QUEUE_ERROR, NULL);
             free(mq_item);
         }
         else

iothub_client/src/uamqp_messaging.c

@@ -1101,29 +1101,25 @@ static int readApplicationPropertiesFromuAMQPMessage(IOTHUB_MESSAGE_HANDLE iothu
                         result = MU_FAILURE;
                     }
 
-                    else if ((result = amqpvalue_get_string(map_key_name, &key_name)) != 0)
+                    else
                     {
-                        LogError("Failed parsing the uAMQP property name (return code %d).", result);
-                        result = MU_FAILURE;
-                    }
-                    else if ((result = amqpvalue_get_string(map_key_value, &key_value)) != 0)
-                    {
-                        LogError("Failed parsing the uAMQP property value (return code %d).", result);
-                        result = MU_FAILURE;
-                    }
-                    else if (Map_AddOrUpdate(iothub_message_properties_map, key_name, key_value) != MAP_OK)
-                    {
-                        LogError("Failed to add/update IoTHub message property map.");
-                        result = MU_FAILURE;
-                    }
+                        if ((result = amqpvalue_get_string(map_key_name, &key_name)) != 0)
+                        {
+                            LogError("Failed parsing the uAMQP property name (return code %d).", result);
+                            result = MU_FAILURE;
+                        }
+                        else if ((result = amqpvalue_get_string(map_key_value, &key_value)) != 0)
+                        {
+                            LogError("Failed parsing the uAMQP property value (return code %d).", result);
+                            result = MU_FAILURE;
+                        }
+                        else if (Map_AddOrUpdate(iothub_message_properties_map, key_name, key_value) != MAP_OK)
+                        {
+                            LogError("Failed to add/update IoTHub message property map.");
+                            result = MU_FAILURE;
+                        }
 
-                    if (map_key_name != NULL)
-                    {
                         amqpvalue_destroy(map_key_name);
-                    }
-
-                    if (map_key_value != NULL)
-                    {
                         amqpvalue_destroy(map_key_value);
                     }
                 }