Update Config Shadow to store config file updates persistently (#490)

* Update Config Shadow to store config file updates persistently

* Add field to trigger config shadow persistance updates

* Update actions/upload-artifact to v4 in github workflows

* Run the clang-format script for formatting

* Config Shadow Documentation updates, checks before updating config files and refactoring

---------

Co-authored-by: Ubuntu <ubuntu@ip-172-31-30-73.us-west-2.compute.internal>
Co-authored-by: DakshitBabbar <dakshitbabbar.iitd@gmail.com>

Author: 3 people

.github/workflows/build.yml

@@ -37,15 +37,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE
       - name: Archive ubuntu Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.ubuntu.x64.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -77,15 +77,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE
       - name: Archive amazonlinux Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.amazonlinux.x64.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -117,15 +117,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE
       - name: Archive rhel-ubi8 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.rhel.x64.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -211,15 +211,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=armhf_cross_mode
       - name: Archive armhf32 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.linux.armhf.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -251,15 +251,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=mips_cross_mode
       - name: Archive mips32 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.linux.mips.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -291,15 +291,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=aarch64_cross_mode
       - name: Archive aarch64 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.linux.aarch64.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -331,15 +331,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=ppc64_cross_mode
       - name: Archive aarch64 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.linux.ppc64.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -371,15 +371,15 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=ppc64le_cross_mode
       - name: Archive aarch64 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
           name: "DC.linux.ppc64le.${{ env.PACKAGE_NAME }}.static"
           path: |
             ./build/${{ env.PACKAGE_NAME }}
       - name: Archive setup files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -411,7 +411,7 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=st_component_mode
       - name: Archive ST Ubuntu aarch64 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -443,7 +443,7 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=st_aarch64_cross_mode
       - name: Archive ST Ubuntu aarch64 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:
@@ -475,7 +475,7 @@ jobs:
           docker pull $DOCKER_IMAGE
           docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF $DOCKER_IMAGE --compile-mode=st_armhf_cross_mode
       - name: Archive st armhf32 Build Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         # Run for main branch only
         if: github.ref == 'refs/heads/main'
         with:

source/ClientBaseNotification.h

@@ -38,7 +38,7 @@ namespace Aws
                 SUBSCRIPTION_FAILED
             };
         } // namespace DeviceClient
-    }     // namespace Iot
+    } // namespace Iot
 } // namespace Aws
 
 #endif // DEVICE_CLIENT_CLIENTBASENOTIFICATION_H

source/ClientBaseNotifier.h

@@ -50,7 +50,7 @@ namespace Aws
                     const std::string &message) = 0;
             };
         } // namespace DeviceClient
-    }     // namespace Iot
+    } // namespace Iot
 } // namespace Aws
 
 #endif // DEVICE_CLIENT_CLIENTBASENOTIFIER_H

source/Feature.h

@@ -53,7 +53,7 @@ namespace Aws
                 virtual std::string getName() = 0;
             };
         } // namespace DeviceClient
-    }     // namespace Iot
+    } // namespace Iot
 } // namespace Aws
 
 #endif // DEVICE_CLIENT_FEATURE_H

source/FeatureRegistry.cpp

@@ -50,9 +50,10 @@ void FeatureRegistry::disable(const std::string &name)
 std::size_t FeatureRegistry::getSize() const
 {
     std::lock_guard<std::mutex> lock(featuresLock);
-    return count_if(features.begin(), features.end(), [](const std::pair<string, std::shared_ptr<Feature>> &feature) {
-        return feature.second != nullptr;
-    });
+    return count_if(
+        features.begin(),
+        features.end(),
+        [](const std::pair<string, std::shared_ptr<Feature>> &feature) { return feature.second != nullptr; });
 }
 
 void FeatureRegistry::stopAll()

source/FeatureRegistry.h

@@ -77,8 +77,8 @@ namespace Aws
                 };
 
             } // namespace Util
-        }     // namespace DeviceClient
-    }         // namespace Iot
+        } // namespace DeviceClient
+    } // namespace Iot
 } // namespace Aws
 
 #endif // AWS_IOT_DEVICE_CLIENT_FEATURE_REGISTRY_H

source/SharedCrtResourceManager.cpp

@@ -227,10 +227,10 @@ int SharedCrtResourceManager::buildClient(const PlainConfig &config)
             "logging::enable-sdk-logging in your configuration file");
     }
 
-    eventLoopGroup = unique_ptr<EventLoopGroup>(
-        new EventLoopGroup(1 // The number of threads used depends on your use-case. IF you have a maximum of less than
-                             // a few hundred connections 1 thread is the ideal threadCount.
-                           ));
+    eventLoopGroup = unique_ptr<EventLoopGroup>(new EventLoopGroup(
+        1 // The number of threads used depends on your use-case. IF you have a maximum of less than
+          // a few hundred connections 1 thread is the ideal threadCount.
+        ));
     if (!eventLoopGroup)
     {
         // cppcheck-suppress nullPointerRedundantCheck
@@ -394,7 +394,8 @@ int SharedCrtResourceManager::establishConnection(const PlainConfig &config)
      * This will execute when an mqtt connect has completed or failed.
      */
     auto onConnectionCompleted = [this, &connectionCompletedPromise](
-                                     const Mqtt::MqttConnection &, int errorCode, Mqtt::ReturnCode returnCode, bool) {
+                                     const Mqtt::MqttConnection &, int errorCode, Mqtt::ReturnCode returnCode, bool)
+    {
         if (errorCode)
         {
             LOGM_ERROR(TAG, "MQTT Connection failed with error: %s", ErrorDebugString(errorCode));
@@ -418,7 +419,8 @@ int SharedCrtResourceManager::establishConnection(const PlainConfig &config)
     /*
      * Invoked when a disconnect message has completed.
      */
-    auto onDisconnect = [this](const Mqtt::MqttConnection & /*conn*/) {
+    auto onDisconnect = [this](const Mqtt::MqttConnection & /*conn*/)
+    {
         {
             LOG_INFO(TAG, "MQTT Connection is now disconnected");
             connectionClosedPromise.set_value();
@@ -428,7 +430,8 @@ int SharedCrtResourceManager::establishConnection(const PlainConfig &config)
     /*
      * Invoked when connection is interrupted.
      */
-    auto OnConnectionInterrupted = [this](const Mqtt::MqttConnection &, int errorCode) {
+    auto OnConnectionInterrupted = [this](const Mqtt::MqttConnection &, int errorCode)
+    {
         {
             if (errorCode)
             {
@@ -444,7 +447,8 @@ int SharedCrtResourceManager::establishConnection(const PlainConfig &config)
     /*
      * Invoked when connection is resumed.
      */
-    auto OnConnectionResumed = [this](const Mqtt::MqttConnection &, int returnCode, bool) {
+    auto OnConnectionResumed = [this](const Mqtt::MqttConnection &, int returnCode, bool)
+    {
         {
             LOGM_INFO(TAG, "MQTT connection resumed with return code: %d", returnCode);
         }

source/SharedCrtResourceManager.h

@@ -103,7 +103,7 @@ namespace Aws
                 void dumpMemTrace();
             };
         } // namespace DeviceClient
-    }     // namespace Iot
+    } // namespace Iot
 } // namespace Aws
 
 #endif // DEVICE_CLIENT_SHAREDCRTRESOURCEMANAGER_H

source/config/Config.cpp

@@ -1883,6 +1883,7 @@ void PlainConfig::SampleShadow::SerializeToObject(Crt::JsonObject &object) const
 
 constexpr char PlainConfig::ConfigShadow::JSON_ENABLE_CONFIG_SHADOW[];
 constexpr char PlainConfig::ConfigShadow::CLI_ENABLE_CONFIG_SHADOW[];
+constexpr char PlainConfig::ConfigShadow::JSON_CONFIG_SHADOW_PERSISTENT_UPDATE[];
 
 bool PlainConfig::ConfigShadow::LoadFromJson(const Crt::JsonView &json)
 {
@@ -1892,6 +1893,12 @@ bool PlainConfig::ConfigShadow::LoadFromJson(const Crt::JsonView &json)
         enabled = json.GetBool(jsonKey);
     }
 
+    jsonKey = JSON_CONFIG_SHADOW_PERSISTENT_UPDATE;
+    if (json.ValueExists(jsonKey))
+    {
+        persistentUpdate = json.GetBool(jsonKey);
+    }
+
     return true;
 }
 
@@ -1913,6 +1920,7 @@ bool PlainConfig::ConfigShadow::Validate() const
 void PlainConfig::ConfigShadow::SerializeToObject(Crt::JsonObject &object) const
 {
     object.WithBool(JSON_ENABLE_CONFIG_SHADOW, enabled);
+    object.WithBool(JSON_CONFIG_SHADOW_PERSISTENT_UPDATE, persistentUpdate);
 }
 
 constexpr char PlainConfig::SecureElement::CLI_ENABLE_SECURE_ELEMENT[];
@@ -2491,6 +2499,7 @@ constexpr char Config::TAG[];
 constexpr char Config::DEFAULT_CONFIG_DIR[];
 constexpr char Config::DEFAULT_KEY_DIR[];
 constexpr char Config::DEFAULT_CONFIG_FILE[];
+constexpr char Config::DEFAULT_SYSTEM_CONFIG_FILE[];
 constexpr char Config::CLI_HELP[];
 constexpr char Config::CLI_VERSION[];
 constexpr char Config::CLI_EXPORT_DEFAULT_SETTINGS[];
@@ -2884,7 +2893,8 @@ bool Config::ParseConfigFile(const string &file, ConfigFileType configFileType)
     }
 
 #if !defined(DISABLE_MQTT)
-    LOGM_INFO(TAG, "Successfully fetched JSON config file: %s", Sanitize(contents).c_str());
+    LOGM_INFO(
+        TAG, "Successfully fetched JSON config file from %s : %s", expandedPath.c_str(), Sanitize(contents).c_str());
 #endif
     setting.close();
 
@@ -3138,6 +3148,7 @@ bool Config::ExportDefaultSetting(const string &file)
         PlainConfig::SampleShadow::JSON_SAMPLE_SHADOW_OUTPUT_FILE,
         PlainConfig::JSON_KEY_CONFIG_SHADOW,
         PlainConfig::ConfigShadow::JSON_ENABLE_CONFIG_SHADOW,
+        PlainConfig::ConfigShadow::JSON_CONFIG_SHADOW_PERSISTENT_UPDATE,
         PlainConfig::JSON_KEY_SECURE_ELEMENT,
         PlainConfig::SecureElement::JSON_ENABLE_SECURE_ELEMENT,
         PlainConfig::SecureElement::JSON_PKCS11_LIB,

source/config/Config.h

@@ -372,8 +372,10 @@ namespace Aws
 
                     static constexpr char CLI_ENABLE_CONFIG_SHADOW[] = "--enable-config-shadow";
                     static constexpr char JSON_ENABLE_CONFIG_SHADOW[] = "enabled";
+                    static constexpr char JSON_CONFIG_SHADOW_PERSISTENT_UPDATE[] = "persistent-update";
 
                     bool enabled{false};
+                    bool persistentUpdate{false};
                 };
                 ConfigShadow configShadow;
 
@@ -486,6 +488,8 @@ namespace Aws
                 static constexpr char DEFAULT_CONFIG_DIR[] = "~/.aws-iot-device-client/";
                 static constexpr char DEFAULT_KEY_DIR[] = "~/.aws-iot-device-client/keys/";
                 static constexpr char DEFAULT_CONFIG_FILE[] = "~/.aws-iot-device-client/aws-iot-device-client.conf";
+                static constexpr char DEFAULT_SYSTEM_CONFIG_FILE[] =
+                    "/etc/.aws-iot-device-client/aws-iot-device-client.conf";
                 static constexpr char DEFAULT_FLEET_PROVISIONING_RUNTIME_CONFIG_FILE[] =
                     "~/.aws-iot-device-client/aws-iot-device-client-runtime.conf";
                 static constexpr char DEFAULT_HTTP_PROXY_CONFIG_FILE[] = "~/.aws-iot-device-client/http-proxy.conf";
@@ -538,7 +542,7 @@ namespace Aws
                 static bool ExportDefaultSetting(const std::string &file);
             };
         } // namespace DeviceClient
-    }     // namespace Iot
+    } // namespace Iot
 } // namespace Aws
 
 #endif // DEVICE_CLIENT_CONFIG_H