Added some niceties around AWS Config

- Added a JSON unwrapping nicety for AWS Config querying convenience.

Author: Mike Grima

.gitignore

@@ -201,5 +201,8 @@ samconfig.toml
 # VS Code:
 .vscode/
 
+# Some kubernetes things
+.kuberlr/
+
 # By default ignore configuration file changes. If this needs to be changed, you need to force add in git:
 src/starfleet/configuration_files/

ATTRIBUTIONS

@@ -0,0 +1,9 @@
+THIS FILE CONTAINS OTHER ATTRIBUTIONS FOR THIS OSS PROJECT:
+
+1. Spaceship Icons:
+    - The spaceship icons are from the *VERY COOL* The Ur-Quan Masters project: https://sc2.sourceforge.net/
+    - The Starfleet open source project has no relation to The Ur-Quan Masters project
+    - The icons were scaled up for size
+    - The content -- … graphics, … -- are copyright (C) 1992, 1993, 2002 Toys for Bob, Inc. or their respective creators.
+      The content may be used freely under the terms of the Creative Commons Attribution-NonCommercial-ShareAlike 2.5 license (available at http://creativecommons.org/licenses/by-nc-sa/2.5/).
+    - The icons are distributed with no warranties of any kind

src/starfleet/worker_ships/niceties.py

@@ -0,0 +1,67 @@
+"""A general set of niceties that Starfleet workers can use to do things that are nice.
+
+This mostly defines some shortcut code utilities that workers can use for a variety of use cases.
+
+:Module: starfleet.worker_ships.niceties
+:Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info
+:License: See the LICENSE file for details
+:Author: Mike Grima <[email protected]>
+"""
+import datetime
+import json
+from typing import Any
+from urllib.parse import unquote_plus
+
+
+def un_wrap_json(json_obj: Any) -> Any:
+    """Helper function to unwrap nested JSON in the AWS Config resource configuration."""
+    # pylint: disable=C0103,W0703,R0911
+    # Is this a field that we can safely return?
+    if isinstance(json_obj, (type(None), int, bool, float)):  # noqa
+        return json_obj
+
+    # Is this a Datetime? Convert it to a string and return it:
+    if isinstance(json_obj, datetime.datetime):
+        return str(json_obj)
+
+    # Is this a Dictionary?
+    if isinstance(json_obj, dict):
+        decoded = {}
+        for k, v in json_obj.items():
+            decoded[k] = un_wrap_json(v)
+
+    # Is this a List?
+    elif isinstance(json_obj, list):
+        decoded = []
+        for x in json_obj:
+            decoded.append(un_wrap_json(x))
+
+        # Yes, try to sort the contents of lists. This is because AWS does not consistently store list ordering for many resource types:
+        try:
+            sorted_list = sorted(decoded)
+            decoded = sorted_list
+        except Exception:  # noqa  # nosec   # If we can't sort then NBD
+            pass
+    else:
+        # Try to load the JSON string:
+        try:
+            # Check if the string starts with a "[" or a "{" (because apparently '123' is a valid JSON 😒😒😒)
+            for check_field in ["{", "[", '"{', '"[']:  # Some of the double-wrapping is really ridiculous 😒
+                if json_obj.startswith(check_field):
+                    decoded = json.loads(json_obj)
+
+                    # If we loaded this properly, then we need to pass the decoded JSON back in for all the nested stuff:
+                    return un_wrap_json(decoded)
+
+            # Check if this string is URL Encoded - if it is, then re-run it through:
+            decoded = unquote_plus(json_obj)
+            if decoded != json_obj:
+                return un_wrap_json(decoded)
+
+            return json_obj
+
+        # If we didn't get a JSON back (exception), then just return the raw value back:
+        except Exception:  # noqa
+            return json_obj
+
+    return decoded

tests/worker_ship_utils/test_niceties.py

@@ -0,0 +1,142 @@
+"""Tests the worker ship niceties.
+
+:Module: starfleet.tests.worker_ship_utils.test_niceties
+:Copyright: (c) 2023 by Gemini Trust Company, LLC., see AUTHORS for more info
+:License: See the LICENSE file for details
+:Author: Mike Grima <[email protected]>
+"""
+# pylint: disable=unused-argument
+import datetime
+
+
+def test_unwrap_json() -> None:
+    """This tests the unwrapping of AWS Config json."""
+    from starfleet.worker_ships.niceties import un_wrap_json
+
+    # Simple AWS policy JSON:
+    test_str = (
+        '{"policyText": "{\\"Version\\":\\"2008-10-17\\",\\"Statement\\":[{\\"Sid\\":\\"AccountReadOnly\\",\\"Effect\\":\\"Allow\\",\\"Principal\\":{'
+        '\\"AWS\\":[\\"arn:aws:iam::000000000001:root\\"]},\\"Action\\":[\\"s3:Get*\\",\\"s3:List*\\"],\\"Resource\\":['
+        '\\"arn:aws:s3:::some-bucket\\",\\"arn:aws:s3:::some-bucket/*\\"]}]}"} '
+    )
+    should_equal = {
+        "policyText": {
+            "Version": "2008-10-17",
+            "Statement": [
+                {
+                    "Sid": "AccountReadOnly",
+                    "Effect": "Allow",
+                    "Principal": {"AWS": ["arn:aws:iam::000000000001:root"]},
+                    "Action": ["s3:Get*", "s3:List*"],
+                    "Resource": ["arn:aws:s3:::some-bucket", "arn:aws:s3:::some-bucket/*"],
+                }
+            ],
+        }
+    }
+    assert un_wrap_json(test_str) == should_equal
+
+    # With URL encoding:
+    test_str_with_url_encoding = (
+        '{"path":"/","roleName":"SomeRole","roleId":"AROAALSKDJFLAKSDJFKLJSDF",'
+        '"arn":"arn:aws:iam::000000000001:role/SomeRole","createDate":"2023-11-12T18:41:33.000Z",'
+        '"assumeRolePolicyDocument":"%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C'
+        '%22Principal%22%3A%7B%22Service%22%3A%22ec2.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D",'
+        '"instanceProfileList":[{"path":"/","instanceProfileName":"SomeRole",'
+        '"instanceProfileId":"AROAALSKDJFLAKSDJFKLJSDF",'
+        '"arn":"arn:aws:iam::000000000001:instance-profile/SomeRole","createDate":"2023-11-12T18:41:33.000Z",'
+        '"roles":[{"path":"/","roleName":"SomeRole","roleId":"AROAALSKDJFLAKSDJFKLJSDF",'
+        '"arn":"arn:aws:iam::000000000001:role/SomeRole","createDate":"2023-11-12T18:41:33.000Z",'
+        '"assumeRolePolicyDocument":"%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C'
+        '%22Principal%22%3A%7B%22Service%22%3A%22ec2.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D",'
+        '"description":null,"maxSessionDuration":null,"permissionsBoundary":null,"tags":[],"roleLastUsed":null}]}],'
+        '"rolePolicyList":[{"policyName":"Config",'
+        '"policyDocument":"%7B%22Statement%22%3A%5B%7B%22Action%22%3A%5B%22config%3Aselectaggregateresourceconfig%22%5D%2C'
+        '%22Effect%22%3A%22Allow%22%2C%22Resource%22%3A%5B%22%2A%22%5D%7D%5D%2C%22Version%22%3A%222012-10-17%22%7D"}],'
+        '"attachedManagedPolicies":[{"policyName":"SomePolicy",'
+        '"policyArn":"arn:aws:iam::000000000001:policy/SomePolicy"}],"permissionsBoundary":null,"tags":[],'
+        '"roleLastUsed":null} '
+    )
+    should_equal = {
+        "path": "/",
+        "roleName": "SomeRole",
+        "roleId": "AROAALSKDJFLAKSDJFKLJSDF",
+        "arn": "arn:aws:iam::000000000001:role/SomeRole",
+        "createDate": "2023-11-12T18:41:33.000Z",
+        "assumeRolePolicyDocument": {
+            "Version": "2012-10-17",
+            "Statement": [{"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole"}],
+        },
+        "instanceProfileList": [
+            {
+                "path": "/",
+                "instanceProfileName": "SomeRole",
+                "instanceProfileId": "AROAALSKDJFLAKSDJFKLJSDF",
+                "arn": "arn:aws:iam::000000000001:instance-profile/SomeRole",
+                "createDate": "2023-11-12T18:41:33.000Z",
+                "roles": [
+                    {
+                        "path": "/",
+                        "roleName": "SomeRole",
+                        "roleId": "AROAALSKDJFLAKSDJFKLJSDF",
+                        "arn": "arn:aws:iam::000000000001:role/SomeRole",
+                        "createDate": "2023-11-12T18:41:33.000Z",
+                        "assumeRolePolicyDocument": {
+                            "Version": "2012-10-17",
+                            "Statement": [{"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole"}],
+                        },
+                        "description": None,
+                        "maxSessionDuration": None,
+                        "permissionsBoundary": None,
+                        "tags": [],
+                        "roleLastUsed": None,
+                    }
+                ],
+            }
+        ],
+        "rolePolicyList": [
+            {
+                "policyName": "Config",
+                "policyDocument": {
+                    "Statement": [{"Action": ["config:selectaggregateresourceconfig"], "Effect": "Allow", "Resource": ["*"]}],
+                    "Version": "2012-10-17",
+                },
+            }
+        ],
+        "attachedManagedPolicies": [{"policyName": "SomePolicy", "policyArn": "arn:aws:iam::000000000001:policy/SomePolicy"}],
+        "permissionsBoundary": None,
+        "tags": [],
+        "roleLastUsed": None,
+    }
+    assert un_wrap_json(test_str_with_url_encoding) == should_equal
+
+    # And again with some strange nesting:
+    test_nested = {
+        "how": [
+            "nested",
+            {
+                "can": '{"we": "really", "really": "[\\"get\\", \\"{\\\\\\"into\\\\\\": \\\\\\"{\\\\\\\\\\\\\\"really\\\\\\\\\\\\\\": '
+                '\\\\\\\\\\\\\\"deep\\\\\\\\\\\\\\"}\\\\\\"}\\"]"}'
+            },
+            94,
+            3.14,
+            {"more": '{"and again": "{\\"this and\\": \\"that\\"}"}'},
+        ]
+    }
+    should_equal = {
+        "how": ["nested", {"can": {"we": "really", "really": ["get", {"into": {"really": "deep"}}]}}, 94, 3.14, {"more": {"and again": {"this and": "that"}}}]
+    }
+    assert un_wrap_json(test_nested) == should_equal
+
+    # And values that are non-JSON:
+    now = datetime.datetime.utcnow()
+    assert un_wrap_json(now) == str(now)
+    assert un_wrap_json(19) == 19
+    assert un_wrap_json(3.14) == 3.14
+    assert un_wrap_json(True) is True
+
+    # Try it with something bizarre, like a function:
+    assert un_wrap_json(test_unwrap_json) == test_unwrap_json  # pylint: disable=comparison-with-callable
+
+    # Test that we are sorting lists:
+    test_sorted = {"a_list": [11, 5.2, 2, 0, 3], "b_list": ["a", "b", "c"], "with_nested_objs": [{"A": "Value"}, {"qwerty": "uiop[]"}]}
+    assert un_wrap_json(test_sorted) == {"a_list": [0, 2, 3, 5.2, 11], "b_list": ["a", "b", "c"], "with_nested_objs": [{"A": "Value"}, {"qwerty": "uiop[]"}]}