refactor of pillar masking simplifying the whole thing

Author: Akm0d

doc/topics/tutorials/pillar.rst

@@ -40,21 +40,6 @@ Arbitrary Data:
     so a key/value store can be defined making it easy to iterate over a group
     of values in sls formulas.
 
-.. versionadded:: 3008
-
-On the minion, compiled pillar is stored using :mod:`salt.utils.secret`
-containers: string and bytes leaves are wrapped with Pydantic ``SecretStr`` and
-``SecretBytes``, and mappings/lists use ``SecretDict`` / ``SecretList`` so later
-in-place updates stay wrapped. State returns and minion logs redact known pillar
-literals where possible. States may set ``no_log: true`` (runtime keyword, not
-passed to the state function) to mask that state's ``comment`` and ``changes``
-in output.
-
-The execution functions ``pillar.raw`` and ``pillar.get`` are exceptions: they
-return plain Python strings, bytes, dicts, and lists (unwrapped) because the
-operator explicitly asked for pillar data. Other code paths use wrapped
-in-memory pillar unless they call those functions.
-
 Pillar is therefore one of the most important systems when using Salt. This
 walkthrough is designed to get a simple Pillar up and running in a few minutes
 and then to dive into the capabilities of Pillar and where the data is

salt/cache/localfs.py

@@ -42,7 +42,7 @@ def get_storage_id(kwargs):
     return ("localfs", __cachedir(kwargs))
 
 
-def store(bank, key, data, cachedir, **kwargs):
+def store(bank, key, data, cachedir):
     """
     Store information in a file.
     """

salt/client/ssh/__init__.py

@@ -44,7 +44,6 @@
 import salt.utils.path
 import salt.utils.platform
 import salt.utils.relenv
-import salt.utils.secret
 import salt.utils.stringutils
 import salt.utils.thin
 import salt.utils.timeutil
@@ -1653,7 +1652,7 @@ def _run_wfunc_thin(self):
         if grains_updates:
             opts["grains"] = {**opts["grains"], **grains_updates}
 
-        opts["pillar"] = salt.utils.secret.expose(data.get("pillar"))
+        opts["pillar"] = data.get("pillar")
 
         # Restore --wipe. Note: Since it is also a CLI option, it should not
         # be read from cache, hence it is restored here. This is currently only
@@ -1691,9 +1690,7 @@ def _run_wfunc_thin(self):
             elif opts["pillar"] and self.fun in opts["pillar"].get(
                 "mine_functions", {}
             ):
-                mine_fun_data = salt.utils.secret.expose(
-                    opts["pillar"]["mine_functions"][self.fun]
-                )
+                mine_fun_data = opts["pillar"]["mine_functions"][self.fun]
             elif self.fun in self.context["master_opts"].get("mine_functions", {}):
                 mine_fun_data = self.context["master_opts"]["mine_functions"][self.fun]
 

salt/client/ssh/wrapper/config.py

@@ -12,7 +12,6 @@
 import salt.utils.data
 import salt.utils.files
 import salt.utils.sdb as sdb
-import salt.utils.secret
 
 log = logging.getLogger(__name__)
 
@@ -192,7 +191,7 @@ def option(
                 return __grains__[value]
         if not omit_pillar:
             if value in __pillar__:
-                return salt.utils.secret.expose(__pillar__[value])
+                return __pillar__[value]
         if not omit_master:
             if value in __pillar__.get("master", {}):
                 return __pillar__["master"][value]
@@ -207,7 +206,7 @@ def option(
         ret = {}
         for omit, data in (
             (omit_master, __pillar__.get("master", {})),
-            (omit_pillar, salt.utils.secret.expose(__pillar__)),
+            (omit_pillar, __pillar__),
             (omit_grains, __grains__),
             (omit_opts, __opts__),
         ):
@@ -254,7 +253,7 @@ def merge(value, default="", omit_opts=False, omit_master=False, omit_pillar=Fal
                 ret = list(ret) + list(tmp)
     if not omit_pillar:
         if value in __pillar__:
-            tmp = salt.utils.secret.expose(__pillar__[value])
+            tmp = __pillar__[value]
             if ret is None:
                 ret = tmp
                 if isinstance(ret, str):
@@ -436,17 +435,14 @@ def get(
 
         if not omit_pillar:
             ret = salt.utils.data.traverse_dict_and_list(
-                salt.utils.secret.expose(__pillar__), key, "_|-", delimiter=delimiter
+                __pillar__, key, "_|-", delimiter=delimiter
             )
             if ret != "_|-":
                 return sdb.sdb_get(ret, __opts__)
 
         if not omit_master:
             ret = salt.utils.data.traverse_dict_and_list(
-                __pillar__.get("master", {}),
-                key,
-                "_|-",
-                delimiter=delimiter,
+                __pillar__.get("master", {}), key, "_|-", delimiter=delimiter
             )
             if ret != "_|-":
                 return sdb.sdb_get(ret, __opts__)
@@ -469,16 +465,10 @@ def get(
 
         data = copy.copy(DEFAULTS)
         data = salt.utils.dictupdate.merge(
-            data,
-            __pillar__.get("master", {}),
-            strategy=merge,
-            merge_lists=merge_lists,
+            data, __pillar__.get("master", {}), strategy=merge, merge_lists=merge_lists
         )
         data = salt.utils.dictupdate.merge(
-            data,
-            salt.utils.secret.expose(__pillar__),
-            strategy=merge,
-            merge_lists=merge_lists,
+            data, __pillar__, strategy=merge, merge_lists=merge_lists
         )
         data = salt.utils.dictupdate.merge(
             data, __grains__, strategy=merge, merge_lists=merge_lists

salt/client/ssh/wrapper/cp.py

@@ -38,7 +38,6 @@
 import salt.client.ssh
 import salt.fileclient
 import salt.utils.files
-import salt.utils.secret
 import salt.utils.stringutils
 import salt.utils.templates
 from salt.exceptions import CommandExecutionError
@@ -206,7 +205,7 @@ def get_template(
     if "salt" not in kwargs:
         kwargs["salt"] = __salt__.value()
     if "pillar" not in kwargs:
-        kwargs["pillar"] = salt.utils.secret.expose(__pillar__.value())
+        kwargs["pillar"] = __pillar__.value()
     if "grains" not in kwargs:
         kwargs["grains"] = __grains__.value()
     if "opts" not in kwargs:
@@ -828,7 +827,7 @@ def _gather_pillar(pillarenv, pillar_override):
         pillar_override=pillar_override,
         pillarenv=pillarenv,
     )
-    return salt.utils.secret.expose(pillar.compile_pillar())
+    return pillar.compile_pillar()
 
 
 def _render_filenames(path, dest, saltenv, template, **kw):
@@ -852,7 +851,7 @@ def _render_filenames(path, dest, saltenv, template, **kw):
         pillarenv = kw.get("pillarenv", __opts__.get("pillarenv"))
         kwargs["pillar"] = _gather_pillar(pillarenv, kw.get("pillar"))
     else:
-        kwargs["pillar"] = salt.utils.secret.expose(__pillar__.value())
+        kwargs["pillar"] = __pillar__.value()
     kwargs["grains"] = __grains__.value()
     kwargs["opts"] = __opts__
     kwargs["saltenv"] = saltenv

salt/fileclient.py

@@ -30,7 +30,6 @@
 import salt.utils.http
 import salt.utils.path
 import salt.utils.platform
-import salt.utils.secret
 import salt.utils.stringutils
 import salt.utils.templates
 import salt.utils.url
@@ -1487,11 +1486,7 @@ def master_tops(self):
         """
         Return the metadata derived from the master_tops system
         """
-        load = {
-            "cmd": "_master_tops",
-            "id": self.opts["id"],
-            "opts": salt.utils.secret.expose(self.opts),
-        }
+        load = {"cmd": "_master_tops", "id": self.opts["id"], "opts": self.opts}
         return self._channel_send(
             load,
         )

salt/master.py

@@ -52,7 +52,6 @@
 import salt.utils.platform
 import salt.utils.process
 import salt.utils.schedule
-import salt.utils.secret
 import salt.utils.ssdp
 import salt.utils.stringutils
 import salt.utils.user
@@ -2105,7 +2104,7 @@ def _pillar(self, load):
                     {"Minion data cache refresh": load["id"]},
                     tagify(load["id"], "refresh", "minion"),
                 )
-        return salt.utils.secret.expose(data)
+        return data
 
     def _minion_event(self, load):
         """

salt/matchers/pillar_exact_match.py

@@ -5,7 +5,6 @@
 import logging
 
 import salt.utils.data
-import salt.utils.secret
 
 log = logging.getLogger(__name__)
 
@@ -32,7 +31,6 @@ def match(tgt, delimiter=":", opts=None, minion_id=None):
     else:
         pillar = {}
 
-    pillar = salt.utils.secret.expose(pillar)
     return salt.utils.data.subdict_match(
         pillar, tgt, delimiter=delimiter, exact_match=True
     )

salt/matchers/pillar_match.py

@@ -5,7 +5,6 @@
 import logging
 
 import salt.utils.data
-import salt.utils.secret
 from salt.defaults import DEFAULT_TARGET_DELIM
 
 log = logging.getLogger(__name__)
@@ -33,5 +32,4 @@ def match(tgt, delimiter=DEFAULT_TARGET_DELIM, opts=None, minion_id=None):
     else:
         pillar = {}
 
-    pillar = salt.utils.secret.expose(pillar)
     return salt.utils.data.subdict_match(pillar, tgt, delimiter=delimiter)

salt/matchers/pillar_pcre_match.py

@@ -34,7 +34,6 @@ def match(tgt, delimiter=DEFAULT_TARGET_DELIM, opts=None, minion_id=None):
     else:
         pillar = {}
 
-    pillar = salt.utils.secret.expose(pillar)
     return salt.utils.data.subdict_match(
         pillar, tgt, delimiter=delimiter, regex_match=True
     )