feat(set all data external options): additional parameters added (#2041)

* feat(set all data external options): parameters added

 "binary" and "base_name" parameters added to set_all_data_external

* feat(set all external binary): doc update

* feat(set all ext options)

Author: scottrp

.docs/Notebooks/mf6_data_tutorial08.py

@@ -1,20 +1,21 @@
 # ---
 # jupyter:
 #   jupytext:
+#     notebook_metadata_filter: metadata
 #     text_representation:
 #       extension: .py
 #       format_name: light
-#       format_version: "1.5"
-#       jupytext_version: 1.5.1
+#       format_version: '1.5'
+#       jupytext_version: 1.14.4
 #   kernelspec:
-#     display_name: Python 3
+#     display_name: Python 3 (ipykernel)
 #     language: python
 #     name: python3
 #   metadata:
 #     section: mf6
 # ---
 
-# # MODFLOW 6: Data Storage Information and Performance Optimization
+# # MODFLOW 6: External Files, Binary Data, and Performance Optimization
 #
 # This tutorial shows the different options for storing MODFLOW data in FloPy.
 # Interaction with a FloPy MODFLOW 6 model is different from other models,
@@ -36,7 +37,7 @@
 # This tutorial focuses on the different storage options for MODFLOW data and
 # how to optimize data storage read/write speed.
 
-# ## Introduction to Data Storage Information
+# ## Introduction to Data Storage Options
 # MODFLOW array and list data can either be stored internally or externally in
 # text or binary files. Additionally array data can have a factor applied to
 # them and can have a format flag/code to define how these data will be
@@ -215,6 +216,10 @@
 print(f"New binary flag for stress period 1:  {spd_record[0]['binary']}")
 print(f"New filename for stress period 2:  {spd_record[1]['filename']}")
 
+# An alternative to individually setting each file to external is to call the set_all_files_external method (there is also a set_all_files_internal method to do the opposite). While this requires less code, it does not give you the ability to set the names of each individual external file. By setting the binary attribute to True, flopy will store data to binary files wherever possible.
+
+sim.set_all_data_external(binary=True)
+
 # ## Optimizing FloPy Performance
 #
 # By default FloPy will perform a number of verification checks on your data

autotest/regression/test_mf6.py

@@ -4240,7 +4240,34 @@ def test045_lake1ss_table(function_tmpdir, example_data_path):
     save_folder = function_tmpdir / "save"
     save_folder.mkdir()
     sim.set_sim_path(save_folder)
+    sim.set_all_data_external(
+        external_data_folder="test_folder",
+        base_name="ext_file",
+        binary=True,
+    )
     sim.write_simulation()
+    # verify external files were written
+    ext_folder = os.path.join(save_folder, "test_folder")
+    files_to_check = [
+        "ext_file_lakeex1b.dis_botm_layer1.bin",
+        "ext_file_lakeex1b.dis_botm_layer2.bin",
+        "ext_file_lakeex1b.dis_botm_layer3.bin",
+        "ext_file_lakeex1b.dis_botm_layer4.bin",
+        "ext_file_lakeex1b.dis_botm_layer5.bin",
+        "ext_file_lakeex1b.npf_k_layer1.bin",
+        "ext_file_lakeex1b.npf_k_layer5.bin",
+        "ext_file_lakeex1b.chd_stress_period_data_1.bin",
+        "ext_file_lakeex1b.lak_connectiondata.txt",
+        "ext_file_lakeex1b.lak_packagedata.txt",
+        "ext_file_lakeex1b.lak_perioddata_1.txt",
+        "ext_file_lakeex1b_table.ref_table.txt",
+        "ext_file_lakeex1b.evt_depth_1.bin",
+        "ext_file_lakeex1b.evt_rate_1.bin",
+        "ext_file_lakeex1b.evt_surface_1.bin",
+    ]
+    for file in files_to_check:
+        data_file_path = os.path.join(ext_folder, file)
+        assert os.path.exists(data_file_path)
 
     # run simulation
     success, buff = sim.run_simulation()

flopy/mf6/data/mfdata.py

@@ -251,7 +251,7 @@ def __init__(
         self._data_type = structure.type
         self._keyword = ""
         if self._simulation_data is not None:
-            self._data_dimensions = DataDimensions(dimensions, structure)
+            self.data_dimensions = DataDimensions(dimensions, structure)
             # build a unique path in the simulation dictionary
             self._org_path = self._path
             index = 0
@@ -380,13 +380,13 @@ def layer_shape(self):
         layers = []
         layer_dims = self.structure.data_item_structures[0].layer_dims
         if len(layer_dims) == 1:
-            layers.append(self._data_dimensions.get_model_grid().num_layers())
+            layers.append(self.data_dimensions.get_model_grid().num_layers())
         else:
             for layer in layer_dims:
                 if layer == "nlay":
                     # get the layer size from the model grid
                     try:
-                        model_grid = self._data_dimensions.get_model_grid()
+                        model_grid = self.data_dimensions.get_model_grid()
                     except Exception as ex:
                         type_, value_, traceback_ = sys.exc_info()
                         raise MFDataException(
@@ -521,13 +521,13 @@ def _get_constant_formatting_string(
             const_val,
             data_type,
             self._simulation_data,
-            self._data_dimensions,
+            self.data_dimensions,
             verify_data=self._simulation_data.verify_data,
         )
         return f"{sim_data.indent_string.join(const_format)}{suffix}"
 
     def _get_aux_var_name(self, aux_var_index):
-        aux_var_names = self._data_dimensions.package_dim.get_aux_variables()
+        aux_var_names = self.data_dimensions.package_dim.get_aux_variables()
         # TODO: Verify that this works for multi-dimensional layering
         return aux_var_names[0][aux_var_index[0] + 1]
 
@@ -608,7 +608,7 @@ def _get_external_formatting_str(
         self, fname, factor, binary, iprn, data_type, ext_file_action
     ):
         file_mgmt = self._simulation_data.mfpath
-        model_name = self._data_dimensions.package_dim.model_dim[0].model_name
+        model_name = self.data_dimensions.package_dim.model_dim[0].model_name
         ext_file_path = file_mgmt.get_updated_path(
             fname, model_name, ext_file_action
         )

flopy/mf6/data/mfdataarray.py

@@ -348,7 +348,7 @@ def supports_layered(self):
         """
 
         try:
-            model_grid = self._data_dimensions.get_model_grid()
+            model_grid = self.data_dimensions.get_model_grid()
         except Exception as ex:
             type_, value_, traceback_ = sys.exc_info()
             raise MFDataException(
@@ -381,7 +381,7 @@ def set_layered_data(self, layered_data):
         """
         if layered_data is True and self.structure.layered is False:
             if (
-                self._data_dimensions.get_model_grid().grid_type()
+                self.data_dimensions.get_model_grid().grid_type()
                 == DiscretizationType.DISU
             ):
                 comment = f"Layered option not available for unstructured grid. {self._path}"
@@ -430,7 +430,7 @@ def make_layered(self):
                 )
         else:
             if (
-                self._data_dimensions.get_model_grid().grid_type()
+                self.data_dimensions.get_model_grid().grid_type()
                 == DiscretizationType.DISU
             ):
                 comment = f"Layered option not available for unstructured grid. {self._path}"
@@ -482,6 +482,7 @@ def store_as_external_file(
                 Whether to replace an existing external file.
             check_data : bool
                 Verify data prior to storing
+
         """
         storage = self._get_storage_obj()
         if storage is None:
@@ -861,11 +862,11 @@ def _set_record(self, data_record):
             )
             type_, value_, traceback_ = sys.exc_info()
             raise MFDataException(
-                self._data_dimensions.structure.get_model(),
-                self._data_dimensions.structure.get_package(),
-                self._data_dimensions.structure.path,
+                self.data_dimensions.structure.get_model(),
+                self.data_dimensions.structure.get_package(),
+                self.data_dimensions.structure.path,
                 "setting record",
-                self._data_dimensions.structure.name,
+                self.data_dimensions.structure.name,
                 inspect.stack()[0][3],
                 type_,
                 value_,
@@ -933,7 +934,7 @@ def _set_data(
             # handle special case of aux variables in an array
             self.layered = True
             aux_var_names = (
-                self._data_dimensions.package_dim.get_aux_variables()
+                self.data_dimensions.package_dim.get_aux_variables()
             )
             if len(aux_data) == len(aux_var_names[0]) - 1:
                 for layer, aux_var_data in enumerate(aux_data):
@@ -980,11 +981,11 @@ def _set_data(
                 )
                 type_, value_, traceback_ = sys.exc_info()
                 raise MFDataException(
-                    self._data_dimensions.structure.get_model(),
-                    self._data_dimensions.structure.get_package(),
-                    self._data_dimensions.structure.path,
+                    self.data_dimensions.structure.get_model(),
+                    self.data_dimensions.structure.get_package(),
+                    self.data_dimensions.structure.path,
                     "setting aux variables",
-                    self._data_dimensions.structure.name,
+                    self.data_dimensions.structure.name,
                     inspect.stack()[0][3],
                     type_,
                     value_,
@@ -1064,7 +1065,7 @@ def load(
         self._resync()
         if self.structure.layered:
             try:
-                model_grid = self._data_dimensions.get_model_grid()
+                model_grid = self.data_dimensions.get_model_grid()
             except Exception as ex:
                 type_, value_, traceback_ = sys.exc_info()
                 raise MFDataException(
@@ -1101,7 +1102,7 @@ def load(
         else:
             file_access = MFFileAccessArray(
                 self.structure,
-                self._data_dimensions,
+                self.data_dimensions,
                 self._simulation_data,
                 self._path,
                 self._current_key,
@@ -1272,7 +1273,7 @@ def _new_storage(
             return DataStorage(
                 self._simulation_data,
                 self._model_or_sim,
-                self._data_dimensions,
+                self.data_dimensions,
                 self._get_file_entry,
                 DataStorageType.internal_array,
                 DataStructureType.ndarray,
@@ -1284,7 +1285,7 @@ def _new_storage(
             return DataStorage(
                 self._simulation_data,
                 self._model_or_sim,
-                self._data_dimensions,
+                self.data_dimensions,
                 self._get_file_entry,
                 DataStorageType.internal_array,
                 DataStructureType.ndarray,
@@ -1402,7 +1403,7 @@ def _get_file_entry_layer(
                     self._simulation_data.debug,
                     ex,
                 )
-            package_dim = self._data_dimensions.package_dim
+            package_dim = self.data_dimensions.package_dim
             model_name = package_dim.model_dim[0].model_name
             self._simulation_data.mfpath.add_ext_file(file_path, model_name)
         return file_entry
@@ -1430,7 +1431,7 @@ def _get_data_layer_string(self, layer, data_indent):
             )
         file_access = MFFileAccessArray(
             self.structure,
-            self._data_dimensions,
+            self.data_dimensions,
             self._simulation_data,
             self._path,
             self._current_key,
@@ -1683,6 +1684,7 @@ def store_as_external_file(
                 Whether to replace an existing external file.
             check_data : bool
                 Verify data prior to storing
+
         """
         # store each stress period in separate file(s)
         for sp in self._data_storage.keys():
@@ -1803,7 +1805,7 @@ def get_record(self, key=None):
         """
         if self._data_storage is not None and len(self._data_storage) > 0:
             if key is None:
-                sim_time = self._data_dimensions.package_dim.model_dim[
+                sim_time = self.data_dimensions.package_dim.model_dim[
                     0
                 ].simulation_time
                 num_sp = sim_time.get_num_stress_periods()
@@ -1825,7 +1827,7 @@ def get_data(self, key=None, apply_mult=True, **kwargs):
         """
         if self._data_storage is not None and len(self._data_storage) > 0:
             if key is None:
-                sim_time = self._data_dimensions.package_dim.model_dim[
+                sim_time = self.data_dimensions.package_dim.model_dim[
                     0
                 ].simulation_time
                 num_sp = sim_time.get_num_stress_periods()