Add option to specify chunk size

replibyte/src/cli.rs

@@ -110,6 +110,9 @@ pub struct DumpCreateArgs {
     /// dump name
     #[clap(short, long)]
     pub name: Option<String>,
+    /// chunk size, default to 100
+    #[clap(short, long)]
+    pub chunk_size: Option<usize>,
 }
 
 #[derive(Args, Debug)]

replibyte/src/commands/dump.rs

@@ -132,6 +132,7 @@ where
                 skip_config: &skip_config,
                 database_subset: &source.database_subset,
                 only_tables: &only_tables_config,
+                chunk_size: args.chunk_size,
             };
 
             match args.source_type.as_ref().map(|x| x.as_str()) {

replibyte/src/source/mod.rs

@@ -29,4 +29,5 @@ pub struct SourceOptions<'a> {
     pub skip_config: &'a Vec<SkipConfig>,
     pub database_subset: &'a Option<DatabaseSubsetConfig>,
     pub only_tables: &'a Vec<OnlyTablesConfig>,
+    pub chunk_size: Option<usize>,
 }

replibyte/src/tasks/full_dump.rs

@@ -69,8 +69,9 @@ where
             Ok(())
         });
 
-        // buffer of 100MB in memory to use and re-use to upload data into datastore
-        let buffer_size = 100 * 1024 * 1024;
+        // buffer default of 100MB (unless specified) in memory to use and re-use to upload data into datastore
+        let chunk_size = self.options.chunk_size.unwrap_or(100);
+        let buffer_size = chunk_size * 1024 * 1024;
         let mut queries = vec![];
         let mut consumed_buffer_size = 0usize;
         let mut total_transferred_bytes = 0usize;