Add option to copy DB file

Author: sfegan

include/diagnostics/stage1.hpp

@@ -62,6 +62,9 @@ class Stage1ParallelEventVisitor:
 
   static calin::ix::diagnostics::stage1::Stage1Config default_config();
 
+  static std::string nectarcam_ancillary_database_filename(const std::string run_filename, uint64_t run_start_time_ns=0, 
+    const std::string forced_filename = "", const std::string forced_directory = "");
+
 private:
   calin::ix::diagnostics::stage1::Stage1Config config_;
 

proto/diagnostics/stage1.proto

@@ -101,7 +101,10 @@ message Stage1Config {
     (CFO).desc = "Configuration of clock regression calculator." ];
 
   string ancillary_database                                = 200 [
-    (CFO).desc = "Location of ancillary database if used." ];
+    (CFO).desc = "Full path to ancillary database including filename if used." ];
+  string ancillary_database_directory                      = 201 [
+    (CFO).desc = "Directory containing ancillary database if used. Overriden by \"ancillary_database\" "
+      "if this is given. An empty value defaults to directory housing the events data file." ];
 };
 
 message NectarCamSpecificStage1 {

proto/scripts/cta_stage1.proto

@@ -62,6 +62,13 @@ message CommandLineOptions {
   int32 nthread                                            = 21 [
     (CFO).desc = "Number of computation threads. A value of -1 disables "
       "threading completely." ];
+  string copy_ancillary_db                                 = 22 [
+    (CFO).desc = "Specify directory to which the ancillary database should be copied "
+      "before processing the data file. If left blank no copy is made, the ancillary "
+      "database will be read from its original place. This option is useful " 
+      "when processing data on NSF filesystems as SQLITE3 perfromance is bad "
+      "on these systems. Autotomatically sents the \"stage1.ancillary_database_directory\" "
+      "option." ];
 
   calin.ix.iact_data.cta_data_source.CTACameraEventDecoderConfig decoder = 30 [
     (CFO).desc = "CTA event decoder options." ];

scripts/cta_stage1.py

@@ -21,6 +21,8 @@
 import traceback
 import numpy
 import time
+import shutil
+import os
 
 import calin.ix.io.zmq_data_source
 import calin.iact_data.event_dispatcher
@@ -90,7 +92,10 @@
 dispatcher = calin.iact_data.event_dispatcher.ParallelEventDispatcher()
 
 # Create the stage1 visitor
-s1pev = calin.diagnostics.stage1.Stage1ParallelEventVisitor(opt.stage1())
+s1cfg = opt.const_stage1().Clone()
+if(opt.const_stage1().ancillary_database_directory() == ""):
+    s1cfg.set_ancillary_database_directory(opt.copy_ancillary_db())
+s1pev = calin.diagnostics.stage1.Stage1ParallelEventVisitor(s1cfg)
 dispatcher.add_visitor(s1pev)
 
 # Open SQL file
@@ -107,6 +112,7 @@
     dispatcher.process_cta_zmq_run(endpoints, cfg)
     sql.insert(opt.db_stage1_table_name(), visitor.stage1_results())
 else:
+    copied_ancillary_db = ''
     first_file = True
     failed_files = []
     nsuccess = 0;
@@ -129,6 +135,25 @@
                     print(f"Deleting old stage1 results from database, OID : {oid}")
                     sql.delete_by_oid(opt.db_stage1_table_name(), oid)
         print("#%d / %d: processing %s"%(ifile+1,nfile,filename))
+        if(opt.copy_ancillary_db() != ''):
+            src_ancillary_db = calin.diagnostics.stage1.Stage1ParallelEventVisitor.nectarcam_ancillary_database_filename(filename,0,
+                opt.const_stage1().ancillary_database(), opt.const_stage1().ancillary_database_directory())
+            dst_ancillary_db = calin.diagnostics.stage1.Stage1ParallelEventVisitor.nectarcam_ancillary_database_filename(filename,0,
+                s1cfg.ancillary_database(), s1cfg.ancillary_database_directory())
+            if(copied_ancillary_db != '' and copied_ancillary_db != dst_ancillary_db):
+                print("Deleting %s"%(copied_ancillary_db))
+                os.unlink(copied_ancillary_db)
+                copied_ancillary_db = ''
+            if(os.path.isfile(src_ancillary_db)):
+                try:
+                    if(not os.path.isfile(dst_ancillary_db)):
+                        print("Copying %s -> %s"%(src_ancillary_db,dst_ancillary_db))
+                        shutil.copyfile(src_ancillary_db, dst_ancillary_db)
+                        copied_ancillary_db = dst_ancillary_db
+                except Exception as x:
+                    traceback.print_exception(*sys.exc_info())
+            else:
+                print("*** Could not copy ancillary DB %s ***"%(src_ancillary_db))
         try:
             calin.util.log.prune_default_protobuf_log()
             calin.provenance.chronicle.prune_the_chronicle()
@@ -148,6 +173,11 @@
             failed_files.append(filename)
             pass
 
+    if(copied_ancillary_db != ''):
+        print("Deleting %s"%(copied_ancillary_db))
+        os.unlink(copied_ancillary_db)
+        copied_ancillary_db = ''
+
     print("")
     print("="*80)
     if(nsuccess > 0):

src/diagnostics/stage1.cpp

@@ -200,30 +200,16 @@ bool Stage1ParallelEventVisitor::leave_telescope_run(
     start_time = std::max(start_time-30, int64_t(0));
     end_time = end_time+30;
 
-    std::string db_filename = config_.ancillary_database();
+    std::string db_filename;
 
     switch(run_config_->camera_layout().camera_type()) {
       case calin::ix::iact_data::instrument_layout::CameraLayout::NECTARCAM:
       case calin::ix::iact_data::instrument_layout::CameraLayout::NECTARCAM_TESTBENCH_19CHANNEL:
       case calin::ix::iact_data::instrument_layout::CameraLayout::NECTARCAM_TESTBENCH_61CHANNEL:
-        if(db_filename.empty()) {
-          db_filename = calin::util::file::dirname(run_config_->filename());
-          std::string utdate = calin::util::file::basename(db_filename);
-          db_filename += "/nectarcam_monitoring_db_";
-          if(utdate.size() == 8) {
-            db_filename += utdate.substr(0,4);
-            db_filename += "-";
-            db_filename += utdate.substr(4,2);
-            db_filename += "-";
-            db_filename += utdate.substr(6,2);
-          } else {
-            db_filename += calin::util::timestamp::
-              Timestamp(run_config_->run_start_time().time_ns()).as_string().substr(0,10);
-          }
-          db_filename += ".sqlite";
-        }
-        if(calin::util::file::is_file(db_filename))
-        {
+        db_filename = nectarcam_ancillary_database_filename(run_config_->filename(), 
+          run_config_->run_start_time().time_ns(), config_.ancillary_database(),
+          config_.ancillary_database_directory());
+        if(calin::util::file::is_file(db_filename)) {
           nectarcam_ancillary_data_ =
             calin::iact_data::nectarcam_ancillary_data::
               retrieve_nectarcam_ancillary_data(db_filename, run_config_->telescope_id(),
@@ -242,6 +228,33 @@ bool Stage1ParallelEventVisitor::leave_telescope_run(
   return good;
 }
 
+std::string Stage1ParallelEventVisitor::
+nectarcam_ancillary_database_filename(const std::string run_filename, uint64_t run_start_time_ns, 
+  const std::string forced_filename, const std::string forced_directory)
+{
+  std::string db_filename = forced_filename;
+  if(db_filename.empty()) {
+    db_filename = forced_directory;
+    if(db_filename.empty()) {
+      db_filename = calin::util::file::dirname(run_filename);
+    }
+    std::string utdate = calin::util::file::basename(calin::util::file::dirname(run_filename));
+    db_filename += "/nectarcam_monitoring_db_";
+    if(utdate.size() == 8) {
+      db_filename += utdate.substr(0,4);
+      db_filename += "-";
+      db_filename += utdate.substr(4,2);
+      db_filename += "-";
+      db_filename += utdate.substr(6,2);
+    } else {
+      db_filename += calin::util::timestamp::
+        Timestamp(run_start_time_ns).as_string().substr(0,10);
+    }
+    db_filename += ".sqlite";
+  }
+  return db_filename;
+}
+
 calin::ix::diagnostics::stage1::Stage1* Stage1ParallelEventVisitor::stage1_results(
   calin::ix::diagnostics::stage1::Stage1* stage1) const
 {