Be more vocal about traces without the proper CPU configuration (#3928)

* Yell louder about improperly configured CPUs.

* Annotate traces when the CPU is not configured correctly but users force recording anyways.

Author: khuey

src/PerfCounters.cc

@@ -44,6 +44,7 @@ namespace rr {
 #define PERF_COUNT_RR 0x72727272L
 
 static bool attributes_initialized;
+static bool cpu_improperly_configured;
 // At some point we might support multiple kinds of ticks for the same CPU arch.
 // At that point this will need to become more complicated.
 struct perf_event_attrs {
@@ -512,6 +513,7 @@ static void check_working_counters(perf_event_attrs &perf_attr) {
 // Returns the ticks minimum period.
 static uint32_t check_for_bugs(perf_event_attrs &perf_attr) {
   DEBUG_ASSERT(!running_under_rr());
+  cpu_improperly_configured = false;
 
   uint32_t min_period = check_for_ioc_period_bug(perf_attr);
   check_working_counters(perf_attr);
@@ -836,6 +838,10 @@ void PerfCounters::start_pt_copy_thread() {
   }
 }
 
+bool PerfCounters::improperly_configured() {
+  return cpu_improperly_configured;
+}
+
 // See https://github.com/intel/libipt/blob/master/doc/howto_capture.md
 void PerfCounters::PTState::open(pid_t tid) {
   static uint32_t event_type = pt_event_type();

src/PerfCounters.h

@@ -177,6 +177,12 @@ class PerfCounters {
    */
   static void start_pt_copy_thread();
 
+  /**
+   * Returns true iff we detected issues with performance counter configuration
+   * and wanted to die but the user forced continuation.
+   */
+  static bool improperly_configured();
+
   /**
    * Try to use BPF to accelerate async signal processing
    */

src/PerfCounters_x86.h

@@ -322,6 +322,8 @@ static void check_for_xen_pmi_bug(const perf_event_attrs &perf_attr) {
              "virtualization bug.\n"
              "Aborting. Retry with -F to override, but it will probably\n"
              "fail.";
+    } else {
+      cpu_improperly_configured = true;
     }
   }
 }
@@ -348,11 +350,13 @@ static void check_for_zen_speclockmap() {
       LOG(debug) << "SpecLockMap is disabled";
     } else {
       LOG(debug) << "SpecLockMap is not disabled";
-      if (!Flags::get().suppress_environment_warnings) {
+      if (!Flags::get().force_things) {
         fprintf(stderr,
                 "On Zen CPUs, rr will not work reliably unless you disable the "
                 "hardware SpecLockMap optimization.\nFor instructions on how to "
                 "do this, see https://github.com/rr-debugger/rr/wiki/Zen\n");
+      } else {
+        cpu_improperly_configured = true;
       }
     }
   }
@@ -376,7 +380,7 @@ static void check_for_freeze_on_smi() {
     LOG(debug) << "freeze_on_smi is set";
   } else if (freeze_on_smi == '0') {
     LOG(warn) << "freeze_on_smi is not set";
-    if (!Flags::get().suppress_environment_warnings) {
+    if (!Flags::get().force_things) {
       fprintf(stderr,
               "Freezing performance counters on SMIs should be enabled for maximum rr\n"
               "reliability on Comet Lake and later CPUs. To manually enable this setting, run\n"
@@ -386,6 +390,8 @@ static void check_for_freeze_on_smi() {
               "to automatically apply this setting on every reboot.\n"
               "See 'man 5 sysfs', 'man 5 tmpfiles.d'.\n"
               "If you are seeing this message, the setting has not been enabled.\n");
+    } else {
+      cpu_improperly_configured = true;
     }
   } else {
     LOG(warn) << "Unrecognized freeze_on_smi value " << freeze_on_smi;

src/ReplaySession.cc

@@ -272,6 +272,18 @@ ReplaySession::ReplaySession(const std::string& dir, const Flags& flags)
   set_intel_pt_enabled(flags.intel_pt_start_checking_event >= 0);
 
   check_virtual_address_size();
+
+  bool cpu_improperly_configured_known, cpu_improperly_configured;
+  cpu_improperly_configured = trace_in.cpu_improperly_configured(&cpu_improperly_configured_known);
+  if (cpu_improperly_configured_known && cpu_improperly_configured) {
+    if (rr::Flags::get().force_things) {
+      LOG(warn) << "CPU was improperly configured at recording but forcing anyways.";
+    } else {
+      CLEAN_FATAL() << "This trace was recorded on a CPU determined to be improperly configured " <<
+        "but rr's automated checks were overridden by the user. If you really want to " <<
+        "replay this trace override those checks again with -F";
+    }
+  }
 }
 
 ReplaySession::ReplaySession(const ReplaySession& other)

src/TraceInfoCommand.cc

@@ -102,6 +102,12 @@ static int dump_trace_info(const string& trace_dir, FILE* out) {
     fprintf(out, "  \"maxVirtualAddressSize\":%d,\n", max_virtual_address_size);
   }
 
+  bool cpu_improperly_configured_known;
+  bool cpu_improperly_configured = trace.cpu_improperly_configured(&cpu_improperly_configured_known);
+  if (cpu_improperly_configured_known) {
+    fprintf(out, "  \"cpuImproperlyConfigured\":%s,\n", cpu_improperly_configured ? "true" : "false");
+  }
+
   if (!trace.uname().sysname.empty()) {
     const auto& uname = trace.uname();
     fputs("  \"uname\":{", out);

src/TraceStream.cc

@@ -1492,6 +1492,7 @@ void TraceWriter::close(CloseStatus status, const TraceUuid* uuid) {
   header.setRuntimePageSize(page_size());
   header.setPreloadLibraryPageSize(PRELOAD_LIBRARY_PAGE_SIZE);
   header.setMaxVirtualAddressSize(max_virtual_address_size);
+  header.setCpuImproperlyConfigured(to_tristate(PerfCounters::improperly_configured()));
 
   {
     struct utsname uname_buf;
@@ -1716,6 +1717,21 @@ TraceReader::TraceReader(const string& dir)
   exclusion_range_ = MemoryRange(remote_ptr<void>(header.getExclusionRangeStart()),
                                  remote_ptr<void>(header.getExclusionRangeEnd()));
 
+  switch (header.getCpuImproperlyConfigured()) {
+    case trace::CpuTriState::UNKNOWN:
+      cpu_improperly_configured_known_ = false;
+      cpu_improperly_configured_ = false;
+      break;
+    case trace::CpuTriState::KNOWN_TRUE:
+      cpu_improperly_configured_known_ = true;
+      cpu_improperly_configured_ = true;
+      break;
+    case trace::CpuTriState::KNOWN_FALSE:
+      cpu_improperly_configured_known_ = true;
+      cpu_improperly_configured_ = false;
+      break;
+  }
+
   const auto& uname = header.getUname();
   uname_.sysname = data_to_str(uname.getSysname());
   uname_.nodename = data_to_str(uname.getNodename());

src/TraceStream.h

@@ -506,6 +506,10 @@ class TraceReader : public TraceStream {
   uint8_t max_virtual_address_size() const {
     return max_virtual_address_size_;
   }
+  bool cpu_improperly_configured(bool* known) const {
+    *known = cpu_improperly_configured_known_;
+    return cpu_improperly_configured_;
+  }
 
   enum TraceQuirks {
     // Whether the /proc/<pid>/mem calls were explicitly recorded in this trace
@@ -547,6 +551,8 @@ class TraceReader : public TraceStream {
   bool chaos_mode_;
   int rrcall_base_;
   uint8_t max_virtual_address_size_;
+  bool cpu_improperly_configured_known_;
+  bool cpu_improperly_configured_;
   uint32_t syscallbuf_fds_disabled_size_;
   uint32_t syscallbuf_hdr_size_;
   int required_forward_compatibility_version_;

src/rr_trace.capnp

@@ -152,6 +152,10 @@ struct Header {
   # size. A value of 0, only present for traces recorded before this was added,
   # means the default value for the relevant arch.
   maxVirtualAddressSize @28 :UInt8 = 0;
+  # One of our pre-flight checks found the CPU has issues (e.g. the Zen SpecLockMap
+  # optimization is not disabled) but the user chose to force recording to
+  # continue regardless.
+  cpuImproperlyConfigured @29 :CpuTriState = unknown;
 }
 
 # A file descriptor belonging to a task