DTrace updates (#1589)

leftwo · Alan Hanson · web-flow · commit 3865ee150d0d · 2025-01-06T08:49:32.000-08:00
Move dtrace scripts to /opt/oxide/dtrace/crucible, changed places that
needed to know about the new paths.

Updated many scripts to use three letter short names for downstairs
state. Updated lookup table to new inline shortcut to reduce code
duplication. Added WQ and RPL where it was missing. At some point I'll
add them all, but waiting for Matt's PR to land where we update how the
states work and I may decide to print things differently.
```
inline string short_state[string ss] =
    ss == "active" ? "ACT" :
    ss == "new" ? "NEW" :
    ss == "replaced" ? "RPL" :
    ss == "live_repair_ready" ? "LRR" :
    ss == "live_repair" ? "LR" :                                          
    ss == "faulted" ? "FLT" :
    ss == "offline" ? "OFL" :                                             
    ss == "reconcile" ? "REC" :
    ss == "wait_quorum" ? "WQ" :
    ss;
```
Fixed column width for PIDs in a number of scripts.

Added example `simple.d` dtrace script from 2024 DTrace conference.

Example output:
get-up-state.sh:
```
BRM42220030 # /opt/oxide/crucible_dtrace/get-up-state.sh
  PID  SESSION DS0 DS1 DS2   NEXT_JOB  DELTA CONN   ELR   ELC   ERR   ERN
    8 00c767d0 NEW NEW NEW       1000      0    0     0     0     0     0
    8 0c5af3b7 NEW NEW NEW       1000      0    0     0     0     0     0
    8 0cd7bba2 ACT ACT ACT       1000      0    3     0     0     0     0
    8 0dd09245 NEW NEW NEW       1000      0    0     0     0     0     0
    8 121dc7ea ACT ACT ACT       1000      0    3     0     0     0     0
17774 03419c44 ACT ACT ACT      46479      0    4    74   294     0     0
17774 f0745404 NEW NEW NEW      32271      0    3     0     0     0     0
18563 0fd262e1 NEW NEW NEW      32285      0    3     0     0     0     0
18563 eb102b5a ACT ACT ACT      46210      0    4    74   294     0     0
19622 4b0ace49 NEW NEW NEW      32271      0    3     0     0     0     0
19622 d994fe0a ACT ACT ACT      46240      0    4    74   294     0     0
```

get-ds-state.sh
```
BRM42220030 # /opt/oxide/crucible_dtrace/get-ds-state.sh 
oxz_crucible_pantry_8e3b64eb     8 00c767d0  WQ NEW  WQ
oxz_crucible_pantry_8e3b64eb     8 00c767d0 NEW NEW NEW
oxz_crucible_pantry_8e3b64eb     8 0c5af3b7 NEW NEW NEW
oxz_crucible_pantry_8e3b64eb     8 0cd7bba2 ACT ACT ACT
oxz_propolis-server_d3bdb604 17774 03419c44 ACT ACT ACT
oxz_propolis-server_d3bdb604 17774 f0745404 NEW NEW NEW
oxz_propolis-server_bfba345a 18563 eb102b5a ACT ACT ACT
oxz_propolis-server_bfba345a 18563 eb102b5a NEW NEW NEW
oxz_propolis-server_335eb83d 19622 4b0ace49 NEW NEW NEW
oxz_propolis-server_335eb83d 19622 d994fe0a ACT ACT ACT
```

Co-authored-by: Alan Hanson &lt;alan@oxide.computer&gt;
diff --git a/package-manifest.toml b/package-manifest.toml
@@ -6,7 +6,7 @@ source.rust.release = true
 source.paths = [
   { from = "agent/smf", to = "/var/svc/manifest/site/crucible" },
   { from = "agent/downstairs_method_script.sh", to = "/opt/oxide/lib/svc/manifest/crucible/downstairs.sh" },
-  { from = "tools/dtrace/downstairs_count.d", to = "/opt/oxide/dtrace/downstairs_count.d" },
+  { from = "tools/dtrace/downstairs_count.d", to = "/opt/oxide/dtrace/crucible/downstairs_count.d" },
 ]
 output.type = "zone"
 
@@ -19,11 +19,12 @@ source.paths = [
   { from = "pantry/smf/pantry.xml", to = "/var/svc/manifest/site/crucible/pantry.xml" },
   { from = "tools/dtrace/upstairs_info.d", to = "/opt/oxide/dtrace/upstairs_info.d" },
   { from = "tools/dtrace/upstairs_repair.d", to = "/opt/oxide/dtrace/upstairs_repair.d" },
-  { from = "tools/dtrace/upstairs_raw.d", to = "/opt/oxide/dtrace/upstairs_raw.d" },
-  { from = "tools/dtrace/get-lr-state.sh", to = "/opt/oxide/dtrace/get-lr-state.sh" },
-  { from = "tools/dtrace/get-ds-state.sh", to = "/opt/oxide/dtrace/get-ds-state.sh" },
-  { from = "tools/dtrace/single_up_info.d", to = "/opt/oxide/dtrace/single_up_info.d" },
-  { from = "tools/dtrace/sled_upstairs_info.d", to = "/opt/oxide/dtrace/sled_upstairs_info.d" },
-  { from = "tools/dtrace/all_downstairs.d", to = "/opt/oxide/dtrace/all_downstairs.d" },
+  { from = "tools/dtrace/upstairs_raw.d", to = "/opt/oxide/dtrace/crucible/upstairs_raw.d" },
+  { from = "tools/dtrace/get-lr-state.sh", to = "/opt/oxide/dtrace/crucible/get-lr-state.sh" },
+  { from = "tools/dtrace/get-ds-state.sh", to = "/opt/oxide/dtrace/crucible/get-ds-state.sh" },
+  { from = "tools/dtrace/single_up_info.d", to = "/opt/oxide/dtrace/crucible/single_up_info.d" },
+  { from = "tools/dtrace/sled_upstairs_info.d", to = "/opt/oxide/dtrace/crucible/sled_upstairs_info.d" },
+  { from = "tools/dtrace/all_downstairs.d", to = "/opt/oxide/dtrace/crucible/all_downstairs.d" },
+  { from = "tools/dtrace/up-info.d", to = "/opt/oxide/dtrace/crucible/up-info.d" },
 ]
 output.type = "zone"
diff --git a/tools/dtrace/get-ds-state.d b/tools/dtrace/get-ds-state.d
@@ -5,16 +5,40 @@
 #pragma D option quiet
 #pragma D option strsize=1k
 
+/*
+ * Translate the longer state string into a shorter version
+ */
+inline string short_state[string ss] =
+    ss == "active" ? "ACT" :
+    ss == "new" ? "NEW" :
+    ss == "replaced" ? "RPL" :
+    ss == "live_repair_ready" ? "LRR" :
+    ss == "live_repair" ? "LR" :
+    ss == "faulted" ? "FLT" :
+    ss == "offline" ? "OFL" :
+    ss == "reconcile" ? "REC" :
+    ss == "wait_quorum" ? "WQ" :
+    ss;
+
 crucible_upstairs*:::up-status
 {
     my_sesh = json(copyinstr(arg1), "ok.session_id");
 
-    printf("%6d %8s %17s %17s %17s\n",
+    this->ds0state = json(copyinstr(arg1), "ok.ds_state[0]");
+    this->d0 = short_state[this->ds0state];
+
+    this->ds1state = json(copyinstr(arg1), "ok.ds_state[1]");
+    this->d1 = short_state[this->ds1state];
+
+    this->ds2state = json(copyinstr(arg1), "ok.ds_state[2]");
+    this->d2 = short_state[this->ds2state];
+
+    printf("%6d %8s %3s %3s %3s\n",
         pid,
         substr(my_sesh, 0, 8),
-        json(copyinstr(arg1), "ok.ds_state[0]"),
-        json(copyinstr(arg1), "ok.ds_state[1]"),
-        json(copyinstr(arg1), "ok.ds_state[2]"));
+        this->d0,
+        this->d1,
+        this->d2);
 }
 
 tick-5s
diff --git a/tools/dtrace/get-ds-state.sh b/tools/dtrace/get-ds-state.sh
@@ -4,13 +4,18 @@
 # it finds running on a system.
 filename='/tmp/get-ds-state.out'
 
+# Clear out any previous state
+echo "" > "$filename"
 # Gather state on all running propolis servers, record summary to a file
-dtrace -s /opt/oxide/crucible_dtrace/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename"
+dtrace -s /opt/oxide/dtrace/crucible/get-ds-state.d | sort -n | uniq | awk 'NF' > "$filename"
 # Walk the lines in the file, append the zone name to each line.
 while read -r p; do
         # For each line in the file, pull out the PID we are looking at and
-        # print the zone that process is running in.
+        # use it to find the zone so we can print the zone name as well.
         pid=$(echo $p | awk '{print $1}')
         zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
-        echo "$zone $p"
+        # Our zone string size is already set from above, force the
+        # rest of the line to take up 26 columns, this prevents PIDs
+        # with fewer than 5 digits from using less columns.
+        printf "%s %26s\n" "$zone" "$p"
 done < "$filename"
diff --git a/tools/dtrace/get-lr-state.sh b/tools/dtrace/get-lr-state.sh
@@ -4,13 +4,18 @@
 # pid/session it finds running on a system.
 filename='/tmp/get-lr-state.out'
 
+# Clear out any previous state
+echo "" > "$filename"
 # Gather state on all running propolis servers, record summary to a file
-dtrace -s /opt/oxide/crucible_dtrace/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename"
+dtrace -s /opt/oxide/dtrace/crucible/get-lr-state.d | sort -n | uniq | awk 'NF' > "$filename"
 # Walk the lines in the file, append the zone name to each line.
 while read -r p; do
         # For each line in the file, pull out the PID we are looking at and
         # print the zone that process is running in.
         pid=$(echo $p | awk '{print $1}')
         zone=$(ps -o zone -p $pid | tail -1 | cut -c 1-28)
-        echo "$zone $p"
+        # Our zone string size is already set from above, force the
+        # rest of the line to take up 26 columns, this prevents PIDs
+        # with fewer than 5 digits from using less columns.
+        printf "%s %26s\n" "$zone" "$p"
 done < "$filename"
diff --git a/tools/dtrace/get-up-state.d b/tools/dtrace/get-up-state.d
@@ -30,6 +30,21 @@ tick-10s
     exit(0);
 }
 
+/*
+ * Translate the longer state string into a shorter version
+ */
+inline string short_state[string ss] =
+    ss == "active" ? "ACT" :
+    ss == "new" ? "NEW" :
+    ss == "replaced" ? "RPL" :
+    ss == "live_repair_ready" ? "LRR" :
+    ss == "live_repair" ? "LR" :
+    ss == "faulted" ? "FLT" :
+    ss == "offline" ? "OFL" :
+    ss == "reconcile" ? "REC" :
+    ss == "wait_quorum" ? "WQ" :
+    ss;
+
 /*
  * All variables should be this->
  * Otherwise, there is a chance another probe will fire and
@@ -38,60 +53,14 @@ tick-10s
 crucible_upstairs*:::up-status
 {
     this->ds0state = json(copyinstr(arg1), "ok.ds_state[0]");
-    if (this->ds0state == "active") {
-        this->d0 = "ACT";
-    } else if (this->ds0state == "new") {
-        this->d0 = "NEW";
-    } else if (this->ds0state == "live_repair_ready") {
-        this->d0 = "LRR";
-    } else if (this->ds0state == "live_repair") {
-        this->d0 = " LR";
-    } else if (this->ds0state == "faulted") {
-        this->d0 = "FLT";
-    } else if (this->ds0state == "offline") {
-        this->d0 = "OFL";
-    } else {
-        this->d0 = this->ds0state;
-    }
+    this->d0 = short_state[this->ds0state];
 
     this->ds1state = json(copyinstr(arg1), "ok.ds_state[1]");
-    if (this->ds1state == "active") {
-        this->d1 = "ACT";
-    } else if (this->ds1state == "new") {
-        this->d1 = "NEW";
-    } else if (this->ds1state == "live_repair_ready") {
-        this->d1 = "LRR";
-    } else if (this->ds1state == "live_repair") {
-        this->d1 = " LR";
-    } else if (this->ds1state == "faulted") {
-        this->d1 = "FLT";
-    } else if (this->ds1state == "offline") {
-        this->d1 = "OFL";
-    } else {
-        this->d1 = this->ds1state;
-    }
+    this->d1 = short_state[this->ds1state];
 
     this->ds2state = json(copyinstr(arg1), "ok.ds_state[2]");
-    if (this->ds2state == "active") {
-        this->d2 = "ACT";
-    } else if (this->ds2state == "new") {
-        this->d2 = "NEW";
-    } else if (this->ds2state == "live_repair_ready") {
-        this->d2 = "LRR";
-    } else if (this->ds2state == "live_repair") {
-        this->d2 = " LR";
-    } else if (this->ds2state == "faulted") {
-        this->d2 = "FLT";
-    } else if (this->ds2state == "offline") {
-        this->d2 = "OFL";
-    } else {
-        this->d2 = this->ds2state;
-    }
+    this->d2 = short_state[this->ds2state];
 
-    /*
-     * All these local variables require the "this->" so the probe firing
-     * from different sessions don't collide with each other.
-     */
     this->full_session_id = json(copyinstr(arg1), "ok.session_id");
     this->session_id = substr(this->full_session_id, 0, 8);
 
diff --git a/tools/dtrace/get-up-state.sh b/tools/dtrace/get-up-state.sh
@@ -7,7 +7,7 @@ final='/tmp/get-up-state.final'
 rm -f $final
 
 # Gather our output first.
-dtrace -s /opt/oxide/crucible_dtrace/get-up-state.d | awk 'NF' > "$filename"
+dtrace -s /opt/oxide/dtrace/crucible/get-up-state.d | awk 'NF' > "$filename"
 if [[ $? -ne 0 ]]; then
     exit 1
 fi
diff --git a/tools/dtrace/simple.d b/tools/dtrace/simple.d
@@ -0,0 +1,100 @@
+#pragma D option quiet
+#pragma D option strsize=1k
+
+dtrace:::BEGIN
+{
+    /*
+     * We have to init something for the associative array last_id.
+     * This means it will be created and later, when we have a
+     * session ID, we can add that element.
+     */
+    last_id["string"] = (int64_t)1;
+}
+
+/*
+ * Print our header at some interval
+ */
+tick-2s
+{
+    printf("%5s %8s ", "PID", "SESSION");
+    printf("%3s %3s %3s", "DS0", "DS1", "DS2");
+    printf(" %10s %6s %4s", "NEXT_JOB", "DELTA", "CONN");
+    printf(" %5s %5s", "ELR", "ELC");
+    printf(" %5s %5s", "ERR", "ERN");
+    printf("\n");
+}
+
+/*
+ * Translate the longer state string into a shorter version
+ */
+inline string short_state[string ss] =
+    ss == "active" ? "ACT" :
+    ss == "new" ? "NEW" :
+    ss == "live_repair_ready" ? "LRR" :
+    ss == "live_repair" ? "LR" :
+    ss == "faulted" ? "FLT" :
+    ss == "offline" ? "OFL" :
+    ss == "reconcile" ? "REC" :
+    ss;
+
+/*
+ * All variables should be this->
+ * Otherwise, there is a chance another probe will fire and
+ * clobber the contents.
+ */
+crucible_upstairs*:::up-status
+{
+    this->ds0state = json(copyinstr(arg1), "ok.ds_state[0]");
+    this->d0 = short_state[this->ds0state];
+
+    this->ds1state = json(copyinstr(arg1), "ok.ds_state[1]");
+    this->d1 = short_state[this->ds1state];
+
+    this->ds2state = json(copyinstr(arg1), "ok.ds_state[2]");
+    this->d2 = short_state[this->ds2state];
+
+    this->full_session_id = json(copyinstr(arg1), "ok.session_id");
+    this->session_id = substr(this->full_session_id, 0, 8);
+
+    this->next_id_str = json(copyinstr(arg1), "ok.next_job_id");
+    this->next_id_value = strtoll(this->next_id_str);
+
+    /*
+     * The first time through, we don't know delta, so start with 0.
+     */
+    if (last_id[this->session_id] == 0) {
+        this->delta = 0;
+        last_id[this->session_id] = this->next_id_value;
+    } else {
+        this->delta = this->next_id_value - last_id[this->session_id];
+        last_id[this->session_id] = this->next_id_value;
+    }
+
+    this->connections = strtoll(json(copyinstr(arg1), "ok.ds_connected[0]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_connected[1]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_connected[2]"));
+
+    /* Total of extents live repaired */
+    this->elr = strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[0]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[1]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_extents_repaired[2]"));
+
+    /* Total of extents not needing repair during live repair */
+    this->elc = strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[0]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[1]")) +
+        strtoll(json(copyinstr(arg1), "ok.ds_extents_confirmed[2]"));
+
+    printf("%5d %8s %3s %3s %3s %10d %6d %4d %5d %5d %5s %5s\n",
+        pid,
+        this->session_id,
+        this->d0,
+        this->d1,
+        this->d2,
+        this->next_id_value,
+        this->delta,
+        this->connections,
+        this->elr,
+        this->elc,
+        json(copyinstr(arg1), "ok.ds_reconciled"),
+        json(copyinstr(arg1), "ok.ds_reconcile_needed"));
+}
diff --git a/tools/dtrace/up-info.d b/tools/dtrace/up-info.d
diff --git a/tools/make-dtrace.sh b/tools/make-dtrace.sh
