Flush logs/stdout/stderr on SIGTERM in client and plugin

client/signal_handler.go

@@ -0,0 +1,69 @@
+//go:build !windows
+
+/***************************************************************
+ *
+ * Copyright (C) 2025, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+package client
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/pelicanplatform/pelican/logging"
+	"github.com/pelicanplatform/pelican/param"
+)
+
+// SetupSignalHandlers sets up signal handlers for SIGTERM to ensure logs are flushed
+// before the process exits. If debug mode is enabled, it will also send SIGQUIT to dump
+// stack traces before exiting.
+func SetupSignalHandlers() {
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGTERM)
+
+	go func() {
+		sig := <-sigChan
+		log.Warnf("Received signal: %v. Flushing logs before exit...", sig)
+
+		// Flush all buffered logs
+		logging.FlushLogs(param.Logging_LogLocation.GetString() != "")
+
+		// Sync stdout and stderr to ensure all output is written
+		if err := os.Stdout.Sync(); err != nil {
+			log.Debugf("Error syncing stdout: %v", err)
+		}
+		if err := os.Stderr.Sync(); err != nil {
+			log.Debugf("Error syncing stderr: %v", err)
+		}
+
+		// If debug mode is enabled, send SIGQUIT to dump stack traces
+		if log.GetLevel() == log.DebugLevel || log.GetLevel() == log.TraceLevel {
+			log.Warnln("Debug mode enabled. Sending SIGQUIT to dump stack traces...")
+			_ = syscall.Kill(os.Getpid(), syscall.SIGQUIT)
+			// Give a moment for the stack trace to be written
+			// Note: SIGQUIT will cause the process to exit with a core dump
+			// so we don't need to explicitly exit here
+			return
+		}
+
+		log.Warnln("Exiting after signal handling...")
+		os.Exit(1)
+	}()
+}

client/signal_handler_test.go

@@ -0,0 +1,95 @@
+//go:build !windows
+
+/***************************************************************
+ *
+ * Copyright (C) 2025, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+package client
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSignalHandlerSetup verifies that SetupSignalHandlers can be called without errors
+func TestSignalHandlerSetup(t *testing.T) {
+	// This test simply verifies that SetupSignalHandlers doesn't panic
+	// The actual signal handling is tested in TestSignalHandlerIntegration
+	SetupSignalHandlers()
+	// Give the goroutine a moment to start
+	time.Sleep(10 * time.Millisecond)
+}
+
+// TestSignalHandlerIntegration is an integration test that spawns a subprocess
+// and sends it a SIGTERM to verify proper log flushing behavior
+func TestSignalHandlerIntegration(t *testing.T) {
+	if os.Getenv("TEST_SIGNAL_HANDLER") == "1" {
+		// This is the subprocess that will receive SIGTERM
+		SetupSignalHandlers()
+		// Write a log message that should be flushed on SIGTERM
+		os.Stderr.WriteString("TEST_LOG_MESSAGE\n")
+		// Wait for signal
+		time.Sleep(10 * time.Second)
+		os.Exit(0)
+		return
+	}
+
+	// This is the parent test that spawns the subprocess
+	cmd := exec.Command(os.Args[0], "-test.run=TestSignalHandlerIntegration")
+	cmd.Env = append(os.Environ(), "TEST_SIGNAL_HANDLER=1")
+
+	// Start the subprocess
+	err := cmd.Start()
+	require.NoError(t, err, "subprocess should start successfully")
+
+	// Give the subprocess time to set up signal handler
+	time.Sleep(100 * time.Millisecond)
+
+	// Send SIGTERM to the subprocess
+	err = cmd.Process.Signal(syscall.SIGTERM)
+	require.NoError(t, err, "should be able to send SIGTERM")
+
+	// Wait for the subprocess to exit
+	err = cmd.Wait()
+	// The process should exit with a non-zero status after receiving SIGTERM
+	assert.Error(t, err, "subprocess should exit with error after SIGTERM")
+
+	t.Log("Successfully verified signal handler responds to SIGTERM")
+}
+
+// TestSignalHandlerSIGTERM verifies basic SIGTERM handling without starting a subprocess
+func TestSignalHandlerSIGTERM(t *testing.T) {
+	// Set up a signal handler
+	SetupSignalHandlers()
+
+	// Give the goroutine time to set up
+	time.Sleep(10 * time.Millisecond)
+
+	// Note: We can't easily test the actual signal handling in a unit test
+	// without spawning a subprocess, as sending a signal to ourselves
+	// would terminate the test process. The integration test above handles
+	// that scenario.
+
+	// This test just verifies the setup doesn't panic
+	assert.True(t, true, "Signal handler setup completed without panic")
+}

client/signal_handler_windows.go

@@ -0,0 +1,27 @@
+//go:build windows
+
+/***************************************************************
+ *
+ * Copyright (C) 2025, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+package client
+
+// SetupSignalHandlers is a no-op on Windows as SIGTERM handling
+// is not implemented for Windows platforms.
+func SetupSignalHandlers() {
+	// No-op on Windows
+}

cmd/object_copy.go

@@ -83,6 +83,9 @@ the client should fallback to discovered caches if all preferred caches fail.`)
 func copyMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	// Need to check just stashcp since it does not go through root, the other modes get checked there
 	if strings.HasPrefix(execName, "stashcp") {
 		if val, err := cmd.Flags().GetBool("debug"); err == nil && val {

cmd/object_delete.go

@@ -63,6 +63,9 @@ func init() {
 func deleteMain(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln("Failed to initialize client:", err)

cmd/object_get.go

@@ -61,6 +61,9 @@ the client should fallback to discovered caches if all preferred caches fail.`)
 func getMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/object_ls.go

@@ -60,6 +60,10 @@ func init() {
 
 func listMain(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
+
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/object_prestage.go

@@ -54,6 +54,9 @@ the client should fallback to discovered caches if all preferred caches fail.`)
 func prestageMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/object_put.go

@@ -126,6 +126,9 @@ func verifyFileChecksum(filePath, expectedChecksum string, alg client.ChecksumTy
 func putMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/object_share.go

@@ -46,6 +46,9 @@ func init() {
 
 func shareMain(cmd *cobra.Command, args []string) error {
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		return errors.Wrap(err, "Failed to initialize the client")

cmd/object_stat.go

@@ -56,6 +56,10 @@ func init() {
 
 func statMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
+
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/object_sync.go

@@ -87,6 +87,9 @@ func isPelicanUrl(input string) bool {
 func syncMain(cmd *cobra.Command, args []string) {
 	ctx := cmd.Context()
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	err := config.InitClient()
 	if err != nil {
 		log.Errorln(err)

cmd/plugin.go

@@ -91,6 +91,9 @@ func init() {
 func stashPluginMain(args []string) {
 	viper.Set(param.Client_IsPlugin.GetName(), true)
 
+	// Set up signal handlers to flush logs on SIGTERM
+	client.SetupSignalHandlers()
+
 	// Handler function to recover from panics
 	defer func() {
 		if r := recover(); r != nil {