Merge pull request #5877 from BOINC/dpa_docker_cleanup

client: remove unused Docker images and containers on startup

client/client_state.cpp

@@ -15,6 +15,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
+// client initialization and main loop
+
 #ifdef _WIN32
 #include "boinc_win.h"
 #else
@@ -729,6 +731,8 @@ int CLIENT_STATE::init() {
 
     process_gpu_exclusions();
 
+    docker_cleanup();
+
     check_clock_reset();
 
     // Check to see if we can write the state file.

client/client_state.h

@@ -378,6 +378,7 @@ struct CLIENT_STATE {
     int app_finished(ACTIVE_TASK&);
     bool handle_finished_apps();
     void check_overdue();
+    void docker_cleanup();
 
     ACTIVE_TASK* get_task(RESULT*);
 

client/cs_apps.cpp

@@ -1,6 +1,6 @@
 // This file is part of BOINC.
-// http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// https://boinc.berkeley.edu
+// Copyright (C) 2024 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -16,7 +16,7 @@
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
 // The "policy" part of task execution is here.
-// The "mechanism" part is in app.C
+// The "mechanism" part is in app.cpp
 //
 
 #include "cpp.h"
@@ -29,11 +29,14 @@
 #include <csignal>
 #endif
 
+#include <algorithm>
+
 #include "error_numbers.h"
 #include "filesys.h"
 #include "md5_file.h"
 #include "shmem.h"
 #include "util.h"
+#include "url.h"
 
 #include "client_msgs.h"
 #include "client_state.h"
@@ -382,3 +385,109 @@ void CLIENT_STATE::check_overdue() {
     active_tasks.report_overdue();
     t = now + 86400;
 }
+
+////////////// DOCKER CLEANUP ///////////////////
+
+// lists of image and container names for active jobs
+//
+struct DOCKER_JOB_INFO {
+    vector<string> images;
+    vector<string> containers;
+    bool image_present(string name) {
+        return std::find(images.begin(), images.end(), name) != images.end();
+    }
+    bool container_present(string name) {
+        return std::find(containers.begin(), containers.end(), name) != containers.end();
+    }
+};
+
+// clean up a Docker installation
+// (Unix: the host; Win: a WSL distro)
+//
+void cleanup_docker(DOCKER_JOB_INFO &info, DOCKER_CONN &dc) {
+    int retval;
+    vector<string> out, out2;
+    char cmd[1024];
+    string name;
+
+    // first containers
+    //
+    retval = dc.command("ps --all", out);
+    if (retval) {
+        fprintf(stderr, "Docker command failed: ps --all\n");
+    } else {
+        for (string line: out) {
+            retval = dc.parse_container_name(line, name);
+            if (retval) continue;
+            if (!docker_is_boinc_name(name.c_str())) continue;
+            if (info.container_present(name)) continue;
+            sprintf(cmd, "rm %s", name.c_str());
+            retval = dc.command(cmd, out);
+            if (retval) {
+                fprintf(stderr, "Docker command failed: %s\n", cmd);
+                continue;
+            }
+            msg_printf(NULL, MSG_INFO,
+                "Removed unused Docker container: %s", name.c_str()
+            );
+        }
+    }
+
+    // then images
+    //
+    retval = dc.command("images", out);
+    if (retval) {
+        fprintf(stderr, "Docker command failed: images\n");
+    } else {
+        for (string line: out) {
+            retval = dc.parse_image_name(line, name);
+            if (retval) continue;
+            if (!docker_is_boinc_name(name.c_str())) continue;
+            if (info.image_present(name)) continue;
+            sprintf(cmd, "image rm %s", name.c_str());
+            retval = dc.command(cmd, out2);
+            if (retval) {
+                fprintf(stderr, "Docker command failed: %s\n", cmd);
+                continue;
+            }
+            msg_printf(NULL, MSG_INFO,
+                "Removed unused Docker image: %s", name.c_str()
+            );
+        }
+    }
+}
+
+// remove old BOINC images and containers from Docker installations
+//
+void CLIENT_STATE::docker_cleanup() {
+    // make lists of the images and containers used by active jobs
+    //
+    DOCKER_JOB_INFO info;
+    for (ACTIVE_TASK *atp: active_tasks.active_tasks) {
+        if (!strstr(atp->app_version->plan_class, "docker")) continue;
+        char buf[256];
+        escape_project_url(atp->wup->project->master_url, buf);
+        string s = docker_image_name(buf, atp->wup->name);
+        info.images.push_back(s);
+        s = docker_container_name(buf, atp->result->name);
+        info.containers.push_back(s);
+    }
+
+    // go through local Docker installations and remove
+    // BOINC images and containers not in the above lists
+    //
+#ifdef _WIN32
+    for (WSL_DISTRO &wd: host_info.wsl_distros.distros) {
+        if (wd.docker_version.empty()) continue;
+        DOCKER_CONN dc;
+        dc.init(wd.docker_type, wd.distro_name);
+        cleanup_docker(info, dc);
+    }
+#else
+    if (strlen(host_info.docker_version)) {
+        DOCKER_CONN dc;
+        dc.init(host_info.docker_type);
+        cleanup_docker(info, dc);
+    }
+#endif
+}

client/hostinfo_linux.cpp

@@ -15,6 +15,10 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
+// Functions for getting the OS name and version of a Linux system.
+// This is included in the Windows build because the Win client
+// needs to get info on WSL distros.
+
 #if   defined(_WIN32) && !defined(__STDWX_H__)
 #include "boinc_win.h"
 #elif defined(_WIN32) && defined(__STDWX_H__)

lib/hostinfo.h

@@ -18,7 +18,7 @@
 #ifndef BOINC_HOSTINFO_H
 #define BOINC_HOSTINFO_H
 
-// Description of a host's hardware and software.
+// struct HOST_INFO describes a host's hardware and software.
 // This is used a few places:
 // - it's part of the client's state file, client_state.xml
 // - it's passed in the reply to the get_host_info GUI RPC

lib/util.cpp

@@ -63,6 +63,7 @@ extern "C" {
 #include "mfile.h"
 #include "miofile.h"
 #include "parse.h"
+#include "hostinfo.h"
 #include "util.h"
 
 using std::min;
@@ -645,7 +646,7 @@ bool process_exists(HANDLE h) {
     return false;
 }
 
-#else
+#else   // _WIN32
 
 // Unix: pthreads doesn't provide an API for getting per-thread CPU time,
 // so just get the process's CPU time
@@ -685,7 +686,9 @@ bool process_exists(int pid) {
     return true;
 }
 
-#endif
+#endif  // _WIN32
+
+#ifndef _USING_FCGI_
 
 string parse_ldd_libc(const char* input) {
     char *q = (char*)strchr(input, '\n');
@@ -698,3 +701,116 @@ string parse_ldd_libc(const char* input) {
     strip_whitespace(s);
     return s;
 }
+
+#ifdef _WIN32
+int DOCKER_CONN::init(DOCKER_TYPE docker_type, string distro_name, bool _verbose) {
+    cli_prog = docker_cli_prog(docker_type);
+    if (docker_type == DOCKER) {
+        int retval = ctl_wc.setup();
+        if (retval) return retval;
+        retval = ctl_wc.run_program_in_wsl(distro_name, "", true);
+        if (retval) return retval;
+    } else if (docker_type == PODMAN) {
+        int retval = ctl_wc.setup_root(distro_name.c_str());
+        if (retval) return retval;
+    } else {
+        return -1;
+    }
+    verbose = _verbose;
+    return 0;
+}
+#else
+int DOCKER_CONN::init(DOCKER_TYPE docker_type, bool _verbose) {
+    cli_prog = docker_cli_prog(docker_type);
+    verbose = _verbose;
+    return 0;
+}
+#endif
+
+int DOCKER_CONN::command(const char* cmd, vector<string> &out) {
+    char buf[1024];
+    int retval;
+    if (verbose) {
+        fprintf(stderr, "running docker command: %s\n", cmd);
+    }
+#ifdef _WIN32
+    string output;
+
+    sprintf(buf, "%s %s; echo EOM\n", cli_prog, cmd);
+    write_to_pipe(ctl_wc.in_write, buf);
+    retval = read_from_pipe(
+        ctl_wc.out_read, ctl_wc.proc_handle, output, TIMEOUT, "EOM"
+    );
+    if (retval) {
+        fprintf(stderr, "read_from_pipe() error: %s\n", boincerror(retval));
+        return retval;
+    }
+    out = split(output, '\n');
+#else
+    sprintf(buf, "%s %s\n", cli_prog, cmd);
+    retval = run_command(buf, out);
+    if (retval) {
+        if (verbose) {
+            fprintf(stderr, "command failed: %s\n", boincerror(retval));
+        }
+        return retval;
+    }
+#endif
+    if (verbose) {
+        fprintf(stderr, "command output:\n");
+        for (string line: out) {
+            fprintf(stderr, "%s\n", line.c_str());
+        }
+    }
+    return 0;
+}
+
+// REPOSITORY                          TAG         IMAGE ID      CREATED       SIZE
+// localhost/boinc__app_test__test_wu  latest      cbc1498dfc49  43 hours ago  121 MB
+
+int DOCKER_CONN::parse_image_name(string line, string &name) {
+    char buf[1024];
+    strcpy(buf, line.c_str());
+    if (strstr(buf, "REPOSITORY")) return -1;
+    if (strstr(buf, "localhost/") != buf) return -1;
+    char *p = buf + strlen("localhost/");
+    char *q = strstr(p, " ");
+    if (!q) return -1;
+    *q = 0;
+    name = (string)p;
+    return 0;
+}
+
+// CONTAINER ID  IMAGE                                      COMMAND               CREATED        STATUS                   PORTS       NAMES
+// 6d4877e0d071  localhost/boinc__app_test__test_wu:latest  /bin/sh -c ./work...  43 hours ago   Exited (0) 21 hours ago              boinc__app_test__test_result
+
+int DOCKER_CONN::parse_container_name(string line, string &name) {
+    char buf[1024];
+    strcpy(buf, line.c_str());
+    if (strstr(buf, "CONTAINER")) return -1;
+    char *p = strrchr(buf, ' ');
+    if (!p) return -1;
+    name = (string)(p+1);
+    return 0;
+}
+
+string docker_image_name(
+    const char* proj_url_esc, const char* wu_name
+) {
+    char buf[1024];
+    sprintf(buf, "boinc__%s__%s", proj_url_esc, wu_name);
+    return string(buf);
+}
+
+string docker_container_name(
+    const char* proj_url_esc, const char* result_name
+){
+    char buf[1024];
+    sprintf(buf, "boinc__%s__%s", proj_url_esc, result_name);
+    return string(buf);
+}
+
+bool docker_is_boinc_name(const char* name) {
+    return strstr(name, "boinc__") == name;
+}
+#endif  // _USING_FCGI

lib/util.h

@@ -25,7 +25,10 @@
 #include <vector>
 #ifdef _WIN32
 #include "boinc_win.h"
+#include "win_util.h"
 #endif
+#include "common_defs.h"
+
 extern double dtime();
 extern double dday();
 extern void boinc_sleep(double);
@@ -142,4 +145,39 @@ extern double simtime;
 #define time(x) ((int)simtime)
 #endif
 
+// represents a connection to a Docker/Podman installation
+// used from docker_wrapper and the client
+//
+struct DOCKER_CONN {
+    DOCKER_TYPE type;
+    const char* cli_prog;
+    bool verbose;
+#ifdef _WIN32
+    WSL_CMD ctl_wc;
+    int init(DOCKER_TYPE type, std::string distro_name, bool verbose=false);
+#else
+    int init(DOCKER_TYPE, bool verbose=false);
+#endif
+    int command(const char* cmd, std::vector<std::string> &out);
+
+    static const int TIMEOUT = 10;    // timeout for docker commands
+
+    // parse a line from "docker images" output; return name
+    int parse_image_name(std::string line, std::string &name);
+
+    // parse a line from "docker ps --all" output; return name
+    int parse_container_name(std::string line, std::string &name);
+};
+
+extern std::string docker_image_name(
+    const char* proj_url_esc,       // escaped project URL
+    const char* wu_name
+);
+extern std::string docker_container_name(
+    const char* proj_url_esc,       // escaped project URL
+    const char* result_name
+);
+// is the name (of a Docker image or container) a BOINC name?
+extern bool docker_is_boinc_name(const char* name);
+
 #endif