llmodel: dont load libs in static initialization

this fixes some issues that were being seen on installed windows builds of 2.5.0

setImplementationsSearchPath did not work if llmodel was initialized before it was called

* do not actually load impl dlls in the static initializer, wait until we're actually trying to load a model for the first time

* rescan on the next model load attempt if the search path has been changed

* only load dlls that actually might be model impl dlls, otherwise we pull all sorts of random junk into the process before it might expect to be

Co-authored-by: cebtenzzre <[email protected]>
Signed-off-by: Aaron Miller <[email protected]>

Author: apage43

gpt4all-backend/llmodel.cpp

@@ -10,6 +10,8 @@
 #include <cassert>
 #include <cstdlib>
 #include <sstream>
+#include <regex>
+#include <iterator>
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
@@ -75,12 +77,20 @@ bool LLModel::Implementation::isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
-const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
+static bool s_scanned = false;
+
+const std::vector<std::unique_ptr<LLModel::Implementation>> &LLModel::Implementation::implementationList() {
+    static std::vector<std::unique_ptr<LLModel::Implementation>> s_impl_libs;
     // NOTE: allocated on heap so we leak intentionally on exit so we have a chance to clean up the
     // individual models without the cleanup of the static list interfering
-    static auto* libs = new std::vector<Implementation>([] () {
-        std::vector<Implementation> fres;
-
+    if(!s_scanned) {
+        std::string impl_name_re = "(bert|llama|gptj|llamamodel-mainline)";
+        if (requires_avxonly()) {
+            impl_name_re += "-avxonly";
+        } else {
+            impl_name_re += "-(default|metal)";
+        }
+        std::regex re(impl_name_re);
         auto search_in_directory = [&](const std::string& paths) {
             std::stringstream ss(paths);
             std::string path;
@@ -90,32 +100,33 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
                 // Iterate over all libraries
                 for (const auto& f : std::filesystem::directory_iterator(fs_path)) {
                     const std::filesystem::path& p = f.path();
+
                     if (p.extension() != LIB_FILE_EXT) continue;
+                    if (!std::regex_search(p.stem().string(), re)) continue;
+
                     // Add to list if model implementation
                     try {
                         Dlhandle dl(p.string());
                         if (!Implementation::isImplementation(dl)) {
                             continue;
                         }
-                        fres.emplace_back(Implementation(std::move(dl)));
+                        s_impl_libs.emplace_back(std::make_unique<Implementation>(std::move(dl)));
                     } catch (...) {}
                 }
             }
         };
 
         search_in_directory(s_implementations_search_path);
-
-        return fres;
-    }());
-    // Return static result
-    return *libs;
+        s_scanned = true;
+    };
+    return s_impl_libs;
 }
 
 const LLModel::Implementation* LLModel::Implementation::implementation(const char *fname, const std::string& buildVariant) {
     for (const auto& i : implementationList()) {
-        if (buildVariant != i.m_buildVariant) continue;
-        if (!i.m_magicMatch(fname)) continue;
-        return &i;
+        if (buildVariant != i->m_buildVariant) continue;
+        if (!i->m_magicMatch(fname)) continue;
+        return i.get();
     }
     return nullptr;
 }
@@ -170,6 +181,7 @@ LLModel *LLModel::Implementation::construct(const std::string &modelPath, std::s
 
 void LLModel::Implementation::setImplementationsSearchPath(const std::string& path) {
     s_implementations_search_path = path;
+    s_scanned = false;
 }
 
 const std::string& LLModel::Implementation::implementationsSearchPath() {

gpt4all-backend/llmodel.h

@@ -8,6 +8,7 @@
 #include <fstream>
 #include <cstdint>
 #include <limits>
+#include <memory>
 
 #define LLMODEL_MAX_PROMPT_BATCH 128
 
@@ -26,7 +27,7 @@ class LLModel {
         std::string_view buildVariant() const { return m_buildVariant; }
 
         static bool isImplementation(const Dlhandle&);
-        static const std::vector<Implementation>& implementationList();
+        static const std::vector<std::unique_ptr<Implementation>>& implementationList();
         static const Implementation *implementation(const char *fname, const std::string& buildVariant);
         static LLModel *construct(const std::string &modelPath, std::string buildVariant = "auto");
         static void setImplementationsSearchPath(const std::string& path);