godotengine · anvilfolk · Aug 19, 2024
diff --git a/editor/doc_tools.cpp b/editor/doc_tools.cpp
@@ -369,6 +369,14 @@ void DocTools::remove_doc(const String &p_class_name) {
 	class_list.erase(p_class_name);
 }
 
+void DocTools::remove_script_doc_by_path(const String &p_path) {
+	for (KeyValue<String, DocData::ClassDoc> &E : class_list) {
+		if (E.value.is_script_doc && E.value.script_path == p_path) {
+			remove_doc(E.key);
+		}
+	}
+}
+
 bool DocTools::has_doc(const String &p_class_name) {
 	if (p_class_name.is_empty()) {
 		return false;

diff --git a/editor/doc_tools.h b/editor/doc_tools.h
@@ -45,6 +45,7 @@ class DocTools {
 	void merge_from(const DocTools &p_data);
 	void add_doc(const DocData::ClassDoc &p_class_doc);
 	void remove_doc(const String &p_class_name);
+	void remove_script_doc_by_path(const String &p_path);
 	bool has_doc(const String &p_class_name);
 	enum GenerateFlags {
 		GENERATE_FLAG_SKIP_BASIC_TYPES = (1 << 0),

diff --git a/editor/editor_file_system.cpp b/editor/editor_file_system.cpp
@@ -1959,6 +1959,7 @@ void EditorFileSystem::_update_script_documentation() {
 
 		if (!efd || index < 0) {
 			// The file was removed
+			EditorHelp::remove_script_doc_by_path(path);
 			continue;
 		}
 
@@ -1979,7 +1980,7 @@ void EditorFileSystem::_update_script_documentation() {
 				}
 				Vector<DocData::ClassDoc> docs = scr->get_documentation();
 				for (int j = 0; j < docs.size(); j++) {
-					EditorHelp::get_doc_data()->add_doc(docs[j]);
+					EditorHelp::add_doc(docs[j]);
 					if (!first_scan) {
 						// Update the documentation in the Script Editor if it is open.
 						ScriptEditor::get_singleton()->update_doc(docs[j].name);

diff --git a/editor/editor_help.cpp b/editor/editor_help.cpp
@@ -34,11 +34,13 @@
 #include "core/core_constants.h"
 #include "core/extension/gdextension.h"
 #include "core/input/input.h"
+#include "core/io/dir_access.h"
 #include "core/object/script_language.h"
 #include "core/os/keyboard.h"
 #include "core/string/string_builder.h"
 #include "core/version_generated.gen.h"
 #include "editor/doc_data_compressed.gen.h"
+#include "editor/editor_file_system.h"
 #include "editor/editor_main_screen.h"
 #include "editor/editor_node.h"
 #include "editor/editor_paths.h"
@@ -112,32 +114,12 @@ const Vector<String> packed_array_types = {
 	"PackedVector4Array",
 };
 
-// TODO: this is sometimes used directly as doc->something, other times as EditorHelp::get_doc_data(), which is thread-safe.
-// Might this be a problem?
 DocTools *EditorHelp::doc = nullptr;
 DocTools *EditorHelp::ext_doc = nullptr;
-
-static bool _attempt_doc_load(const String &p_class) {
-	// Docgen always happens in the outer-most class: it also generates docs for inner classes.
-	String outer_class = p_class.get_slice(".", 0);
-	if (!ScriptServer::is_global_class(outer_class)) {
-		return false;
-	}
-
-	// ResourceLoader is used in order to have a script-agnostic way to load scripts.
-	// This forces GDScript to compile the code, which is unnecessary for docgen, but it's a good compromise right now.
-	Ref<Script> script = ResourceLoader::load(ScriptServer::get_global_class_path(outer_class), outer_class);
-	if (script.is_valid()) {
-		Vector<DocData::ClassDoc> docs = script->get_documentation();
-		for (int j = 0; j < docs.size(); j++) {
-			const DocData::ClassDoc &doc = docs.get(j);
-			EditorHelp::get_doc_data()->add_doc(doc);
-		}
-		return true;
-	}
-
-	return false;
-}
+bool EditorHelp::script_docs_loaded = false;
+Vector<DocData::ClassDoc> EditorHelp::_docs_to_add;
+Vector<String> EditorHelp::_docs_to_remove;
+Vector<String> EditorHelp::_docs_to_remove_by_path;
 
 // Removes unnecessary prefix from p_class_specifier when within the p_edited_class context
 static String _contextualize_class_specifier(const String &p_class_specifier, const String &p_edited_class) {
@@ -685,8 +667,7 @@ void EditorHelp::_pop_code_font() {
 }
 
 Error EditorHelp::_goto_desc(const String &p_class) {
-	// If class doesn't have docs listed, attempt on-demand docgen
-	if (!doc->class_list.has(p_class) && !_attempt_doc_load(p_class)) {
+	if (!doc->class_list.has(p_class)) {
 		return ERR_DOES_NOT_EXIST;
 	}
 
@@ -2898,6 +2879,43 @@ String EditorHelp::get_cache_full_path() {
 	return EditorPaths::get_singleton()->get_cache_dir().path_join(vformat("editor_doc_cache-%d.%d.res", VERSION_MAJOR, VERSION_MINOR));
 }
 
+String EditorHelp::get_script_doc_cache_full_path() {
+	return EditorPaths::get_singleton()->get_project_settings_dir().path_join("editor_script_doc_cache.res");
+}
+
+void EditorHelp::add_doc(const DocData::ClassDoc &p_class_doc) {
+	if (!script_docs_loaded) {
+		_docs_to_add.append(p_class_doc);
+		return;
+	}
+
+	get_doc_data()->add_doc(p_class_doc);
+}
+
+void EditorHelp::remove_doc(const String &p_class_name) {
+	if (!script_docs_loaded) {
+		_docs_to_remove.append(p_class_name);
+		return;
+	}
+
+	DocTools *dt = get_doc_data();
+	if (dt->has_doc(p_class_name)) {
+		dt->remove_doc(p_class_name);
+	}
+}
+
+void EditorHelp::remove_script_doc_by_path(const String &p_path) {
+	if (!script_docs_loaded) {
+		_docs_to_remove_by_path.append(p_path);
+		return;
+	}
+	get_doc_data()->remove_script_doc_by_path(p_path);
+}
+
+bool EditorHelp::has_doc(const String &p_class_name) {
+	return get_doc_data()->has_doc(p_class_name);
+}
+
 void EditorHelp::load_xml_buffer(const uint8_t *p_buffer, int p_size) {
 	if (!ext_doc) {
 		ext_doc = memnew(DocTools);
@@ -2916,23 +2934,27 @@ void EditorHelp::remove_class(const String &p_class) {
 	}
 
 	if (doc && doc->has_doc(p_class)) {
-		doc->remove_doc(p_class);
+		remove_doc(p_class);
 	}
 }
 
 void EditorHelp::_load_doc_thread(void *p_udata) {
+	bool use_script_cache = (bool)p_udata;
 	Ref<Resource> cache_res = ResourceLoader::load(get_cache_full_path());
 	if (cache_res.is_valid() && cache_res->get_meta("version_hash", "") == doc_version_hash) {
 		Array classes = cache_res->get_meta("classes", Array());
 		for (int i = 0; i < classes.size(); i++) {
 			doc->add_doc(DocData::ClassDoc::from_dict(classes[i]));
 		}
 
+		if (use_script_cache) {
+			callable_mp_static(&EditorHelp::load_script_doc_cache).call_deferred();
+		}
 		// Extensions' docs are not cached. Generate them now (on the main thread).
 		callable_mp_static(&EditorHelp::_gen_extensions_docs).call_deferred();
 	} else {
 		// We have to go back to the main thread to start from scratch, bypassing any possibly existing cache.
-		callable_mp_static(&EditorHelp::generate_doc).call_deferred(false);
+		callable_mp_static(&EditorHelp::generate_doc).call_deferred(false, use_script_cache);
 	}
 
 	OS::get_singleton()->benchmark_end_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
@@ -2962,6 +2984,12 @@ void EditorHelp::_gen_doc_thread(void *p_udata) {
 		ERR_PRINT("Cannot save editor help cache (" + get_cache_full_path() + ").");
 	}
 
+	// Load script docs after native ones are cached so native cache doesn't contain script docs.
+	bool use_script_cache = (bool)p_udata;
+	if (use_script_cache) {
+		callable_mp_static(&EditorHelp::load_script_doc_cache).call_deferred();
+	}
+
 	OS::get_singleton()->benchmark_end_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
 }
 
@@ -2974,7 +3002,138 @@ void EditorHelp::_gen_extensions_docs() {
 	}
 }
 
-void EditorHelp::generate_doc(bool p_use_cache) {
+void EditorHelp::load_script_doc_cache() {
+	if (!ProjectSettings::get_singleton()->is_project_loaded()) {
+		print_verbose("Skipping loading script doc cache since no project is open.");
+		return;
+	}
+
+	if (!ResourceLoader::exists(get_script_doc_cache_full_path())) {
+		print_verbose("Script documentation cache not found. Regenerating it may take a while for projects with many scripts.");
+		regenerate_script_doc_cache();
+		return;
+	}
+
+	if (EditorFileSystem::get_singleton()->is_scanning()) {
+		// This is assuming EditorFileSystem is performing first scan. We must wait until it is done.
+		EditorFileSystem::get_singleton()->connect("filesystem_changed", callable_mp_static(EditorHelp::load_script_doc_cache), CONNECT_ONE_SHOT);
+		return;
+	}
+
+	EditorHelp::_wait_for_thread();
+	worker_thread.start(_load_script_doc_cache_thread, nullptr);
+}
+
+void EditorHelp::_load_script_doc_cache_thread(void *p_udata) {
+	ERR_FAIL_COND_MSG(!ProjectSettings::get_singleton()->is_project_loaded(), "Error: cannot load script doc cache without a project.");
+	ERR_FAIL_COND_MSG(!ResourceLoader::exists(get_script_doc_cache_full_path()), "Error: cannot load script doc cache from inexistent file.");
+
+	Ref<Resource> script_doc_cache_res = ResourceLoader::load(get_script_doc_cache_full_path());
+	if (!script_doc_cache_res.is_valid()) {
+		print_verbose("Script doc cache is corrupted. Regenerating it instead.");
+		_delete_script_doc_cache();
+		callable_mp_static(EditorHelp::regenerate_script_doc_cache).call_deferred();
+		return;
+	}
+
+	Array classes = script_doc_cache_res->get_meta("classes", Array());
+	for (const Dictionary dict : classes) {
+		doc->add_doc(DocData::ClassDoc::from_dict(dict));
+	}
+
+	// Process postponed doc changes, likely added by EditorFileSystem's scans while the cache was loading in EditorHelp::worker_thread.
+	for (const DocData::ClassDoc &cd : _docs_to_add) {
+		doc->add_doc(cd);
+	}
+	for (const String &class_name : _docs_to_remove) {
+		doc->remove_doc(class_name);
+	}
+	for (const String &path : _docs_to_remove_by_path) {
+		doc->remove_script_doc_by_path(path);
+	}
+	_docs_to_add.clear();
+	_docs_to_remove.clear();
+	_docs_to_remove_by_path.clear();
+
+	// Always delete the doc cache after successful load: most uses of editor will change a script, invalidating cache.
+	_delete_script_doc_cache();
+	script_docs_loaded = true;
+}
+
+// For use only during editor startup. Won't track filesystem changes when called at other times.
+void EditorHelp::regenerate_script_doc_cache() {
+	if (EditorFileSystem::get_singleton()->is_scanning()) {
+		// Wait for filesystem to finish scanning before starting worker thread, which will need updated filesystem data.
+		EditorFileSystem::get_singleton()->connect("filesystem_changed", callable_mp_static(EditorHelp::regenerate_script_doc_cache), CONNECT_ONE_SHOT);
+		return;
+	}
+
+	EditorHelp::_wait_for_thread();
+	worker_thread.start(_regen_script_doc_thread, EditorFileSystem::get_singleton()->get_filesystem());
+}
+
+void EditorHelp::_regen_script_doc_thread(void *p_udata) {
+	OS::get_singleton()->benchmark_begin_measure("EditorHelp", "Generate Script Documentation");
+
+	EditorFileSystemDirectory *dir = static_cast<EditorFileSystemDirectory *>(p_udata);
+	script_docs_loaded = false;
+
+	_reload_scripts_documentation(dir);
+	// Ignore changes from filesystem scan since scripts scripts were just regenerated.
+	_docs_to_add.clear();
+	_docs_to_remove.clear();
+	_docs_to_remove_by_path.clear();
+
+	script_docs_loaded = true;
+
+	OS::get_singleton()->benchmark_end_measure("EditorHelp", "Generate Script Documentation");
+}
+
+void EditorHelp::_reload_scripts_documentation(EditorFileSystemDirectory *p_dir) {
+	// Recursively force compile all scripts, which should generate their documentation.
+	for (int i = 0; i < p_dir->get_subdir_count(); i++) {
+		_reload_scripts_documentation(p_dir->get_subdir(i));
+	}
+
+	for (int i = 0; i < p_dir->get_file_count(); i++) {
+		if (ClassDB::is_parent_class(p_dir->get_file_type(i), SNAME("Script"))) {
+			Ref<Script> scr = ResourceLoader::load(p_dir->get_file_path(i));
+			if (scr.is_valid()) {
+				for (const DocData::ClassDoc &cd : scr->get_documentation()) {
+					doc->add_doc(cd);
+				}
+			}
+		}
+	}
+}
+
+void EditorHelp::_delete_script_doc_cache() {
+	if (FileAccess::exists(get_script_doc_cache_full_path())) {
+		DirAccess::remove_file_or_error(ProjectSettings::get_singleton()->globalize_path(get_script_doc_cache_full_path()));
+	}
+}
+
+void EditorHelp::save_script_doc_cache() {
+	if (!script_docs_loaded) {
+		print_verbose("Script docs haven't been properly loaded or regenerated, so don't save them to disk.");
+		return;
+	}
+
+	Ref<Resource> cache_res;
+	cache_res.instantiate();
+	Array classes;
+	for (const KeyValue<String, DocData::ClassDoc> &E : doc->class_list) {
+		if (E.value.is_script_doc) {
+			classes.push_back(DocData::ClassDoc::to_dict(E.value));
+		}
+	}
+
+	cache_res->set_meta("classes", classes);
+	Error err = ResourceSaver::save(cache_res, get_script_doc_cache_full_path(), ResourceSaver::FLAG_COMPRESS);
+	ERR_FAIL_COND_MSG(err != OK, vformat("Cannot save script documentation cache in %s.", get_script_doc_cache_full_path()));
+}
+
+void EditorHelp::generate_doc(bool p_use_cache, bool p_use_script_cache) {
 	doc_generation_count++;
 	OS::get_singleton()->benchmark_begin_measure("EditorHelp", vformat("Generate Documentation (Run %d)", doc_generation_count));
 
@@ -2990,11 +3149,11 @@ void EditorHelp::generate_doc(bool p_use_cache) {
 	}
 
 	if (p_use_cache && FileAccess::exists(get_cache_full_path())) {
-		worker_thread.start(_load_doc_thread, nullptr);
+		worker_thread.start(_load_doc_thread, (void *)p_use_script_cache);
 	} else {
 		print_verbose("Regenerating editor help cache");
 		doc->generate();
-		worker_thread.start(_gen_doc_thread, nullptr);
+		worker_thread.start(_gen_doc_thread, (void *)p_use_script_cache);
 	}
 }
 

diff --git a/editor/editor_help.h b/editor/editor_help.h
@@ -85,6 +85,8 @@ class FindBar : public HBoxContainer {
 	FindBar();
 };
 
+class EditorFileSystemDirectory;
+
 class EditorHelp : public VBoxContainer {
 	GDCLASS(EditorHelp, VBoxContainer);
 
@@ -195,10 +197,19 @@ class EditorHelp : public VBoxContainer {
 	static String doc_version_hash;
 	static Thread worker_thread;
 
+	static bool script_docs_loaded;
+	static Vector<DocData::ClassDoc> _docs_to_add;
+	static Vector<String> _docs_to_remove;
+	static Vector<String> _docs_to_remove_by_path;
+
 	static void _wait_for_thread();
 	static void _load_doc_thread(void *p_udata);
 	static void _gen_doc_thread(void *p_udata);
 	static void _gen_extensions_docs();
+	static void _load_script_doc_cache_thread(void *p_udata);
+	static void _regen_script_doc_thread(void *p_udata);
+	static void _reload_scripts_documentation(EditorFileSystemDirectory *p_dir);
+	static void _delete_script_doc_cache();
 	static void _compute_doc_version_hash();
 
 	struct PropertyCompare {
@@ -218,10 +229,21 @@ class EditorHelp : public VBoxContainer {
 	static void _bind_methods();
 
 public:
-	static void generate_doc(bool p_use_cache = true);
-	static DocTools *get_doc_data();
+	static void generate_doc(bool p_use_cache = true, bool p_use_script_cache = true);
 	static void cleanup_doc();
+	static void load_script_doc_cache();
+	static void regenerate_script_doc_cache();
+	static void save_script_doc_cache();
 	static String get_cache_full_path();
+	static String get_script_doc_cache_full_path();
+
+	// Adding scripts to DocData directly may make script doc cache inconsistent. Use methods below when adding script docs.
+	static DocTools *get_doc_data();
+	// Method forwarding to underlying DocTools to keep script doc cache consistent.
+	static void add_doc(const DocData::ClassDoc &p_class_doc);
+	static void remove_doc(const String &p_class_name);
+	static void remove_script_doc_by_path(const String &p_path);
+	static bool has_doc(const String &p_class_name);
 
 	static void load_xml_buffer(const uint8_t *p_buffer, int p_size);
 	static void remove_class(const String &p_class);