From 7026706256a7dbf52893fceb38a56529c8e5e770 Mon Sep 17 00:00:00 2001 From: Jesse van Herk Date: Sat, 20 Apr 2024 11:27:28 -0600 Subject: [PATCH] Update Godot docs to include v4.2 and fix older version scraping Godot 4.2 was released on Nov 2023. Since v3.5, godot upstream docs have used some new HTML layouts and file structures. Since older versions still need to be scrapable, this introduces another set of versioned filters for godot. This change also makes small changes to the filters to handle the current website markup for the previous godot doc versions. The @gdscript and @globalscope entries can't currently be browsed, because of an encoding mismatch between the frontend and backend; I've identified a possible fix for that but will PR that separately. --- lib/docs/filters/godot/clean_html.rb | 23 ++++++++-- lib/docs/filters/godot/clean_html_v2.rb | 2 +- lib/docs/filters/godot/clean_html_v3.rb | 27 ++++++++++++ lib/docs/filters/godot/entries.rb | 6 ++- lib/docs/filters/godot/entries_v3.rb | 39 +++++++++++++++++ lib/docs/scrapers/godot.rb | 57 +++++++++++++++---------- 6 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 lib/docs/filters/godot/clean_html_v3.rb create mode 100644 lib/docs/filters/godot/entries_v3.rb diff --git a/lib/docs/filters/godot/clean_html.rb b/lib/docs/filters/godot/clean_html.rb index 9543f102f8..54ace4df87 100644 --- a/lib/docs/filters/godot/clean_html.rb +++ b/lib/docs/filters/godot/clean_html.rb @@ -6,12 +6,13 @@ def call at_css('h1').content = 'Godot Engine' at_css('.admonition.note').remove end + css('.admonition-grid').remove - css('ul[id].simple li:first-child:last-child').each do |node| + css('p[id]').each do |node| heading = Nokogiri::XML::Node.new 'h3', doc.document - heading['id'] = node.parent['id'] + heading['id'] = node['id'] heading.children = node.children - node.parent.before(heading).remove + node.before(heading).remove end css('h3 strong').each do |node| @@ -20,6 +21,22 @@ def call css('a.reference').remove_attr('class') + # flatten gdscript+C# example blocks and add language name. + css('div[role="tabpanel"]').each do |node| + language_label = Nokogiri::XML::Node.new 'strong', doc.document + language_name = 'GDScript' if node.at_css('div.highlight-gdscript') + language_name = 'C#' if node.at_css('div.highlight-csharp') + language_label.content = language_name.to_s + + node.before(language_label) + node.before(node.children).remove + end + + css('div.sphinx-tabs [role="tablist"]').remove + + # remove the remotely hosted "percent-translated" badge + css('a[href^="https://hosted.weblate"]').remove if root_page? + doc end end diff --git a/lib/docs/filters/godot/clean_html_v2.rb b/lib/docs/filters/godot/clean_html_v2.rb index 1379818a61..9245b6ccf0 100644 --- a/lib/docs/filters/godot/clean_html_v2.rb +++ b/lib/docs/filters/godot/clean_html_v2.rb @@ -4,7 +4,7 @@ class CleanHtmlV2Filter < Filter def call if root_page? at_css('h1').content = 'Godot Engine' - at_css('.admonition.tip').remove + at_css('.admonition.caution').remove end css('ul[id].simple li:first-child:last-child').each do |node| diff --git a/lib/docs/filters/godot/clean_html_v3.rb b/lib/docs/filters/godot/clean_html_v3.rb new file mode 100644 index 0000000000..819066f743 --- /dev/null +++ b/lib/docs/filters/godot/clean_html_v3.rb @@ -0,0 +1,27 @@ +module Docs + class Godot + class CleanHtmlV3Filter < Filter + def call + if root_page? + at_css('h1').content = 'Godot Engine' + at_css('.admonition.caution').remove + end + + css('ul[id].simple li:first-child:last-child').each do |node| + heading = Nokogiri::XML::Node.new 'h3', doc.document + heading['id'] = node.parent['id'] + heading.children = node.children + node.parent.before(heading).remove + end + + css('h3 strong').each do |node| + node.before(node.children).remove + end + + css('a.reference').remove_attr('class') + + doc + end + end + end +end diff --git a/lib/docs/filters/godot/entries.rb b/lib/docs/filters/godot/entries.rb index 202c46e582..9cda5afa27 100644 --- a/lib/docs/filters/godot/entries.rb +++ b/lib/docs/filters/godot/entries.rb @@ -11,7 +11,7 @@ def get_type if slug.start_with?('getting_started') # Getting started sections are different even between different minor # versions from v3 so we're programmatically generating them instead. - "Getting started: " + slug.split('/')[1].tr_s('_', ' ').capitalize + 'Getting started: ' + slug.split('/')[1].tr_s('_', ' ').capitalize else name end @@ -20,9 +20,10 @@ def get_type def additional_entries return [] unless slug.start_with?('classes') - css('.simple[id]').each_with_object [] do |node, entries| + css('p[id]').each_with_object [] do |node, entries| name = node.at_css('strong').content next if name == self.name + name.prepend "#{self.name}." name << '()' entries << [name, node['id']] unless entries.any? { |entry| entry[0] == name } @@ -32,6 +33,7 @@ def additional_entries def include_default_entry? return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html') return false if subpath == 'classes/index.html' + true end end diff --git a/lib/docs/filters/godot/entries_v3.rb b/lib/docs/filters/godot/entries_v3.rb new file mode 100644 index 0000000000..44d851469b --- /dev/null +++ b/lib/docs/filters/godot/entries_v3.rb @@ -0,0 +1,39 @@ +module Docs + class Godot + class EntriesV3Filter < Docs::EntriesFilter + def get_name + name = at_css('h1').content + name.remove! "\u{00B6}" # Remove the pilcrow + name + end + + def get_type + if slug.start_with?('getting_started') + # Getting started sections are different even between different minor + # versions from v3 so we're programmatically generating them instead. + "Getting started: " + slug.split('/')[1].tr_s('_', ' ').capitalize + else + name + end + end + + def additional_entries + return [] unless slug.start_with?('classes') + + css('.simple[id]').each_with_object [] do |node, entries| + name = node.at_css('strong').content + next if name == self.name + name.prepend "#{self.name}." + name << '()' + entries << [name, node['id']] unless entries.any? { |entry| entry[0] == name } + end + end + + def include_default_entry? + return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html') + return false if subpath == 'classes/index.html' + true + end + end + end +end diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb index f3cfb08b6a..3c7995479a 100644 --- a/lib/docs/scrapers/godot.rb +++ b/lib/docs/scrapers/godot.rb @@ -5,59 +5,72 @@ class Godot < UrlScraper home: 'https://godotengine.org/', code: 'https://github.com/godotengine/godot' } + # godot docs since 3.5 don't link everything from the index. + self.initial_paths = %w[ + getting_started/introduction/index.html + getting_started/step_by_step/index.html + classes/index.html + ] - options[:container] = '.document .section' - + options[:container] = '.document > [itemprop="articleBody"]' options[:download_images] = false - options[:only_patterns] = [/\Agetting_started\//, /\Aclasses\//] + options[:only_patterns] = [%r{\Agetting_started/}, %r{\Aclasses/}] + + options[:attribution] = <<-HTML + © 2014–present Juan Linietsky, Ariel Manzur and the Godot community
+ Licensed under the Creative Commons Attribution Unported License v3.0. + HTML - options[:attribution] = ->(filter) do - if filter.subpath.start_with?('classes') - <<-HTML - © 2014–2022 Juan Linietsky, Ariel Manzur, Godot Engine contributors
- Licensed under the MIT License. - HTML - else - <<-HTML - © 2014–2022 Juan Linietsky, Ariel Manzur and the Godot community
- Licensed under the Creative Commons Attribution Unported License v3.0. - HTML - end + version '4.2' do + self.release = '4.2.2' + self.base_url = "https://docs.godotengine.org/en/#{self.version}/" + html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html' end version '3.5' do - self.release = '3.5.1' + self.release = '3.5.3' self.base_url = "https://docs.godotengine.org/en/#{self.version}/" - options[:container] = '.document > [itemprop="articleBody"] > section[id]' + + # godot 3.5 upstream docs are formatted like godot4 html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html' end version '3.4' do self.release = '3.4.5' self.base_url = "https://docs.godotengine.org/en/#{self.version}/" + options[:container] = '.document > [itemprop="articleBody"] > section[id]' - html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html' + html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html' end version '3.3' do self.release = '3.3.0' self.base_url = "https://docs.godotengine.org/en/#{self.version}/" - html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html' + self.initial_paths = %w[/index.html] + + options[:only_patterns] = [%r{\Aclasses/}] + options[:container] = '.document .section' + html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html' end version '3.2' do self.release = '3.2.3' self.base_url = "https://docs.godotengine.org/en/#{self.version}/" - html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html' + self.initial_paths = %w[/index.html] + + options[:only_patterns] = [%r{\Aclasses/}] + options[:container] = '.document .section' + html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html' end version '2.1' do self.release = '2.1.6' self.base_url = "https://docs.godotengine.org/en/#{self.version}/" + self.initial_paths = %w[/index.html] options[:skip] = %w(classes/class_@global\ scope.html) - options[:only_patterns] = [/\Alearning\//, /\Aclasses\//] - + options[:only_patterns] = [%r{\Alearning/}, %r{\Aclasses/}] + options[:container] = '.document .section' html_filters.push 'godot/entries_v2', 'godot/clean_html_v2', 'sphinx/clean_html' end