Integrate commitfest status

This commit adds a script to scrape the commitfest website, and mirror
some data in our database. This script is executed each day for open
commitfests, and once manually at the beginning for all.

Based on this info, we can link the commitfest patch pages and display
commitfest status even on the topic index.

Author: dutow

Gemfile

@@ -21,6 +21,7 @@ gem "stimulus-rails"
 gem "jbuilder"
 gem "slim"
 gem "kaminari"
+gem "nokogiri"
 
 # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword]
 gem "bcrypt", "~> 3.1"

Gemfile.lock

@@ -493,6 +493,7 @@ DEPENDENCIES
   kaminari
   letter_opener
   letter_opener_web
+  nokogiri
   omniauth
   omniauth-google-oauth2
   omniauth-rails_csrf_protection
@@ -517,4 +518,4 @@ DEPENDENCIES
   web-console
 
 BUNDLED WITH
-   2.6.9
+  4.0.3

app/assets/stylesheets/components/sidebar.css

@@ -107,3 +107,66 @@
 .sidebar .attachment-names {
   color: var(--color-text-secondary);
 }
+
+.sidebar .commitfest-list {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-3);
+}
+
+.sidebar .commitfest-entry-header {
+  display: flex;
+  justify-content: space-between;
+  gap: var(--spacing-2);
+  align-items: baseline;
+}
+
+.sidebar .commitfest-link,
+.sidebar .commitfest-patch-link {
+  color: var(--color-text-link);
+  text-decoration: none;
+  font-weight: var(--font-weight-semibold);
+}
+
+.sidebar .commitfest-link:hover,
+.sidebar .commitfest-patch-link:hover {
+  text-decoration: underline;
+}
+
+.sidebar .commitfest-status {
+  color: var(--color-text-secondary);
+  font-size: var(--font-size-sm);
+  white-space: nowrap;
+}
+
+.sidebar .commitfest-patch {
+  margin-top: var(--spacing-1);
+  color: var(--color-text-secondary);
+}
+
+.sidebar .commitfest-tags {
+  display: flex;
+  flex-wrap: wrap;
+  gap: var(--spacing-2);
+  margin-top: var(--spacing-2);
+}
+
+.sidebar .commitfest-tag {
+  background: var(--color-gray-100);
+  padding: 2px 8px;
+  border-radius: var(--border-radius-sm);
+  font-size: var(--font-size-xs);
+  color: var(--color-text-secondary);
+}
+
+.sidebar .commitfest-reviewers,
+.sidebar .commitfest-committers {
+  list-style: none;
+  padding-left: 0;
+  margin: 0;
+}
+
+.sidebar .commitfest-reviewers li + li,
+.sidebar .commitfest-committers li + li {
+  margin-top: var(--spacing-1);
+}

app/assets/stylesheets/components/topics.css

@@ -237,6 +237,61 @@
   margin-bottom: 0;
 }
 
+.commitfest-icon {
+  background: var(--color-bg-icon);
+  color: var(--color-text-secondary);
+}
+
+.commitfest-icon.commitfest-committed {
+  color: var(--color-success);
+}
+
+.commitfest-hover {
+  display: flex;
+  flex-direction: column;
+  gap: var(--spacing-2);
+}
+
+.commitfest-hover-title {
+  display: flex;
+  justify-content: space-between;
+  gap: var(--spacing-2);
+  font-weight: var(--font-weight-semibold);
+}
+
+.commitfest-hover-title a {
+  color: var(--color-text-link);
+  text-decoration: none;
+}
+
+.commitfest-hover-title a:hover {
+  text-decoration: underline;
+}
+
+.commitfest-hover-status {
+  color: var(--color-text-secondary);
+  font-size: var(--font-size-sm);
+  white-space: nowrap;
+}
+
+.commitfest-hover-row {
+  display: flex;
+  gap: var(--spacing-2);
+  align-items: baseline;
+}
+
+.commitfest-hover-label {
+  color: var(--color-text-muted);
+  font-size: var(--font-size-xs);
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+}
+
+.commitfest-hover-value {
+  color: var(--color-text-primary);
+  font-size: var(--font-size-sm);
+}
+
 .topic-meta {
   & span {
     padding-right: var(--spacing-2);

app/controllers/topics_controller.rb

@@ -7,6 +7,7 @@ def index
     base_query = apply_filters(Topic.includes(:creator))
 
     apply_cursor_pagination(base_query)
+    preload_commitfest_summaries
     @new_topics_count = 0
     @page_cache_key = topics_page_cache_key
 
@@ -42,6 +43,7 @@ def show
 
     build_participants_sidebar_data(messages_scope)
     build_thread_outline(messages_scope)
+    load_commitfest_sidebar
     load_notes if user_signed_in?
   end
 
@@ -101,6 +103,7 @@ def search
       @new_topics_count = 0
     end
 
+    preload_commitfest_summaries
     preload_participation_flags if user_signed_in?
 
     respond_to do |format|
@@ -157,6 +160,7 @@ def user_state_frame
     preload_topic_states
     preload_note_counts
     preload_participation_flags
+    preload_commitfest_summaries
 
     respond_to do |format|
       format.turbo_stream
@@ -700,6 +704,37 @@ def load_cached_search_results
     @new_topics_count = 0
   end
 
+  def preload_commitfest_summaries
+    topic_ids = @topics.map(&:id)
+    @commitfest_summaries = Topic.commitfest_summaries(topic_ids)
+  end
+
+  def load_commitfest_sidebar
+    entries = CommitfestPatchCommitfest
+      .joins(commitfest_patch: :commitfest_patch_topics)
+      .includes(:commitfest, commitfest_patch: :commitfest_tags)
+      .where(commitfest_patch_topics: { topic_id: @topic.id })
+      .order("commitfests.end_date DESC, commitfests.start_date DESC")
+
+    @commitfest_sidebar_entries = entries.map do |entry|
+      patch = entry.commitfest_patch
+      {
+        commitfest_name: entry.commitfest.name,
+        commitfest_external_id: entry.commitfest.external_id,
+        patch_external_id: patch.external_id,
+        patch_title: patch.title,
+        status: entry.status,
+        tags: patch.commitfest_tags.map(&:name)
+      }
+    end
+
+    reviewers = entries.flat_map { |entry| Topic.parse_csv_list(entry.commitfest_patch.reviewers) }
+    @commitfest_reviewers = reviewers.uniq
+
+    committers = entries.map { |entry| entry.commitfest_patch.committer.to_s.strip }.reject(&:blank?).uniq
+    @commitfest_committers = committers
+  end
+
   def slice_cached_entries(entries, cursor_param)
     return { entries: entries.first(25) } unless cursor_param.present?
 

app/helpers/application_helper.rb

@@ -36,6 +36,24 @@ def message_id_anchor(message)
     "message-id-#{CGI.escape(message.message_id)}"
   end
 
+  def commitfest_ci_label(summary)
+    case summary[:ci_status]
+    when "not_processed"
+      "Not processed"
+    when "needs_rebase"
+      "Needs rebase"
+    when "score"
+      "CI score: #{summary[:ci_score].to_i}/10"
+    end
+  end
+
+  def commitfest_icon_class(summary)
+    return "fa-circle-check" if summary[:committed]
+    return "fa-magnifying-glass" if summary[:status] == "Needs review"
+
+    "fa-code-branch"
+  end
+
   def read_visibility_seconds
     5
   end

app/models/commitfest.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Commitfest < ApplicationRecord
+  has_many :commitfest_patch_commitfests, dependent: :destroy
+  has_many :commitfest_patches, through: :commitfest_patch_commitfests
+
+  validates :external_id, :name, :status, :start_date, :end_date, presence: true
+end

app/models/commitfest_patch.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CommitfestPatch < ApplicationRecord
+  has_many :commitfest_patch_commitfests, dependent: :destroy
+  has_many :commitfests, through: :commitfest_patch_commitfests
+  has_many :commitfest_patch_tags, dependent: :destroy
+  has_many :commitfest_tags, through: :commitfest_patch_tags
+  has_many :commitfest_patch_messages, dependent: :destroy
+  has_many :commitfest_patch_topics, dependent: :destroy
+  has_many :topics, through: :commitfest_patch_topics
+
+  validates :external_id, :title, presence: true
+end

app/models/commitfest_patch_commitfest.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class CommitfestPatchCommitfest < ApplicationRecord
+  belongs_to :commitfest
+  belongs_to :commitfest_patch
+
+  validates :status, presence: true
+end

app/models/commitfest_patch_message.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class CommitfestPatchMessage < ApplicationRecord
+  belongs_to :commitfest_patch
+  belongs_to :message, optional: true, foreign_key: :message_record_id
+
+  validates :message_id, presence: true
+end