[IMP] runbot: allow to link non done batch

To make the propagation faster, mostly for the freeze and to test
forwardport chains, allowing to link non done batch (preparing, ready)
would allow to start two batch in two version at the same time and cross
link them.

It is only problematic for upgrades where the template build will be
used for upgrade and needs to be finished.
The latest changes allows to link a pending template build to an upgrade
one and the upgrade waits for the template to finish before starting.
The leader also waits for the template in the current batch to be
created before starting the configure upgrade.

The next part to do is to wait for the template to be created in other
batches before starting the configure.

Author: Xavier-Do

runbot/models/batch.py

@@ -180,16 +180,20 @@ def _create_build(self, params, slot):
         slot.build_id = build
         build._prepare_github_status()
 
-    def _get_latest_batch_per_version(self, skip_versions):
+    def _get_latest_batch_per_version(self, skip_versions, done=True):
+        domain = [
+            ('bundle_id.project_id', '=', self.bundle_id.project_id.id),
+            ('bundle_id.is_base', '=', True),
+            ('bundle_id.sticky', '=', True),
+            ('category_id', '=', self.category_id.id),
+            ('bundle_id.version_id', 'not in', skip_versions.ids),
+        ]
+
+        if done:
+            domain += [('state', '=', 'done')]
+
         return self.env['runbot.batch'].browse(result[1] for result in self.env['runbot.batch']._read_group(
-            domain=[
-                ('state', '=', 'done'),
-                ('bundle_id.project_id', '=', self.bundle_id.project_id.id),
-                ('bundle_id.is_base', '=', True),
-                ('bundle_id.sticky', '=', True),
-                ('category_id', '=', self.category_id.id),
-                ('bundle_id.version_id', 'not in', skip_versions.ids),
-            ],
+            domain=domain,
             groupby=['bundle_id'],
             aggregates=['id:max'],
         ))
@@ -294,7 +298,7 @@ def _fill_missing(branch_commits, match_type):
         if bundle.is_base or auto_rebase:
             existing = self.reference_batch_ids
             existing_versions = existing.mapped('bundle_id.version_id')
-            self.reference_batch_ids = self.reference_batch_ids | self._get_latest_batch_per_version(skip_versions=existing_versions)
+            self.reference_batch_ids = self.reference_batch_ids | self._get_latest_batch_per_version(skip_versions=existing_versions, done=False)
         if not bundle.is_base:
             merge_base_commits = self.commit_link_ids.mapped('merge_base_commit_id')
             if self.base_reference_batch_id:
@@ -483,6 +487,7 @@ def _start_builds(self):
                 continue
             trigger = slot.trigger_id
             trigger_custom = trigger_customs.get(trigger, self.env['runbot.bundle.trigger.custom'])
+
             if trigger.starts_after_pending:
                 missing_triggers = trigger.starts_after_ids - started_trigger
             elif trigger.starts_after_failure:
@@ -492,6 +497,21 @@ def _start_builds(self):
             if missing_triggers:
                 if not trigger_custom or (missing_triggers - disabled_triggers):
                     continue
+
+            if trigger.upgrade_dumps_trigger_id and trigger.config_id.uses_referenced_batches:
+                missing_template = False
+                for ref_batch in self.reference_batch_ids | self:
+                    if ref_batch.state in ('done', 'skipped'):
+                        continue
+                    if ref_batch.state == 'preparing':
+                        missing_template = True
+                        break
+                    needed_slot = ref_batch.slot_ids.filtered(lambda s: s.trigger_id in trigger.upgrade_dumps_trigger_id)
+                    if not needed_slot or not needed_slot.build_id:
+                        missing_template = True
+                        break
+                if missing_template:
+                    continue
             force_trigger = trigger_custom and trigger_custom.start_mode == 'force'
             skip_trigger = (trigger_custom and trigger_custom.start_mode == 'disabled') or trigger.manual
             should_start = slot.trigger_id.id in should_start_triggers_ids

runbot/models/build_config.py

@@ -144,6 +144,12 @@ class Config(models.Model):
     dynamic_config_extension = fields.Text('Dynamic Config Extend File', tracking=True)
 
     use_extra_slot = fields.Boolean('Use extra slot', default=False, tracking=True, help="Allow to use an extra slot for this config, if available")
+    uses_referenced_batches = fields.Boolean('Uses references builds', compute='_compute_uses_referenced_batches', store=True)
+
+    @api.depends('step_order_ids.step_id.uses_referenced_batches')
+    def _compute_uses_referenced_batches(self):
+        for record in self:
+            record.uses_referenced_batches = any(step.uses_referenced_batches for step in record.step_order_ids.step_id)
 
     @api.constrains('default_dynamic_config', 'dynamic_config_extension')
     def _check_dynamic_config(self):
@@ -508,7 +514,7 @@ class ConfigStep(models.Model):
     file_limit = fields.Integer('File limit', default=450)
     break_before_if_ko = fields.Boolean('Break before this step if build is ko')
     break_after_if_ko = fields.Boolean('Break after this step if build is ko')
-
+    uses_referenced_batches = fields.Boolean('Uses references builds', compute='_compute_uses_referenced_batches', store=True, readonly=False)
 
     @api.constrains('python_code')
     def _check_python_code(self):
@@ -536,6 +542,11 @@ def _compute_db_name(self):
         for step in self:
             step.db_name = step.custom_db_name or step.name
 
+    @api.depends('job_type')
+    def _compute_uses_referenced_batches(self):
+        for record in self:
+            record.uses_referenced_batches = record.job_type == 'configure_upgrade'
+
     def _get_db_name(self, build):
         db_name = self.custom_db_name or self.name
         return re.sub(r'[^a-z0-9\-_]', '_', db_name.lower())

runbot/views/config_views.xml

@@ -110,6 +110,7 @@
                       <field name="allow_similar_build_quick_result"/>
                       <field name="allow_build_link"/>
                       <field name="upgrade_matrix_id"/>
+                      <field name="uses_referenced_batches"/>
                     </group>
                     <group invisible="job_type not in ('python', 'configure_upgrade', 'dynamic')">
                       <group class="col" string="Upgrade matrix settings"  invisible="not upgrade_matrix_id">