[IMP] runbot: refactor build error models

The initial idea to link an error to another one was a quick solution
to group them if they where related, but this became challenging
to copute metada regarding errors.

- The displayed error message was not always consistent with the real
root cause/the error that lead here.
- The aggregates (lets says, linked buils ids) could be the one of the
error, or from all error messages. Same for the versions, first seen, ..
This is confusing to knwo what is the leist we are managing and what is
the expecte result to display

Main motivation:
on a standard error page (will be changed to "assignment"), we want to
have the list of error message that is related to this one. We want to
know for each message (a real build error) what is the version,
first seen, ...
This will give more flexibility on the display,

The assigned person/team/test-tags, ... are moved to this model
The appearance data remains on the build error but are aggregate on the
assignation.

Author: Xavier-Do

runbot/__manifest__.py

@@ -6,7 +6,7 @@
     'author': "Odoo SA",
     'website': "http://runbot.odoo.com",
     'category': 'Website',
-    'version': '5.7',
+    'version': '5.8',
     'application': True,
     'depends': ['base', 'base_automation', 'website'],
     'data': [

runbot/controllers/frontend.py

@@ -30,7 +30,7 @@ def response_wrap(*args, **kwargs):
             keep_search = request.httprequest.cookies.get('keep_search', False) == '1'
             cookie_search = request.httprequest.cookies.get('search', '')
             refresh = kwargs.get('refresh', False)
-            nb_build_errors = request.env['runbot.build.error'].search_count([('random', '=', True), ('parent_id', '=', False)])
+            nb_build_errors = request.env['runbot.build.error'].search_count([])
             nb_assigned_errors = request.env['runbot.build.error'].search_count([('responsible', '=', request.env.user.id)])
             nb_team_errors = request.env['runbot.build.error'].search_count([('responsible', '=', False), ('team_id', 'in', request.env.user.runbot_team_ids.ids)])
             kwargs['more'] = more
@@ -459,7 +459,7 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs):
             ('responsible', '=', False),
             ('team_id', 'in', request.env.user.runbot_team_ids.ids)
         ], order='last_seen_date desc, build_count desc')
-        domain = [('parent_id', '=', False), ('responsible', '!=', request.env.user.id), ('build_count', '>', 1)]
+        domain = [('responsible', '!=', request.env.user.id), ('build_count', '>', 1)]
         build_errors_count = request.env['runbot.build.error'].search_count(domain)
         url_args = {}
         url_args['sort'] = sort
@@ -481,7 +481,7 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs):
     @route(['/runbot/teams', '/runbot/teams/<model("runbot.team"):team>',], type='http', auth='user', website=True, sitemap=False)
     def team_dashboards(self, team=None, hide_empty=False, **kwargs):
         teams = request.env['runbot.team'].search([]) if not team else None
-        domain = [('id', 'in', team.build_error_ids.ids)] if team else []
+        domain = [('id', 'in', team.assignment_ids.ids)] if team else []
 
         # Sort & Filter
         sortby = kwargs.get('sortby', 'count')
@@ -496,7 +496,7 @@ def team_dashboards(self, team=None, hide_empty=False, **kwargs):
             'not_one': {'label': 'Seen more than once', 'domain': [('build_count', '>', 1)]},
         }
 
-        for trigger in team.build_error_ids.trigger_ids if team else []:
+        for trigger in team.assignment_ids.trigger_ids if team else []:
             k = f'trigger_{trigger.name.lower().replace(" ", "_")}'
             searchbar_filters.update(
                 {k: {'label': f'Trigger {trigger.name}', 'domain': [('trigger_ids', '=', trigger.id)]}}
@@ -510,7 +510,7 @@ def team_dashboards(self, team=None, hide_empty=False, **kwargs):
         qctx = {
             'team': team,
             'teams': teams,
-            'build_error_ids': request.env['runbot.build.error'].search(domain, order=order),
+            'build_assignment_ids': request.env['runbot.build.assignment'].search(domain, order=order),
             'hide_empty': bool(hide_empty),
             'searchbar_sortings': searchbar_sortings,
             'sortby': sortby,

runbot/data/error_link.xml

@@ -9,6 +9,16 @@
             records.action_link_errors()
         </field>
     </record>
+    <record model="ir.actions.server" id="action_link_build_errors_contents">
+        <field name="name">Link build errors contents</field>
+        <field name="model_id" ref="runbot.model_runbot_build_error" />
+        <field name="binding_model_id" ref="runbot.model_runbot_build_error" />
+        <field name="type">ir.actions.server</field>
+        <field name="state">code</field>
+        <field name="code">
+            records.action_link_errors_content()
+        </field>
+    </record>
     <record model="ir.actions.server" id="action_clean_build_errors">
         <field name="name">Re-clean build errors</field>
         <field name="model_id" ref="runbot.model_runbot_build_error" />

runbot/migrations/17.0.5.8/post-migration.py

@@ -0,0 +1,123 @@
+import logging
+
+_logger = logging.getLogger(__name__)
+
+
+def migrate(cr, version):
+
+    # get seen infos
+    cr.execute("SELECT error_content_id, min(build_id), min(log_date), max(build_id), max(log_date), count(DISTINCT build_id) FROM runbot_build_error_link GROUP BY error_content_id")
+    vals_by_error = {error: vals for error, *vals in cr.fetchall()}
+
+    # first_seen_build_id was not stored, lets fill it and update all values for good mesure
+    for error, vals in vals_by_error.items():
+        cr.execute('UPDATE runbot_build_error_content SET first_seen_build_id = %s, first_seen_date = %s, last_seen_build_id = %s, last_seen_date = %s WHERE id=%s', (vals[0], vals[1], vals[2], vals[3], error))
+
+    # generate flattened error hierarchy
+    cr.execute('''SELECT
+                    id,
+                    parent_id
+                FROM runbot_build_error_content
+                ORDER BY id
+                ''')
+
+    error_by_parent = {}
+    for error_id, parent_id in cr.fetchall():
+        if parent_id:
+            error_by_parent.setdefault(parent_id, []).append(error_id)
+    stable = False
+    while not stable:
+        stable = True
+        for parent, child_ids in error_by_parent.items():
+            for child_id in child_ids:
+                if parent == child_id:
+                    continue
+                sub_childrens = error_by_parent.get(child_id)
+                if sub_childrens:
+                    error_by_parent[parent] = error_by_parent[parent] + sub_childrens
+                    error_by_parent[child_id] = []
+                    stable = False
+    for parent, child_ids in error_by_parent.items():
+        if parent in child_ids:
+            _logger.info('Breaking cycle parent on %s', parent)
+            error_by_parent[parent] = [c for c in child_ids if c != parent]
+            cr.execute('UPDATE runbot_build_error_content SET parent_id = null WHERE id=%s', (parent,))
+    error_by_parent = {parent: chilren for parent, chilren in error_by_parent.items() if chilren}
+
+    cr.execute('''SELECT
+                    id,
+                    active,
+                    parent_id
+                    random,
+                    content,
+                    test_tags,
+                    tags_min_version_id,
+                    tags_max_version_id,
+                    team_id,
+                    responsible,
+                    customer,
+                    fixing_commit,
+                    fixing_pr_id
+                FROM runbot_build_error_content
+                WHERE parent_id IS null
+                ORDER BY id
+                ''')
+    errors = cr.fetchall()
+    nb_groups = len(error_by_parent)
+    _logger.info('Creating %s errors', nb_groups)
+    for error in errors:
+        error_id, *values = error
+        children = error_by_parent.get(error_id, [])
+        assert not error_id in children
+        all_errors = [error_id, *children]
+        error_count = len(all_errors)
+
+        # vals_by_error order: min(build_id), min(log_date), max(build_id), max(log_date)
+        build_count = 0
+        first_seen_build_id = first_seen_date = last_seen_build_id = last_seen_date = None
+        if error_id in vals_by_error:
+            error_vals = [vals_by_error[error_id] for error_id in all_errors]
+            first_seen_build_id = min(vals[0] for vals in error_vals)
+            first_seen_date = min(vals[1] for vals in error_vals)
+            last_seen_build_id = max(vals[2] for vals in error_vals)
+            last_seen_date = max(vals[3] for vals in error_vals)
+            build_count = sum(vals[4] for vals in error_vals)  # not correct for distinct but close enough
+            assert first_seen_date <= last_seen_date
+            assert first_seen_build_id <= last_seen_build_id
+        name = values[2].split('\n')[0]
+
+        values = [error_id, *values, last_seen_build_id, first_seen_build_id, last_seen_date, first_seen_date, build_count, error_count, name]
+
+        cr.execute('''
+        INSERT INTO runbot_build_error (
+            id,
+            active,
+            random,
+            description,
+            test_tags,
+            tags_min_version_id,
+            tags_max_version_id,
+            team_id,
+            responsible,
+            customer,
+            fixing_commit,
+            fixing_pr_id,
+            last_seen_build_id,
+            first_seen_build_id,
+            last_seen_date,
+            first_seen_date,
+            build_count,
+            error_count,
+            name
+        )
+        VALUES (%s)
+        RETURNING id
+        ''' % ', '.join(['%s'] * len(values)), values)  # noqa: S608
+
+        error_id = cr.fetchone()
+        cr.execute('UPDATE runbot_build_error_content SET error_id = %s WHERE id in %s', (error_id, tuple(all_errors)))
+
+    cr.execute('ALTER TABLE runbot_build_error_content ALTER COLUMN error_id SET NOT NULL')
+    cr.execute('SELECT max(id) from runbot_build_error')
+    cr.execute("SELECT SETVAL('runbot_build_error_id_seq', %s)", (cr.fetchone()[0] + 1,))
+    _logger.info('Done')

runbot/migrations/17.0.5.8/pre-migration.py

@@ -0,0 +1,4 @@
+def migrate(cr, version):
+    cr.execute('ALTER TABLE runbot_build_error RENAME TO runbot_build_error_content')
+    cr.execute('ALTER TABLE runbot_build_error_content ADD COLUMN first_seen_build_id INT')
+    cr.execute('ALTER TABLE runbot_build_error_link RENAME COLUMN build_error_id TO error_content_id')

runbot/models/batch.py

@@ -237,8 +237,9 @@ def _fill_missing(branch_commits, match_type):
         #  use last not preparing batch to define previous repos_heads instead of branches heads:
         #  Will allow to have a diff info on base bundle, compare with previous bundle
         last_base_batch = self.env['runbot.batch'].search([('bundle_id', '=', bundle.base_id.id), ('state', '!=', 'preparing'), ('category_id', '=', self.category_id.id), ('id', '!=', self.id)], order='id desc', limit=1)
-        base_head_per_repo = {commit.repo_id.id: commit for commit in last_base_batch.commit_ids}
-        self._update_commits_infos(base_head_per_repo)  # set base_commit, diff infos, ...
+        if last_base_batch:
+            base_head_per_repo = {commit.repo_id.id: commit for commit in last_base_batch.commit_ids}
+            self._update_commits_infos(base_head_per_repo)  # set base_commit, diff infos, ...
 
         # 2. FIND missing commit in a compatible base bundle
         if bundle.is_base or auto_rebase:
@@ -496,7 +497,6 @@ class BatchSlot(models.Model):
     _description = 'Link between a bundle batch and a build'
     _order = 'trigger_id,id'
 
-
     batch_id = fields.Many2one('runbot.batch', index=True)
     trigger_id = fields.Many2one('runbot.trigger', index=True)
     build_id = fields.Many2one('runbot.build', index=True)

runbot/models/build.py

@@ -316,7 +316,7 @@ def _compute_global_result(self):
     @api.depends('build_error_link_ids')
     def _compute_build_error_ids(self):
         for record in self:
-            record.build_error_ids = record.build_error_link_ids.mapped('build_error_id')
+            record.build_error_ids = record.build_error_link_ids.error_content_id.error_id
 
     def _get_worst_result(self, results, max_res=False):
         results = [result for result in results if result]  # filter Falsy values
@@ -1182,11 +1182,10 @@ def _get_py_version(self):
 
     def _parse_logs(self):
         """ Parse build logs to classify errors """
-        BuildError = self.env['runbot.build.error']
         # only parse logs from builds in error and not already scanned
         builds_to_scan = self.search([('id', 'in', self.ids), ('local_result', 'in', ('ko', 'killed', 'warn')), ('build_error_link_ids', '=', False)])
         ir_logs = self.env['ir.logging'].search([('level', 'in', ('ERROR', 'WARNING', 'CRITICAL')), ('type', '=', 'server'), ('build_id', 'in', builds_to_scan.ids)])
-        return BuildError._parse_logs(ir_logs)
+        return self.env['runbot.build.error']._parse_logs(ir_logs)
 
     def _is_file(self, file, mode='r'):
         file_path = self._path(file)