From de10cddcb60cdeb3064a7c85513c4baab693b945 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Nov 2024 10:46:18 +0100 Subject: [PATCH 1/3] rpm_dep_tree: graph of (Build)Requires for a rpm --- scripts/rpmwatcher/repoquery.py | 153 ++++++++++++++++++++++++++++++++ scripts/rpmwatcher/rpm_dep_tree | 100 +++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 scripts/rpmwatcher/repoquery.py create mode 100755 scripts/rpmwatcher/rpm_dep_tree diff --git a/scripts/rpmwatcher/repoquery.py b/scripts/rpmwatcher/repoquery.py new file mode 100644 index 0000000..46264b2 --- /dev/null +++ b/scripts/rpmwatcher/repoquery.py @@ -0,0 +1,153 @@ +import logging +import os +import re +import subprocess +from typing import Iterable, Sequence + +XCPNG_YUMREPO_TMPL = """ +[xcpng-{section}{suffix}] +name=xcpng - {section}{suffix} +baseurl=https://updates.xcp-ng.org/8/{version}/{section}/{rpmarch}/ +gpgkey=https://xcp-ng.org/RPM-GPG-KEY-xcpng +failovermethod=priority +skip_if_unavailable=False +""" + +# DNF v4 adds an implicit trailing newline to --qf format, but v5 does not +dnf_version = subprocess.check_output(['dnf', '--version'], universal_newlines=True).strip().split('.') +if int(dnf_version[0]) >= 5: + QFNL = "\n" +else: + QFNL = "" + +def setup_xcpng_yum_repos(*, yum_repo_d: str, sections: Iterable[str], + bin_arch: str | None, version: str) -> None: + with open(os.path.join(yum_repo_d, "xcpng.repo"), "w") as yumrepoconf: + for section in sections: + # binaries + block = XCPNG_YUMREPO_TMPL.format(rpmarch=bin_arch, + section=section, + version=version, + suffix='', + ) + yumrepoconf.write(block) + # sources + block = XCPNG_YUMREPO_TMPL.format(rpmarch='Source', + section=section, + version=version, + suffix='-src', + ) + yumrepoconf.write(block) + +DNF_BASE_CMD = None +def dnf_setup(*, dnf_conf: str, yum_repo_d: str) -> None: + global DNF_BASE_CMD + DNF_BASE_CMD = ['dnf', '--quiet', + '--releasever', 'WTF', + '--config', dnf_conf, + f'--setopt=reposdir={yum_repo_d}', + ] + +BINRPM_SOURCE_CACHE: dict[str, str] = {} +def rpm_source_package(rpmname: str) -> str: + return BINRPM_SOURCE_CACHE[rpmname] + +def run_repoquery(args: list[str], split: bool = True) -> str | Sequence[str]: + assert DNF_BASE_CMD is not None + cmd = DNF_BASE_CMD + ['repoquery'] + args + logging.debug('$ %s', ' '.join(cmd)) + output = subprocess.check_output(cmd, universal_newlines=True).strip() + logging.debug('> %s', output) + return output.split() if split else output + +SRPM_BINRPMS_CACHE: dict[str, set[str]] = {} # binrpm-nevr -> srpm-nevr +def fill_srpm_binrpms_cache() -> None: + # HACK: get nevr for what dnf outputs as %{sourcerpm} + logging.debug("get epoch info for SRPMs") + args = [ + '--disablerepo=*', '--enablerepo=*-src', '*', + '--qf', '%{name}-%{version}-%{release}.src.rpm,%{name}-%{evr}' + QFNL, + '--latest-limit=1', + ] + SRPM_NEVR_CACHE = { # sourcerpm -> srpm-nevr + sourcerpm: nevr + for sourcerpm, nevr in (line.split(',') + for line in run_repoquery(args)) + } + + # binary -> source mapping + logging.debug("get binary to source mapping") + global SRPM_BINRPMS_CACHE, BINRPM_SOURCE_CACHE + args = [ + '--disablerepo=*-src', '*', + '--qf', '%{name}-%{evr},%{sourcerpm}' + QFNL, # FIXME no epoch in sourcerpm, why does it work? + '--latest-limit=1', + ] + BINRPM_SOURCE_CACHE = { + # packages without source are not in SRPM_NEVR_CACHE, fallback to sourcerpm + binrpm: SRPM_NEVR_CACHE.get(sourcerpm, srpm_strip_src_rpm(sourcerpm)) + for binrpm, sourcerpm in (line.split(',') + for line in run_repoquery(args)) + } + + # reverse mapping source -> binaries + SRPM_BINRPMS_CACHE = {} + for binrpm, srpm in BINRPM_SOURCE_CACHE.items(): + binrpms = SRPM_BINRPMS_CACHE.get(srpm, set()) + if not binrpms: + SRPM_BINRPMS_CACHE[srpm] = binrpms + binrpms.add(binrpm) + +def srpm_nevr(rpmname: str) -> str: + args = [ + '--disablerepo=*', '--enablerepo=*-src', + '--qf=%{name}-%{evr}' + QFNL, # to get the epoch only when non-zero + '--latest-limit=1', + rpmname, + ] + ret = run_repoquery(args) + assert ret, f"Found no SRPM named {rpmname}" + assert len(ret) == 1 # ensured by --latest-limit=1 ? + return ret[0] + +# dnf insists on spitting .src.rpm names it cannot take as input itself +def srpm_strip_src_rpm(srpmname: str) -> str: + SUFFIX = ".src.rpm" + assert srpmname.endswith(SUFFIX), f"{srpmname} does not end in .src.rpm" + nrv = srpmname[:-len(SUFFIX)] + return nrv + +def rpm_requires(rpmname: str) -> Sequence[str]: + args = [ + '--disablerepo=*-src', # else requires of same-name SRPM are included + '--qf=%{name}-%{evr}' + QFNL, # to avoid getting the arch and explicit zero epoch + '--resolve', + '--requires', rpmname, + ] + ret = run_repoquery(args) + return ret + +def srpm_requires(srpmname: str) -> set[str]: + args = [ + '--qf=%{name}-%{evr}' + QFNL, # to avoid getting the arch + '--resolve', + '--requires', f"{srpmname}.src", + ] + ret = set(run_repoquery(args)) + return ret + +def srpm_binrpms(srpmname: str) -> set[str]: + ret = SRPM_BINRPMS_CACHE.get(srpmname, None) + if ret is None: # FIXME should not happen + logging.error("%r not found in cache", srpmname) + assert False + return [] + logging.debug("binrpms for %s: %s", srpmname, ret) + return ret + +UPSTREAM_REGEX = re.compile(r'\.el[0-9]+(_[0-9]+)?(\..*|)$') +RPM_NVR_SPLIT_REGEX = re.compile(r'^(.+)-([^-]+)-([^-]+)$') +def is_pristine_upstream(rpmname:str) -> bool: + if re.search(UPSTREAM_REGEX, rpmname): + return True + return False diff --git a/scripts/rpmwatcher/rpm_dep_tree b/scripts/rpmwatcher/rpm_dep_tree new file mode 100755 index 0000000..c0e3cf4 --- /dev/null +++ b/scripts/rpmwatcher/rpm_dep_tree @@ -0,0 +1,100 @@ +#! /usr/bin/env python3 + +import atexit +import logging +import re +import sys +import tempfile + +import repoquery + +ARCH = "x86_64" +SHOW_BOUNDARY = False + +# Tell if package is pristine upstream, or part of well-kown list of +# packages we want to consider as "upstream" rather than forks +def is_upstream(rpmname: str) -> bool: + if repoquery.is_pristine_upstream(rpmname): + return True + m = re.match(repoquery.RPM_NVR_SPLIT_REGEX, rpmname) + assert m, f"{rpmname!r} does not match {repoquery.RPM_NVR_SPLIT_REGEX!r}" + if m.group(1) in ['systemd', 'util-linux', 'ncurses', + #'xapi', + 'devtoolset-11-gcc', 'devtoolset-11-binutils']: + return True + return False + +def main() -> int: + logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG) + + this_exe, version, root_srpm = sys.argv + + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod, + open(f"{root_srpm}-{version}.dot", "w") as dotfile): + + repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod, + sections=['base', 'updates'], + bin_arch=ARCH, + version=version) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + + repoquery.fill_srpm_binrpms_cache() + # print([x for x in sorted(SRPM_BINRPMS_CACHE.keys()) if x.startswith("openssl-")]) + # return 0 + + print("digraph packages {", file=dotfile) + srpms_seen: set[str] = set() + new_srpms = {repoquery.srpm_nevr(root_srpm)} + while new_srpms: + next_srpms = set() # preparing next round's new_srpms + logging.info("seen: %s, new: %s", len(srpms_seen), len(new_srpms)) + logging.debug(" new: %s", new_srpms) + for srpm in new_srpms: + # draw source packages themselves + if is_upstream(srpm): + if SHOW_BOUNDARY: + print(f'"{srpm}" [color=grey];', file=dotfile) + logging.debug("skipping upstream %s", srpm) + continue # we don't rebuild upstream rpms + elif ".xcpng8.3.": + print(f'"{srpm}";', file=dotfile) + else: + print(f'"{srpm}" [color=red];', file=dotfile) + + # build reqs + breqs = {repoquery.rpm_source_package(breq) + for breq in repoquery.srpm_requires(srpm)} + logging.debug("%s req sources: %s", len(breqs), breqs) + + # reqs of binary rpms produced + reqs = set() + for binrpm in repoquery.srpm_binrpms(srpm): + reqs.update({repoquery.rpm_source_package(req) + for req in repoquery.rpm_requires(binrpm)}) + + # draw breqs, plain + for breq in breqs: + if (not SHOW_BOUNDARY) and is_upstream(breq): + continue + print(f'"{srpm}" -> "{breq}";', file=dotfile) + # draw additional runtime reqs, dotted + for req in reqs.difference(breqs): + if (not SHOW_BOUNDARY) and is_upstream(req): + continue + if srpm == req: + continue # dependency between RPMs of this SRPM + print(f'"{srpm}" -> "{req}" [style=dotted];', file=dotfile) + + # accumulate + srpms_seen.update(new_srpms) + next_srpms.update(breqs.difference(srpms_seen)) + next_srpms.update(reqs.difference(srpms_seen)) + + new_srpms = next_srpms + + print("}", file=dotfile) + return 0 + +if __name__ == "__main__": + sys.exit(main()) From 3b6331e580ae71a4c03db540e62a9eaf74166892 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 10 Feb 2025 17:38:46 +0100 Subject: [PATCH 2/3] yum_repo_query: repoquery for xs8 and xcpng8.3 repos --- scripts/rpmwatcher/yum_repo_query | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 scripts/rpmwatcher/yum_repo_query diff --git a/scripts/rpmwatcher/yum_repo_query b/scripts/rpmwatcher/yum_repo_query new file mode 100755 index 0000000..f0e5803 --- /dev/null +++ b/scripts/rpmwatcher/yum_repo_query @@ -0,0 +1,35 @@ +#! /usr/bin/env python3 + +import logging +import sys +import tempfile + +import repoquery + +ARCH = "x86_64" +XCP_VERSION = "8.3" + +# Use `repoquery` on xs8 and xcpng8.3 repos +def main() -> int: + logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO) + + args = sys.argv[1:] + + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod, + sections=['base', 'updates'], + bin_arch=ARCH, + version=XCP_VERSION) + repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod, + sections=['base', 'normal', 'earlyaccess'], + ) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + + print(repoquery.run_repoquery(args, split=False)) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) From 8a9b2c39655908f19a599e9bccdf7fc94cf373f7 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 7 Feb 2025 17:30:22 +0100 Subject: [PATCH 3/3] WIP yum_repo_status: generate a table showing how much outdated 8.3 is from xs8 FIXME: - include successive xs8 update waves - migrate infos from Sam's wiki page to package_status.csv - show binrpms? - include "current" el7 versions - link proprietary packages replaced by ours - should take import_reason from provenance.csv as tooltip? - include upstream releases from GH? --- scripts/rpmwatcher/package_status.csv | 60 ++++++ scripts/rpmwatcher/repoquery.py | 92 +++++++-- scripts/rpmwatcher/style.css | 57 ++++++ scripts/rpmwatcher/yum_repo_status | 259 ++++++++++++++++++++++++++ 4 files changed, 457 insertions(+), 11 deletions(-) create mode 100644 scripts/rpmwatcher/package_status.csv create mode 100644 scripts/rpmwatcher/style.css create mode 100755 scripts/rpmwatcher/yum_repo_status diff --git a/scripts/rpmwatcher/package_status.csv b/scripts/rpmwatcher/package_status.csv new file mode 100644 index 0000000..5782bb3 --- /dev/null +++ b/scripts/rpmwatcher/package_status.csv @@ -0,0 +1,60 @@ +SRPM_name;status;comment +auto-cert-kit;ignored;unused, why? +automake16;ignored;unused, why? +bpftool;ignored;unused, why? +capstone;ignored;unused, why? +citrix-crypto-module;ignored;proprietary, patched FIPS openssl +compiler-rt18;ignored;unused, why? +dlm;ignored;previous dependency for corosync +emu-manager;ignored;proprietary, replaced by xcp-emu-manager +epel-release;ignored;unused, why? +forkexecd;ignored;now in xapi +fuse;;breq for e2fsprogs 1.47 +gfs2-utils;ignored;unsupported fs +glib2;ignored;same version as el7, why? +golang;ignored;for newer xe-guest-utilities +hcp_nss;ignored;unused, “enforce any permitted user login as root” +hwloc;ignored;unused, why? +libbpf;ignored;unused, why? +libcgroup;ignored;unused, same version as el7, why? +libhbalinux;;unused? el7 fork, “Fix crash in fcoeadm/elxhbamgr on certain machines” +libnbd;ignored;unused, dep for xapi-storage-plugins +linuxconsoletools;ignored;unused, same version as el7, why? +mbootpack;;for secureboot? +message-switch;ignored;now in xapi +mpdecimal;ignored;unused, why? +ninja-build;ignored;unused +pbis-open;ignored;unused, likely linked to upgrade-pbis-to-winbind +pbis-open-upgrade;ignored;unused, likely linked to upgrade-pbis-to-winbind +pvsproxy;ignored;proprietary +python-monotonic;ignored;previous dependency for sm +python-tqdm;ignored;unused, dependency for pvsproxy +rrdd-plugins;ignored;now in xapi +ruby;ignored;unused, why? +sbd;;unused? "storage-based death functionality" +sm-cli;ignored;now in xapi +secureboot-certificates;ignored;proprietary, needs alternative? +security-tools;ignored;proprietary, pool_secret tool +sm-transport-lib;ignored;proprietary +squeezed;ignored;now in xapi +tix;ignored;unused, why? +upgrade-pbis-to-winbind;ignored;proprietary +v6d;ignored;proprietary, replaced by xcp-featured +varstored-guard;ignored;now in xapi +vendor-update-keys;ignored;proprietary +vgpu;ignored;proprietary +vhd-tool;ignored;now in xapi +wsproxy;ignored;now in xapi +xapi-clusterd;ignored;proprietary +xapi-nbd;ignored;now in xapi +xapi-storage;ignored;now in xapi +xapi-storage-plugins;ignored;proprietarized, forked as xcp-ng-xapi-storage +xapi-storage-script;ignored;now in xapi +xcp-networkd;ignored;now in xapi +xcp-rrdd;ignored;now in xapi +xencert;;"automated testkit for certifying storage hardware with XenServer" +xenopsd;ignored;now in xapi +xenserver-release;forked;xcp-ng-release +xenserver-snmp-agent;ignored;proprietary, SNMP MIB +xenserver-telemetry;ignored;proprietary, xapi plugin +xs-clipboardd;ignored;proprietary, replaced by xcp-clipboardd diff --git a/scripts/rpmwatcher/repoquery.py b/scripts/rpmwatcher/repoquery.py index 46264b2..51e6ce4 100644 --- a/scripts/rpmwatcher/repoquery.py +++ b/scripts/rpmwatcher/repoquery.py @@ -13,6 +13,15 @@ skip_if_unavailable=False """ +XCPNG_YUMREPO_USER_TMPL = """ +[xcpng-{section}{suffix}] +name=xcpng - {section}{suffix} +baseurl=https://koji.xcp-ng.org/repos/user/8/{version}/{section}/{rpmarch}/ +gpgkey=https://xcp-ng.org/RPM-GPG-KEY-xcpng +failovermethod=priority +skip_if_unavailable=False +""" + # DNF v4 adds an implicit trailing newline to --qf format, but v5 does not dnf_version = subprocess.check_output(['dnf', '--version'], universal_newlines=True).strip().split('.') if int(dnf_version[0]) >= 5: @@ -24,19 +33,47 @@ def setup_xcpng_yum_repos(*, yum_repo_d: str, sections: Iterable[str], bin_arch: str | None, version: str) -> None: with open(os.path.join(yum_repo_d, "xcpng.repo"), "w") as yumrepoconf: for section in sections: + # HACK: use USER_TMPL if section ends with a number + if section[-1].isdigit(): + tmpl = XCPNG_YUMREPO_USER_TMPL + else: + tmpl = XCPNG_YUMREPO_TMPL + # binaries - block = XCPNG_YUMREPO_TMPL.format(rpmarch=bin_arch, - section=section, - version=version, - suffix='', - ) - yumrepoconf.write(block) + if bin_arch: + block = tmpl.format(rpmarch=bin_arch, + section=section, + version=version, + suffix='', + ) + yumrepoconf.write(block) # sources - block = XCPNG_YUMREPO_TMPL.format(rpmarch='Source', - section=section, - version=version, - suffix='-src', - ) + block = tmpl.format(rpmarch='Source', + section=section, + version=version, + suffix='-src', + ) + yumrepoconf.write(block) + + +XS8_YUMREPO_TMPL = """ +[xs8-{section}] +name=XS8 - {section} +baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}/ +failovermethod=priority +skip_if_unavailable=False + +[xs8-{section}-src] +name=XS8 - {section} source +baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}-source/ +failovermethod=priority +skip_if_unavailable=False +""" + +def setup_xs8_yum_repos(*, yum_repo_d: str, sections: Iterable[str])-> None: + with open(os.path.join(yum_repo_d, "xs8.repo"), "w") as yumrepoconf: + for section in sections: + block = XS8_YUMREPO_TMPL.format(section=section) yumrepoconf.write(block) DNF_BASE_CMD = None @@ -151,3 +188,36 @@ def is_pristine_upstream(rpmname:str) -> bool: if re.search(UPSTREAM_REGEX, rpmname): return True return False + +def rpm_parse_nevr(nevr: str, suffix: str) -> tuple[str, str, str, str]: + "Parse into (name, epoch:version, release) stripping suffix from release" + m = re.match(RPM_NVR_SPLIT_REGEX, nevr) + assert m, f"{nevr} does not match NEVR pattern" + n, ev, r = m.groups() + if ":" in ev: + e, v = ev.split(":") + else: + e, v = "0", ev + if r.endswith(suffix): + r = r[:-len(suffix)] + return (n, e, v, r) + +def all_binrpms() -> set[str]: + args = [ + '--disablerepo=*-src', + '--qf=%{name}-%{evr}' + QFNL, # to avoid getting the arch + '--latest-limit=1', # only most recent for each package + '*', + ] + ret = set(run_repoquery(args)) + return ret + +def all_srpms() -> set[str]: + args = [ + '--disablerepo=*', '--enablerepo=*-src', + '--qf=%{name}-%{evr}' + QFNL, # to avoid getting the arch + '--latest-limit=1', # only most recent for each package + '*', + ] + ret = set(run_repoquery(args)) + return ret diff --git a/scripts/rpmwatcher/style.css b/scripts/rpmwatcher/style.css new file mode 100644 index 0000000..aeca09f --- /dev/null +++ b/scripts/rpmwatcher/style.css @@ -0,0 +1,57 @@ +table { + border-collapse: collapse; + border-spacing: 0; +} +td { + padding: 2px 5px; + /* background-color: lightblue; */ +} + +tr.header { + vertical-align: top; +} +tr.ignored { + color: #888888; + background-color: #cccccc; +} +tr.notused { /* still to check */ + color: red; + background-color: #cccccc; +} + +th.xs { + background-color: #ddddff; +} +th.xcp { + background-color: #ddffdd; +} + +td.pkgname { + white-space: nowrap; +} + +.nosource { + color: red; + background-color: black; +} + +.unexpected { + color: red; +} + +.outdated { + background-color: red; +} + +.uptodate { + background-color: lightgreen; +} + +.better { + background-color: #77cc00; +} + +.upstream { + /* must not be background-color, can exist in outdated/uptodate/better */ + color: blue; +} diff --git a/scripts/rpmwatcher/yum_repo_status b/scripts/rpmwatcher/yum_repo_status new file mode 100755 index 0000000..843830d --- /dev/null +++ b/scripts/rpmwatcher/yum_repo_status @@ -0,0 +1,259 @@ +#! /usr/bin/env python3 + +import argparse +from collections import namedtuple, OrderedDict +import csv +import logging +import os +import rpm # type: ignore +import sys +import tempfile +from typing import Iterable, Iterator, Literal, Sequence + +import repoquery + +ARCH = "x86_64" +XCP_VERSION = "8.3" +FILTER_UPSTREAM = False + +def evr_format(evr: tuple[str, str, str] | None | Literal['=']) -> str: + if evr == "=": # dirty special case + return evr + if evr is None: + return "-" + return (f"{evr[0]}:" if evr[0] != "0" else "") + "-".join(evr[1:]) + +# Filters an iterator of (n, e, v, r) for newest evr of each `n`. +# Older versions are allowed to appear before the newer ones. +def filter_best_evr(nevrs: Iterable[tuple[str, str, str, str]] + ) -> Iterator[tuple[str, str, str, str]]: + best: dict[str, tuple[str, str, str]] = {} + for (n, e, v, r) in nevrs: + if n not in best or rpm.labelCompare(best[n], (e, v, r)) < 0: + best[n] = (e, v, r) + yield (n, e, v, r) + # else (e, v, r) is older than a previously-seen version, drop + +def collect_data_xcpng() -> OrderedDict: + xcp_sets = OrderedDict() + for (label, sections) in (("released", ['base', 'updates']), + ("candidates", ["candidates"]), + ("testing", ["testing"]), + ("ci", ["ci"]), + ("incoming", ["incoming"]), + #("ydi1", ["ydi1"]), + ("dtt1", ["dtt1"]), + ): + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod, + sections=sections, + bin_arch=None, + version=XCP_VERSION) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + #repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XCP-ng %s SRPMs", label) + xcp_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xcp_sets[label] = { + n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xcpng{XCP_VERSION}") + for nevr in xcp_srpms)} + + logging.info(f"{label}: {len(xcp_sets[label])}") + + return xcp_sets + +def collect_data_xs8(): + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod, + sections=['base', 'normal'], + ) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + logging.debug("fill cache with XS info") + repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XS SRPMs") + xs8_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + xs8_rpms_sources = {nevr for nevr in repoquery.SRPM_BINRPMS_CACHE.keys() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xs8_srpms_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8_srpms)} + xs8_rpms_sources_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8_rpms_sources)} + + logging.info(f"xs8 src: {len(xs8_srpms_set)}") + logging.info(f"xs8 bin: {len(xs8_rpms_sources_set)}") + + return (xs8_srpms_set, xs8_rpms_sources_set) + +def collect_data_xs8ea(): + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod, + sections=['earlyaccess'], + ) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + logging.debug("fill cache with XS EA info") + repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XS EA SRPMs") + xs8ea_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xs8ea_srpms_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8ea_srpms)} + + logging.info(f"xs8 EA src: {len(xs8ea_srpms_set)}") + + return xs8ea_srpms_set + +def read_package_status_metadata(): + with open('package_status.csv', newline='') as csvfile: + csvreader = csv.reader(csvfile, delimiter=';', quotechar='|') + headers = next(csvreader) + assert headers == ["SRPM_name", "status", "comment"], f"unexpected headers {headers!r}" + PackageStatus = namedtuple("PackageStatus", headers[1:]) # type: ignore[misc] + return {row[0]: PackageStatus(*row[1:]) + for row in csvreader} + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count', default=0) + args = parser.parse_args() + if args.verbose > 2: + args.verbose = 2 + + loglevel = {0: logging.WARNING, + 1: logging.INFO, + 2: logging.DEBUG, + }[args.verbose] + logging.basicConfig(format='[%(levelname)s] %(message)s', level=loglevel) + + PACKAGE_STATUS = read_package_status_metadata() + + # collect data from repos + # ... xcp-ng + xcp_sets = collect_data_xcpng() + # ... xs8 + (xs8_srpms_set, xs8_rpms_sources_set) = collect_data_xs8() + # ... xs8 earlyaccess + xs8ea_srpms_set = collect_data_xs8ea() + + # ... collect + + all_srpms = set(xs8_srpms_set.keys()) | xs8_rpms_sources_set.keys() | xs8ea_srpms_set.keys() + for label, srpms in xcp_sets.items(): + all_srpms |= srpms.keys() + + logging.info(f"all: {len(all_srpms)}") + + # output + + with open(f"repo-status-{XCP_VERSION}.html", "w") as outfile: + print('', file=outfile) + print(f''' + + + XCP-ng {XCP_VERSION} RPM flow + + + + + + + ''', file=outfile) + print('', file=outfile) + + for srpm in sorted(all_srpms): + xs_ver_src = xs8_srpms_set.get(srpm, None) + xsea_ver_src: tuple[str, str, str] | None | Literal["="] + xsea_ver_src = xs8ea_srpms_set.get(srpm, None) + if xs_ver_src and xsea_ver_src == xs_ver_src: + xsea_ver_src = "=" # don't overload with useless info # FIXME + xs_ver_bin = xs8_rpms_sources_set.get(srpm, None) + xcp_vers = {label: xcp_sets[label].get(srpm, None) + for label in xcp_sets} + + srpm_status = PACKAGE_STATUS.get(srpm, None) + + if srpm_status: + tooltip = srpm_status.comment + else: + tooltip = "" + + if srpm_status and srpm_status.status == "ignored": + row_classes = "ignored" + elif all(ver is None for ver in xcp_vers.values()) and xs_ver_bin is None: + row_classes = "notused" + else: + row_classes = "" + + # XS source + xss_classes = "xs" + if srpm in xs8_rpms_sources_set and srpm not in xs8_srpms_set: + xss_classes += " nosource" + + # XS binary + xsb_classes = "xs" + if xs_ver_bin and xs_ver_src != xs_ver_bin: + xsb_classes += " nosource" + + print(f'', file=outfile) + print('
package + XenServer 8 + XCP-ng {XCP_VERSION} +
srcrpmbinrpmEA srcrpm', file=outfile) + for label in xcp_sets: + print(f'{label}', file=outfile) + print(f'
{srpm}' + f'{evr_format(xs_ver_src)}' + f'{evr_format(xs_ver_bin)}', + f'{evr_format(xsea_ver_src)}', + file=outfile) + + # XCP-ng + ref_xs_ver = xs_ver_bin if xs_ver_bin else xs_ver_src + for label in xcp_sets: + xcp_ver = xcp_vers[label] + xcp_ver_str = evr_format(xcp_ver) + classes = "xcp" + if repoquery.is_pristine_upstream(xcp_ver_str): + classes += " upstream" + if not xcp_ver: + pass + elif xcp_ver and xcp_ver == ref_xs_ver: + classes += " uptodate" + elif not ref_xs_ver: + classes += " better" + elif not ref_xs_ver: + pass + elif rpm.labelCompare(xcp_ver, ref_xs_ver) < 0: + classes += " outdated" + else: # xcp_ver > ref_xs_ver + classes += " better" + if row_classes == "notused" and xcp_ver: + classes += " unexpected" + + print(f'{xcp_ver_str}', file=outfile) + print(f'
', file=outfile) + + return 0 + +if __name__ == "__main__": + sys.exit(main())