build-buildroot

#!/usr/bin/env python3

import os
import pathlib
import shutil
import subprocess
import sys
import time
import re

import common
from shell_helpers import LF

class Main(common.BuildCliFunction):
    def __init__(self):
        super().__init__(
            description='''\
Build Buildroot. This includes, notably: the userland GCC cross-toolchain,
and the root filesystem.
''')
        self.add_argument(
            '--build-linux', default=False,
            help='''\
See https://cirosantilli.com/linux-kernel-module-cheat#buildroot-vanilla-kernel
'''
        )
        self.add_argument(
            '--baseline', default=False,
            help='''Do a default-ish Buildroot defconfig build, without any of our extra options.
Mostly to track how much slower we are than a basic build.
'''
        )
        self.add_argument(
            '--config', default=[], action='append',
            help='''Add a single Buildroot config to the current build.
Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
Can be used multiple times to add multiple configs.
Takes precedence over any Buildroot config files.
'''
        )
        self.add_argument(
            '--config-fragment', default=[], action='append',
            help='''Also use the given Buildroot configuration fragment file.
Pass multiple times to use multiple fragment files.
'''
        )
        self.add_argument(
            '--no-all', default=False,
            help='''\
Don't build the all target which normally gets build by default.
That target builds the root filesystem and all its dependencies.
'''
        )
        self.add_argument(
            '--no-overlay', default=False,
            help='''\
Don't add our overlay which contains all files we build without going through Buildroot.
This prevents us from overwriting certain Buildroot files. Remember however that you must
still rebuild the Buildroot package that provides those files to actually put the Buildroot
files on the root filesystem.
'''
        )
        self._add_argument('--force-rebuild')
        self._add_argument('extra_make_args')

    def build(self):
        build_dir = self.get_build_dir()
        os.makedirs(self.env['out_dir'], exist_ok=True)
        extra_make_args = self.sh.add_newlines(self.env['extra_make_args'])
        if self.env['build_linux']:
            extra_make_args.extend(['linux-reconfigure', LF])
        if self.env['arch'] == 'x86_64':
            defconfig = 'qemu_x86_64_defconfig'
        elif self.env['arch'] == 'arm':
            defconfig = 'qemu_arm_vexpress_defconfig'
        elif self.env['arch'] == 'aarch64':
            defconfig = 'qemu_aarch64_virt_defconfig'
        br2_external_dirs = []
        for package_dir in sorted(os.listdir(self.env['packages_dir'])):
            package_dir_abs = os.path.join(self.env['packages_dir'], package_dir)
            if os.path.isdir(package_dir_abs):
                br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
        br2_external_str = ':'.join(br2_external_dirs)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'BR2_EXTERNAL={}'.format(br2_external_str), LF,
                defconfig, LF,
            ],
            cwd=self.env['buildroot_source_dir'],
        )
        configs = self.env['config']
        configs.extend([
            'BR2_JLEVEL={}'.format(self.env['nproc']),
            'BR2_DL_DIR="{}"'.format(self.env['buildroot_download_dir']),
        ])
        if not self.env['build_linux']:
            configs.extend([
                '# BR2_LINUX_KERNEL is not set',
            ])
        config_fragments = []
        if not self.env['baseline']:
            configs.extend([
                'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'patches', 'global'))
                ),
                'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'busybox_config_fragment'))
                ),
                'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'buildroot_override'))
                ),
                'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'rootfs-post-build-script'))
                ),
                'BR2_ROOTFS_USERS_TABLES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'user_table'))
                ),
            ])
            if not self.env['no_overlay']:
                configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
                    self._path_relative_to_buildroot(self.env['out_rootfs_overlay_dir'])
                ))
            config_fragments = [
                os.path.join(self.env['root_dir'], 'buildroot_config', 'default')
            ] + self.env['config_fragment']
        if self.env['initrd'] or self.env['initramfs']:
            configs.append('BR2_TARGET_ROOTFS_CPIO=y')
        # TODO Can't get rid of these for now with nice fragments on Buildroot:
        # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
        self.sh.write_configs(self.env['buildroot_config_file'], configs, config_fragments)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'olddefconfig', LF,
            ],
            cwd=self.env['buildroot_source_dir'],
        )
        self.make_build_dirs()
        if self.env['force_rebuild']:
            extra_make_args.extend(['-B', LF])
        if not self.env['no_all']:
            extra_make_args.extend(['all', LF])
        self.sh.run_cmd(
            [
                'make', LF,
                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_source_dir']), LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'V={}'.format(int(self.env['verbose'])), LF,
            ] +
            extra_make_args
            ,
            cwd=self.env['buildroot_source_dir'],
            delete_env=['LD_LIBRARY_PATH', 'PERL_MM_OPT'],
            extra_env={
                # In Docker, >>> host-tar 1.29 Configuring
                # checking whether mknod can create fifo without root privileges... configure: error: in `/root/lkmc/out.docker/buildroot/build/default/aarch64/build/host-tar-1.29':
                # configure: error: you should not run configure as root (set FORCE_UNSAFE_CONFIGURE=1 in environment to bypass this check)
                'FORCE_UNSAFE_CONFIGURE': '1',
            },
            out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'),
        )
        # Create the qcow2 from ext2.
        # Skip if qemu is not present, because gem5 does not need the qcow2.
        # so we don't force a QEMU build for gem5.
        if not self.env['no_all'] and os.path.exists(self.env['qemu_img_executable']):
            self.raw_to_qcow2()

    def get_build_dir(self):
        return self.env['buildroot_build_dir']

    def _path_relative_to_buildroot(self, abspath):
        return os.path.relpath(abspath, self.env['buildroot_source_dir'])

if __name__ == '__main__':
    Main().cli()