Added support for build_image.py in ssh mode to

Robert Foley · Robert Foley · commit 8faac3b16897 · 2020-04-01T12:06:47.000-04:00
automatically determine the .yml name in the
case where the image was created as part of
install_kernel.py.

Also split out some common routines into a
separate base command class.
diff --git a/external/qemu b/external/qemu
@@ -1 +1 @@
-Subproject commit f163db8245935900fd3c4d10971bb9bb886c5085
+Subproject commit 1ecaa642dac0c6f436998fbad74d39d8d4043c81
diff --git a/scripts/base_cmd.py b/scripts/base_cmd.py
@@ -0,0 +1,98 @@
+#
+# Copyright 2019 Linaro
+#
+# Base class for our command objects.
+#
+
+import sys
+import os
+import stat
+import shutil
+import subprocess
+from subprocess import Popen,PIPE
+import argparse
+import yaml
+
+class BaseCmd:
+    
+    def __init__(self):
+        self.kernel_ver = None
+        self.kernel_ver_minor = None
+        self._debug = False
+        self._dry_run = False
+
+    def set_debug(self, debug):
+        self._debug = debug
+
+    def set_dry_run(self, dry_run):
+        self._dry_run = dry_run
+
+    def print(self, trace, debug=False):
+        if not debug or self._debug:
+            print("{}: {}".format(sys.argv[0], trace))
+            
+    def terminate(self, err):
+        if not self.continue_on_error:
+            exit(err)
+            
+    def issue_cmd(self, cmd, show_cmd=False, fail_on_err=True, 
+                  err_msg=None, enable_stdout=True, no_capture=False):
+        rc, output = self.run_command(cmd, show_cmd, enable_stdout=enable_stdout, no_capture=no_capture)
+        if fail_on_err and rc != 0:
+            self.print("cmd failed with status: {} cmd: {}".format(rc, cmd))
+            if (err_msg):
+                self.print(err_msg)
+            self.terminate(1)
+        return rc, output
+    
+    def run_command(self, command, show_cmd=False, enable_stdout=True, no_capture=False):
+        output_lines = []
+        if show_cmd or self._debug:
+            print("{}: {} ".format(sys.argv[0], command))
+        if self._dry_run:
+            print("")
+            return 0, output_lines
+        if no_capture:
+            rc = subprocess.call(command, shell=True)
+            return rc, output_lines
+        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) #shlex.split(command)
+        while True:
+            output = process.stdout.readline()
+            if (not output or output == '') and process.poll() is not None:
+                break
+            if output and enable_stdout:
+                self.print(str(output, 'utf-8').strip())
+            output_lines.append(str(output, 'utf-8'))
+        rc = process.poll()
+        return rc, output_lines
+
+    def get_kernel_img_version(self, image):
+        if self.kernel_ver:
+            return
+        entries = image.split("-")
+        if len(entries) > 2:
+            self.kernel_ver_minor = "{}-{}".format(entries[1], entries[2])
+            self.kernel_ver = entries[1]
+        if self.kernel_ver == None:
+            raise Exception("Unable to determine kernel version, "\
+                            "please use --kernel_ver argument.")
+        self.print("Kernel version is:"\
+                   " {} ({})".format(self.kernel_ver, self.kernel_ver_minor),                   
+                   debug=True)
+
+    def get_kernel_pkg_version(self, kernel_pkg_path):
+        if self.kernel_ver:
+            return
+        cmd = "dpkg --info {}".format(kernel_pkg_path)
+        rc, output = self.issue_cmd(cmd)
+        for line in output:
+            if "Version: " in line:
+                entries = line.split(" ")
+                if len(entries) > 2:
+                    self.kernel_ver_minor = entries[2].rstrip()
+                    self.kernel_ver = entries[2].split("-")[0].rstrip()
+        if self.kernel_ver == None:
+            raise Exception("Unable to determine kernel version, "\
+                            "please use --kernel_ver argument.")
+        self.print("Kernel version is:"\
+                   " {} ({})".format(self.kernel_ver, self.kernel_ver_minor),debug=True)
diff --git a/scripts/build_image.py b/scripts/build_image.py
@@ -21,8 +21,9 @@
 import argparse
 from argparse import RawTextHelpFormatter
 import yaml
+import base_cmd
 
-class build_image:
+class BuildImage(base_cmd.BaseCmd):
     qemu_path_rel = "external/qemu"
     qemu_build_path_rel = "external/qemu/build"
     build_path_rel = "build"
@@ -34,6 +35,7 @@ class build_image:
     key_files = ["id_rsa", "id_rsa.pub"]
     
     def __init__(self, ssh=False):
+        super(BuildImage, self).__init__()
         self.script_path = os.path.dirname(os.path.realpath(__file__))
         self.root_path = os.path.realpath(os.path.join(self.script_path, "../"))
         self.orig_default_config_path = os.path.realpath(os.path.join(self.root_path, 
@@ -46,6 +48,8 @@ def __init__(self, ssh=False):
         else:
             self.default_config_path = self.orig_default_config_path
         self.parse_args()
+        self.set_debug(self._args.debug)
+        self.set_dry_run(self._args.dry_run)
         self.build_path = os.path.realpath(os.path.join(self.root_path, self.build_path_rel))
         self.qemu_build_path = os.path.realpath(os.path.join(self.root_path, 
                                                              self.qemu_build_path_rel))
@@ -64,6 +68,16 @@ def __init__(self, ssh=False):
         self.print("config file: {}".format(self.config_path), debug=True)
         self.continue_on_error = self._args.debug
         self.vm_config_path = os.path.join(self.image_dir_path, "conf.yml")
+        image_name = os.path.basename(self.image_path)
+        if ".kernel-" in image_name:
+            self.get_kernel_img_version(image_name)
+            kernel_config_file = os.path.join(self.image_dir_path, 
+                         "conf-kernel-{}.yml".format(self.kernel_ver_minor))
+            if os.path.exists(kernel_config_file):
+                self.print("using kernel config: {}".format(kernel_config_file),
+                           debug=True)
+                self.vm_config_path = kernel_config_file
+                
         self.src_ssh_key = os.path.join(self.def_key_path, "id_rsa")
         self.dest_ssh_key = os.path.join(self.image_dir_path, "id_rsa")
         self.src_ssh_pub_key = os.path.join(self.def_key_path, "id_rsa.pub")
@@ -81,45 +95,6 @@ def __init__(self, ssh=False):
         if self.start_ssh and not self.building_image and \
            self._args.config != self.orig_default_config_path:
             self.vm_config_path = self._args.config
-            
-            
-    def print(self, trace, debug=False):
-        if not debug or self._args.debug:
-            print("{}: {}".format(sys.argv[0], trace))
-            
-    def terminate(self, err):
-        if not self.continue_on_error:
-            exit(err)
-            
-    def issue_cmd(self, cmd, show_cmd=False, fail_on_err=True, err_msg=None, enable_stdout=True, no_capture=False):
-        rc, output = self.run_command(cmd, show_cmd, enable_stdout=enable_stdout, no_capture=no_capture)
-        if fail_on_err and rc != 0:
-            self.print("cmd failed with status: {} cmd: {}".format(rc, cmd))
-            if (err_msg):
-                self.print(err_msg)
-            self.terminate(1)
-        return rc, output
-    
-    def run_command(self, command, show_cmd=False, enable_stdout=True, no_capture=False):
-        output_lines = []
-        if show_cmd or self._args.debug:
-            print("{}: {} ".format(sys.argv[0], command))
-        if self._args.dry_run:
-            print("")
-            return 0, output_lines
-        if no_capture:
-            rc = subprocess.call(command, shell=True)
-            return rc, output_lines
-        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) #shlex.split(command)
-        while True:
-            output = process.stdout.readline()
-            if (not output or output == '') and process.poll() is not None:
-                break
-            if output and enable_stdout:
-                self.print(str(output, 'utf-8').strip())
-            output_lines.append(str(output, 'utf-8'))
-        rc = process.poll()
-        return rc, output_lines
     
     def parse_args(self):
         parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter,
@@ -280,6 +255,8 @@ def ssh(self):
             debug = "--debug"
         else:
             debug = ""
+        print("Launching Image.  Please be patient, this may take several minutes...")
+        print("To enable more verbose tracing of each step, please use the --debug option.\n")
         cmd = self.launch_cmd.format(env_vars, self._args.image_type, self.image_path, debug, "/bin/bash")
         self.issue_cmd(cmd, no_capture=True)
         
@@ -308,5 +285,5 @@ def run(self):
             self.ssh()
         
 if __name__ == "__main__":
-    inst_obj = build_image()    
+    inst_obj = BuildImage()    
     inst_obj.run()
diff --git a/scripts/install_kernel.py b/scripts/install_kernel.py
@@ -25,9 +25,10 @@
 import traceback
 import re
 import yaml
+import base_cmd
 
                 
-class install_kernel:
+class InstallKernel(base_cmd.BaseCmd):
     mount_path = "./mnt"
     host_tmp = "/tmp"
     mount_tmp = os.path.join(mount_path, "tmp")
@@ -49,6 +50,7 @@ class install_kernel:
     build_path_rel = "build"
     
     def __init__(self):
+        super(InstallKernel, self).__init__()
         self._image_mounted = False
         
         self.device = None
@@ -67,6 +69,8 @@ def __init__(self):
         self._image_dir_path = os.path.join(self.build_path, "VM-" + "ubuntu.aarch64")
         self._default_image_path = os.path.join(self._image_dir_path, self.default_image_name)
         self.parse_args()
+        self.set_debug(self._args.debug)
+        self.set_dry_run(self._args.dry_run)
         self._image_path = os.path.abspath(getattr(self._args, 'image'))
         self._image_dir_path = os.path.dirname(self._image_path)
         self.vm_config_path = os.path.join(self._image_dir_path, "conf.yml")
@@ -75,54 +79,17 @@ def __init__(self):
         self.kernel_ver = self._args.kernel_ver
         self._kernel_pkg_name = os.path.basename(self._args.kernel_pkg)
         self._kernel_pkg_path = os.path.abspath(self._args.kernel_pkg)
-        self.get_kernel_version()
+        self.get_kernel_pkg_version(self._kernel_pkg_path)
         self.kernel_config_path = os.path.join(self._image_dir_path,
-                                               "conf-{}.yml".format(self.kernel_ver_minor))
-        self._output_image_path = self._image_path + '.kernel-' + self.kernel_ver
+                                               "conf-kernel-{}.yml".format(self.kernel_ver_minor))
+        self._output_image_path = self._image_path + '.kernel-' + self.kernel_ver_minor
         self._config_path = os.path.abspath(self._args.config)
         self._install_pkg_vm_path =os.path.join(self.install_pkg_vm_path, self._kernel_pkg_name)
         self.chroot_cmd = "chroot {} {}".format(self._mount_path, 
                                                 os.path.join(self.qemu_static_path, 
                                                              self.qemu_static_name))
         self.print("image_path: " + self._image_path)
         self.print("kernel_pkg_name: " + self._kernel_pkg_name)
-        
-    def print(self, trace):
-        print("{}: {}".format(sys.argv[0], trace))
-            
-    def terminate(self, err):
-        if not self.continue_on_error:
-            exit(err)
-            
-    def issue_cmd(self, cmd, fail_on_err=True, err_msg=None, enable_stdout=True, no_capture=False):
-        rc, output = self.run_command(cmd, enable_stdout=enable_stdout, no_capture=no_capture)
-        if fail_on_err and rc != 0:
-            self.print("cmd failed with status: {} cmd: {}".format(rc, cmd))
-            if (err_msg):
-                self.print(err_msg)
-            self.terminate(1)
-        return rc, output
-    
-    def run_command(self, command, enable_stdout=True, no_capture=False):
-        output_lines = []
-        if self._args.debug:
-            print("{}: {}".format(sys.argv[0], command))
-        if self._args.dry_run:
-            print("")
-            return 0, output_lines
-        if no_capture:
-            rc = subprocess.call(command, shell=True)
-            return rc, output_lines
-        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) #shlex.split(command)
-        while True:
-            output = process.stdout.readline()
-            if (not output or output == '') and process.poll() is not None:
-                break
-            if output and enable_stdout:
-                self.print(str(output, 'utf-8').strip())
-            output_lines.append(str(output, 'utf-8'))
-        rc = process.poll()
-        return rc, output_lines
     
     def parse_args(self):
         parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter,
@@ -234,28 +201,11 @@ def move_old_kernels(self, root_path="/"):
         for pattern in copy_patterns:
             for file in glob.glob(os.path.join(src_path, pattern)):
                 if (self.kernel_ver not in file):
-                    self.print("move {} to {}".format(file, dest_path))
+                    self.print("move {} to {}".format(file, dest_path),
+                               debug=True)
                     cmd = "mv {} {}".format(file, dest_path)
                     self.issue_cmd(cmd)
 
-    def get_kernel_version(self):
-        if self.kernel_ver:
-            return
-         #Description: Linux kernel, version 5.4.0+
-        cmd = "dpkg --info {}".format(self._kernel_pkg_path)
-        rc, output = self.issue_cmd(cmd)
-        self.kernel_ver = None
-        self.kernel_ver_minor = None
-        for line in output:
-            if "Version: " in line:
-                entries = line.split(" ")
-                if len(entries) > 2:
-                    self.kernel_ver_minor = entries[2].rstrip()
-                    self.kernel_ver = entries[2].split("-")[0].rstrip()
-        if self.kernel_ver == None:
-            raise Exception("Unable to determine kernel version, please use --kernel_ver argument.")
-        self.print("Kernel version is: {} ({})".format(self.kernel_ver, self.kernel_ver_minor))
-
     def install_pkg(self):
         cmd = self.kernel_pkg_cpy_cmd.format(self._kernel_pkg_path)
         self.issue_cmd(cmd)
@@ -296,7 +246,7 @@ def read_config(self):
             print("default config file {} does not exist.  Continuing.")
             return None
         with open(self.vm_config_path) as f:
-            print("parse {}".format(self.vm_config_path))
+            self.print("parse {}".format(self.vm_config_path))
             yaml_dict = yaml.safe_load(f)
         if 'qemu-conf' in yaml_dict:
             return yaml_dict
@@ -310,7 +260,6 @@ def get_qemu_args_for_kernel(self, existing_args):
                                    "initrd.img-{}".format(self.kernel_ver_minor))
         args = "-kernel {} --initrd {}".format(vmlinuz_path, initrd_path)
         args += ' -append "root=/dev/vda1 nokaslr console=ttyAMA0"'
-        print(args)
         return existing_args + " " + args
 
     def create_config_file(self):
@@ -324,7 +273,7 @@ def create_config_file(self):
         yaml_dict['qemu-conf']['qemu_args'] = new_args
         with open(self.kernel_config_path, 'w') as f:
             yaml_dict = yaml.dump(yaml_dict, f)
-            print("config file {} written".format(self.kernel_config_path))
+            self.print("config file {} written".format(self.kernel_config_path))
 
     def run_cmd_in_vm(self):
         env_vars = "QEMU=./aarch64-softmmu/qemu-system-aarch64 "
@@ -392,5 +341,5 @@ def run(self):
             self.cleanup()
         
 if __name__ == "__main__":
-    inst_obj = install_kernel()    
+    inst_obj = InstallKernel()    
     inst_obj.run()
diff --git a/scripts/launch_image.py b/scripts/launch_image.py
@@ -6,5 +6,5 @@
 import build_image
 
 if __name__ == "__main__":
-    inst_obj = build_image.build_image(ssh=True)    
+    inst_obj = build_image.BuildImage(ssh=True)    
     inst_obj.run()
