Move first RISC-V builder over to cross-compiler + execute tests under qemu-system setup

buildbot/osuosl/master/config/builders.py

@@ -3244,16 +3244,20 @@
                         'CXX': 'clang++',
                     })},
 
-    ## RISC-V RVA20 profile check-all 2-stage
+    ## RISC-V RVA20 profile check-all 2-stage. The second stage is
+    # cross-compiled on the x86 host and then lit runs under a qemu-system image
+    # using the just-built artifacts.
     {'name' : "clang-riscv-rva20-2stage",
     'tags'  : ["clang"],
     'workernames' : ["rise-clang-riscv-rva20-2stage"],
     'builddir':"clang-riscv-rva20-2stage",
     'factory' : ClangBuilder.getClangCMakeBuildFactory(
-                clean=False,
+                clean=True,
                 useTwoStage=True,
                 runTestSuite=False,
                 testStage1=False,
+                checkout_compiler_rt=False,
+                checkout_zorg=True,
                 extra_cmake_args=[
                     "-DCMAKE_C_COMPILER=clang",
                     "-DCMAKE_CXX_COMPILER=clang++",
@@ -3262,9 +3266,26 @@
                     "-DCMAKE_C_COMPILER_LAUNCHER=ccache",
                     "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"],
                 extra_stage2_cmake_args=[
-                    "-DLLVM_ENABLE_LLD=True",
-                    "-DCMAKE_C_FLAGS='-march=rva20u64'",
-                    "-DCMAKE_CXX_FLAGS='-march=rva20u64'"]
+                    util.Interpolate("-DLLVM_NATIVE_TOOL_DIR=%(prop:builddir)s/stage1.install/bin"),
+                    "-DLLVM_BUILD_TESTS=True",
+                    util.Interpolate("-DLLVM_EXTERNAL_LIT=%(prop:builddir)s/llvm-zorg/buildbot/riscv-rise/lit-on-qemu")],
+                stage2_toolchain_options=[
+                    "set(CMAKE_SYSTEM_NAME Linux)",
+                    "set(CMAKE_SYSROOT %(prop:builddir)s/../rvsysroot)",
+                    "set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)",
+                    "set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)",
+                    "set(CMAKE_C_FLAGS_INIT '-march=rva20u64')",
+                    "set(CMAKE_CXX_FLAGS_INIT '-march=rva20u64')",
+                    "set(CMAKE_LINKER_TYPE LLD)",
+                    "set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)",
+                    "set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)",
+                    "set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)",
+                    "set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)"],
+                env={
+                    "BB_IMG_DIR": util.Interpolate("%(prop:builddir)s/.."),
+                    "BB_QEMU_CPU": "rv64,zfa=false,zba=false,zbb=false,zbc=false,zbs=false",
+                    "BB_QEMU_SMP": "32",
+                    "BB_QEMU_MEM": "64G"}
                 )},
 
     ## RISC-V RVA23 profile check-all 2-stage

buildbot/riscv-rise/create-sid-riscv-appliance-image.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+#===----------------------------------------------------------------------===//
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===//
+
+TGT=riscv-sid-for-qemu
+# Uses <https://github.com/muxup/medley/blob/main/rootless-debootstrap-wrapper>
+~/rootless-debootstrap-wrapper \
+  --arch=riscv64 \
+  --suite=sid \
+  --cache-dir="$HOME/debcache" \
+  --target-dir=$TGT \
+  --include=linux-image-riscv64,zstd,dbus,adduser,python3,python3-psutil,git
+cat - <<EOF > $TGT/etc/resolv.conf
+nameserver 1.1.1.1
+EOF
+"$TGT/_enter" sh -e <<'EOF'
+ln -sf /dev/null /etc/udev/rules.d/80-net-setup-link.rules # disable persistent network names
+cat - <<INNER_EOF > /etc/systemd/network/10-eth0.network
+[Match]
+Name=eth0
+
+[Network]
+DHCP=yes
+INNER_EOF
+systemctl enable systemd-networkd
+echo root:root | chpasswd
+adduser --gecos ",,," --disabled-password user
+echo user:user | chpasswd
+ln -sf /dev/null /etc/systemd/system/[email protected]
+ln -sf /dev/null /etc/systemd/system/[email protected]
+
+cat - <<'INNER_EOF' > /opt/on-boot-logic.sh
+#!/bin/sh
+error() {
+  printf "!!!!!!!!!! Error: %s !!!!!!!!!!\n" "$*" >&2
+  exit 1
+}
+
+mkdir -p /mnt/hgcomm
+mount -t 9p -o trans=virtio,version=9p2000.L hgcomm /mnt/hgcomm || error "Failed to mount hgcomm"
+if [ -e /mnt/hgcomm/debug-mode-on ]; then
+  echo "debug-mode-on file present: Not executing exec-on-boot and instead starting getty"
+  systemctl unmask [email protected]
+  systemctl start [email protected]
+  systemctl mask [email protected]
+  exit 0
+fi
+[ -f /mnt/hgcomm/exec-on-boot ] || error "exec-on-boot doesn't exist"
+[ -x /mnt/hgcomm/exec-on-boot ] || error "exec-on-boot isn't executable"
+/mnt/hgcomm/exec-on-boot
+echo "$?" > /mnt/hgcomm/exec-on-boot.exitcode
+poweroff
+INNER_EOF
+chmod +x /opt/on-boot-logic.sh
+
+cat - <<'INNER_EOF' > /etc/systemd/system/appliance.service
+[Unit]
+Description=Execute on boot logic
+After=multi-user.target
+
+[Service]
+Type=oneshot
+StandardOutput=tty
+TTYPath=/dev/ttyS0
+ExecStart=/opt/on-boot-logic.sh
+ExecStopPost=/bin/sh -c '[ "$EXIT_STATUS" != 0 ] && poweroff'
+
+[Install]
+WantedBy=multi-user.target
+INNER_EOF
+systemctl enable appliance.service
+echo "Finished rootfs config"
+EOF
+
+fakeroot -i $TGT/.fakeroot.env sh <<EOF
+ln -L $TGT/vmlinuz kernel
+ln -L $TGT/initrd.img initrd
+fallocate -l 30GiB rootfs.img
+mkfs.ext4 -d $TGT rootfs.img
+EOF

buildbot/riscv-rise/lit-on-qemu

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+
+# Note:
+# * Builders always use the latest version of this script, checking out the
+# llvm-zorg repository. This means you can't rely on the script being updated
+# in lockstep with the buildmaster being re-deployed.
+# * This script expects to be run from within an LLVM build directory. e.g.
+# llvm-project/build/foo.
+# * The approach in this script could easily be applied to other
+# architectures. It may or may not make sense to generalise this script in
+# order to do so. This is intentionally left for evaluation when/if someone
+# applies seeks to apply the same approach on another target.
+
+
+def error(message):
+    print(f"!!!!!!!!!! Error: {message} !!!!!!!!!!")
+    sys.exit(1)
+
+
+# Validate environment variables
+for var in ["BB_IMG_DIR", "BB_QEMU_CPU", "BB_QEMU_SMP", "BB_QEMU_MEM"]:
+    if not os.getenv(var):
+        error(f"{var} not set")
+
+# Create appropriate exec-on-boot script
+current_path = pathlib.Path.cwd()
+build_dir_name = current_path.name
+hgcomm_path = current_path / "hgcomm"
+
+if hgcomm_path.exists():
+    shutil.rmtree(hgcomm_path)
+hgcomm_path.mkdir()
+
+target_uid = 1000
+target_gid = 1000
+base_mount_path = current_path.parent
+
+args_string = " ".join(sys.argv[1:])
+exec_on_boot_content = f"""#!/bin/sh
+error() {{
+  printf "!!!!!!!!!! Error: %s !!!!!!!!!!\\n" "$*" >&2
+  exit 1
+}}
+[ -e "{base_mount_path}" ] && error "Can't mount path, already exists"
+mkdir -p "{base_mount_path}" || error "Can't make mount path"
+chown {target_uid}:{target_gid} "{base_mount_path}" || error "Chown failed"
+mount -t ext4 /dev/vdb "{base_mount_path}" || error "Mount failed"
+cd "{current_path}"
+su user -c "./bin/llvm-lit {args_string}"
+"""
+exec_on_boot_path = hgcomm_path / "exec-on-boot"
+exec_on_boot_path.write_text(exec_on_boot_content)
+exec_on_boot_path.chmod(0o755)
+
+# Create ext4 filesystem containing the LLVM build directory and LLVM source
+# tree. Two layouts are supported:
+# 1) A typical layout used in local development, with build directories within
+# build/ in the monorepo checkout.
+# 2) The layout used by ClangBuilder.py in CI, with build directories as
+# siblings to the monorepo checkout (named 'llvm').
+print("@@@@@@@@@@ Creating ext4 filesystem with LLVM build directory @@@@@@@@")
+subprocess.run(["fallocate", "-l", "25GiB", "llvm-project.img"], check=True)
+
+if (current_path.parent.parent / ".git").is_dir():
+    print("Note: 'Local dev' layout detected (build/build_dir nested in LLVM checkout)")
+    extra_tar_args = [
+        f"--exclude=build/{p.name} "
+        for p in current_path.parent.iterdir()
+        if p.is_dir() and p.name != build_dir_name
+    ]
+    extra_tar_args.append("--exclude=.git")
+    extra_tar_args.append(f"--exclude=build/{build_dir_name}/llvm-project.img")
+    paths_to_tar = "."
+    change_to_dir = "../.."
+elif (current_path.parent / "llvm" / ".git").is_dir():
+    print("Note: 'CI style' layout detected (llvm checkout and build_dir as siblings)")
+    extra_tar_args = [
+        "--exclude=llvm/.git",
+        f"--exclude={build_dir_name}/llvm-project.img"
+    ]
+    paths_to_tar = f"llvm {build_dir_name}"
+    change_to_dir = ".."
+else:
+    error("Unrecognized repo/build layout")
+
+parent_dir = current_path.parent
+tar_command = (
+    f"tar --create --file=- --owner={target_uid} --group={target_gid} "
+    f"{' '.join(extra_tar_args)} "
+    f"-C {change_to_dir} {paths_to_tar} | mkfs.ext4 -d - llvm-project.img"
+)
+subprocess.run(tar_command, shell=True, check=True)
+
+# Launch qemu-system appliance
+print("@@@@@@@@@@ Pivoting execution to qemu-system @@@@@@@@")
+# fmt: off
+qemu_command = [
+    "qemu-system-riscv64",
+    "-machine", "virt",
+    "-cpu", os.getenv("BB_QEMU_CPU"),
+    "-smp", os.getenv("BB_QEMU_SMP"),
+    "-m", os.getenv("BB_QEMU_MEM"),
+    "-device", "virtio-blk-device,drive=hd",
+    "-drive", f"file={os.getenv('BB_IMG_DIR')}/rootfs.img,if=none,id=hd,format=raw",
+    "-virtfs", "local,path=hgcomm,mount_tag=hgcomm,security_model=none,id=hgcomm",
+    "-device", "virtio-blk-device,drive=hdb",
+    "-drive", "file=llvm-project.img,format=raw,if=none,id=hdb",
+    "-device", "virtio-net-device,netdev=net",
+    "-netdev", "user,id=net,hostfwd=tcp:127.0.0.1:10222-:22",
+    "-bios", "/usr/share/qemu/opensbi-riscv64-generic-fw_dynamic.bin",
+    "-kernel", f"{os.getenv('BB_IMG_DIR')}/kernel",
+    "-initrd", f"{os.getenv('BB_IMG_DIR')}/initrd",
+    "-object", "rng-random,filename=/dev/urandom,id=rng",
+    "-device", "virtio-rng-device,rng=rng",
+    "-nographic",
+    "-append", "rw quiet root=/dev/vda console=ttyS0",
+]
+# fmt: on
+subprocess.run(qemu_command, check=True)
+print("@@@@@@@@@@ qemu-system execution finished @@@@@@@@")
+
+exit_code_file = hgcomm_path / "exec-on-boot.exitcode"
+if exit_code_file.is_file():
+    sys.exit(int(exit_code_file.read_text().strip()))
+else:
+    sys.exit(111)

zorg/buildbot/builders/ClangBuilder.py

@@ -4,6 +4,7 @@
 from buildbot.plugins import util
 from buildbot.steps.shell import ShellCommand, SetProperty
 from buildbot.steps.shell import WarningCountingShellCommand
+from buildbot.steps.transfer import StringDownload
 
 import zorg.buildbot.builders.Util as builders_util
 
@@ -162,13 +163,18 @@ def getClangCMakeBuildFactory(
             # CMake arguments to use for stage2 instead of extra_cmake_args.
             extra_stage2_cmake_args=None,
 
+            # If set, use a toolchains file for the stage 2 build and add
+            # these options to it.
+            stage2_toolchain_options=None,
+
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_flang=False,
             checkout_test_suite=False,
+            checkout_zorg=False,
 
             enable_runtimes="auto"):
     return _getClangCMakeBuildFactory(
@@ -180,12 +186,14 @@ def getClangCMakeBuildFactory(
                submitURL=submitURL, testerName=testerName,
                env=env, extra_cmake_args=extra_cmake_args,
                extra_stage2_cmake_args=extra_stage2_cmake_args,
+               stage2_toolchain_options=stage2_toolchain_options,
                checkout_clang_tools_extra=checkout_clang_tools_extra,
                checkout_lld=checkout_lld,
                checkout_compiler_rt=checkout_compiler_rt,
                checkout_libcxx=checkout_libcxx,
                checkout_flang=checkout_flang,
                checkout_test_suite=checkout_test_suite,
+               checkout_zorg=checkout_zorg,
                enable_runtimes=enable_runtimes)
 
 def _getClangCMakeBuildFactory(
@@ -219,13 +227,18 @@ def _getClangCMakeBuildFactory(
             # CMake arguments to use for stage2 instead of extra_cmake_args.
             extra_stage2_cmake_args=None,
 
+            # If set, use a toolchains file for the stage 2 build and add
+            # these options to it.
+            stage2_toolchain_options=None,
+
             # Extra repositories
             checkout_clang_tools_extra=True,
             checkout_compiler_rt=True,
             checkout_lld=True,
             checkout_libcxx=False,
             checkout_test_suite=False,
             checkout_flang=False,
+            checkout_zorg=False,
 
             enable_runtimes="auto",
 
@@ -294,6 +307,13 @@ def _getClangCMakeBuildFactory(
             src_dir='test/test-suite',
             alwaysUseLatest=True)
 
+    if checkout_zorg:
+        f.addGetSourcecodeForProject(
+            project='zorg',
+            src_dir='llvm-zorg',
+            alwaysUseLatest=True,
+            repourl="https://github.com/asb/llvm-zorg.git")
+
     # Then get the LLVM source code revision this particular build is for.
     f.addGetSourcecodeSteps()
 
@@ -427,10 +447,28 @@ def _getClangCMakeBuildFactory(
         # Absolute paths to just built compilers.
         # Note: Backslash path separators do not work well with cmake and ninja.
         # Forward slash path separator works on Windows as well.
-        stage1_cc = InterpolateToPosixPath(
-                        f"-DCMAKE_C_COMPILER=%(prop:builddir)s/{stage1_install}/bin/{cc}")
-        stage1_cxx = InterpolateToPosixPath(
-                        f"-DCMAKE_CXX_COMPILER=%(prop:builddir)s/{stage1_install}/bin/{cxx}")
+        stage1_cc = f"%(prop:builddir)s/{stage1_install}/bin/{cc}"
+        stage1_cxx = f"%(prop:builddir)s/{stage1_install}/bin/{cxx}"
+
+        # If stage2_toolchain_options is set then we'll use a toolchain file
+        # to specify the compiler being used (the just-built stage1) and add
+        # any stage2_toolchain_options to it. Otherwise, just set
+        # -DCMAKE_{C,CXX}_COMPILER.
+        if stage2_toolchain_options is None:
+            compiler_args = [
+                    InterpolateToPosixPath(f"-DCMAKE_C_COMPILER={stage1_cc}"),
+                    InterpolateToPosixPath(f"-DCMAKE_CXX_COMPILER={stage1_cxx}"),
+            ]
+        else:
+            toolchain_file = f"%(prop:builddir)s/{stage2_build}/stage1-toolchain.cmake"
+            toolchain_file_contents = "\n".join([
+                    f"set(CMAKE_C_COMPILER {stage1_cc})",
+                    f"set(CMAKE_CXX_COMPILER {stage1_cxx})",
+                    ] + stage2_toolchain_options)
+            f.addStep(StringDownload(util.Interpolate(toolchain_file_contents),
+                                     workerdest=InterpolateToPosixPath(toolchain_file)))
+            compiler_args = [InterpolateToPosixPath(f"-DCMAKE_TOOLCHAIN_FILE={toolchain_file}")]
+
 
         # If we have a separate stage2 cmake arg list, then ensure we re-apply
         # enable_projects and enable_runtimes if necessary.
@@ -446,13 +484,12 @@ def _getClangCMakeBuildFactory(
 
         rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage2_build)
         cmake_cmd2 = [cmake, "-G", "Ninja", rel_src_dir,
-                      stage1_cc,
-                      stage1_cxx,
                       f"-DCMAKE_BUILD_TYPE={stage2_config}",
                       "-DLLVM_ENABLE_ASSERTIONS=True",
                       f"-DLLVM_LIT_ARGS={lit_args}",
                       f"-DCMAKE_INSTALL_PREFIX=../{stage2_install}"
-                     ] + (extra_stage2_cmake_args or extra_cmake_args)
+                     ] + (extra_stage2_cmake_args or extra_cmake_args) \
+                       + compiler_args
 
         f.addStep(ShellCommand(name='cmake stage 2',
                                command=cmake_cmd2,