From 335570dfd315855c4449271cf1b5800f6c089550 Mon Sep 17 00:00:00 2001 From: spectranator Date: Sat, 12 Apr 2025 21:18:30 +0200 Subject: [PATCH] In containerized AppImage build use Clang and allow thin/fat LTO for smaller and more optimized AppImages --- AppImage-build-debian-inner.sh | 52 ++++++++++++++++++++++--------- AppImage-build-debian.sh | 57 +++++++++++++++++++++++++--------- CMakeLists.txt | 10 ++++++ externals/CMakeLists.txt | 2 +- src/CMakeLists.txt | 7 +++-- src/video_core/CMakeLists.txt | 6 +++- 6 files changed, 101 insertions(+), 33 deletions(-) diff --git a/AppImage-build-debian-inner.sh b/AppImage-build-debian-inner.sh index 7716c3296..4bbcaef7e 100755 --- a/AppImage-build-debian-inner.sh +++ b/AppImage-build-debian-inner.sh @@ -2,29 +2,53 @@ set -e # Make sure script is called from inside our container -test -e /tmp/torzu-src || (echo "Script MUST NOT be called directly!" ; exit 1) +test -e /tmp/torzu-src-ro || (echo "Script MUST NOT be called directly!" ; exit 1) + +# Set up environment +export LANG=C.UTF-8 +export LC_ALL=C.UTF-8 +unset LC_ADDRESS LC_NAME LC_MONETARY LC_PAPER LC_TELEPHONE LC_MEASUREMENT LC_TIME + +# Raise max open files count +ulimit -n 50000 # Install dependencies -apt -y install cmake ninja-build build-essential pkg-config locales wget git file -apt -y install libfmt-dev libenet-dev liblz4-dev nlohmann-json3-dev zlib1g-dev libopus-dev libsimpleini-dev libstb-dev libzstd-dev libusb-1.0-0-dev libcubeb-dev libcpp-jwt-dev libvulkan-dev gamemode-dev libasound2-dev libglu1-mesa-dev libxext-dev mesa-common-dev qtbase5-dev qtmultimedia5-dev qtbase5-private-dev libva-dev glslang-tools libavcodec-dev libavfilter-dev libavutil-dev libswscale-dev +apt -y install cmake ninja-build build-essential autoconf pkg-config locales wget git file mold libtool lsb-release wget software-properties-common gnupg \ + qtbase5-dev qtmultimedia5-dev qtbase5-private-dev glslang-tools libssl-dev libavcodec-dev libavfilter-dev libavutil-dev libswscale-dev libpulse-dev libalsaplayer-dev -# Install correct version of boost +# Install Clang +if ! clang-19 --version; then + cd /tmp + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh 19 + rm llvm.sh +fi + +# Mount Torzu sources with temporary overlay cd /tmp -wget https://archives.boost.io/release/1.88.0/source/boost_1_88_0.tar.bz2 -echo "Extracting Boost sources..." -tar xf boost_1_88_0.tar.bz2 -cd boost_1_88_0 -./bootstrap.sh -./b2 install --with-{headers,context} link=static -cd .. -rm -rf boost_1_88_0 +mkdir torzu-src-upper torzu-src-work torzu-src +mount -t overlay overlay -olowerdir=torzu-src-ro,upperdir=torzu-src-upper,workdir=torzu-src-work torzu-src + +# Get extra compile options +EXTRA_COMPILE_FLAGS="" +if [ "$BUILD_USE_THIN_LTO" = 1 ]; then + EXTRA_COMPILE_FLAGS="-flto=thin" +fi +if [ "$BUILD_USE_FAT_LTO" = 1 ]; then + EXTRA_COMPILE_FLAGS="-flto=full" +fi # Build Torzu cd /tmp mkdir torzu-build cd torzu-build -cmake /tmp/torzu-src -GNinja -DCMAKE_BUILD_TYPE=Release -DYUZU_TESTS=OFF -DENABLE_QT_TRANSLATION=OFF -DSPIRV_WERROR=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" -DSPIRV-Headers_SOURCE_DIR=/tmp/torzu-src/externals/SPIRV-Headers -ninja +cmake /tmp/torzu-src -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-19 -DCMAKE_CXX_COMPILER=clang++-19 -DYUZU_TESTS=OFF -DENABLE_QT_TRANSLATION=OFF -DSPIRV_WERROR=OFF -DYUZU_USE_CPM=ON -DCMAKE_FIND_LIBRARY_SUFFIXES=".a;.so" -DSPIRV-Headers_SOURCE_DIR=/tmp/torzu-src/externals/SPIRV-Headers -DCMAKE_{C,CXX}_FLAGS="${EXTRA_COMPILE_FLAGS} -fdata-sections -ffunction-sections" -DCMAKE_{EXE,SHARED}_LINKER_FLAGS="-Wl,--gc-sections" +ninja || ( + echo "Compilation has failed. Dropping you into a shell so you can inspect the situation. Run 'ninja' to retry and exit shell once compilation has finished successfully." + echo "Note that any changes made here will not be reflected to the host environment, but changes made from the host environment will be reflected here." + bash +) # Generate AppImage cp -rv /tmp/torzu-src/AppImageBuilder /tmp/AppImageBuilder diff --git a/AppImage-build-debian.sh b/AppImage-build-debian.sh index 773d1cb4b..af6bf82bc 100755 --- a/AppImage-build-debian.sh +++ b/AppImage-build-debian.sh @@ -1,28 +1,55 @@ #! /bin/bash set -e +# Parse options +for i in "$@" +do +case $i in + -o|--thin-lto) + export BUILD_USE_THIN_LTO=1 + echo "-> Thin link time optimization enabled." + ;; + -O|--fat-lto) + export BUILD_USE_FAT_LTO=1 + echo "-> Fat link time optimization enabled." + ;; + -k|--keep-rootfs) + BUILD_KEEP_ROOTFS=1 + echo "-> Not deleting rootfs after successful build." + ;; + *) + echo "Usage: $0 [--thin-lto/-o] [--fat-lto/-O] [--keep-rootfs/-k]" + exit 1 + ;; +esac +done + # Get torzu source dir TORZU_SOURCE_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -echo "Source dir is $TORZU_SOURCE_DIR" +echo "-> Source dir is $TORZU_SOURCE_DIR" rm -rf "$TORZU_SOURCE_DIR/AppImageBuilder/build" # Generate debian rootfs cd /tmp echo "Cleaning up before build..." -rm -rf torzu-debian-appimage-rootfs rootfs-torzu-appimage-build -debootstrap stable rootfs-torzu-appimage-build http://deb.debian.org/debian/ +rm -rf torzu-debian-appimage-rootfs +[ -d rootfs-torzu-appimage-build ] || + debootstrap stable rootfs-torzu-appimage-build http://deb.debian.org/debian/ bwrap --bind rootfs-torzu-appimage-build / \ - --unshare-pid \ - --dev-bind /dev /dev --proc /proc --tmpfs /tmp --ro-bind /sys /sys --dev-bind /run /run \ - --tmpfs /var/tmp \ - --chmod 1777 /tmp \ - --ro-bind /etc/resolv.conf /etc/resolv.conf \ - --ro-bind "$TORZU_SOURCE_DIR" /tmp/torzu-src \ - --chdir / \ - --tmpfs /home \ - --setenv HOME /home \ - --bind /tmp /tmp/hosttmp \ - /tmp/torzu-src/AppImage-build-debian-inner.sh + --unshare-pid \ + --dev-bind /dev /dev --proc /proc --tmpfs /tmp --ro-bind /sys /sys --dev-bind /run /run \ + --tmpfs /var/tmp \ + --chmod 1777 /tmp \ + --ro-bind /etc/resolv.conf /etc/resolv.conf \ + --ro-bind "$TORZU_SOURCE_DIR" /tmp/torzu-src-ro \ + --chdir / \ + --tmpfs /home \ + --setenv HOME /home \ + --bind /tmp /tmp/hosttmp \ + /tmp/torzu-src-ro/AppImage-build-debian-inner.sh appimagetool torzu-debian-appimage-rootfs torzu.AppImage echo "AppImage generated at /tmp/torzu.AppImage! Cleaning up..." -exec rm -rf torzu-debian-appimage-rootfs rootfs-torzu-appimage-build +rm -rf torzu-debian-appimage-rootfs +if [ ! "$BUILD_KEEP_ROOTFS" = 1 ]; then + rm -rf rootfs-torzu-appimage-build +fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 669c8156f..f1034e3d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,12 +366,22 @@ else() # Disable tests in all externals supporting the standard option name set(BUILD_TESTING OFF) + # Some externals take these options instead + set(BUILD_TESTS OFF) + set(BUILD_TOOLS OFF) + # Build only static externals set(BUILD_SHARED_LIBS OFF) # Do not attempt to use Brotli in httplib since we're not downloading it set(HTTPLIB_USE_BROTLI_IF_AVAILABLE OFF) + # Configure Opus to not generate as much overhead + set(OPUS_BUILD_TESTING OFF) + set(OPUS_BUILD_PROGRAMS OFF) + set(OPUS_INSTALL_PKG_CONFIG_MODULE OFF) + set(OPUS_INSTALL_CMAKE_CONFIG_MODULE OFF) + message(STATUS "Downloading and extracting boost library sources. This will take some time...") CPMAddPackage( NAME boost diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index 10645cf38..4a45430f2 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -146,7 +146,7 @@ endif() if (ENABLE_WEB_SERVICE AND NOT TARGET cpp-jwt::cpp-jwt) set(CPP_JWT_BUILD_EXAMPLES OFF) set(CPP_JWT_BUILD_TESTS OFF) - set(CPP_JWT_USE_VENDORED_NLOHMANN_JSON OFF) + set(CPP_JWT_USE_VENDORED_NLOHMANN_JSON ${YUZU_USE_CPM}) add_subdirectory(cpp-jwt) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 720b2f2fc..bc8ea21a2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -134,9 +134,12 @@ else() -Wno-braced-scalar-init -Wno-unused-private-field -Wno-nullability-completeness - -Werror=shadow-uncaptured-local - -Werror=implicit-fallthrough -Werror=type-limits + # Boost... + #-Werror=shadow-uncaptured-local + #-Werror=implicit-fallthrough + -Wshadow-uncaptured-local + -Wimplicit-fallthrough ) endif() diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 4d6da757f..ce2ca1daf 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -347,7 +347,11 @@ if (NOT MSVC) # error: implicit conversion loses integer precision: 'int' to 'boost::icl::bound_type' (aka 'unsigned char') target_compile_options(video_core PRIVATE -Wno-shadow -Wno-unused-local-typedef) else() - target_compile_options(video_core PRIVATE -Werror=conversion) + target_compile_options(video_core PRIVATE + # Boost is breaking this so don't make it an error. + #-Werror=conversion + -Wconversion + ) endif() target_compile_options(video_core PRIVATE