File PKGBUILD of Package llvm-god-mlgo

# Maintainer: neycrol <330578697@qq.com>

pkgname=llvm-god-mlgo
pkgver=0
pkgrel=13
pkgdesc="LLVM/Clang/LLD (master) with 4-stage PGO + Polly + MLGO, installed under /opt/llvm-god"
arch=('x86_64')
url="https://github.com/llvm/llvm-project"
license=('custom:Apache-2.0 WITH LLVM-exception')
depends=('zlib' 'zstd' 'libxml2' 'ncurses' 'libedit' 'libffi' 'abseil-cpp' 'google-glog' 'gflags' 'protobuf' 'libelf' 'openssl')
optdepends=('tensorflow: optional for custom MLGO training workflows')
makedepends=('cmake' 'ninja' 'python' 'git' 'gcc' 'abseil-cpp' 'google-glog' 'gflags' 'protobuf' 'gtest' 'libelf' 'openssl')
options=('!strip' '!debug' '!lto')
source=('llvm-project.tar.gz' 'autofdo.tar.gz' 'perf_data_converter.tar.gz' 'autofdo-system-llvm.patch')
sha256sums=('SKIP' 'SKIP' 'SKIP' 'SKIP')

_find_llvm_src() {
  local d
  for d in "${srcdir}"/llvm-project*; do
    if [[ -d "${d}" && -f "${d}/llvm/CMakeLists.txt" ]]; then
      echo "${d}"
      return 0
    fi
  done
  return 1
}

_find_autofdo_src() {
  local d
  for d in "${srcdir}"/autofdo*; do
    if [[ -d "${d}" && -f "${d}/CMakeLists.txt" ]]; then
      echo "${d}"
      return 0
    fi
  done
  return 1
}

_find_perf_data_src() {
  local d
  for d in "${srcdir}"/perf_data_converter*; do
    if [[ -d "${d}" && -f "${d}/src/quipper/perf_reader.cc" ]]; then
      echo "${d}"
      return 0
    fi
  done
  return 1
}

_llvm_base_ver() {
  local _src _cm major minor patch
  _src=$(_find_llvm_src) || return 1
  _cm="${_src}/llvm/CMakeLists.txt"

  major=$(sed -n 's/^set(LLVM_VERSION_MAJOR[[:space:]]\+\([0-9]\+\)).*/\1/p' "${_cm}" | head -n1)
  minor=$(sed -n 's/^set(LLVM_VERSION_MINOR[[:space:]]\+\([0-9]\+\)).*/\1/p' "${_cm}" | head -n1)
  patch=$(sed -n 's/^set(LLVM_VERSION_PATCH[[:space:]]\+\([0-9]\+\)).*/\1/p' "${_cm}" | head -n1)

  if [[ -n "${major}" && -n "${minor}" && -n "${patch}" ]]; then
    echo "${major}.${minor}.${patch}"
  else
    echo "0.0.0"
  fi
}

pkgver() {
  local _base _date _rev
  _base=$(_llvm_base_ver)
  _date=$(date -u +%Y%m%d)

  if [[ -f "${srcdir}/._servicedata" ]]; then
    _rev=$(sed -n 's:.*<revision>\([0-9a-f]\+\)</revision>.*:\1:p' "${srcdir}/._servicedata" | head -n1)
    if [[ -z "${_rev}" ]]; then
      _rev=$(sed -n 's:.*revision="\([0-9a-f]\+\)".*:\1:p' "${srcdir}/._servicedata" | head -n1)
    fi
  fi

  if [[ -n "${_rev}" ]]; then
    echo "${_base}.${_date}.g${_rev:0:8}"
  else
    echo "${_base}.${_date}"
  fi
}

prepare() {
  local _src _autofdo _perf _f
  _src=$(_find_llvm_src) || {
    echo "llvm-project sources not found"
    return 1
  }
  _autofdo=$(_find_autofdo_src) || {
    echo "autofdo sources not found"
    return 1
  }
  _perf=$(_find_perf_data_src) || {
    echo "perf_data_converter sources not found"
    return 1
  }

  rm -rf "${srcdir}/build-stage1" \
         "${srcdir}/build-instrumented" \
         "${srcdir}/build-final" \
         "${srcdir}/build-autofdo" \
         "${srcdir}/profiles" \
         "${srcdir}/merged.profdata"

  rm -rf "${_autofdo}/third_party/perf_data_converter"
  cp -a "${_perf}" "${_autofdo}/third_party/perf_data_converter"

  # Upstream source uses third_party abseil include prefixes; switch to
  # system abseil headers to avoid protobuf/abseil API skew.
  while IFS= read -r -d '' _f; do
    sed -i 's#"third_party/abseil/absl/#"absl/#g' "${_f}"
  done < <(
    find "${_autofdo}" -type f ! -path "${_autofdo}/third_party/*" -print0 \
      | xargs -0 -r grep -lZ 'third_party/abseil/absl/' || true
  )

  cat > "${_autofdo}/absl_nonnull_compat.h" <<'EOF'
#ifndef AUTOFDO_ABSL_NONNULL_COMPAT_H_
#define AUTOFDO_ABSL_NONNULL_COMPAT_H_
#include "absl/base/nullability.h"
namespace absl {
template <typename T>
using Nonnull = T;
}
#endif  // AUTOFDO_ABSL_NONNULL_COMPAT_H_
EOF

  cat > "${_autofdo}/base/commandlineflags.h" <<'EOF'
#ifndef AUTOFDO_BASE_COMMANDLINEFLAGS_H_
#define AUTOFDO_BASE_COMMANDLINEFLAGS_H_
#include <gflags/gflags.h>
#endif  // AUTOFDO_BASE_COMMANDLINEFLAGS_H_
EOF

  # protobuf >= 33 removed Arena::CreateMessage from public API.
  sed -i 's/Arena::CreateMessage<PerfDataProto>(&arena_)/Arena::Create<PerfDataProto>(\&arena_)/' \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/perf_reader.cc"

  # perf_data_converter expects include roots that differ from this integration.
  sed -i 's#"src/quipper/perf_data.pb.h"#"perf_data.pb.h"#g' \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/compat/proto.h" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/address_context.h" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/address_context.cc" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/perf_data_utils.cc"
  sed -i 's#"quipper/perf_data.pb.h"#"perf_data.pb.h"#g' \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/compat/proto.h" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/address_context.h" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/address_context.cc" \
    "${_autofdo}/third_party/perf_data_converter/src/quipper/perf_data_utils.cc"

  # LLVM trunk APIs now prefer llvm::Triple over std::string triple forms.
  sed -i 's/triple.getTriple()/triple/g' \
    "${_autofdo}/mini_disassembler.cc"

  patch -d "${_autofdo}" -Np0 -i "${srcdir}/autofdo-system-llvm.patch"
}

build() {
  local _src _autofdo _prefix _stage1 _inst _final _profile_dir _profile_data _autofdo_build
  local _host_cc _host_cxx _gcc_install_prefix _jobs _nproc _mem_kb _mem_jobs
  local _stage1_profile_glob _stage1_profile_lib _profile_tgt
  local _stage1_llvm_tblgen _stage1_clang_tblgen
  local _base_args _stage1_args _stage2_args _final_args
  local _god_cflags _god_ldflags

  _src=$(_find_llvm_src) || {
    echo "llvm-project sources not found"
    return 1
  }
  _autofdo=$(_find_autofdo_src) || {
    echo "autofdo sources not found"
    return 1
  }

  _prefix="/opt/llvm-god"
  _stage1="${srcdir}/build-stage1"
  _inst="${srcdir}/build-instrumented"
  _final="${srcdir}/build-final"
  _autofdo_build="${srcdir}/build-autofdo"
  _profile_dir="${srcdir}/profiles"
  _profile_data="${srcdir}/merged.profdata"
  _stage1_profile_glob="${_stage1}/lib/clang/*/lib/*/libclang_rt.profile.a"

  if [[ -x /opt/gcc-git-god/bin/gcc && -x /opt/gcc-git-god/bin/g++ ]]; then
    _host_cc="/opt/gcc-git-god/bin/gcc"
    _host_cxx="/opt/gcc-git-god/bin/g++"
    _gcc_install_prefix="/opt/gcc-git-god"
  else
    _host_cc="/usr/bin/gcc"
    _host_cxx="/usr/bin/g++"
    _gcc_install_prefix="/usr"
  fi

  _nproc=$(/usr/bin/getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
  _mem_kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null)
  if [[ -n "${_mem_kb}" && "${_mem_kb}" -gt 0 ]]; then
    # LLVM full builds are memory-heavy; reserve ~2.5 GiB per compile job.
    _mem_jobs=$(( _mem_kb / 2500000 ))
    (( _mem_jobs < 1 )) && _mem_jobs=1
  else
    _mem_jobs=${_nproc}
  fi
  _jobs=${_nproc}
  (( _jobs > _mem_jobs )) && _jobs=${_mem_jobs}

  _base_args=(
    -G Ninja
    -DCMAKE_BUILD_TYPE=Release
    -DCMAKE_INSTALL_PREFIX="${_prefix}"
    -DLLVM_TARGETS_TO_BUILD="X86;AArch64;ARM;RISCV"
    -DLLVM_ENABLE_RTTI=ON
    -DLLVM_ENABLE_FFI=ON
    -DLLVM_ENABLE_ZLIB=FORCE_ON
    -DLLVM_ENABLE_ZSTD=FORCE_ON
    -DLLVM_INCLUDE_TESTS=OFF
    -DLLVM_INCLUDE_BENCHMARKS=OFF
    -DLLVM_INCLUDE_EXAMPLES=OFF
    -DLLVM_ENABLE_BINDINGS=OFF
    -DCLANG_DEFAULT_PIE_ON_LINUX=ON
    -DLLVM_ENABLE_PROJECTS="clang;lld;polly;bolt"
    -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind"
  )

  # Stage 1: Bootstrap clang/llvm-profdata with GCC God toolchain.
  _stage1_args=(
    -DCMAKE_C_COMPILER="${_host_cc}"
    -DCMAKE_CXX_COMPILER="${_host_cxx}"
    -DCMAKE_C_FLAGS="-O2 -pipe"
    -DCMAKE_CXX_FLAGS="-O2 -pipe"
    -DGCC_INSTALL_PREFIX="${_gcc_install_prefix}"
    -DUSE_DEPRECATED_GCC_INSTALL_PREFIX=ON
  )

  cmake -S "${_src}/llvm" -B "${_stage1}" "${_base_args[@]}" "${_stage1_args[@]}"
  # Stage1 must also provide compiler-rt profile runtime for Stage2
  # (-fprofile-generate links against libclang_rt.profile.a).
  ninja -C "${_stage1}" -j"${_jobs}" clang llvm-profdata runtimes

  _stage1_llvm_tblgen="${_stage1}/bin/llvm-tblgen"
  _stage1_clang_tblgen="${_stage1}/bin/clang-tblgen"
  if [[ ! -x "${_stage1_llvm_tblgen}" || ! -x "${_stage1_clang_tblgen}" ]]; then
    ninja -C "${_stage1}" -j"${_jobs}" llvm-tblgen clang-tblgen
  fi
  if [[ ! -x "${_stage1_llvm_tblgen}" || ! -x "${_stage1_clang_tblgen}" ]]; then
    echo "Stage 1 did not produce host tablegen tools"
    return 1
  fi

  if ! compgen -G "${_stage1_profile_glob}" > /dev/null; then
    _profile_tgt=$(ninja -C "${_stage1}" -t targets all \
      | sed -n 's/^\(clang_rt\.profile[^:[:space:]]*\):.*/\1/p' \
      | head -n1)
    if [[ -n "${_profile_tgt}" ]]; then
      ninja -C "${_stage1}" -j"${_jobs}" "${_profile_tgt}"
    fi
  fi

  _stage1_profile_lib=$(compgen -G "${_stage1_profile_glob}" | head -n1 || true)
  if [[ -z "${_stage1_profile_lib}" ]]; then
    echo "Stage 1 did not produce libclang_rt.profile.a (compiler-rt profile runtime)"
    return 1
  fi

  # Stage 2: Instrumented build for PGO profile generation.
  mkdir -p "${_profile_dir}"
  _stage2_args=(
    -DCMAKE_C_COMPILER="${_stage1}/bin/clang"
    -DCMAKE_CXX_COMPILER="${_stage1}/bin/clang++"
    "-DCMAKE_C_FLAGS=-O2 -pipe -fprofile-generate=${_profile_dir}"
    "-DCMAKE_CXX_FLAGS=-O2 -pipe -fprofile-generate=${_profile_dir}"
    -DGCC_INSTALL_PREFIX="${_gcc_install_prefix}"
    -DUSE_DEPRECATED_GCC_INSTALL_PREFIX=ON
  )

  cmake -S "${_src}/llvm" -B "${_inst}" "${_base_args[@]}" "${_stage2_args[@]}"
  ninja -C "${_inst}" -j"${_jobs}"

  # Stage 3: Minimal training pass to produce profile data for clang itself.
  # Use synthetic translation units instead of LLVM sources so this step
  # does not depend on generated headers/layout changes in-tree.
  rm -f "${_profile_data}"
  rm -f "${_profile_dir}"/*.profraw
  export LLVM_PROFILE_FILE="${_profile_dir}/code-%m.profraw"
  cat > "${srcdir}/.mlgo-train.c" <<'EOF'
int add(int a, int b) { return a + b; }
int main(void) { return add(1, 2) - 3; }
EOF
  cat > "${srcdir}/.mlgo-train.cpp" <<'EOF'
template <typename T>
T clamp_like(T v, T lo, T hi) {
  return (v < lo) ? lo : (v > hi ? hi : v);
}
int main() { return clamp_like<int>(7, 1, 5) - 5; }
EOF

  "${_inst}/bin/clang" -O2 -c "${srcdir}/.mlgo-train.c" -o /dev/null
  "${_inst}/bin/clang++" -O2 -std=gnu++20 -c "${srcdir}/.mlgo-train.cpp" -o /dev/null
  rm -f "${srcdir}/.mlgo-train.c" "${srcdir}/.mlgo-train.cpp"

  shopt -s nullglob
  local _raw=("${_profile_dir}"/*.profraw)
  shopt -u nullglob
  if (( ${#_raw[@]} == 0 )); then
    echo "PGO training produced no .profraw files"
    return 1
  fi
  "${_inst}/bin/llvm-profdata" merge -output="${_profile_data}" "${_raw[@]}"

  # Stage 4: Final optimized build (PGO + Polly + MLGO), LTO intentionally OFF.
  _god_cflags="-O3 -march=alderlake -mtune=alderlake -pipe -fno-semantic-interposition -mllvm -polly -mllvm -polly-vectorizer=stripmine"
  _god_ldflags="-Wl,--as-needed"
  local _libstdcpp_a _libgcc_a
  _libstdcpp_a="$("${_host_cxx}" -print-file-name=libstdc++.a 2>/dev/null || true)"
  _libgcc_a="$("${_host_cc}" -print-file-name=libgcc.a 2>/dev/null || true)"

  if [[ -n "${_libstdcpp_a}" && "${_libstdcpp_a}" != "libstdc++.a" && -f "${_libstdcpp_a}" ]]; then
    _god_ldflags="-static-libstdc++ ${_god_ldflags}"
  else
    msg2 "libstdc++.a not found for ${_host_cxx}; disabling -static-libstdc++ in stage4"
  fi
  if [[ -n "${_libgcc_a}" && "${_libgcc_a}" != "libgcc.a" && -f "${_libgcc_a}" ]]; then
    _god_ldflags="-static-libgcc ${_god_ldflags}"
  else
    msg2 "libgcc.a not found for ${_host_cc}; disabling -static-libgcc in stage4"
  fi
  _final_args=(
    # Emergency stability guard: final-stage runtimes bootstrap can fail on
    # some OBS workers (compiler ABI detection breaks under the self-host path).
    # Keep producing clang/lld/polly first; runtimes can be re-enabled after
    # allocator regression fix is fully validated.
    -DLLVM_ENABLE_RUNTIMES=
    -DLLVM_USE_HOST_TOOLS=ON
    -DLLVM_NATIVE_TOOL_DIR="${_stage1}/bin"
    -DLLVM_TABLEGEN="${_stage1_llvm_tblgen}"
    -DCLANG_TABLEGEN="${_stage1_clang_tblgen}"
    -DCMAKE_C_COMPILER="${_stage1}/bin/clang"
    -DCMAKE_CXX_COMPILER="${_stage1}/bin/clang++"
    "-DCMAKE_C_FLAGS=${_god_cflags}"
    "-DCMAKE_CXX_FLAGS=${_god_cflags}"
    "-DCMAKE_EXE_LINKER_FLAGS=${_god_ldflags}"
    "-DCMAKE_SHARED_LINKER_FLAGS=${_god_ldflags}"
    -DLLVM_PROFDATA_FILE="${_profile_data}"
    -DLLVM_ENABLE_LTO=OFF
    -DGCC_INSTALL_PREFIX="${_gcc_install_prefix}"
    -DUSE_DEPRECATED_GCC_INSTALL_PREFIX=ON
    -DLLVM_INLINE_ADVISOR_MODEL=Release
    -DLLVM_RA_EVICTION_ADVISOR_MODEL=Release
  )

  cmake -S "${_src}/llvm" -B "${_final}" "${_base_args[@]}" "${_final_args[@]}"
  ninja -C "${_final}" -j"${_jobs}"

  # Build AutoFDO's create_llvm_prof against the just-built LLVM tree.
  cmake -S "${_autofdo}" -B "${_autofdo_build}" \
    -G Ninja \
    -DCMAKE_BUILD_TYPE=Release \
    -DENABLE_TOOL=llvm \
    -DBUILD_SHARED=TRUE \
    -DLLVM_DIR="${_final}/lib/cmake/llvm" \
    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
  ninja -C "${_autofdo_build}" -j"${_jobs}" create_llvm_prof
}

package() {
  local _src _autofdo _final _autofdo_build
  _src=$(_find_llvm_src) || {
    echo "llvm-project sources not found"
    return 1
  }
  _autofdo=$(_find_autofdo_src) || {
    echo "autofdo sources not found"
    return 1
  }
  _final="${srcdir}/build-final"
  _autofdo_build="${srcdir}/build-autofdo"

  DESTDIR="${pkgdir}" ninja -C "${_final}" install
  install -Dm644 "${_src}/LICENSE.TXT" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
  install -Dm644 "${_autofdo}/LICENSE" "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE.autofdo"

  install -Dm755 "${_autofdo_build}/create_llvm_prof" "${pkgdir}/opt/llvm-god/bin/create_llvm_prof"
  install -Dm755 /dev/stdin "${pkgdir}/opt/llvm-god/bin/llvm_propeller_profile" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
exec "$(dirname "$0")/create_llvm_prof" --format=propeller "$@"
EOF
}