Overview

File milvus.spec of Package milvus

#
# spec file for package milvus
#
# Copyright (c) 2025 SUSE LLC
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
# upon. The license for this file, and modifications and additions to the
# file, is the same license as for the pristine package itself (unless the
# license for the pristine package is not an Open Source License, in which
# case the license is the MIT License). An "Open Source License" is a
# license that conforms to the Open Source Definition (Version 1.9)
# published by the Open Source Initiative.

# Please submit bugfixes or comments via https://bugs.opensuse.org/
#


%global flavor @BUILD_FLAVOR@%{nil}
%bcond_without allarches

%if "%{flavor}" == "cppcpu"
%bcond_without cppcpu
%bcond_without cpp
%else
%{nil}
%endif

%if "%{flavor}" == "cppgpu"
%bcond_without cppgpu
%bcond_without cpp
%bcond_without cppcpu
%endif

%if "%{flavor}" == "gpu"
%bcond_without gpu
%endif

%define milvus_proto_vers 2.5.4
%define milvus_storage_vers 2e81a4c
%define milvus_storage_vers_full 2e81a4c0bc45143a59659acda00fb51dbea42b07
%define knowhere_vers 2.5.2
%define rapids_vers 24.10
%define raft_vers 24.10
%define cccl_vers 2.2.0
%define rmm_vers 24.04.00
%define cutlass_vers 2.10.0
%define cucollections_vers 56c53beb6fb0cafd265b7fcc3df78ae487811b22
%define hnswlib_vers 0.6.2
# 0.7.0
%define rapidscmake_vers 24.04.00
%define cpm_vers 0.40.2

Name:           milvus
Version:        2.5.4
Release:        0
Summary:        Cloud-native vector database, storage for next generation AI applications
License:        Apache-2.0
URL:            https://milvus.io/
Source0:        %{name}-%{version}.tar
# see comment in _service in how to create the vendor directory
Source1:        vendor.tar.xz
Source2:        https://github.com/milvus-io/milvus-proto/archive/v%{milvus_proto_vers}.tar.gz#/milvus-proto-%{milvus_proto_vers}.tar.gz
Source3:        https://github.com/milvus-io/milvus-storage/archive/%{milvus_storage_vers}.tar.gz#/milvus-storage-%{milvus_storage_vers}.tar.gz
Source4:        https://github.com/zilliztech/knowhere/archive/v%{knowhere_vers}.tar.gz#/knowhere-%{knowhere_vers}.tar.gz
Source5:        https://github.com/rapidsai/rapids-cmake/archive/v%{rapidscmake_vers}.tar.gz#/rapdiscmake-%{rapidscmake_vers}.tar.gz
Source6:        https://github.com/rapidsai/rmm/archive/v%{rmm_vers}.tar.gz#/rmm-%{rmm_vers}.tar.gz
Source7:        https://github.com/milvus-io/raft/archive/branch-%{raft_vers}.tar.gz#/raft-branch-%{raft_vers}.tar.gz
Source8:        https://github.com/NVIDIA/cccl/archive/v%{cccl_vers}.tar.gz#/cccl-%{cccl_vers}.tar.gz
Source9:        https://github.com/NVIDIA/cutlass/archive/v%{cutlass_vers}.tar.gz#/cutlass-%{cutlass_vers}.tar.gz
Source10:       https://github.com/NVIDIA/cuCollections/archive/%{cucollections_vers}.tar.gz#/cuCollections-%{cucollections_vers}.tar.gz
Source11:       https://github.com/nmslib/hnswlib/archive/v%{hnswlib_vers}.tar.gz#/hnswlib-%{hnswlib_vers}.tar.gz
Source15:       CMakeLists.txt-milvus-storage
Source20:       tantivy-config.toml
Source21:       tantivy-vendor.tar.xz
Source22:       https://github.com/cpm-cmake/CPM.cmake/releases/download/v0.38.5/CPM.cmake#/CPM_0.38.5.cmake
# patch for vendored third party stuff, I hate them so much
Source50:       knowhere-opentelemetry.patch
Source60:       CMakeLists.txt
Source61:       conanfile.py.in
Patch0:         conan-adjustments.patch
Patch1:         systemlibs-in-cmake.patch
Patch2:         fix-simdjson.h.patch
Patch3:         include-gflags-if-they-are-used.patch
Patch4:         aws-patches.patch
Patch5:         fmt-format-fixes.patch
Patch6:         tantivity-local-install.patch
Patch7:         additional-libraries.patch

%if 0%{?sle_version} == 150600
BuildRequires:  gcc12
BuildRequires:  gcc12-c++
%else
# use gcc12 also for tw
BuildRequires:  gcc
BuildRequires:  gcc-c++
%endif
BuildRequires:  glog-4-devel
BuildRequires:  go >= 1.21
BuildRequires:  librdkafka-devel
BuildRequires:  opentelemetry-cpp_19-devel
BuildRequires:  pkgconfig
BuildRequires:  protobuf21-devel
BuildRequires:  protoc-gen-go
BuildRequires:  protoc-gen-go-grpc
BuildRequires:  rocksdb6-devel
BuildRequires:  tbb-devel
%if %{without cppcpu}
%if %{without gpu}
Requires:       milvus-cppcpu
BuildRequires:  milvus-cppcpu
BuildRequires:  milvus-cppcpu-devel
%else
Requires:       milvus-cppgpu
BuildRequires:  libcublas-12-5
BuildRequires:  libcusolver-12-5
BuildRequires:  libcusparse-12-5
BuildRequires:  libnvjitlink-12-5
BuildRequires:  milvus-cppgpu
BuildRequires:  milvus-cppgpu-devel
%endif
%else
BuildRequires:  aws-c-auth-devel
BuildRequires:  aws-c-common-devel
BuildRequires:  aws-c-http-devel
BuildRequires:  aws-c-io-devel
BuildRequires:  aws-c-mqtt-devel
BuildRequires:  aws-c-s3-devel
BuildRequires:  aws-c-sdkutils-devel
BuildRequires:  aws-crt-cpp-devel
BuildRequires:  aws-sdk-cpp-devel
BuildRequires:  aws-sdk-cpp-libs
%if 0%{?sle_version} == 150600
BuildRequires:  cargo
BuildRequires:  rust
%else
BuildRequires:  cargo1.79
BuildRequires:  rust1.79
%endif
BuildRequires:  cblas-devel
BuildRequires:  cmake
BuildRequires:  conan
BuildRequires:  double-conversion-devel
BuildRequires:  fmt-devel
BuildRequires:  folly-devel
BuildRequires:  gflags-devel
BuildRequires:  git
BuildRequires:  glog-4-devel
BuildRequires:  gmock
BuildRequires:  google-cloud-cpp25-devel
BuildRequires:  gtest
BuildRequires:  jemalloc-devel
BuildRequires:  libboost_context-devel
BuildRequires:  libboost_filesystem-devel
BuildRequires:  libboost_program_options-devel
BuildRequires:  libboost_regex-devel
BuildRequires:  libboost_system-devel
BuildRequires:  libboost_thread-devel
BuildRequires:  libdwarf-devel
BuildRequires:  libevent-devel
BuildRequires:  libgoogle_cloud_cpp252
BuildRequires:  libuuid-devel
BuildRequires:  libzstd-devel
BuildRequires:  marisa-devel
BuildRequires:  openblas-devel
BuildRequires:  pkgconfig
BuildRequires:  protoc-gen-go
BuildRequires:  rapidjson-devel
BuildRequires:  roaring-devel
BuildRequires:  s2n-devel
BuildRequires:  simdjson-devel
BuildRequires:  xz
BuildRequires:  zlib-devel
BuildRequires:  cmake(Catch2)
BuildRequires:  cmake(Snappy)
BuildRequires:  cmake(arrow)
BuildRequires:  cmake(aws-c-cal)
BuildRequires:  cmake(aws-c-compression)
BuildRequires:  cmake(aws-c-event-stream)
BuildRequires:  cmake(aws-checksums)
BuildRequires:  cmake(nlohmann_json)
BuildRequires:  cmake(prometheus-cpp)
BuildRequires:  cmake(re2)
BuildRequires:  cmake(yaml-cpp)
BuildRequires:  pkgconfig(bzip2)
BuildRequires:  pkgconfig(liblz4)
%if %{with cppgpu}
BuildRequires:  cuda-cudart-devel-12-5
BuildRequires:  cuda-nvcc-12-5
BuildRequires:  libcublas-devel-12-5
BuildRequires:  libcurand-devel-12-5
BuildRequires:  libcusolver-devel-12-5
BuildRequires:  libcusparse-devel-12-5
BuildRequires:  libnvjitlink-12-5
BuildRequires:  spdlog-devel
%endif
%endif

%description
Milvus is an open-source vector database built to power embedding similarity
search and AI applications. Milvus makes unstructured data search more
accessible, and provides a consistent user experience regardless of the
deployment environment.

%if %{with gpu}
%package gpu
Summary:        Cloud-native vector database, storage for next generation GPU AI applications
Requires:       milvus-cppgpu = %version

%description gpu
Milvus is an open-source vector database built to power embedding similarity
search and AI applications. Milvus makes unstructured data search more
accessible, and provides a consistent user experience regardless of the
deployment environment.
%endif

%if %{with cppcpu}
%package cppcpu
Summary:        The cpp core of milvus, including knowhere

%description cppcpu
Milvus is an open-source vector database built to power embedding similarity
search and AI applications. Milvus makes unstructured data search more
accessible, and provides a consistent user experience regardless of the
deployment environment.

%package cppcpu-devel
Summary:        The cpp core of milvus, including knowhere
Requires:       milvus-cppcpu = %{version}

%description cppcpu-devel
Milvus is an open-source vector database built to power embedding similarity
search and AI applications. Milvus makes unstructured data search more
accessible, and provides a consistent user experience regardless of the
deployment environment.
%endif
%if %{with cppgpu}
%package -n milvus-cppgpu
Summary:        The cpp core of milvus, including knowhere with GPU support

%description -n milvus-cppgpu
Milvus is an open-source vector database built to power embedding similarity
search and AI applications. Milvus makes unstructured data search more
accessible, and provides a consistent user experience regardless of the
deployment environment.

%package -n milvus-cppgpu-devel
Summary:        Devel files for GPU milus, shouldn't be installed
Requires:       milvus-cppgpu = %{version}

%description -n milvus-cppgpu-devel
as a lot of third party package are in milvus, their depency files are
dumped in this package and will so conflict with other packages

%endif

%prep
%define make_depend_src() test -e %{?2}%{!?2:$(basename %1| sed 's/-.*//')} && rmdir %{?2}%{!?2:$(basename %1| sed 's/-.*//')}; tar xzf %1; mv $(basename %{1} | sed 's/\.tar\.gz//' )* %{?2}%{!?2:$(basename %1| sed 's/-.*//')}
pushd .
cd %{_sourcedir}
tar xzf %{S:3} # milvus-storage
cp %{S:15} milvus-storage-%{milvus_storage_vers_full}/cpp/CMakeLists.txt
sed -i 's/arrow::internal::Uri/arrow::util::Uri/g' milvus-storage-%{milvus_storage_vers_full}/cpp/src/common/fs_util.cpp
tar xzf %{S:4} #knowhere
cd knowhere-%{knowhere_vers}
patch -p1 < %{S:50}
cd ..
%if %{with cppgpu}
tar xzf %{S:5} # rapdiscmake
# hardway patching
%if %{without allarches }
sed 's@set(supported_archs "70" "75" "80" "86" "90")@set(supported_archs "90")@' -i %{_sourcedir}/rapids-cmake-%{rapidscmake_vers}/rapids-cmake/cuda/set_architectures.cmake
%endif
tar xzf %{S:6} # rmm
cp %{_sourcedir}/rapids-cmake-%{rapidscmake_vers}/RAPIDS.cmake %{_sourcedir}/rmm-%{rmm_vers}/RMM_RAPIDS-20.04.cmake
tar xzf %{S:7} # raft
cp %{_sourcedir}/rapids-cmake-%{rapidscmake_vers}/RAPIDS.cmake %{_sourcedir}/raft-branch-%{raft_vers}/RAFT_RAPIDS-20.04.cmake
cp %{_sourcedir}/rapids-cmake-%{rapidscmake_vers}/RAPIDS.cmake %{_sourcedir}/raft-branch-%{raft_vers}/cpp/RAFT_RAPIDS.cmake
tar xzf %{S:8} # cccl
tar xzf %{S:9} # cutlass
tar xzf %{S:10} # cuCollections
tar xzf %{S:11} # hnswlib
patch -p1 -d %{_sourcedir}/hnswlib-%hnswlib_vers < %{_sourcedir}/raft-branch-%{raft_vers}/cpp/cmake/patches/hnswlib.diff
%endif
popd
%autosetup -p1
tar xJf %{S:1}
mkdir -p cmake_build/thirdparty
pushd .
cd cmake_build/thirdparty
%make_depend_src %{SOURCE2} milvus-proto
popd
# setup rust vendor stuff for tantivy
pushd .
mkdir -p ~/.cargo/bin/
cd internal/core/thirdparty/tantivy/tantivy-binding
mkdir -p .cargo
cp %{SOURCE20} .cargo/config.toml
tar xJf %{SOURCE21}
# tantivy bindings are severly br0ken
mkdir -p ~/usr/include \
         ~/usr/lib \
         %{nil}
popd
# some more weired thing to setup testing of tantivity
mkdir -p ~/usr/lib ~/usr/lib

%build
# create protobuf for milvus core
mkdir -p cmake_build/bin
ln -s $(which protoc) cmake_build/bin/protoc
./scripts/generate_proto.sh
# recreate other protobuf files
#pushd .
#cd cmake_build/thirdparty/milvus-proto
#rm -rf cmake-build
##bash ./scripts/core_build.sh
#bash ./scripts/proto_gen_go.sh $(pwd)/bin
#popd
# proto files from milvus-storage
#make -C internal/core/thirdparty/milvus-storage/milvus-storage-src/cpp/ proto
make -C %{_sourcedir}/milvus-storage-%{milvus_storage_vers_full}/cpp/ proto
%if %{with cppgpu}
# Copy RAPIDS.cmake into knowhere-build
mkdir -p internal/core/build/thirdparty/knowhere/knowhere-build
cp %{_sourcedir}/rapids-cmake-%{rapidscmake_vers}/RAPIDS.cmake internal/core/build/thirdparty/knowhere/knowhere-build/
# Copy CPM.cmake version 0.38.5 into code to prevent download attempts
mkdir -p internal/core/build/cmake/
cp %{S:22} internal/core/build/cmake/
%endif
%if %{with cpp}
# now at first deal with conan
pushd .
rm -rf ~/.conan2 ~/conan-system # silly hack as this directory isn't cleaned up
conan profile detect
mkdir ~/conan-system
cd ~/conan-system
cp %{S:60} %{S:61} .
systemlibs="rocksdb;zlib;libunwind;liblzma;roaring"
cmake -DLIBRARIES=$systemlibs .
cat default >> ~/.conan2/profiles/default
for lib in $(echo $systemlibs | tr ';' ' '); do
  conan create ./${lib} --user abuild --channel obs
done
popd
%endif

# build the c++ core of mivlus
%if 0%{?sle_version} == 150600
%define CC_VER 12
export CXX=g++-%{CC_VER}
export CC=gcc-%{CC_VER}
%endif
%if %{with cppgpu}
export CUDACXX=/usr/local/cuda-12.5/bin/nvcc
export CUDAARCHS=80
# --verbose   -std=c++20
export NVCC_PREPEND_FLAGS='-ccbin g++-12'
%endif
%if %{with cppcpu}
export CXXFLAGS="-Wno-error=return-type "
export CXXFLAGS="${CXXFLAGS} -march=x86-64-v3" # this is realy ugly, parts of milvus sue -msse4.2 what
                                               # will make the code only to runable on x86-64-v3 so set
                                               # set this explicitly. Also folly needs this
export CXXFLAGS="${CXXFLAGS} -DGLOG_USE_GLOG_EXPORT" # milvus doesn't start glog correctly
export CXXFLAGS="${CXXFLAGS} -DHAVE_ABSEIL" # avoid opentelemetry-cpp to use its own deprecated abseil
export CXXFLAGS="${CXXFLAGS} -DHAVE_CPP_STDLIB -DHAVE_GSL" # add some more flags, which would be fixed in
                                                           # newer opentelemetry-cpp version, which deprectated
                                                           # Jaeger so we can't use them

pushd .
cd internal/core
conan install . --output-folder=conan --build=missing
%cmake \
  -DCMAKE_TOOLCHAIN_FILE=conan/conan_toolchain.cmake \
  -DCMAKE_POLICY_DEFAULT_CMP0167=OLD \
  -DCMAKE_CXX_STANDARD=17 \
  -DOFFLINEBUILD=ON \
  -DMILVUS_DEPENDENCY_SOURCE=SYSTEM \
  -DWITH_COVERAGE=OFF \
  -DWITH_UT=OFF \
  -DBUILD_STATIC=OFF \
  -DFETCHCONTENT_TRY_FIND_PACKAGE_MODE:BOOL=ALWAYS \
  -DFETCHCONTENT_UPDATES_DISCONNECTED=ON \
  -DFETCHCONTENT_SOURCE_DIR_MILVUS-STORAGE=%{_sourcedir}/milvus-storage-%{milvus_storage_vers_full} \
  -DFETCHCONTENT_SOURCE_DIR_KNOWHERE=%{_sourcedir}/knowhere-%{knowhere_vers} \
  -DTBB_ENABLE_IPO=OFF \
  %if %{with cppgpu}
  -DCUTLASS_NVCC_ARCHS="80" \
  -DCMAKE_CUDA_ARCHITECTURES="80" \
  -DCCCL_ENABLE_TESTING=OFF \
  -DMILVUS_GPU_VERSION=ON \
  -DFETCHCONTENT_SOURCE_DIR_RMM=%{_sourcedir}/rmm-%{rmm_vers} \
  -DFETCHCONTENT_SOURCE_DIR_RAFT=%{_sourcedir}/raft-branch-%{raft_vers} \
  -DFETCHCONTENT_SOURCE_DIR_CCCL=%{_sourcedir}/cccl-%{cccl_vers} \
  -DFETCHCONTENT_SOURCE_DIR_NVIDIACUTLASS=%{_sourcedir}/cutlass-%{cutlass_vers} \
  -DFETCHCONTENT_SOURCE_DIR_CUCO=%{_sourcedir}/cuCollections-%{cucollections_vers} \
  -DFETCHCONTENT_SOURCE_DIR_HNSWLIB=%{_sourcedir}/hnswlib-%{hnswlib_vers} \
  -DSPDLOG_BUILD_SHARED=OFF \
  -DCPM_DOWNLOAD_ALL=OFF \
  -DCPM_USE_LOCAL_PACKAGES=ON \
  -DFETCHCONTENT_SOURCE_DIR_RAPIDS-CMAKE=%{_sourcedir}/rapids-cmake-%{rapidscmake_vers} \
  %endif
  %{nil}

%cmake_build
popd
%else
# now the go build
export CGO_LDFLAGS="-ltbb"
export OBJPREFIX="github.com/milvus-io/milvus/cmd/milvus"
%if %{without gpu}
export BUILD_TAGS="%{version}-%{release}-suse-%{suse_version}"
%else
export BUILD_TAGS="suse-%{suse_version}-%{version}-gpu"
%endif
export BUILD_DATE="$(date --utc --date="@${SOURCE_DATE_EPOCH:-$(date +%s)}" +%Y-%m-%d)"
export GOFLAGS="-buildmode=pie"
export GO_VERSION="$(go version | cut -f 3 -d ' ')"
CGO_ENABLED="1" GO111MODULE=on go build \
  -ldflags="-X=${OBJPREFIX}.BuildTime=${BUILD_DATE} -X=${OBJPREFIX}.BuildTags=${BUILD_TAGS} -X=${OBJPREFIX}.GitCommit=%{version}-%{release}-%{suse_version} -X ${OBJPREFIX}.GoVersion=${GO_VERSION}" \
  -tags dynamic -mod vendor -o milvus ./cmd/main.go
%endif

%install
%if %{without cppcpu}
mkdir -p %{buildroot}/usr/bin/
cat <<EOF  > %{buildroot}/usr/bin/milvus
#!/bin/bash
export MILVUSCONF=/etc/milvus/configs/
milvus-server \$@
EOF
chmod 755  %{buildroot}/usr/bin/milvus
install -Dm 755 milvus %{buildroot}/usr/bin/milvus-server
install -Dm 755 configs/milvus.yaml %{buildroot}/etc/milvus/configs/milvus.yaml
install -Dm 755 configs/advanced/etcd.yaml %{buildroot}/etc/milvus/configs/advanced/etcd.yaml
install -Dm 644 build/rpm/services/milvus.service %{buildroot}%{_unitdir}/milvus.service
mkdir -p %{buildroot}%{_sbindir}/
ln -s %{_sbindir}/service %{buildroot}%{_sbindir}/rcmilvus
%else
cd internal/core
mkdir build/lib
%cmake_install
mkdir -p %{buildroot}%{_libdir}
mv -v %{buildroot}%{_prefix}/lib/libknowhere.so %{buildroot}%{_prefix}/lib/libmilvus_core.so %{buildroot}%{_prefix}/lib/pkgconfig %{buildroot}%{_libdir}
%endif
%if %{with cppgpu}
# remove static raft
rm -rv %{buildroot}%{_libdir}/libraft.a \
       %{buildroot}%{_prefix}/test \
  %{nil}
%endif

%pre
%service_add_pre milvus.service

%post
%service_add_post milvus.service

%preun
%service_del_preun milvus.service

%postun
%service_del_postun milvus.service

%if %{without cpp}
%if %{without gpu}
%files
%else

%files gpu
%endif
%license LICENSE
%doc README.md
%{_bindir}/milvus
%{_bindir}/milvus-server
%{_sysconfdir}/milvus
%{_sbindir}/rcmilvus
%{_unitdir}/milvus.service
%endif

%if %{with cpp}
%if %{with cppgpu}
%files -n milvus-cppgpu
%{_libdir}/libraft.so
%else

%files cppcpu
%endif
%license LICENSE
%doc README.md
%{_libdir}/libknowhere.so
%{_libdir}/libmilvus_core.so

%if %{with cpp}
%if %{with cppgpu}
%files -n milvus-cppgpu-devel
%else

%files cppcpu-devel
%endif
%{_includedir}/storage
%{_includedir}/segcore
%{_includedir}/knowhere
%{_includedir}/indexbuilder
%{_includedir}/common
%{_includedir}/clustering
%{_includedir}/exec
%{_includedir}/futures
%{_includedir}/monitor
%{_libdir}/pkgconfig/milvus_core.pc
%{_libdir}/pkgconfig/knowhere.pc
%if %{with cppgpu}
%{_includedir}/cuco
%{_includedir}/cutlass
%{_includedir}/hnswlib
%{_includedir}/raft
%{_includedir}/raft_runtime
%{_includedir}/rapids
%{_libdir}/rapids
%dir %{_libdir}/rapids
%{_includedir}/rmm
%{_libdir}/cmake/NvidiaCutlassConfig.cmake
%{_libdir}/cmake/NvidiaCutlassTargets.cmake
%{_libdir}/cmake/cuco
%dir %{_libdir}/cmake/cuco
%{_libdir}/cmake/hnswlib
%dir %{_libdir}/cmake/hnswlib
%{_libdir}/cmake/raft
%dir %{_libdir}/cmake/raft
%{_libdir}/cmake/rmm
%dir %{_libdir}/cmake/rmm
%endif
%endif
%endif

%changelog
openSUSE Build Service is sponsored by
Places