File refresh-sources.sh of Package openlist
#!/usr/bin/env bash
set -euo pipefail
# Refresh sources and build RPM packages
#
# This script automates the maintenance workflow:
# 1. Downloads Source0 (upstream tarball) using spectool
# 2. Generates Source1 (vendor tarball) with go mod vendor
# 3. Updates Debian packaging files (.dsc, debian.changelog)
# 4. Copies both sources to ~/rpmbuild/SOURCES and project root
# 5. Builds source RPM (rpmbuild -bs) or binary RPM (rpmbuild -bb)
usage() {
cat <<'EOF'
Usage:
bash refresh-sources.sh [OPTIONS]
Options:
--spec <path> Path to spec file (default: auto-detect in current/parent dir)
--no-srpm Skip rpmbuild -bs step
--binary Also build binary RPM (rpmbuild -bb) after SRPM
--force Re-download Source0 and regenerate vendor tarball
--no-debian Skip updating Debian packaging files
--bump Increment Debian release number (default: keep current)
--tags <tags> Override build tags (comma-separated)
-h, --help Show this help
Examples:
# Standard workflow: refresh sources and build SRPM
bash refresh-sources.sh
# Only refresh sources without building
bash refresh-sources.sh --no-srpm
# Build both source and binary RPM
bash refresh-sources.sh --binary
# Force re-download of Source0 and regenerate vendor
bash refresh-sources.sh --force
# Bump Debian release number
bash refresh-sources.sh --bump
EOF
}
# Auto-detect spec file
find_spec_file() {
local search_dirs=("$PWD" "$(dirname "$PWD")")
for dir in "${search_dirs[@]}"; do
local specs=( "$dir"/*.spec )
if [[ ${#specs[@]} -eq 1 && -f "${specs[0]}" ]]; then
echo "${specs[0]}"
return 0
fi
done
return 1
}
# Generate vendor tarball from Source0
generate_vendor_tarball() {
local source_tar="$1"
local out_tar="$2"
local tags="$3"
local patches_dir="$4"
local workdir
workdir="$(mktemp -d)"
cleanup_vendor() {
# Go module cache files are often read-only by design; make them writable so
# the temp directory can be removed cleanly.
chmod -R u+rwX "$workdir" 2>/dev/null || true
rm -rf "$workdir" 2>/dev/null || true
}
trap cleanup_vendor RETURN
# Determine top-level directory in the tarball.
local root_dir
root_dir="$(tar -tf "$source_tar" | head -n 1 | cut -d/ -f1 || true)"
if [[ -z "$root_dir" ]]; then
echo "ERROR: unable to determine top-level dir in $source_tar" >&2
return 1
fi
tar -xf "$source_tar" -C "$workdir"
local srcdir="$workdir/$root_dir"
if [[ ! -d "$srcdir" ]]; then
echo "ERROR: extracted source dir not found: $srcdir" >&2
return 1
fi
cd "$srcdir"
if [[ ! -f go.mod ]]; then
echo "ERROR: go.mod not found in extracted source" >&2
return 1
fi
# Apply patches before vendoring (important for go.mod/go.sum changes)
if [[ -n "$patches_dir" ]]; then
local patches
mapfile -t patches < <(find "$patches_dir" -maxdepth 1 -name 'patch-*.patch' -type f | sort)
if [[ ${#patches[@]} -gt 0 ]]; then
echo " Applying ${#patches[@]} patch(es) before vendoring..."
for patch_file in "${patches[@]}"; do
local patch_name
patch_name="$(basename "$patch_file")"
echo " - $patch_name"
if ! patch -p1 -i "$patch_file" --no-backup-if-mismatch -s; then
echo "ERROR: failed to apply patch: $patch_name" >&2
return 1
fi
done
fi
fi
# Use build tags from spec
export GOFLAGS="-tags=${tags}"
# Keep module cache local to avoid polluting user cache.
export GOMODCACHE="$workdir/.gomodcache"
# Ensure go.mod and go.sum are consistent before vendoring.
echo " Running: go mod tidy"
go mod tidy
# Generate vendor/ directory
rm -rf vendor
echo " Running: go mod vendor (tags=jsoniter)"
go mod vendor
if [[ ! -d vendor ]]; then
echo "ERROR: vendor/ directory was not created" >&2
return 1
fi
# Create tarball with vendor/, go.mod, and go.sum
rm -f "$out_tar"
echo " Writing: $out_tar"
tar -czf "$out_tar" -C "$srcdir" vendor go.mod go.sum
return 0
}
# Apply patches to extracted source and repackage
apply_patches_to_source() {
local source_tar="$1"
local out_tar="$2"
local patches_dir="$3"
local workdir
workdir="$(mktemp -d)"
cleanup_patch() {
rm -rf "$workdir" 2>/dev/null || true
}
trap cleanup_patch RETURN
# Determine top-level directory in tarball
local root_dir
root_dir="$(tar -tf "$source_tar" | head -n 1 | cut -d/ -f1 || true)"
if [[ -z "$root_dir" ]]; then
echo "ERROR: unable to determine top-level dir in $source_tar" >&2
return 1
fi
echo " Extracting: $source_tar"
tar -xf "$source_tar" -C "$workdir"
local srcdir="$workdir/$root_dir"
if [[ ! -d "$srcdir" ]]; then
echo "ERROR: extracted source dir not found: $srcdir" >&2
return 1
fi
cd "$srcdir"
# Find and apply patches in order
local patches
mapfile -t patches < <(find "$patches_dir" -maxdepth 1 -name 'patch-*.patch' -type f | sort)
if [[ ${#patches[@]} -eq 0 ]]; then
echo " No patches found in $patches_dir"
return 0
fi
echo " Found ${#patches[@]} patch(es), applying..."
for patch_file in "${patches[@]}"; do
local patch_name
patch_name="$(basename "$patch_file")"
echo " Applying: $patch_name"
if ! patch -p1 -i "$patch_file" --no-backup-if-mismatch; then
echo "ERROR: failed to apply patch: $patch_name" >&2
return 1
fi
done
# Repackage the patched source
cd "$workdir"
rm -f "$out_tar"
echo " Repackaging: $out_tar"
tar -czf "$out_tar" "$root_dir"
return 0
}
# Update spec file with Patch tags
update_spec_patches() {
local spec_file="$1"
local patches_dir="$2"
# Find all patch files
local patches
mapfile -t patches < <(find "$patches_dir" -maxdepth 1 -name 'patch-*.patch' -type f | sort)
if [[ ${#patches[@]} -eq 0 ]]; then
# Remove existing Patch tags if no patches
sed -i '/^Patch[0-9]*:/d' "$spec_file"
return 0
fi
echo "Updating spec with ${#patches[@]} patch(es)"
# Remove old Patch tags
sed -i '/^Patch[0-9]*:/d' "$spec_file"
# Insert Patch tags after Source1 line
local patch_tags=""
local idx=0
for patch_file in "${patches[@]}"; do
local patch_name
patch_name="$(basename "$patch_file")"
patch_tags+="Patch${idx}: ${patch_name}\n"
((idx++)) || true
done
# Insert after Source1 line
sed -i "/^Source1:/a\\${patch_tags}" "$spec_file"
# Ensure %autosetup is used with -p1 in %prep section (remove -q if present)
if grep -q '^%autosetup' "$spec_file"; then
# Update existing %autosetup: keep -n parameter but ensure -p1 and remove -q
# Extract the -n parameter if it exists
local current_autosetup
current_autosetup=$(grep '^%autosetup' "$spec_file" | head -1)
# Check if -n parameter exists
if echo "$current_autosetup" | grep -q -- '-n'; then
# Keep the existing -n parameter, add -p1, remove -q
sed -i 's/^%autosetup.*/%autosetup -p1 -n %{software_name}-%{upstream_version}/' "$spec_file"
else
# Just add -p1
sed -i 's/^%autosetup.*/%autosetup -p1/' "$spec_file"
fi
else
# Replace %setup with %autosetup if present
if grep -q '^%setup' "$spec_file"; then
sed -i 's/^%setup.*/%autosetup -p1 -n %{software_name}-%{upstream_version}/' "$spec_file"
fi
fi
return 0
}
# Generate debian.series for patch application
generate_debian_series() {
local series_file="$1"
local patches_dir="$2"
# Find all patch files
local patches
mapfile -t patches < <(find "$patches_dir" -maxdepth 1 -name 'patch-*.patch' -type f | sort)
if [[ ${#patches[@]} -eq 0 ]]; then
# Remove series file if no patches
rm -f "$series_file"
return 0
fi
echo "Generating debian.series with ${#patches[@]} patch(es)"
# Create series file with patch filenames
: > "$series_file"
for patch_file in "${patches[@]}"; do
basename "$patch_file" >> "$series_file"
done
return 0
}
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
spec_path=""
no_srpm=false
build_binary=false
force_download=false
no_debian=false
bump_release=false
override_tags=""
while [[ $# -gt 0 ]]; do
case "$1" in
--spec)
spec_path="$2"; shift 2 ;;
--no-srpm)
no_srpm=true; shift ;;
--binary)
build_binary=true; shift ;;
--force)
force_download=true; shift ;;
--no-debian)
no_debian=true; shift ;;
--tags)
override_tags="$2"; shift 2 ;;
--bump)
bump_release=true; shift ;;
-h|--help)
usage; exit 0 ;;
*)
echo "Unknown argument: $1" >&2
usage; exit 2 ;;
esac
done
# Auto-detect spec file if not provided
if [[ -z "$spec_path" ]]; then
if ! spec_path="$(find_spec_file)"; then
echo "ERROR: no .spec file found in current or parent directory" >&2
echo "Use --spec to specify the spec file path" >&2
exit 1
fi
echo "Auto-detected spec: $spec_path"
fi
if [[ ! -f "$spec_path" ]]; then
echo "ERROR: spec file not found: $spec_path" >&2
exit 1
fi
# Resolve absolute paths
spec_path="$(realpath "$spec_path")"
project_root="$(dirname "$spec_path")"
# Parse spec variables
software_name="$(awk '/^%global[[:space:]]+software_name[[:space:]]+/ {print $3; exit}' "$spec_path" || true)"
upstream_version="$(awk '/^%global[[:space:]]+upstream_version[[:space:]]+/ {print $3; exit}' "$spec_path" || true)"
if [[ -z "$software_name" || -z "$upstream_version" ]]; then
echo "ERROR: failed to parse software_name/upstream_version from spec" >&2
exit 1
fi
# Parse build tags from spec
spec_tags="$(awk '/^%global[[:space:]]+build_tags[[:space:]]+/ {print $3; exit}' "$spec_path" || true)"
if [[ -z "$spec_tags" ]]; then
echo "ERROR: failed to parse build_tags from spec. Ensure '%global build_tags ...' exists in spec." >&2
exit 1
fi
tags="${override_tags:-$spec_tags}"
echo "==> Project: $software_name"
echo "==> Version: $upstream_version"
echo "==> Spec: $spec_path"
echo "==> Tags: $tags"
echo
# Check required tools
for cmd in spectool rpmbuild tar go; do
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "ERROR: required command not found: $cmd" >&2
if [[ "$cmd" == "spectool" ]]; then
echo "Hint: install rpmdevtools for spectool" >&2
fi
exit 1
fi
done
# Setup rpmbuild directory structure
rpmbuild_root="${HOME}/rpmbuild"
rpmbuild_sources="${rpmbuild_root}/SOURCES"
rpmbuild_srpms="${rpmbuild_root}/SRPMS"
for dir in SOURCES SPECS SRPMS BUILD BUILDROOT; do
mkdir -p "${rpmbuild_root}/${dir}"
done
echo "==> Step 1: Downloading Source0 (upstream tarball)"
echo
source0_name="v${upstream_version}.tar.gz"
source0_path_project="$project_root/$source0_name"
source0_path_rpmbuild="$rpmbuild_sources/$source0_name"
# Download to project root first
cd "$project_root"
if [[ -f "$source0_name" && "$force_download" != true ]]; then
echo "Source0 already exists: $source0_name"
echo "Use --force to re-download"
else
# Remove old tarball if forcing
rm -f "$source0_name"
echo "Running: spectool -g $spec_path"
if ! spectool -g "$spec_path"; then
echo "ERROR: spectool failed to download Source0" >&2
echo "Hint: check if Source0 URL in spec is correct" >&2
exit 1
fi
if [[ ! -f "$source0_name" ]]; then
echo "ERROR: Source0 was not downloaded: $source0_name" >&2
exit 1
fi
echo "Downloaded: $source0_name"
fi
echo
echo "==> Step 2: Generating Source1 (vendor tarball)"
echo
source1_name="${software_name}-${upstream_version}-vendor.tar.gz"
source1_path_project="$project_root/$source1_name"
source1_path_rpmbuild="$rpmbuild_sources/$source1_name"
cd "$project_root"
if [[ -f "$source1_name" && "$force_download" != true ]]; then
echo "Source1 already exists: $source1_name"
echo "Use --force to regenerate"
else
# Remove old vendor tarball if forcing
rm -f "$source1_name"
if ! generate_vendor_tarball "$project_root/$source0_name" "$source1_path_project" "$tags" "$project_root"; then
echo "ERROR: failed to generate vendor tarball" >&2
exit 1
fi
# Return to project root after vendor generation (function changes cwd)
cd "$project_root"
if [[ ! -f "$source1_path_project" ]]; then
echo "ERROR: vendor tarball was not created: $source1_name" >&2
exit 1
fi
echo "Generated: $source1_name"
fi
echo
echo "==> Step 2.2: Downloading Source2 (frontend tarball)"
echo
source2_name="${software_name}-frontend-${upstream_version}.tar.gz"
source2_path_project="$project_root/$source2_name"
source2_path_rpmbuild="$rpmbuild_sources/$source2_name"
frontend_url="https://github.com/OpenListTeam/OpenList-Frontend/releases/download/v${upstream_version}/openlist-frontend-dist-lite-v${upstream_version}.tar.gz"
cd "$project_root"
if [[ -f "$source2_name" && "$force_download" != true ]]; then
echo "Source2 already exists: $source2_name"
echo "Use --force to re-download"
else
# Remove old frontend tarball if forcing
rm -f "$source2_name"
echo "Downloading frontend from: $frontend_url"
if ! curl -fsSL --max-time 30 "$frontend_url" -o "$source2_name"; then
echo "ERROR: failed to download frontend tarball" >&2
echo "URL: $frontend_url" >&2
exit 1
fi
if [[ ! -f "$source2_path_project" ]]; then
echo "ERROR: frontend tarball was not downloaded: $source2_name" >&2
exit 1
fi
echo "Downloaded: $source2_name"
fi
echo
# =============================================================================
# Debian packaging files update
# =============================================================================
if [[ "$no_debian" != true ]]; then
echo "==> Step 2.5: Updating Debian packaging files"
echo
dsc_file="$project_root/${software_name}.dsc"
changelog_file="$project_root/debian.changelog"
# Source tarball names for Debian
deb_source0_name="v${upstream_version}.tar.gz"
deb_source1_name="${software_name}-${upstream_version}-vendor.tar.gz"
deb_source2_name="${software_name}-frontend-${upstream_version}.tar.gz"
# Debian version is same as upstream
debian_upstream_version="${upstream_version}"
if [[ -f "$dsc_file" ]]; then
echo "Updating: $dsc_file"
# Calculate checksums for all source files
echo " Calculating checksums..."
source0_size=$(stat -c%s "$project_root/$deb_source0_name")
source0_md5=$(md5sum "$project_root/$deb_source0_name" | awk '{print $1}')
source0_sha256=$(sha256sum "$project_root/$deb_source0_name" | awk '{print $1}')
source1_size=$(stat -c%s "$project_root/$deb_source1_name")
source1_md5=$(md5sum "$project_root/$deb_source1_name" | awk '{print $1}')
source1_sha256=$(sha256sum "$project_root/$deb_source1_name" | awk '{print $1}')
source2_size=$(stat -c%s "$project_root/$deb_source2_name")
source2_md5=$(md5sum "$project_root/$deb_source2_name" | awk '{print $1}')
source2_sha256=$(sha256sum "$project_root/$deb_source2_name" | awk '{print $1}')
# Update Version
sed -i -E "s/^Version:.*/Version: ${debian_upstream_version}/" "$dsc_file"
# Update DEBTRANSFORM-TAR (uppercase, only main source)
sed -i -E "s|^DEBTRANSFORM-TAR:.*|DEBTRANSFORM-TAR: ${deb_source0_name}|" "$dsc_file"
# Remove old Files and Checksums-Sha256 sections
sed -i '/^Files:/,$d' "$dsc_file"
# Append new Files and Checksums sections (all sources declared for OBS)
cat >> "$dsc_file" <<EOF
Files:
${source0_md5} ${source0_size} ${deb_source0_name}
${source1_md5} ${source1_size} ${deb_source1_name}
${source2_md5} ${source2_size} ${deb_source2_name}
Checksums-Sha256:
${source0_sha256} ${source0_size} ${deb_source0_name}
${source1_sha256} ${source1_size} ${deb_source1_name}
${source2_sha256} ${source2_size} ${deb_source2_name}
EOF
echo " Updated Version: ${debian_upstream_version}"
echo " Updated checksums for all 3 source files"
else
echo "WARNING: .dsc file not found: $dsc_file"
echo " Please create ${software_name}.dsc manually"
fi
# Update debian.changelog with new version
cat > "$changelog_file" <<EOF
${software_name} (${debian_upstream_version}-1) unstable; urgency=medium
* Update to ${debian_upstream_version}
-- reF1nd <ref1nd@duck.com> $(date -R)
EOF
echo "Updated: ${changelog_file}"
echo
fi
echo "==> Step 2.8: Managing patches for RPM/Debian"
echo
# Update spec file with Patch tags
if ! update_spec_patches "$spec_path" "$project_root"; then
echo "ERROR: failed to update spec patches" >&2
exit 1
fi
# Generate debian.series
debian_series="$project_root/debian.series"
if ! generate_debian_series "$debian_series" "$project_root"; then
echo "ERROR: failed to generate debian.series" >&2
exit 1
fi
if [[ -f "$debian_series" ]]; then
echo "Generated: debian.series"
fi
echo
echo "==> Step 3: Copying sources to ~/rpmbuild/SOURCES"
echo
cp -v "$source0_name" "$rpmbuild_sources/"
cp -v "$source1_name" "$rpmbuild_sources/"
cp -v "$source2_name" "$rpmbuild_sources/"
cp -v "$spec_path" "${rpmbuild_root}/SPECS/"
# Copy additional source files (service, sysusers, rpmlintrc)
for extra_file in "${software_name}.service" "${software_name}@.service" "${software_name}.sysusers" "${software_name}-rpmlintrc"; do
if [[ -f "$project_root/$extra_file" ]]; then
cp -v "$project_root/$extra_file" "$rpmbuild_sources/"
else
echo "WARNING: $extra_file not found in project root"
fi
done
# Copy patch files to rpmbuild/SOURCES
patch_count=0
while IFS= read -r -d '' patch_file; do
cp -v "$patch_file" "$rpmbuild_sources/"
((patch_count++)) || true
done < <(find "$project_root" -maxdepth 1 -name 'patch-*.patch' -type f -print0 2>/dev/null)
if [[ $patch_count -gt 0 ]]; then
echo "Copied $patch_count patch file(s)"
fi
echo
echo "==> Step 3.1: Cleaning up old tarballs"
echo
# Clean old Source0 tarballs in project root (keep current version)
old_source0_count=0
while IFS= read -r -d '' old_tar; do
if [[ "$(basename "$old_tar")" != "$source0_name" ]]; then
echo "Removing old Source0 from project: $(basename "$old_tar")"
rm -f "$old_tar"
((old_source0_count++)) || true
fi
done < <(find "$project_root" -maxdepth 1 -name "v*.tar.gz" -type f -print0 2>/dev/null)
# Clean old Source1 tarballs in project root (keep current version)
old_source1_count=0
while IFS= read -r -d '' old_tar; do
if [[ "$(basename "$old_tar")" != "$source1_name" ]]; then
echo "Removing old Source1 from project: $(basename "$old_tar")"
rm -f "$old_tar"
((old_source1_count++)) || true
fi
done < <(find "$project_root" -maxdepth 1 -name "${software_name}-*-vendor.tar.gz" -type f -print0 2>/dev/null)
# Clean old Source2 (frontend) tarballs in project root (keep current version)
old_source2_count=0
while IFS= read -r -d '' old_tar; do
if [[ "$(basename "$old_tar")" != "$source2_name" ]]; then
echo "Removing old Source2 from project: $(basename "$old_tar")"
rm -f "$old_tar"
((old_source2_count++)) || true
fi
done < <(find "$project_root" -maxdepth 1 -name "${software_name}-frontend-*.tar.gz" -type f -print0 2>/dev/null)
# Clean old tarballs in ~/rpmbuild/SOURCES (keep current versions)
old_rpmbuild_count=0
for pattern in "v*.tar.gz" "${software_name}-*-vendor.tar.gz" "${software_name}-frontend-*.tar.gz"; do
while IFS= read -r -d '' old_tar; do
basename_tar="$(basename "$old_tar")"
if [[ "$basename_tar" != "$source0_name" && "$basename_tar" != "$source1_name" && "$basename_tar" != "$source2_name" ]]; then
echo "Removing old tarball from rpmbuild: $basename_tar"
rm -f "$old_tar"
((old_rpmbuild_count++)) || true
fi
done < <(find "$rpmbuild_sources" -maxdepth 1 -name "$pattern" -type f -print0 2>/dev/null)
done
total_cleaned=$((old_source0_count + old_source1_count + old_source2_count + old_rpmbuild_count))
if [[ $total_cleaned -gt 0 ]]; then
echo "Cleaned up $total_cleaned old tarball(s)"
else
echo "No old tarballs to clean"
fi
echo
if [[ "$no_srpm" == true ]]; then
echo "==> Skipping rpmbuild (--no-srpm specified)"
echo
echo "Sources ready in:"
echo " - Project: $project_root"
echo " - rpmbuild: $rpmbuild_sources"
exit 0
fi
echo "==> Step 4: Building source RPM"
echo
spec_name="$(basename "$spec_path")"
cd "${rpmbuild_root}/SPECS"
echo "Running: rpmbuild -bs $spec_name"
if ! rpmbuild -bs "$spec_name"; then
echo "ERROR: rpmbuild -bs failed" >&2
exit 1
fi
echo
# Find the generated SRPM (use wildcard to match any package name)
srpm_file=$(find "$rpmbuild_srpms" -name "*.src.rpm" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-)
if [[ -n "$srpm_file" && -f "$srpm_file" ]]; then
echo "==> Source RPM built successfully!"
echo " $srpm_file"
echo
if [[ "$build_binary" == true ]]; then
echo "==> Step 5: Building binary RPM"
echo
rpmbuild_rpms="${rpmbuild_root}/RPMS"
echo "Running: rpmbuild -bb $spec_name"
if ! rpmbuild -bb "$spec_name"; then
echo "ERROR: rpmbuild -bb failed" >&2
exit 1
fi
echo
echo "==> Binary RPM(s) built successfully!"
# List generated binary RPMs
rpm_files=$(find "$rpmbuild_rpms" -name "*.rpm" -type f -printf '%T@ %p\n' | sort -rn | head -10 | cut -d' ' -f2-)
if [[ -n "$rpm_files" ]]; then
echo "$rpm_files" | while read -r rpm; do
echo " $rpm"
done
fi
echo
fi
echo "==> SUCCESS!"
echo
echo "Sources:"
echo " Source0: $source0_path_rpmbuild"
echo " Source1: $source1_path_rpmbuild"
echo " Source2: $source2_path_rpmbuild"
echo
echo "Next steps for OBS:"
echo " 1. osc co <project> <package>"
echo " 2. cd <package>"
echo " 3. cp $source0_path_rpmbuild ."
echo " 4. cp $source1_path_rpmbuild ."
echo " 5. cp $spec_path ."
echo " 6. osc addremove"
echo " 7. osc commit -m 'Update to $upstream_version'"
else
echo "WARNING: SRPM file not found in $rpmbuild_srpms"
fi