File obs_mirror_project of Package obs-server

#!/usr/bin/ruby

# This script mirrors a base distribution from the opensuse.org 
# Build Service. You can use it to create initial base projects
# in your build service to build for.
#
# This script does mirror only build packages, not the sources. 
# Yes, it is just a fast hack for now, please submit cleanups :)
#
# Missing: * switch off building of mirror projects
#            (baserepos are disabled anyway by default)
#          * mirroring of package meta data, needed to build packages
#          * use mirrors by default for downloading

if ARGV.length < 3 || ARGV.length > 5
  puts "\
Call this script with <project> <repository> <architecture> [-v] arguments.
To mirror openSUSE 10.2 as base distro, please call:

  obs_mirror_project openSUSE:10.2 standard i586

"
  exit(1)
end

project=ARGV[0]
repository=ARGV[1]
architecture=ARGV[2]
verbose=false
verbose=1 if ARGV[3] == "-v"

rootdir="/srv/obs/" # package default
ownership="obsrun"

projectdir = rootdir + "/projects/" 

destinationdir = rootdir + "build/" + project + "/" + repository + "/" + architecture + "/:full/"
puts "Destination dir: #{destinationdir}\n" if verbose
downloadurl = "http://download.opensuse.org/repositories/" + project + "/" + repository
puts "Download URL: #{downloadurl}\n" if verbose

require 'rexml/document'
include REXML

# prepare project
puts "Creating directories: #{destinationdir} #{projectdir}\n" if verbose
system("mkdir -p #{destinationdir} #{projectdir}")

# create meta data
puts "Creating project meta data: osc meta prj #{project} > #{projectdir}/#{project}.xml\n" if verbose
system("osc meta prj #{project} > #{projectdir}/#{project}.xml")
puts "Creating project configuration data: osc meta prjconf #{project} > #{projectdir}/#{project}.conf\n" if verbose
system("osc meta prjconf #{project} > #{projectdir}/#{project}.conf")

# download all binary packages
puts "Downloading: osc req -m GET https://api.opensuse.org/build/#{project}/#{repository}/#{architecture}/_repository\n" if verbose

process = File::popen("osc req -m GET https://api.opensuse.org/build/#{project}/#{repository}/#{architecture}/_repository", "r")
filelist = Document.new process
filelist.elements.each("binarylist/binary") { |binary| 
  fname = binary.attributes["filename"];

  # skip src and nosrc rpms
  inform_user_about_hacky_reget=false
  unless fname[-7..-1] == "src.rpm"
    if File.file?("#{destinationdir}/#{fname}")

      puts "package #{destinationdir}/#{fname} already exists."
      inform_user_about_hacky_reget=true

      next

      # TODO: maybe get server side file size or md5 hash or both into
      # _repository, then test if file size matches?  Or maybe even
      # have reget, resuming at file size, then check md5?

      # Checking the integrity of the file, removing it if it's broken
      # to refetch it from the server like this takes aaages, no fun.

      #if system("rpm --checksig #{destinationdir}/#{fname}")
      #  next
      #else
      #  puts "Removing broken download #{destinationdir}/#{fname}"
      #  File.unlink("#{destinationdir}/#{fname}")
      #end

    end
    if verbose
      puts "Downloading: osc req -m GET https://api.opensuse.org/build/#{project}/#{repository}/#{architecture}/_repository/#{fname}"
    else
      print "Download " + fname + "\n"
    end
    # FIXME: we can not use the mirrors atm, since the full expanded package name is unknown
    #    unless system("wget -q -N -O #{destinationdir}/#{fname} #{downloadurl}/#{archdir}/#{fname}")
    # fallback to api download
    #      print "failed, try via api for #{fname} \n"
    
    unless system("osc req -m GET https://api.opensuse.org/build/#{project}/#{repository}/#{architecture}/_repository/#{fname} > #{destinationdir}/#{fname}")
      print "Unable to download #{fname} from mirror AND api\n"
      exit(1)
    end
  end

  if inform_user_about_hacky_reget
    puts "\

I skipped the download of packages that already exist.  I did not
check wether those files were correctly or completely downloaded
before.

When you restart the scheduler it will tell you about broken packages
in it's log file in /srv/obs/log/ .  Simply remove the broken package
and re-run this script to update the broken file.

The paranoid may run rpm --checksig, like this:
  rpm --checksig #{destinationdir}/*.rpm

"
  end
}

# fix ownership in case this script did run as root
puts "Changing ownerchip: chown -R #{ownership} #{rootdir}/build #{projectdir}" if verbose
system("chown -R #{ownership} #{rootdir}/build #{projectdir}")

puts "\
Mirroring succeeded :)
Please restart the scheduler to rescan your projects:

  rcobsscheduler restart"