Overview

File yup12.pl of Package yup

#!/usr/bin/perl -w
use strict;
use LWP;
use XML::Bare;

my $downloaddir='/abuild/meissner/yup/';

# from e.g. /etc/zypp/credentials.d/SCCcredentials
my $sccuser="";
my $sccpass="";

my @products = (
	'SLES12_x86_64',
	'SLES12_s390x',
	'SLES12-SP1_x86_64',
	'SLES12-SP2_x86_64',
	'SLES12-SP2_RPI_aarch64',
	'SLES12-SP3_x86_64',
	'SLES12-SP4_x86_64',

	'SDK12_x86_64',
	'SDK12_s390x',
	'SDK12-SP1_x86_64',
	'SDK12-SP2_x86_64',
	'SDK12-SP3_x86_64',
	'SDK12-SP4_x86_64',

	'PkgHub12-SP1_x86_64',
	'PkgHub12-SP2_x86_64',
	'PkgHub12-SP3_x86_64',
	'PkgHub12-SP4_x86_64',

	'ASMM12',

	'LEGACY12_x86_64',
	'LEGACY12_s390x',

	'CERTIFICATIONS12_x86_64',

	'WEBSCRIPT12_x86_64',
	'WEBSCRIPT12_s390x',

	'HAE_x86_64',
	'HAE_s390x',

	'HAE_GEO_x86_64',
	'HAE_GEO_s390x',

	'PUBLICCLOUD12_x86_64',
	'PUBLICCLOUD12_s390x',
    
	'HPC12_x86_64',

	'CONTAINERS12_x86_64',

	'TOOLCHAIN12_x86_64',

	'SLEWE12',
	'SLED12',
	'SLED12-SP1',
);

# Change to 1 if you want to download the full released product GA trees aka "pool channels"
my $downloadpool = 0;

# Change to 1 if you want the debuginfo trees
my $downloaddebuginfo = 0;

# Debugging
my $debug = 0;

############# Internal ##############

my $ua = LWP::UserAgent->new;
$ua->agent("yup12/1.0 ");

$ua->credentials("scc.suse.com:443","SCC Connect API",$sccuser,$sccpass);

my %product2scc = (
	'SLES12_x86_64'		=> 'https://scc.suse.com/access/services/1106?credentials=SUSE_Linux_Enterprise_Server_12_x86_64',
	'SLES12_s390x' 		=> 'https://scc.suse.com/access/services/1104?credentials=SUSE_Linux_Enterprise_Server_12_s390x',
	'SLES12-SP1_x86_64'	=> 'https://scc.suse.com/access/services/1311?credentials=SUSE_Linux_Enterprise_Server_12_SP1_x86_64',
	'SLES12-SP2_x86_64'	=> 'https://scc.suse.com/access/services/1346?credentials=SUSE_Linux_Enterprise_Server_12_SP2_x86_64',
	'SLES12-SP2_RPI_aarch64'=> 'https://scc.suse.com/access/services/1403?credentials=SUSE_Linux_Enterprise_Server_for_the_Raspberry_Pi_12_SP2_aarch64',
	'SLES12-SP3_x86_64'	=> 'https://scc.suse.com/access/services/1406?credentials=SUSE_Linux_Enterprise_Server_12_SP3_x86_64',
	'SLES12-SP4_x86_64'	=> 'https://scc.suse.com/access/services/1610?credentials=SUSE_Linux_Enterprise_Server_12_SP4_x86_64',

	'SDK12_x86_64' 		=> 'https://scc.suse.com/access/services/1206?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_x86_64',
	'SDK12_s390x' 		=> 'https://scc.suse.com/access/services/1136?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_s390x',
	'SDK12-SP1_x86_64'	=> 'https://scc.suse.com/access/services/1312?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_x86_64',
	'SDK12-SP2_x86_64'	=> 'https://scc.suse.com/access/services/1355?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_x86_64',
	'SDK12-SP3_x86_64'	=> 'https://scc.suse.com/access/services/1412?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_SP3_x86_64',
	'SDK12-SP4_x86_64'	=> 'https://scc.suse.com/access/services/1615?credentials=SUSE_Linux_Enterprise_Software_Development_Kit_12_SP4_x86_64',

	'PkgHub12-SP1_x86_64'	=> 'https://scc.suse.com/access/services/1461?credentials=SUSE_Package_Hub_12_SP1_x86_64',
	'PkgHub12-SP2_x86_64'	=> 'https://scc.suse.com/access/services/1464?credentials=SUSE_Package_Hub_12_SP2_x86_64',
	'PkgHub12-SP3_x86_64'	=> 'https://scc.suse.com/access/services/1514?credentials=SUSE_Package_Hub_12_SP3_x86_64',
	'PkgHub12-SP4_x86_64'	=> 'https://scc.suse.com/access/services/1798?credentials=SUSE_Package_Hub_12_SP4_x86_64',

	'ASMM12'		=> 'https://scc.suse.com/access/services/1195?credentials=Advanced_Systems_Management_Module_12_x86_64',

	'LEGACY12_x86_64'	=> 'https://scc.suse.com/access/services/1140?credentials=Legacy_Module_12_x86_64',
	'LEGACY12_s390x'	=> 'https://scc.suse.com/access/services/1139?credentials=Legacy_Module_12_s390x',

	'CERTIFICATIONS12_x86_64' => 'https://scc.suse.com/access/services/1357?credentials=Certifications_Module_12_x86_64',

	'WEBSCRIPT12_x86_64'	=> 'https://scc.suse.com/access/services/1143?credentials=Web_and_Scripting_Module_12_x86_64',
	'WEBSCRIPT12_s390x'	=> 'https://scc.suse.com/access/services/1142?credentials=Web_and_Scripting_Module_12_s390x',

	'PUBLICCLOUD12_x86_64'	=> 'https://scc.suse.com/access/services/1203?credentials=Public_Cloud_Module_12_x86_64',
	'PUBLICCLOUD12_s390x'	=> 'https://scc.suse.com/access/services/1202?credentials=Public_Cloud_Module_12_s390x',

	'HPC12_x86_64'		=> 'https://scc.suse.com/access/services/1425?credentials=HPC_Module_12_x86_64',

	'HAE_x86_64'		=> 'https://scc.suse.com/access/services/1241?credentials=SUSE_Linux_Enterprise_High_Availability_Extension_12_x86_64',
	'HAE_s390x'			=> 'https://scc.suse.com/access/services/1240?credentials=SUSE_Linux_Enterprise_High_Availability_Extension_12_s390x',

	'HAE_GEO_x86_64'	=> 'https://scc.suse.com/access/services/1147?credentials=SUSE_Linux_Enterprise_High_Availability_GEO_Extension_12_x86_64',
	'HAE_GEO_s390x'		=> 'https://scc.suse.com/access/services/1146?credentials=SUSE_Linux_Enterprise_High_Availability_GEO_Extension_12_s390x',

	'CONTAINERS12_x86_64'	=> 'https://scc.suse.com/access/services/1321?credentials=Containers_Module_12_x86_64',

	'TOOLCHAIN12_x86_64'	=> 'https://scc.suse.com/access/services/1330?credentials=Toolchain_Module_12_x86_64',

	# These are and will be x86_64 only for SLE12.
	'SLEWE12' 	=> 'https://scc.suse.com/access/services/1205?credentials=SUSE_Linux_Enterprise_Workstation_Extension_12_x86_64',
	'SLED12' 	=> 'https://scc.suse.com/access/services/1108?credentials=SUSE_Linux_Enterprise_Desktop_12_x86_64',
	'SLED12-SP1' 	=> 'https://scc.suse.com/access/services/1322?credentials=SUSE_Linux_Enterprise_Desktop_12_SP1_x86_64',
);

my $url;
my $req;
my $res;

sub download_file_checksum($$$$$$) {
	my ($baseurl,$token,$alias,$file,$shatype,$checksumneeded) = @_;

	if (($shatype ne "") && (-f "$downloaddir/$alias/$file")) {
		open(CHECKSUM,"${shatype}sum <$downloaddir/$alias/$file|");
		my $checksum = <CHECKSUM>;
		close(CHECKSUM);
		chomp $checksum;
		$checksum =~ s/[ -]*$//;
		if ($checksum eq $checksumneeded) {
			print STDERR "$file - in cache with valid checksum.\n";
			return 1;
		}
		print STDERR "$downloaddir/$alias/$file existing checksum mismatch: |$checksum| calculated vs |" . $checksumneeded . "| in repomd.xml\n";
		unlink("$downloaddir/$alias/$file");
	}
	print "DOWNLOADING $file\n";

	if ($file =~ /\//) {
		my $dir = $file;
		$dir =~ s/\/[^\/]*$//;
		mkdir("$downloaddir/$alias/$dir");
	}

	$url = "$baseurl/$file?$token";
	my $req = HTTP::Request->new(GET => $url);
	my $res = $ua->request($req);

	if (!$res->is_success) {
		print STDERR "failed to download $url, reason " , $res->status_line, "\n";
		return 0;
	}
	if (!open(CONTENT,">$downloaddir/$alias/$file")) {
		warn "$downloaddir/$alias/$file:$!\n";
		return 0;
	}
	print CONTENT $res->content;
	close(CONTENT);
	if ($checksumneeded ne "") {
		open(CHECKSUM,"${shatype}sum <$downloaddir/$alias/$file|");
		my $checksum = <CHECKSUM>;
		close(CHECKSUM);
		chomp $checksum;
		$checksum =~ s/[ -]*$//;
		if ($checksum ne $checksumneeded) {
			print STDERR "$downloaddir/$alias/$file checksum mismatch: |$checksum| calculated vs |" . $checksumneeded . "| in repomd.xml\n";
			unlink("$downloaddir/$alias/$file");
			return 0;
		}
	}
	return 1;
}

# we cannot use this as there is no checksum protection
sub download_updateinfo($$$$) {
	my ($baseurl,$token,$alias,$text) = @_;

	my $updatexml = new XML::Bare( text => $text );
	my $updateroot = $updatexml->parse();

	# will only work if we have more than 1 update ... EDONTCARE
	if ("ARRAY" eq ref($updateroot->{'updates'}->{'update'})) {
		for (my $u=0;defined($updateroot->{'updates'}->{'update'}->[$u]);$u++) {
			my $update = $updateroot->{'updates'}->{'update'}->[$u];

			if ("ARRAY" eq ref($update->{'pkglist'}->{'collection'}->{'package'})) {
				for (my $p=0;defined($update->{'pkglist'}->{'collection'}->{'package'}->[$p]);$p++) {
					my $package = $update->{'pkglist'}->{'collection'}->{'package'}->[$p];

					# print "updateinfo filename " . $package->{'src'}->{'value'} . "\n";
				}
			} else {
				my $package = $update->{'pkglist'}->{'collection'}->{'package'};

				#print "updateinfo filename " . $package->{'src'}->{'value'} . "\n";
			}
		}
	} else {
		# channel with just 1 update
		my $update = $updateroot->{'updates'}->{'update'};

		if ("ARRAY" eq ref($update->{'pkglist'}->{'collection'}->{'package'})) {
			for (my $p=0;defined($update->{'pkglist'}->{'collection'}->{'package'}->[$p]);$p++) {
				my $package = $update->{'pkglist'}->{'collection'}->{'package'}->[$p];

				# print "updateinfo filename " . $package->{'src'}->{'value'} . "\n";
			}
		} else {
			my $package = $update->{'pkglist'}->{'collection'}->{'package'};

			#print "updateinfo filename " . $package->{'src'}->{'value'} . "\n";
		}
	}
}

sub download_deltainfo($$$$) {
	my ($baseurl,$token,$alias,$text) = @_;

	my $deltaxml = new XML::Bare( text => $text );
	my $deltaroot = $deltaxml->parse();

	if ("ARRAY" eq ref($deltaroot->{'deltainfo'}->{'newpackage'})) {
		for (my $di=0;defined($deltaroot->{'deltainfo'}->{'newpackage'}->[$di]);$di++) {
			my $update = $deltaroot->{'deltainfo'}->{'newpackage'}->[$di];

			# print STDERR "delta is " . ref($update->{'delta'}) . "\n";

			if ("ARRAY" eq ref($update->{'delta'})) {
				for (my $d=0;defined($update->{'delta'}->[$d]);$d++) {
					my $package = $update->{'delta'}->[$d];

					# print "filename " . $package->{'filename'}->{'value'} . "\n";
					download_file_checksum($baseurl,$token,$alias,
							$package->{'filename'}->{'value'},
							$package->{'checksum'}->{'type'}->{'value'},
							$package->{'checksum'}->{'value'}
					);
				}
			} else {
				my $package = $update->{'delta'};

				# print "filename " . $package->{'filename'}->{'value'} . "\n";
				download_file_checksum($baseurl,$token,$alias,
					$package->{'filename'}->{'value'},
					$package->{'checksum'}->{'type'}->{'value'},
					$package->{'checksum'}->{'value'}
				);
			}
		}
	} else {
		my $update = $deltaroot->{'deltainfo'}->{'newpackage'};

		# print STDERR "delta is " . ref($update->{'delta'}) . "\n";

		if ("ARRAY" eq ref($update->{'delta'})) {
			for (my $d=0;defined($update->{'delta'}->[$d]);$d++) {
				my $package = $update->{'delta'}->[$d];

				# print "filename " . $package->{'filename'}->{'value'} . "\n";
				download_file_checksum($baseurl,$token,$alias,
						$package->{'filename'}->{'value'},
						$package->{'checksum'}->{'type'}->{'value'},
						$package->{'checksum'}->{'value'}
				);
			}
		} else {
			my $package = $update->{'delta'};

			# print "filename " . $package->{'filename'}->{'value'} . "\n";
			download_file_checksum($baseurl,$token,$alias,
				$package->{'filename'}->{'value'},
				$package->{'checksum'}->{'type'}->{'value'},
				$package->{'checksum'}->{'value'}
			);
		}
	}
}

sub download_primary($$$$) {
	my ($baseurl,$token,$alias,$text) = @_;

	my $packagexml = new XML::Bare( text => $text );
	my $packageroot = $packagexml->parse();

	# will only work if we have more than 1 update ... EDONTCARE
	for (my $p=0;defined($packageroot->{'metadata'}->{'package'}->[$p]);$p++) {
		my $package = $packageroot->{'metadata'}->{'package'}->[$p];

		# print "filename " . $package->{'filename'}->{'value'} . "\n" if ($debug);
		download_file_checksum($baseurl,$token,$alias,
			$package->{'location'}->{'href'}->{'value'},
			$package->{'checksum'}->{'type'}->{'value'},
			$package->{'checksum'}->{'value'}
		);
	}
}


sub download_repodata($$) {
	my ($downloadurl,$alias) = @_;

	my ($baseurl,$token) = split(/\?/,$downloadurl);

	print STDERR "$alias downloading $baseurl\n";

	mkdir("$downloaddir/$alias");
	mkdir("$downloaddir/$alias/repodata");

	download_file_checksum($baseurl,$token,$alias,"repodata/repomd.xml.asc","","");
	download_file_checksum($baseurl,$token,$alias,"repodata/repomd.xml.key","","");
	return unless download_file_checksum($baseurl,$token,$alias,"repodata/repomd.xml","","");

	my $repomdxml = new XML::Bare( file => "$downloaddir/$alias/repodata/repomd.xml");
	my $reporoot = $repomdxml->parse();

	for (my $j=0;defined($reporoot->{'repomd'}->{'data'}->[$j]);$j++) {
		my $repodataxml = $reporoot->{'repomd'}->{'data'}->[$j];
		my $type = $repodataxml->{'type'}->{'value'};
		my $location = $repodataxml->{'location'}->{'href'}->{'value'};

		# print STDERR "type = " . $type . "\n";
		# print STDERR "location = " . $repodataxml->{'location'}->{'href'}->{'value'} . "\n";
		# print STDERR "checksum = " . $repodataxml->{'checksum'}->{'value'} . "\n";
		# print STDERR "checksum type = " . $repodataxml->{'checksum'}->{'type'}->{'value'} . "\n";
		# print STDERR "timestamp = " . $repodataxml->{'timestamp'}->{'value'} . "\n";
		# print STDERR "size = " . $repodataxml->{'size'}->{'value'} . "\n";
		# print STDERR "open-size = " . $repodataxml->{'open-size'}->{'value'} . "\n";
		# print STDERR "open-checksum = " . $repodataxml->{'open-checksum'}->{'value'} . "\n";
#  <data type="susedata">
#    <location href="repodata/a5b9125090f4487980e3bf94d469ae3a3c5c6e47016b18a0ddf26e9a7106bce7-susedata.xml.gz"/>
#    <checksum type="sha256">a5b9125090f4487980e3bf94d469ae3a3c5c6e47016b18a0ddf26e9a7106bce7</checksum>
#    <timestamp>1413282656</timestamp>
#    <size>122</size>
#    <open-size>123</open-size>
#    <open-checksum type="sha256">743071394bef1495867b673ef088a678ce105740ee18b0cac194c5a267e1117d</open-checksum>
#  </data>
		# basic sanity
		if ($location !~ /[a-z0-9\-.\/]/) {
			print STDERR "bad character in location $location\n";
			next;
		}
		next unless download_file_checksum($baseurl,$token,$alias,$location,
			$repodataxml->{'checksum'}->{'type'}->{'value'},
			$repodataxml->{'checksum'}->{'value'}
		);

		next if ($type eq "other");
		next if ($type eq "susedata");

		print STDERR "parsing type $type\n" if ($debug);
		if ($location =~ /.gz$/) {
			open(UNCOMPRESSED,"zcat $downloaddir/$alias/$location|")||die "zcat downloaddir/$alias/$location:$!";
		} else {
			open(UNCOMPRESSED,"<$downloaddir/$alias/$location");
		}
		my $text = join("",<UNCOMPRESSED>);
		close(UNCOMPRESSED);

		if ($type eq "updateinfo") {
			download_updateinfo($baseurl,$token,$alias,$text);
			next;
		}
		if ($type eq "deltainfo") {
			download_deltainfo($baseurl,$token,$alias,$text);
			next;
		}
		if ($type eq "primary") {
			download_primary($baseurl,$token,$alias,$text);
			next;
		}
	}
}

if (! -d $downloaddir) {
	die "Download directory $downloaddir does not exist yet, please adjust directory and/or create it.\n";
}

foreach my $product (@products) {
	if (!defined($product2scc{$product})) {
		print STDERR "no scc url mapped for $product, check / adjust \%product2scc\n";
		next;
	}
	$url = $product2scc{$product};
	my $req = HTTP::Request->new(GET => $url);

	my $res = $ua->request($req);

	if (!$res->is_success) {
		print STDERR "failed to download $url, reason " , $res->status_line, "\n";
		next;
	}

	# print STDERR $res->content;

	my $productxml = new XML::Bare( text => $res->content);
	my $productroot = $productxml->parse();

	for (my $i=0;defined($productroot->{'repoindex'}->{'repo'}->[$i]);$i++) {
		my $repoxml = $productroot->{'repoindex'}->{'repo'}->[$i];

		print STDERR "url = " . $repoxml->{'url'}->{'value'} . "\n" if ($debug);
		print STDERR "alias = " . $repoxml->{'alias'}->{'value'} . "\n" if ($debug);
		print STDERR "name = " . $repoxml->{'name'}->{'value'} . "\n" if ($debug);
		print STDERR "autorefresh = " . $repoxml->{'autorefresh'}->{'value'} . "\n" if ($debug);
		print STDERR "enabled = " . $repoxml->{'enabled'}->{'value'} . "\n" if ($debug);
		print STDERR "distro_target = " . $repoxml->{'distro_target'}->{'value'} . "\n" if ($debug);
#  <repo url="https://updates.suse.com/SUSE/Updates/SLE-SERVER/12/x86_64/update?5bc3650926e7f0c7ef485805725f135153d25d5137743176f0365397681adec894b25c0c6d9784d8d5c81258b2be6b675d80082e4457d838b355b45994851352e3767717c245c806d7fa1f071272021f90f6a865728b337161" alias="SLES12-Updates" name="SLES12-Updates" autorefresh="true" enabled="true" distro_target="sle-12-x86_64"/>

		if ($repoxml->{'alias'}->{'value'} =~ /Pool/i && !$downloadpool) {
			print "SKIP " . $repoxml->{'alias'}->{'value'} . " - pool channel not selected for downloading\n";
			next;
		}
		if ($repoxml->{'alias'}->{'value'} =~ /debuginfo/i && !$downloaddebuginfo) {
			print "SKIP " . $repoxml->{'alias'}->{'value'} . " - debuginfo channel not selected for downloading\n";
			next;
		}
		download_repodata($repoxml->{'url'}->{'value'},$repoxml->{'alias'}->{'value'});
	}
}
openSUSE Build Service is sponsored by
Places