File upstream-collect.sh of Package translation-update-upstream

#!/bin/bash

# This program collects all upstream translations defined in
# configuration and merges them together.

# If you want to use po files for completing of translations, you can remove --no-fuzzy-matching.
# It will run much slower, but you could start with fuzzy matches.

WORK_DIR=$PWD
DEBUG=false

set -o errexit
shopt -s nullglob

source ${0%.sh}.conf

function rpmprep {
    RPMDIR=$HOME/.var.rpmpatch$$
    rm -rf BUILD $HOME/.var.rpmpatch$$
    trap "rm -rf $RPMDIR" 0
    mkdir -p BUILD $RPMDIR
    # Add "%BUILD_ORIG 1" to support original source translation:
    cat >$RPMDIR/macros <<EOF
%_sourcedir      $PWD
%_builddir       $PWD/BUILD
EOF

    eval rpmbuild --macros=/usr/lib/rpm/macros:/usr/lib/rpm/suse_macros:/usr/lib/rpm/platform/$(uname -i)-linux/macros:/etc/rpm/\\\*:$RPMDIR/macros --nodeps -bp ${*:-*.spec}
    rm -rf $RPMDIR
    trap - 0
}

ONLY_PACKAGE=
FULL_PROCESS=true
SNAPSHOT=$(LC_ALL=C LANG=C date +%Y%m%d)
case $1 in
    --help )
	# "only_new" is tricky: Processing new is the default. "only_new" has no match => only_new.
	echo "Usage: $0 [ package | only_new ]"
	exit
	;;
    # no arguments: process everything
    "" )
	;;
    * )
	ONLY_PACKAGE="$1"
	FULL_PROCESS=false
	;;
esac

rm -rf UPSTREAM
mkdir UPSTREAM
if ! test -d STAMPS ; then
    mkdir PACKAGES UPDATE STAMPS
    rm -f upstream-collect.log
fi

# wd may contain ":" in the name, use ~/ instead:
mkdir -p ~/.upstream-collect.tmp
cat >~/.upstream-collect.tmp/translation-update-upstream <<EOF
#!/bin/sh
echo "Dummy translation-update-upstream for upstream-collect.sh. Skipping merge of old translations."
if test -z "\$1" ; then
    touch po/.translation-update-upstream-implemented
else
    touch "\$1"/.translation-update-upstream-implemented
fi
EOF
chmod +x ~/.upstream-collect.tmp/translation-update-upstream
export PATH=~/.upstream-collect.tmp:$PATH

if ! $FULL_PROCESS ; then
    # more tarballs are available => use the latest one
    # FIXME: Fix 20090213.10 < 20090213.9
    # (but it should not happen for people who update and submit)
    for ARCHIVE_ in translation-update-upstream-*.tar.bz2 ; do
	ARCHIVE=$ARCHIVE_
    done
    if ! test -d UPDATE_OLD ; then
	mkdir UPDATE_OLD
	cd UPDATE_OLD
	tar -jxf ../$ARCHIVE
	# If it is not a full process, increment only release
	# SNAPSHOT 20090213 -> 20090213.1, 20090213.1 -> 20090213.2
    fi
    SNAPSHOT=${ARCHIVE#translation-update-upstream-}
    SNAPSHOT=${SNAPSHOT%.tar.bz2}
    SNAPSHOT_PART1=${SNAPSHOT%.*}
    SNAPSHOT_PART2=${SNAPSHOT#*.}
    if test "$SNAPSHOT_PART1" = "$SNAPSHOT" ; then
	SNAPSHOT_PART2=1
    else
	let SNAPSHOT_PART2++
    fi
    SNAPSHOT=$SNAPSHOT_PART1.$SNAPSHOT_PART2
    cd ..
fi

for TLST in *.tlst ; do
    exec <$WORK_DIR/$TLST

    SERIAL=0
    while read PACKAGE DOMAIN METHOD REPO DIR BRANCH ; do

    	# Continue for empty lines and comments
	if test "${PACKAGE###}" != "$PACKAGE" ; then
	    continue
	fi
	if test -z "$PACKAGE" ; then
	    continue
	fi

	echo
	echo "$(tput setf 3)Processing: package=$PACKAGE gettext-package=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:-(default)}$(tput init)"

	# NOTE: Force a limitation: tlst rules for one package must be placed on contiguous line sequence
	if ! $DEBUG ; then
	    if test "$OLD_PACKAGE" != "$PACKAGE" ; then
		if test -n "$RPMPKGDIR" ; then
		    rm -rf $RPMPKGDIR
		fi
	    fi
	    OLD_PACKAGE=$PACKAGE
	fi

	if test -d $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__} ; then
	    echo "   Already successfully processed. Skipping..."
	    continue
	fi

	if test -n "$ONLY_PACKAGE" ; then
	    if test "$ONLY_PACKAGE" != "$PACKAGE" ; then
		if test -d $WORK_DIR/UPDATE_OLD/po/$DOMAIN ; then
		    if test -d $WORK_DIR/UPDATE/po/$DOMAIN ; then
			echo "Should not happen. Internal error."
			exit 255
		    else
			echo "   Not scheduled to process. Recycling old update..."
			mkdir -p $WORK_DIR/UPDATE/po
			 mv $WORK_DIR/UPDATE_OLD/po/$DOMAIN $WORK_DIR/UPDATE/po/
		    fi
		else
		    if test -d $WORK_DIR/UPDATE/po/$DOMAIN ; then
			echo "   Already recycled old update..."
		    else
			echo "   Not scheduled to process. No update available..."
		    fi
		fi
		mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
		touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
		continue
	    fi
	fi

	cd $WORK_DIR/PACKAGES

	RPMPKGDIR=$(echo $WORK_DIR/PACKAGES/$OSC_REPOSITORY/$PACKAGE)

	if ! test -f $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok ; then
	    if ! test -d "$RPMPKGDIR" ; then
		osc ${OSC_APIURL:+--apisrv=$OSC_APIURL} checkout --expand-link $OSC_REPOSITORY $PACKAGE
	    else
		rm -rf "$RPMPKGDIR" $WORK_DIR/STAMPS/$PACKAGE
		echo "$(tput setf 4)Removed possibly incorrect temporary files from previous runs. Please re-run $0 now.$(tput init)"
		exit 1
	    fi
	    RPMPKGDIR=$(echo $WORK_DIR/PACKAGES/$OSC_REPOSITORY/$PACKAGE)
	    cd $RPMPKGDIR
	    rpmprep $PACKAGE.spec
	else
	# During processing, builddir may contain incomplete po files:
	    rm $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
	fi
	REPODIR=$DIR
	RPMPODIR=$(echo $RPMPKGDIR/BUILD/*/${DIR#*/})

	if test -f $WORK_DIR/${TLST%.tlst}.hook ; then
	    source $WORK_DIR/${TLST%.tlst}.hook
	fi

	cd $RPMPODIR
	if ! test -f .translation-update-upstream-implemented ; then
	    echo "$RPMPODIR: Missing or incorrect translation-update-upstream in the spec file."
	    echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE gettext-package=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: packaging error, package does not call translation-update-upstream properly"
	fi
	if ! intltool-update --gettext-package=$DOMAIN --pot ; then
	    if test -f $DOMAIN.pot ; then
		echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE gettext-package=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: intltool-update error, continuing with original $DOMAIN.pot"
	    else
		echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE gettext-package=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: intltool-update error, no way to update"
		mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$REPODIR/${BRANCH:-__HEAD__}
		continue
	    fi
	fi

	cd $WORK_DIR/UPSTREAM
	let SERIAL++ || :
	mkdir $SERIAL
	cd $SERIAL

	case "$METHOD" in
	    cvs )
		cvs -z3 -d:pserver:anoncvs@$REPO co $REPODIR $BRANCH
		cd $REPODIR
		;;
	    svn )
		if test -z "$BRANCH" ; then
		    svn co $REPO/${REPODIR%%/*}/trunk/${REPODIR#*/}
		else
		    svn co $REPO/${REPODIR%%/*}/branches/$BRANCH/${REPODIR#*/}
		fi
		cd ${REPODIR##*/}
		;;
	    git )
		if ! test -d $WORK_DIR/GIT/${REPO//[\/:.]/_} ; then
		    mkdir -p $WORK_DIR/GIT/${REPO//[\/:.]/_}
		    cd $WORK_DIR/GIT/${REPO//[\/:.]/_}
		    git clone $REPO
		    cd $OLDPWD
		fi
		cp -a $WORK_DIR/GIT/${REPO//[\/:.]/_}/* .
		if test -n "$BRANCH" ; then
		    cd *
		    git checkout remotes/origin/$BRANCH
		    cd $OLDPWD
		fi
		cd $REPODIR
		;;
	    # Web-based Git repository viewer makes possible to download particular file.
	    cgit )
		# Some tricks to be able to recycle git:// URI
		CGIT_URI=$REPO
		CGIT_URI=${CGIT_URI/git:\/\/anongit./http://cgit.}
		CGIT_URI=${CGIT_URI/git:\/\//http://}
		CGIT_BRANCH=${BRANCH:+?id=$BRANCH}
		CGIT_SERVER=${CGIT_URI#http://}
		CGIT_SERVER=${CGIT_SERVER%%/*}
		curl $CGIT_URI/tree/${REPODIR#*/}$CGIT_BRANCH | sed -n 's:^.*class='\''ls-blob'\'' href='\''\([^'\'']*\)'\''.*$:\1:p' |
		    while read ; do
			wget -N http://$CGIT_SERVER${REPLY/\/tree\///plain/}
		    done
		;;
	    # standard http directory with po files (BRANCH is not supported)
	    http )
		wget -N -r --no-parent --level=1 http://$REPO/$REPODIR/$DIR
		cd $REPO/$REPODIR/$DIR
		;;
	    # GNOME Translation project l10n directory
	    gtp )
		GTP_NAME_BASE=${REPODIR%%/*}
		# Projects with multiple domains have custom handling in GTP.
		case $DOMAIN in
		    gimp20-libgimp ) GTP_NAME_BASE=gimp-libgimp ;;
		    gimp20-python ) GTP_NAME_BASE=gimp-python ;;
		    gimp20-script-fu ) GTP_NAME_BASE=gimp-script-fu ;;
		    gimp20-std-plug-ins ) GTP_NAME_BASE=gimp-plug-ins ;;
		    gimp20-tags ) GTP_NAME_BASE=gimp-tags ;;
		    gimp20-tips ) GTP_NAME_BASE=gimp-tips ;;
		    gnumeric-functions ) GTP_NAME_BASE=gnumeric-functions ;;
		esac
		curl http://$REPO/${REPODIR%%/*}.${BRANCH:-master}/ | sed -n 's:^.*href="\([^"]*\.po\)".*$:\1:p' |
		    while read ; do
			case $REPLY in
			    $GTP_NAME_BASE.${BRANCH:-master}.*)
				wget -N http://$REPO/${REPODIR%%/*}.${BRANCH:-master}/$REPLY
				mv $REPLY ${REPLY#$GTP_NAME_BASE.${BRANCH:-master}.}
				;;
			esac
		    done
		;;
	    tbz )
		wget -N $REPO
		tar -jxf ${REPO##*/}
		cd $REPODIR
		;;
	    tgz )
		wget -N $REPO
		tar -zxf ${REPO##*/}
		cd $REPODIR
		;;
	    * )
		echo "$PACKAGE: Unknown update method $METHOD"
		exit 1
		;;
	esac

	for PO in *.po ; do
	    # step 0: Merge new po file into old project. Removes unused (too new) translations.
	    if ! msgmerge --no-fuzzy-matching $PO $RPMPODIR/$DOMAIN.pot -o ${PO%.po}-backport.po ; then
		echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE gettext-package=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgmerge error"
		continue
	    fi
	    if test -f $RPMPODIR/$PO ; then
	    # step 1: Clean the po file to be safe.
		if ! msgmerge --no-fuzzy-matching $RPMPODIR/$PO $RPMPODIR/$DOMAIN.pot -o $RPMPODIR/${PO%.po}-clean.po ; then
		    echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE gettext-package=$DOMAIN directory=$RPMPODIR po=$PO: package msgmerge error"
		    # Failed initial msgmerge is fatal. There is no way to update. Build may fail.
		    continue
		fi
		# Do the magic:
		# step 2: Merge new po and previous updates (if any).
		if test -f $RPMPODIR/${PO%.po}-updates.po ; then
		    $WORK_DIR/msgheadermerge $RPMPODIR/${PO%.po}-updates.po $PO ${PO%.po}-uheader.po
		    msgcat --force-po --use-first ${PO%.po}-uheader.po ${PO%.po}-backport.po $RPMPODIR/${PO%.po}-updates.po -o ${PO%.po}-join.po
		else
		    cp -a ${PO%.po}-backport.po ${PO%.po}-join.po
		fi
		# step 3: Join both translations, without --use-first string changes will disappear as fuzzy.
		msgcat --force-po ${PO%.po}-join.po $RPMPODIR/${PO%.po}-clean.po -o ${PO%.po}-allfz.po
		msgcat --use-first --force-po ${PO%.po}-join.po $RPMPODIR/${PO%.po}-clean.po -o ${PO%.po}-all.po
		# step 4: Find string fixes (existed before, now different).
		msgcat --force-po --unique ${PO%.po}-all.po ${PO%.po}-allfz.po -o ${PO%.po}-fixes.po
		# step 5: Find newly translated strings (translation removal is not supported).
		msgcat --force-po --unique $RPMPODIR/${PO%.po}-clean.po ${PO%.po}-all.po -o ${PO%.po}-additions.po
		# step 6: Join both to collect all known fixes.
		msgcat ${PO%.po}-fixes.po ${PO%.po}-additions.po -o $RPMPODIR/${PO%.po}-updatesraw.po
		# Are there any updated? If no, game over.
		if test -f $RPMPODIR/${PO%.po}-updatesraw.po ; then
		    # step 7: Compose the best po file header.
		    $WORK_DIR/msgheadermerge $RPMPODIR/$PO ${PO%.po}-join.po ${PO%.po}-header.po --newdate
		    # step 8: And yet another ugly game to get rid commented out dust.
		    sed '/#~/d' <$RPMPODIR/${PO%.po}-updatesraw.po >$RPMPODIR/${PO%.po}-updates.po~
		    # step 9: Merge correct header to the updates file.
		    msgcat --no-location --use-first ${PO%.po}-header.po $RPMPODIR/${PO%.po}-updates.po~ -o $RPMPODIR/${PO%.po}-updates.po
		fi
	    else
		# step 1: Merge new po and previous updates (if any).
		if test -f ${PO%.po}-updates.po ; then
		    msgcat --force-po --use-first ${PO%.po}-backport.po $RPMPODIR/${PO%.po}-updates.po -o ${PO%.po}-updates.po~
		    mv ${PO%.po}-updates.po~ ${PO%.po}-updates.po
		else
		    cp -a ${PO%.po}-backport.po ${PO%.po}-updates.po
		fi
	    fi
	done

	mkdir -p $WORK_DIR/UPDATE/po/$DOMAIN
	cd $RPMPODIR
	for POX in *-updates.po ; do
	    PO=${POX/-updates/}
	    cp -a $POX $WORK_DIR/UPDATE/po/$DOMAIN/$PO
	done

	mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
	touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok

	if ! $DEBUG ; then
	    rm -rf $WORK_DIR/UPSTREAM/$SERIAL
	fi

    done

done

if ! $DEBUG ; then
    if test -n "$RPMPKGDIR" ; then
	rm -rf $RPMPKGDIR
    fi
fi

cd $WORK_DIR/UPDATE
tar -j -c -f $WORK_DIR/translation-update-upstream-$SNAPSHOT.tar.bz2 po

cd $WORK_DIR
if ! $DEBUG ; then
    rm -rf UPSTREAM PACKAGES UPDATE UPDATE_OLD STAMPS BIN
fi
rm -rf ~/.upstream-collect.tmp

echo ""
echo "$(tput setf 2)Done. Please update version date in the spec file.$(tput init)"