File upstream-collect.sh of Package translation-update-upstream
#!/bin/bash
# This program collects all upstream translations defined in
# configuration and merges them together.
# If you want to use po files for completing of translations, you can remove --no-fuzzy-matching.
# It will run much slower, but you could start with fuzzy matches.
# Debug mode. Set to true if you want to keep working directories.
DEBUG=false
# Set to false to generate only pot files, true to collect po files.
COLLECT_UPSTREAM=true
# Number of CPUs. When > 1, then parallel processing of more po files is possible.
CPUS=$(cat /proc/cpuinfo | grep processor | wc -l)
WORK_DIR=$PWD
set -o errexit
shopt -s nullglob
source ${0%.sh}.conf
function get_pot_name {
POT=
for POT in *.pot ; do
:
done
}
function validate_po_with_plural_check {
local LOG=$HOME/.validate${1##*/}.log$$
trap "rm -f $LOG" 0
if LC_ALL=C LANG=C msgfmt -c -o /dev/null "$1" 2>$LOG ; then
RC=0
else
RC=1
fi
PLURAL_FAILURE=false
if grep -q 'plural form' $LOG ; then
PLURAL_FAILURE=true
fi
rm $LOG
trap - 0
return $RC
}
function validate_po {
LC_ALL=C LANG=C msgfmt -c -o /dev/null "$1"
}
function rpmprep {
RPMDIR=$HOME/.var.rpmpatch$$
rm -rf BUILD $HOME/.var.rpmpatch$$
trap "rm -rf $RPMDIR" 0
mkdir -p BUILD $RPMDIR
# Add "%BUILD_ORIG 1" to support original source translation:
cat >$RPMDIR/macros <<EOF
%_sourcedir $PWD
%_builddir $PWD/BUILD
EOF
sed -i '
# Remove <RELEASE> tag from Release and delete shell escape comments.
/^Release:/s/[<>]//g
# Remove bash shell escape comments (gstreamer, calls autoreconf that can fail).
/#%(bash/d
# Remove negative expressions in BuildRequires.
:1
s/^\(BuildRequires:.*[[:space:]]\)-[^[:space:]]*/\1/
t1
/^BuildRequires:[[:space:]]*$/d
# Modify some requirements to work without relevant packages installed.
s/%{xulrunner_version}/dummy/g
s/%{mozilla_ver}/dummy/g
s/%kde4_runtime_requires/Requires: dummy/g
' *.spec
eval rpmbuild --macros=/usr/lib/rpm/macros:/usr/lib/rpm/suse_macros:/usr/lib/rpm/platform/$(uname -i)-linux/macros:/etc/rpm/\\\*:$RPMDIR/macros --nodeps -bp ${*:-*.spec}
rm -rf $RPMDIR
trap - 0
}
ONLY_PACKAGES=()
FULL_PROCESS=true
SNAPSHOT=$(LC_ALL=C LANG=C date +%Y%m%d)
MSNAPSHOT=$SNAPSHOT
case $1 in
--help )
# "only_new" is tricky: Processing new is the default. "only_new" has no match => only_new.
echo "Usage: $0 [ package | only_new ]"
exit
;;
# no arguments: process everything
"" )
;;
* )
ONLY_PACKAGES=("$@")
FULL_PROCESS=false
;;
esac
rm -rf UPSTREAM
mkdir UPSTREAM
if ! test -d STAMPS ; then
mkdir -p pot pot-tuu pot-diff po-full po-mandatory-full
mkdir OSC PACKAGES UPDATE STAMPS
rm -f upstream-collect.log
rm -f upstream-collect.domain-map.tmp
fi
# wd may contain ":" in the name, use ~/ instead:
mkdir -p ~/.upstream-collect.tmp
cat >~/.upstream-collect.tmp/translation-update-upstream <<EOF
#!/bin/sh
echo "Dummy translation-update-upstream for upstream-collect.sh. Skipping merge of old translations."
echo \${3:-intltool-update\${2+ --gettext-package=\$2} --pot} >\${1:-po}/.translation-update-upstream-implemented
cd \${1:-po}
# Generate and save a copy of the pot file now and compare later.
eval \${3:-intltool-update\${2+ --gettext-package=\$2} --pot}
mkdir -p tuu
for POT in *.pot ; do
cp -a \$POT tuu/
done
EOF
chmod +x ~/.upstream-collect.tmp/translation-update-upstream
# executable flag does not survive some build systems
chmod +x msgheadermerge msgheadermerge-compose msgheadermerge-parse upstream-collect.sh create-tlst-step*.sh translation-update-upstream-to-translation-update.sh
# Strings in installed instance of gnome-patch-translation may interfere
# with upstream-collect.sh. Use of dummies allows to import upstream
# strings, that are part of openSUSE patches.
for FILE in gnome-patch-translation-prepare gnome-patch-translation-update ; do
cat >~/.upstream-collect.tmp/$FILE <<EOF
#!/bin/sh
set -x
echo "Dummy $FILE for upstream-collect.sh. Skipping gnome-patch-translation."
# gnome-patch-translation may be a symlink (libgweather), that is why we have to check even with mkdir -p.
if ! test -L gnome-patch-translation ; then
mkdir -p gnome-patch-translation
fi
touch po/.gnome-patch-translation-implemented
EOF
chmod +x ~/.upstream-collect.tmp/$FILE
done
export PATH=~/.upstream-collect.tmp:$PATH
if ! $FULL_PROCESS ; then
# more tarballs are available => use the latest one
# FIXME: Fix 20090213.10 < 20090213.9
# (but it should not happen for people who update and submit)
for ARCHIVE_ in translation-update-upstream-*.tar.bz2 ; do
ARCHIVE=$ARCHIVE_
done
if ! test -f STAMPS/UPDATE_old_tarball ; then
cd UPDATE
tar -jxf ../$ARCHIVE
# If it is not a full process, increment only release
# SNAPSHOT 20090213 -> 20090213.1, 20090213.1 -> 20090213.2
cd ..
touch STAMPS/UPDATE_old_tarball
fi
SNAPSHOT=${ARCHIVE#translation-update-upstream-}
SNAPSHOT=${SNAPSHOT%.tar.bz2}
SNAPSHOT_PART1=${SNAPSHOT%.*}
SNAPSHOT_PART2=${SNAPSHOT#*.}
if test "$SNAPSHOT_PART1" = "$SNAPSHOT" ; then
SNAPSHOT_PART2=1
else
let SNAPSHOT_PART2++
fi
SNAPSHOT=$SNAPSHOT_PART1.$SNAPSHOT_PART2
for ARCHIVE_ in translation-update-mandatory-*.tar.bz2 ; do
ARCHIVE=$ARCHIVE_
done
if ! test -f STAMPS/UPDATE_old_mtarball ; then
cd UPDATE
tar -jxf ../$ARCHIVE
# If it is not a full process, increment only release
# SNAPSHOT 20090213 -> 20090213.1, 20090213.1 -> 20090213.2
cd ..
touch STAMPS/UPDATE_old_mtarball
fi
MSNAPSHOT=${ARCHIVE#translation-update-mandatory-}
MSNAPSHOT=${MSNAPSHOT%.tar.bz2}
SNAPSHOT_PART1=${MSNAPSHOT%.*}
SNAPSHOT_PART2=${MSNAPSHOT#*.}
if test "$SNAPSHOT_PART1" = "$MSNAPSHOT" ; then
SNAPSHOT_PART2=1
else
let SNAPSHOT_PART2++
fi
MSNAPSHOT=$SNAPSHOT_PART1.$SNAPSHOT_PART2
fi
SERIAL=0
for TLST in *.tlst ; do
exec <$WORK_DIR/$TLST
while read PACKAGE DOMAIN METHOD REPO DIR BRANCH MANDATORY ; do
# Continue for empty lines and comments
if test "${PACKAGE###}" != "$PACKAGE" ; then
continue
fi
if test -z "$PACKAGE" ; then
continue
fi
if test "$MANDATORY" = "mandatory" ; then
MANDATORY=true
else
MANDATORY=false
fi
echo
echo "$(tput setf 3)Processing: package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:-(default)} mandatory=$MANDATORY$(tput init)"
STATUS=OK
# NOTE: Force a limitation: tlst rules for one package must be placed on contiguous line sequence
if ! $DEBUG ; then
if test "$OLD_PACKAGE" != "$PACKAGE" ; then
if test -n "$RPMPKGDIR" ; then
rm -rf $RPMPKGDIR $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
fi
OLD_PACKAGE=$PACKAGE
fi
if test -d $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__} ; then
echo " Already successfully processed. Skipping..."
continue
fi
if test "${#ONLY_PACKAGES}" -gt 0 ; then
SKIP_PACKAGE=true
for ONLY_PACKAGE in "${ONLY_PACKAGES[@]}" ; do
if test "$ONLY_PACKAGE" = "$PACKAGE" ; then
SKIP_PACKAGE=false
break
fi
done
if $SKIP_PACKAGE ; then
echo " Not scheduled to process. Recycling old update..."
#FIXME mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
# touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
continue
fi
fi
cd $WORK_DIR/OSC
RPMPKGDIR=$WORK_DIR/PACKAGES/$PACKAGE
if ! test -f $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok ; then
if ! test -d "$RPMPKGDIR" ; then
for OSC_REPOSITORY in "${OSC_REPOSITORIES[@]}" ; do
echo "Trying to check-out PACKAGES $PACKAGE from $OSC_REPOSITORY..."
# FIXME: When obs-service-download_url appears in Factory, use --source-service-files
if osc ${OSC_APIURL:+--apiurl=$OSC_APIURL} checkout --server-side-source-service-files --expand-link $OSC_REPOSITORY $PACKAGE >gnome-patch-translation-collect-tmp.log 2>&1 ; then
mv $OSC_REPOSITORY/$PACKAGE $WORK_DIR/PACKAGES/
for FILE in $WORK_DIR/PACKAGES/$PACKAGE/_service\:download_url\:* ; do
mv "$FILE" "${FILE/_service:download_url:/}"
done
break
fi
if ! grep -q "HTTP Error 404" gnome-patch-translation-collect-tmp.log ; then
cat gnome-patch-translation-collect-tmp.log
rm gnome-patch-translation-collect-tmp.log
echo "ERROR: Checkout failed!"
exit 1
fi
done
rm gnome-patch-translation-collect-tmp.log
else
rm -rf "$RPMPKGDIR" $WORK_DIR/STAMPS/$PACKAGE
echo "$(tput setf 4)Removed possibly incorrect temporary files from previous runs. Please re-run $0 now.$(tput init)"
exit 1
fi
cd $RPMPKGDIR
rpmprep $PACKAGE.spec
else
# During processing, builddir may contain incomplete po files:
rm $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
REPODIR=$DIR
RPMPODIR=$(echo $RPMPKGDIR/BUILD/*/${DIR#*/})
if test -f $WORK_DIR/${TLST%.tlst}.hook ; then
source $WORK_DIR/${TLST%.tlst}.hook
fi
cd $RPMPODIR
get_pot_name
REAL_DOMAIN=${POT%.pot}
if test -f .gnome-patch-translation-implemented ; then
echo $PACKAGE >>$WORK_DIR/gnome-patch-translation.lst
fi
if test -f .translation-update-upstream-implemented ; then
if bash ./.translation-update-upstream-implemented ; then
get_pot_name
REAL_DOMAIN=${POT%.pot}
cp -a $REAL_DOMAIN.pot $WORK_DIR/pot/
# pot-tuu DOMAIN is the external domain - LCN may use different domain
cp -a tuu/$REAL_DOMAIN.pot $WORK_DIR/pot-tuu/$DOMAIN.pot
# Verify that patches don't introduce new strings.
msgcomm --uniq $REAL_DOMAIN.pot tuu/$REAL_DOMAIN.pot -o $WORK_DIR/pot-diff/$DOMAIN.pot
if ! test -f .gnome-patch-translation-implemented ; then
if test -f $WORK_DIR/pot-diff/$DOMAIN.pot ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: new pot file contains unique strings, please check gnome-patch-translation"
STATUS=OK_INCOMPLETE
fi
fi
else
# translation-update-upstream is implemented but fails:
get_pot_name
REAL_DOMAIN=${POT%.pot}
if test -f $REAL_DOMAIN.pot ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: pot file update error, continuing with original $DOMAIN.pot"
STATUS=OK_OUTDATED
cp -a $DOMAIN.pot $WORK_DIR/pot/
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: pot file update error, no way to update"
STATUS=POT_ERROR
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$REPODIR/${BRANCH:-__HEAD__}
continue
fi
fi
else
echo "$RPMPODIR: Missing or incorrect translation-update-upstream in the spec file."
# translation-update-upstream is implemented, try the default with the upstream domain:
if intltool-update --gettext-package=$DOMAIN --pot ; then
get_pot_name
REAL_DOMAIN=${POT%.pot}
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: packaging error, package does not call translation-update-upstream properly"
STATUS=TUU_NOT_CALLED
cp -a $DOMAIN.pot $WORK_DIR/pot/
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: packaging error, package does not call translation-update-upstream properly and intltool-update fails, no way to update"
STATUS=TUU_BROKEN
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$REPODIR/${BRANCH:-__HEAD__}
continue
fi
fi
echo $PACKAGE $REAL_DOMAIN $STATUS >>$WORK_DIR/upstream-collect.domain-map.tmp
if $COLLECT_UPSTREAM ; then
cd $WORK_DIR/UPSTREAM
let SERIAL++ || :
mkdir $SERIAL
cd $SERIAL
echo >.info "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}"
case "$METHOD" in
cvs )
cvs -z3 -d:pserver:anoncvs@$REPO co $REPODIR $BRANCH
cd $REPODIR
;;
svn )
if test -z "$BRANCH" ; then
svn co $REPO/${REPODIR%%/*}/trunk/${REPODIR#*/}
else
svn co $REPO/${REPODIR%%/*}/branches/$BRANCH/${REPODIR#*/}
fi
cd ${REPODIR##*/}
;;
git )
if ! test -d $WORK_DIR/GIT/${REPO//[\/:.]/_} ; then
mkdir -p $WORK_DIR/GIT/${REPO//[\/:.]/_}
cd $WORK_DIR/GIT/${REPO//[\/:.]/_}
git clone $REPO
cd $OLDPWD
fi
cp -a $WORK_DIR/GIT/${REPO//[\/:.]/_}/* .
if test -n "$BRANCH" ; then
cd *
git checkout remotes/origin/$BRANCH
cd $OLDPWD
fi
cd $REPODIR
;;
# Web-based Git repository viewer makes possible to download particular file.
cgit )
# Some tricks to be able to recycle git:// URI
CGIT_URI=$REPO
CGIT_URI=${CGIT_URI/git:\/\/anongit./http://cgit.}
CGIT_URI=${CGIT_URI/git:\/\//http://}
CGIT_BRANCH=${BRANCH:+?id=$BRANCH}
CGIT_SERVER=${CGIT_URI#http://}
CGIT_SERVER=${CGIT_SERVER%%/*}
curl $CGIT_URI/tree/${REPODIR#*/}$CGIT_BRANCH | sed -n 's:^.*src="/icons/text\.gif" .* href="\([^"]*\)".*$:\1:p' |
while read ; do
wget -N http://$CGIT_SERVER${REPLY/\/tree\///plain/}
done
;;
# standard http directory with po files (BRANCH is not supported)
http )
wget -N -r --no-parent --level=1 http://$REPO/$REPODIR/$DIR
cd $REPO/$REPODIR/$DIR
;;
# GNOME Translation project l10n directory
gtp )
GTP_NAME_BASE=${REPODIR%%/*}
# Projects with multiple domains have custom handling in GTP.
case $DOMAIN in
gimp20-libgimp ) GTP_NAME_BASE=gimp-libgimp ;;
gimp20-python ) GTP_NAME_BASE=gimp-python ;;
gimp20-script-fu ) GTP_NAME_BASE=gimp-script-fu ;;
gimp20-std-plug-ins ) GTP_NAME_BASE=gimp-plug-ins ;;
gimp20-tags ) GTP_NAME_BASE=gimp-tags ;;
gimp20-tips ) GTP_NAME_BASE=gimp-tips ;;
gnumeric-functions ) GTP_NAME_BASE=gnumeric-functions ;;
gtk20-properties ) GTP_NAME_BASE=gtk+-properties ;;
libgweather-locations ) GTP_NAME_BASE=locations ;;
esac
curl https://$REPO/${REPODIR%%/*}.${BRANCH:-master}/ | sed -n 's:^.*href="\([^"]*\.po\)".*$:\1:p' |
while read ; do
case $REPLY in
*.reduced.po )
;;
$GTP_NAME_BASE.${BRANCH:-master}.*)
wget -N https://$REPO/${REPODIR%%/*}.${BRANCH:-master}/$REPLY
mv $REPLY ${REPLY#$GTP_NAME_BASE.${BRANCH:-master}.}
;;
esac
done
;;
tbz )
wget -N $REPO
tar -jxf ${REPO##*/}
cd $REPODIR
;;
tgz )
wget -N $REPO
tar -zxf ${REPO##*/}
cd $REPODIR
;;
static )
if ! test -d $WORK_DIR/translation-update-static ; then
cd $WORK_DIR
tar -jxf translation-update-static.tar.bz2
cd -
fi
cp -a $WORK_DIR/translation-update-static/$DOMAIN .
cd $DOMAIN
;;
lcn )
if ! test -d $WORK_DIR/LCN-${BRANCH:-trunk} ; then
mkdir $WORK_DIR/LCN-${BRANCH:-trunk}
cd $WORK_DIR/LCN-${BRANCH:-trunk}
if test "${BRANCH:-trunk}" = "trunk" ; then
BRANCH_PATH="${BRANCH:-trunk}"
else
BRANCH_PATH="branches/$BRANCH"
fi
svn co https://svn.opensuse.org/svn/opensuse-i18n/$BRANCH_PATH/lcn
for PO in lcn/*/po/*.po ; do
LCN_LANG=${PO%/*}
LCN_LANG=${LCN_LANG#lcn/}
LCN_LANG=${LCN_LANG%%/*}
LCN_DOMAIN=${PO##*/}
LCN_DOMAIN=${LCN_DOMAIN%.$LCN_LANG.po}
mkdir -p data/$LCN_DOMAIN
ln $PO data/$LCN_DOMAIN/$LCN_LANG.po
done
cd -
fi
cp -a $WORK_DIR/LCN-${BRANCH:-trunk}/data/${REPODIR%/po} .
cd ${REPODIR%/po}
;;
* )
echo "$PACKAGE: Unknown update method $METHOD"
exit 1
;;
esac
#
# Files created in this section:
#
# upstream dir:
# foo-backport.po: raw upstream translation backported to the downstream
# foo-uheader.po: header with the upstream date from the branch modified as last
# foo-upstream.po: backport + updates from all upstreams processed before (date from the branch modified as last)
# foo-allfz.po: downstream + joined updates. Strings that were modified create fuzzy multi-match.
# foo-all.po: downstream + joined updates. Strings that were modified use the last processed instance.
# foo-fixes.po: Strings that are different while comparing downstream and joined updates.
# foo-additions.po: Strings that are newly introduced in joined updates while comparing with downstream.
# foo-header.po: header with the lastest date, either upstream date from the branch modified as last or the downstream date
# foo-fixes-clean.po: fixes with a nice header.
#
# downstream dir:
# foo-downstream.po: cleaned and complete downstream po file
# foo-updatesraw.po: fixes + additions in a single file, raw form
# foo-updates.po: fixes + additions in a single file, clean form with an useful header (this file will be copied to UPDATE/)
#
for PO in *.po ; do
(
if $MANDATORY ; then
if ! validate_po $PO ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: mandatory validation error!"
exit
fi
# Mandatory sources: copy the whole source to the mandatory po directory. Strings must not be skipped.
mkdir -p $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN
if test -f $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO ; then
msgcat --use-first $PO $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO -o $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO~
mv $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO~ $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO
else
msgcat $PO -o $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO
fi
mkdir -p $WORK_DIR/po-mandatory-full/$REAL_DOMAIN
if test -f $RPMPODIR/$PO ; then
# FIXME: Downstream po file may be invalid and cause invalid po-full po file. It does not
# create any regression, but it would be nice to try harder to fix brokenness.
if ! msgmerge --no-fuzzy-matching $RPMPODIR/$PO $RPMPODIR/$REAL_DOMAIN.pot -o $RPMPODIR/${PO%.po}-downstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: package msgmerge error"
# Failed initial msgmerge is fatal. There is no way to update. Build may fail.
exit
fi
msgcat --use-first --force-po $PO $RPMPODIR/${PO%.po}-downstream.po -o $RPMPODIR/${PO%.po}-merged.po
msgattrib --no-obsolete $RPMPODIR/${PO%.po}-merged.po -o $WORK_DIR/po-mandatory-full/$REAL_DOMAIN/$PO
else
msgattrib --no-obsolete $PO -o $WORK_DIR/po-mandatory-full/$REAL_DOMAIN/$PO
fi
else
# step 0: Merge new po file into old project. Removes unused (too new) translations.
if ! msgmerge --no-fuzzy-matching $PO $RPMPODIR/$REAL_DOMAIN.pot -o ${PO%.po}-backport.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgmerge error"
exit
fi
if ! validate_po ${PO%.po}-backport.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: validation error (backported po file)"
exit
fi
if test -f $RPMPODIR/$PO ; then
# step 1: Clean the RPM po file to be safe.
if ! msgmerge --no-fuzzy-matching $RPMPODIR/$PO $RPMPODIR/$REAL_DOMAIN.pot -o $RPMPODIR/${PO%.po}-downstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: package msgmerge error"
# Failed initial msgmerge is fatal. There is no way to update. Build may fail.
exit
fi
# Do the magic:
# step 2: Merge new upstream po and previous upstream updates to RPM po (if any).
OLD_UPDATE=false
PLURAL_FAILURE=false
if test -f $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO ; then
if $WORK_DIR/msgheadermerge $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO $PO ${PO%.po}-uheader.po --mergemode --continue ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: old po file, skipping fixes"
OLD_UPDATE=true
fi
msgcat --force-po --use-first ${PO%.po}-uheader.po ${PO%.po}-backport.po~ $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO -o ${PO%.po}-upstream.po
if ! validate_po_with_plural_check ${PO%.po}-upstream.po ; then
if $PLURAL_FAILURE ; then
# Try to use downstream plural forms.
$WORK_DIR/msgheadermerge $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO $PO ${PO%.po}-uheader_opf.po --mergemode --continue --old-plural-forms
msgcat --force-po --use-first ${PO%.po}-uheader_opf.po ${PO%.po}-backport.po~ $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO -o ${PO%.po}-upstream.po
if ! validate_po ${PO%.po}-upstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: validation error (merged translation), possible plural forms clash"
exit
fi
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: validation error (merged translation)"
exit
fi
fi
else
cp -a ${PO%.po}-backport.po~ ${PO%.po}-upstream.po
fi
# step 3: Join both translations, without --use-first string changes will disappear as fuzzy.
msgcat --force-po ${PO%.po}-upstream.po $RPMPODIR/${PO%.po}-downstream.po -o ${PO%.po}-allfz.po
msgcat --use-first --force-po ${PO%.po}-upstream.po $RPMPODIR/${PO%.po}-downstream.po -o ${PO%.po}-all.po
# step 4: Find string fixes (existed before, now different).
msgcat --force-po --unique ${PO%.po}-all.po ${PO%.po}-allfz.po -o ${PO%.po}-fixes.po~
# step 5: Find newly translated strings (translation removal is not supported).
msgcat --force-po --unique $RPMPODIR/${PO%.po}-downstream.po ${PO%.po}-all.po -o ${PO%.po}-additions.po~
# step 6: Join both to collect all known fixes.
if $OLD_UPDATE ; then
# If the update has an old time stamp, don't include fixes. Use just additions.
msgcat ${PO%.po}-additions.po~ -o $RPMPODIR/${PO%.po}-updatesraw.po
else
msgcat ${PO%.po}-fixes.po~ ${PO%.po}-additions.po~ -o $RPMPODIR/${PO%.po}-updatesraw.po
fi
# Are there any updates? If no, game over.
if test -f $RPMPODIR/${PO%.po}-updatesraw.po ; then
# step 7: Compose the best po file header.
if $PLURAL_FAILURE ; then
$WORK_DIR/msgheadermerge $RPMPODIR/$PO ${PO%.po}-upstream.po ${PO%.po}-header.po --newdate "" --old-plural-forms
else
$WORK_DIR/msgheadermerge $RPMPODIR/$PO ${PO%.po}-upstream.po ${PO%.po}-header.po --newdate
fi
# step 8: And yet another ugly game to get rid commented out garbage.
msgattrib --no-obsolete --force-po $RPMPODIR/${PO%.po}-updatesraw.po -o $RPMPODIR/${PO%.po}-updates.po~
# step 9: Merge correct header to the updates file.
msgcat --no-location --use-first ${PO%.po}-header.po $RPMPODIR/${PO%.po}-updates.po~ -o $RPMPODIR/${PO%.po}-updates.po
fi
# Prepare po-full file.
mkdir -p $WORK_DIR/po-full/$REAL_DOMAIN
if $OLD_UPDATE ; then
$WORK_DIR/msgheadermerge ${PO%.po}-upstream.po $RPMPODIR/$PO ${PO%.po}-dheader.po --mergemode --continue
msgcat --use-first --force-po ${PO%.po}-dheader.po $RPMPODIR/${PO%.po}-downstream.po ${PO%.po}-upstream.po -o ${PO%.po}-alldown.po
msgattrib --no-obsolete ${PO%.po}-alldown.po -o $WORK_DIR/po-full/$REAL_DOMAIN/$PO
else
msgattrib --no-obsolete ${PO%.po}-all.po -o $WORK_DIR/po-full/$REAL_DOMAIN/$PO
fi
# step 10: Prepare texts for review. We created them in previous steps, but files need cleanup.
if test -f ${PO%.po}-header.po ; then
if test -f ${PO%.po}-additions.po~ ; then
msgattrib --no-obsolete --force-po ${PO%.po}-additions.po~ -o ${PO%.po}-additions.po
mkdir -p $WORK_DIR/po-review/${PO%.po}/additions
msgcat --use-first ${PO%.po}-header.po ${PO%.po}-additions.po -o $WORK_DIR/po-review/${PO%.po}/additions/$REAL_DOMAIN.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/additions
fi
if test -f ${PO%.po}-fixes.po~ ; then
msgattrib --no-obsolete --force-po ${PO%.po}-fixes.po~ -o ${PO%.po}-fixes.po
msgcat --use-first ${PO%.po}-header.po ${PO%.po}-fixes.po -o ${PO%.po}-fixes-clean.po
fi
if test -f ${PO%.po}-fixes-clean.po ; then
msgmerge ${PO%.po}-allfz.po ${PO%.po}-fixes-clean.po -o ${PO%.po}-fixes-review.po~
msgattrib --no-obsolete --force-po ${PO%.po}-fixes-review.po~ -o ${PO%.po}-fixes-review.po~~
msgcat ${PO%.po}-fixes-review.po~~ -o ${PO%.po}-fixes-review.po
if $OLD_UPDATE ; then
mkdir -p $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR
cp -a ${PO%.po}-fixes-clean.po $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR/$REAL_DOMAIN.po
cp -a ${PO%.po}-fixes-review.po $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR/$REAL_DOMAIN-review.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR
if test -d $WORK_DIR/po-review/${PO%.po}/excluded-changes ; then
echo -e "Excluded changes contains changes introduced by upstream po files with\ntime stamp older than our package." >$WORK_DIR/po-review/${PO%.po}/excluded-changes/README
fi
else
mkdir -p $WORK_DIR/po-review/${PO%.po}/changes
cp -a ${PO%.po}-fixes-clean.po $WORK_DIR/po-review/${PO%.po}/changes/$REAL_DOMAIN.po
cp -a ${PO%.po}-fixes-review.po $WORK_DIR/po-review/${PO%.po}/changes/$REAL_DOMAIN-review.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/changes
fi
fi
fi
else
# Test is not needed in current msgmerge, file is generated even if there is nothing inside.
if test -f ${PO%.po}-backport.po~ ; then
# step 1: Merge new po and previous updates (if any).
msgattrib --no-obsolete --no-fuzzy --translated ${PO%.po}-backport.po~ -o ${PO%.po}-backport.po
if ! test -f ${PO%.po}-backport.po ; then
# backport file does not contain anything useful
exit
fi
if test -f $RPMPODIR/${PO%.po}-updates.po ; then
if ! msgcat --force-po --use-first ${PO%.po}-backport.po $RPMPODIR/${PO%.po}-updates.po -o $RPMPODIR/${PO%.po}-updates.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgcat error"
exit
fi
mv $RPMPODIR/${PO%.po}-updates.po~ $RPMPODIR/${PO%.po}-updates.po
else
# To get surely a valid po file, use msgcat instead of cp.
if ! msgcat ${PO%.po}-backport.po -o $RPMPODIR/${PO%.po}-updates.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgcat error"
exit
fi
fi
# step 2: Prepare texts for review.
mkdir -p $WORK_DIR/po-review/${PO%.po}/new-files
cp -a $RPMPODIR/${PO%.po}-updates.po $WORK_DIR/po-review/${PO%.po}/new-files/$REAL_DOMAIN.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/new-files
mkdir -p $WORK_DIR/po-full/$REAL_DOMAIN
cp -a $RPMPODIR/${PO%.po}-updates.po $WORK_DIR/po-full/$REAL_DOMAIN/$PO
fi
fi
fi
) &
if test $CPUS -le 1 ; then
wait
else
echo "JOBS: New job launched."
SHOWME=true
while test $(jobs -p | wc -l) -ge $CPUS ; do
if $SHOWME ; then
echo -n "JOBS: $CPUS jobs running. Will check later~"
SHOWME=false
else
echo -n "~"
fi
sleep 3
done
fi
if ! $SHOWME ; then
echo "JOBS: Releasing."
fi
done
if test $CPUS -gt 1 ; then
echo "JOBS: All tasks launched. Waiting for results."
wait
fi
mkdir -p $WORK_DIR/UPDATE/po/$REAL_DOMAIN
cd $RPMPODIR
for POX in *-updates.po ; do
PO=${POX/-updates/}
cp -a $POX $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO
done
if ! $DEBUG ; then
rm -rf $WORK_DIR/UPSTREAM/$SERIAL
fi
fi
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
done
done
if ! $DEBUG ; then
if test -n "$RPMPKGDIR" ; then
rm -rf $RPMPKGDIR $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
fi
if $COLLECT_UPSTREAM ; then
cd $WORK_DIR/UPDATE
if test -d po ; then
tar -j -c -f $WORK_DIR/translation-update-upstream-$SNAPSHOT.tar.bz2 po
fi
if test -d po-mandatory ; then
tar -j -c -f $WORK_DIR/translation-update-mandatory-$MSNAPSHOT.tar.bz2 po-mandatory
fi
fi
if $FULL_PROCESS ; then
# FIXME: Partial process should be able to update corresponding parts of domain-map.
echo >$WORK_DIR/upstream-collect.domain-map "# This file was generated $(LANG=C LC_ALL=C date) by upstream-collect.sh."
LC_ALL=C LANG=C sort -u <$WORK_DIR/upstream-collect.domain-map.tmp >>$WORK_DIR/upstream-collect.domain-map
rm $WORK_DIR/upstream-collect.domain-map.tmp
fi
cd $WORK_DIR
if ! $DEBUG ; then
rm -rf UPSTREAM OSC PACKAGES UPDATE UPDATE_OLD STAMPS BIN translation-update-static
fi
rm -rf ~/.upstream-collect.tmp
echo ""
echo "$(tput setf 2)Done. Please update version date in the spec file.$(tput init)"
