File upstream-collect.sh of Package translation-update-upstream
#!/bin/bash
# This program collects all upstream translations defined in
# configuration and merges them together.
# If you want to use po files for completing of translations, you can remove --no-fuzzy-matching.
# It will run much slower, but you could start with fuzzy matches.
# Debug mode. Set to true if you want to keep working directories.
DEBUG=false
# Incremental mode. With exception of bootstrapping or resetting the memory, it should be true.
# Non-incremental update cannot process a single package.
# FIXME: The incremental mode does not drop obsolete packages. You
# need to delete them manually afterwards using time stamp.
INCREMENTAL=true
# Set to false to generate only pot files, true to collect po files.
COLLECT_UPSTREAM=true
# Number of CPUs. When > 1, then parallel processing of more po files is possible.
CPUS=$(cat /proc/cpuinfo | grep processor | wc -l)
WORK_DIR=$PWD
set -o errexit
shopt -s nullglob
source ${0%.sh}.conf
function get_pot_name {
POT=
for POT in *.pot ; do
:
done
}
function validate_po_with_plural_check {
local LOG=$HOME/.validate${1##*/}.log$$
trap "rm -f $LOG" 0
if LC_ALL=C LANG=C msgfmt -c -o /dev/null "$1" 2>$LOG ; then
RC=0
else
RC=1
fi
PLURAL_FAILURE=false
if grep -q 'plural form' $LOG ; then
PLURAL_FAILURE=true
fi
rm $LOG
trap - 0
return $RC
}
function validate_po {
LC_ALL=C LANG=C msgfmt -c -o /dev/null "$1"
}
function rpmprep {
RPMDIR=$HOME/.var.rpmpatch$$
rm -rf BUILD $HOME/.var.rpmpatch$$
trap "rm -rf $RPMDIR" 0
mkdir -p BUILD $RPMDIR
# Add "%BUILD_ORIG 1" to support original source translation:
cat >$RPMDIR/macros <<EOF
%_sourcedir $PWD
%_builddir $PWD/BUILD
EOF
sed -i '
# Remove <RELEASE> tag from Release and delete shell escape comments.
/^Release:/s/[<>]//g
# Remove bash shell escape comments (gstreamer, calls autoreconf that can fail).
/#%(bash/d
# Remove negative expressions in BuildRequires.
:1
s/^\(BuildRequires:.*[[:space:]]\)-[^[:space:]]*/\1/
t1
/^BuildRequires:[[:space:]]*$/d
# Modify some requirements to work without relevant packages installed.
s/%{xulrunner_version}/dummy/g
s/%{mozilla_ver}/dummy/g
s/%kde4_runtime_requires/Requires: dummy/g
s/%systemd_requires/Requires: dummy/g
' *.spec
eval rpmbuild --macros=/usr/lib/rpm/macros:/usr/lib/rpm/suse_macros:/usr/lib/rpm/suse/macros:/usr/lib/rpm/macros.d/macros.systemd:/usr/lib/rpm/platform/$(uname -i)-linux/macros:/etc/rpm/\\\*:$RPMDIR/macros --nodeps -bp ${*:-*.spec}
rm -rf $RPMDIR
trap - 0
}
ONLY_PACKAGES=()
FULL_PROCESS=true
SNAPSHOT=$(LC_ALL=C LANG=C date +%Y%m%d)
MSNAPSHOT=$SNAPSHOT
case $1 in
--help )
# "only_new" is tricky: Processing new is the default. "only_new" has no match => only_new.
echo "Usage: $0 [ package | only_new ]"
exit
;;
# no arguments: process everything
"" )
;;
* )
ONLY_PACKAGES=("$@")
FULL_PROCESS=false
;;
esac
rm -rf UPSTREAM
mkdir UPSTREAM
if ! test -d STAMPS ; then
mkdir -p pot pot-tuu pot-diff
mkdir OSC PACKAGES UPDATE STAMPS
rm -f upstream-collect.log
rm -f upstream-collect.domain-map.tmp
fi
# wd may contain ":" in the name, use ~/ instead:
mkdir -p ~/.upstream-collect.tmp
cat >~/.upstream-collect.tmp/translation-update-upstream <<EOF
#!/bin/sh
echo "Dummy translation-update-upstream for upstream-collect.sh. Skipping merge of old translations."
USE_MESON=false
if test -f meson.build -a ! \( -f \${1:-po}/Makefile.in.in -o -f \${1:-po}/Makefile.in -o -f \${1:-po}/Makefile \); then
echo "Switching to meson style pot file extraction."
if test -n "\$2" ; then
DOMAIN="\$2"
else
MESON_PROJECT="\$(sed -n "/^project(/,+1{1{h;d};2{x;G}};s/^project[[:space:]]*([[:space:]]*'\([^']*\)'.*/\1/p" <meson.build)"
DOMAIN="\$(sed -n "s/meson.project_name[[:space:]]*([[:space:]]*)/'\$MESON_PROJECT'/g;s/.*\\\\.set_quoted[[:space:]]*('GETTEXT_PACKAGE',[[:space:]]'\([^']*\)').*/\1/p" <meson.build)"
fi
if test -f \${1:-po}/POTFILES ; then
POTFILES="\$PWD/\${1:-po}/POTFILES"
else
POTFILES="\$PWD/\${1:-po}/POTFILES.in"
fi
echo "cd \\"\$PWD\\"
xgettext --package-name=\\"\$DOMAIN\\"\\\\
-p \\"\$PWD\\"\\\\
-f \\"\$POTFILES\\"\\\\
-D \\"\$PWD\\"\\\\
-k_ -o \\"\$PWD/\${1:-po}/\$DOMAIN.pot\\"\\\\
--keyword=NC_:1c,2\\\\
--flag=g_strdup_printf:1:c-format\\\\
--flag=g_set_error:4:c-format\\\\
--flag=g_dngettext:2:pass-c-format\\\\
--flag=g_string_printf:2:c-format\\\\
--add-comments\\\\
--from-code=UTF-8\\\\
--keyword=C_:1c,2\\\\
--flag=N_:1:pass-c-format\\\\
--flag=g_string_append_printf:2:c-format\\\\
--flag=C_:2:pass-c-format\\\\
--keyword=N_\\\\
--flag=g_error_new:3:c-format\\\\
--flag=NC_:2:pass-c-format\\\\
--keyword=g_dngettext:2,3\\\\
--keyword=g_dpgettext2:2c,3\\\\
--keyword=_\\\\
--keyword=g_dcgettext:2" >\${1:-po}/.translation-update-upstream-implemented
else
echo \${3:-intltool-update\${2+ --gettext-package=\$2} --pot} >\${1:-po}/.translation-update-upstream-implemented
fi
cd \${1:-po}
# Generate and save a copy of the pot file now and compare later.
bash ./.translation-update-upstream-implemented
mkdir -p tuu
for POT in *.pot ; do
cp -a \$POT tuu/
done
EOF
chmod +x ~/.upstream-collect.tmp/translation-update-upstream
# executable flag does not survive some build systems
chmod +x msgheadermerge msgheadermerge-compose msgheadermerge-parse upstream-collect.sh create-tlst-step*.sh translation-update-upstream-to-translation-update.sh
# Strings in installed instance of gnome-patch-translation may interfere
# with upstream-collect.sh. Use of dummies allows to import upstream
# strings, that are part of openSUSE patches.
for FILE in gnome-patch-translation-prepare gnome-patch-translation-update ; do
cat >~/.upstream-collect.tmp/$FILE <<EOF
#!/bin/sh
set -x
echo "Dummy $FILE for upstream-collect.sh. Skipping gnome-patch-translation."
# gnome-patch-translation may be a symlink (libgweather), that is why we have to check even with mkdir -p.
if ! test -L gnome-patch-translation ; then
mkdir -p gnome-patch-translation
fi
touch po/.gnome-patch-translation-implemented
EOF
chmod +x ~/.upstream-collect.tmp/$FILE
done
export PATH=~/.upstream-collect.tmp:$PATH
# FIXME: Even in incremental mode, we should drop all updates that generate empty update file after the run. These files are obsolete, included into package, and may even degrade translation quality.
if $INCREMENTAL ; then
# more tarballs are available => use the latest one
# FIXME: Fix 20090213.10 < 20090213.9
# (but it should not happen for people who update and submit)
for ARCHIVE_ in translation-update-upstream-*.tar.bz2 ; do
ARCHIVE=$ARCHIVE_
done
if ! test -f STAMPS/UPDATE_old_tarball ; then
cd UPDATE
echo "$(tput setf 3)Unpacking: $ARCHIVE$(tput init)"
tar -jxf ../$ARCHIVE
# If it is not a full process, increment only release
# SNAPSHOT 20090213 -> 20090213.1, 20090213.1 -> 20090213.2
cd ..
touch STAMPS/UPDATE_old_tarball
fi
SNAPSHOT_RELEASE=${ARCHIVE#translation-update-upstream-}
SNAPSHOT_RELEASE=${SNAPSHOT_RELEASE%.tar.bz2}
SNAPSHOT_RELEASE_PART1=${SNAPSHOT_RELEASE%.*}
SNAPSHOT_RELEASE_PART2=${SNAPSHOT_RELEASE#*.}
if test "$SNAPSHOT_RELEASE_PART1" = "$SNAPSHOT_RELEASE" ; then
SNAPSHOT_RELEASE_PART2=1
else
let SNAPSHOT_RELEASE_PART2++
fi
SNAPSHOT_RELEASE=$SNAPSHOT_RELEASE_PART1.$SNAPSHOT_RELEASE_PART2
for ARCHIVE_ in translation-update-mandatory-*.tar.bz2 ; do
ARCHIVE=$ARCHIVE_
done
if ! test -f STAMPS/UPDATE_old_mtarball ; then
cd UPDATE
echo "$(tput setf 3)Unpacking: $ARCHIVE$(tput init)"
tar -jxf ../$ARCHIVE
# If it is not a full process, increment only release
# SNAPSHOT 20090213 -> 20090213.1, 20090213.1 -> 20090213.2
cd ..
touch STAMPS/UPDATE_old_mtarball
fi
MSNAPSHOT_RELEASE=${ARCHIVE#translation-update-mandatory-}
MSNAPSHOT_RELEASE=${MSNAPSHOT_RELEASE%.tar.bz2}
SNAPSHOT_RELEASE_PART1=${MSNAPSHOT_RELEASE%.*}
SNAPSHOT_RELEASE_PART2=${MSNAPSHOT_RELEASE#*.}
if test "$SNAPSHOT_RELEASE_PART1" = "$MSNAPSHOT_RELEASE" ; then
SNAPSHOT_RELEASE_PART2=1
else
let SNAPSHOT_RELEASE_PART2++
fi
MSNAPSHOT_RELEASE=$SNAPSHOT_RELEASE_PART1.$SNAPSHOT_RELEASE_PART2
fi
if ! $FULL_PROCESS ; then
SNAPSHOT=$SNAPSHOT_RELEASE
MSNAPSHOT=$MSNAPSHOT_RELEASE
fi
SERIAL=0
for TLST in *.tlst ; do
exec <$WORK_DIR/$TLST
while read PACKAGE DOMAIN METHOD REPO DIR BRANCH MANDATORY ; do
# Continue for empty lines and comments
if test "${PACKAGE###}" != "$PACKAGE" ; then
continue
fi
if test -z "$PACKAGE" ; then
continue
fi
if test "$MANDATORY" = "mandatory" ; then
MANDATORY=true
else
MANDATORY=false
fi
echo
echo "$(tput setf 3)Processing: package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:-(default)} mandatory=$MANDATORY$(tput init)"
STATUS=OK
# NOTE: Force a limitation: tlst rules for one package must be placed on contiguous line sequence
if ! $DEBUG ; then
if test "$OLD_PACKAGE" != "$PACKAGE" ; then
if test -n "$RPMPKGDIR" ; then
rm -rf $RPMPKGDIR $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
fi
OLD_PACKAGE=$PACKAGE
fi
if test -d $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__} ; then
echo " Already successfully processed. Skipping..."
continue
fi
if test "${#ONLY_PACKAGES}" -gt 0 ; then
SKIP_PACKAGE=true
for ONLY_PACKAGE in "${ONLY_PACKAGES[@]}" ; do
if test "$ONLY_PACKAGE" = "$PACKAGE" ; then
SKIP_PACKAGE=false
break
fi
done
if $SKIP_PACKAGE ; then
echo " Not scheduled to process. Recycling old update..."
#FIXME mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
# touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
continue
fi
fi
cd $WORK_DIR/OSC
RPMPKGDIR=$WORK_DIR/PACKAGES/$PACKAGE
if ! test -f $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok ; then
if ! test -d "$RPMPKGDIR" ; then
for OSC_REPOSITORY in "${OSC_REPOSITORIES[@]}" ; do
echo "Trying to check-out PACKAGES $PACKAGE from $OSC_REPOSITORY..."
# FIXME: When obs-service-download_url appears in Factory, use --source-service-files
if osc ${OSC_APIURL:+--apiurl=$OSC_APIURL} checkout --server-side-source-service-files --expand-link $OSC_REPOSITORY $PACKAGE >gnome-patch-translation-collect-tmp.log 2>&1 ; then
mv $OSC_REPOSITORY/$PACKAGE $WORK_DIR/PACKAGES/
for FILE in $WORK_DIR/PACKAGES/$PACKAGE/_service\:download_url\:* ; do
mv "$FILE" "${FILE/_service:download_url:/}"
done
break
fi
if ! grep -q "HTTP Error 404" gnome-patch-translation-collect-tmp.log ; then
cat gnome-patch-translation-collect-tmp.log
rm gnome-patch-translation-collect-tmp.log
echo "ERROR: Checkout failed!"
exit 1
fi
done
rm gnome-patch-translation-collect-tmp.log
else
rm -rf "$RPMPKGDIR" $WORK_DIR/STAMPS/$PACKAGE
echo "$(tput setf 4)Removed possibly incorrect temporary files from previous runs. Please re-run $0 now.$(tput init)"
exit 1
fi
cd $RPMPKGDIR
rpmprep $PACKAGE.spec
else
# During processing, builddir may contain incomplete po files:
rm $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
REPODIR=$DIR
RPMPODIR=$(echo $RPMPKGDIR/BUILD/*/${DIR#*/})
if test -f $WORK_DIR/${TLST%.tlst}.hook ; then
source $WORK_DIR/${TLST%.tlst}.hook
fi
cd $RPMPODIR
get_pot_name
REAL_DOMAIN=${POT%.pot}
if test -f .gnome-patch-translation-implemented ; then
echo $PACKAGE >>$WORK_DIR/gnome-patch-translation.lst
fi
if test -f .translation-update-upstream-implemented ; then
if bash ./.translation-update-upstream-implemented ; then
get_pot_name
REAL_DOMAIN=${POT%.pot}
cp -a $REAL_DOMAIN.pot $WORK_DIR/pot/
# pot-tuu DOMAIN is the external domain - LCN may use different domain
cp -a tuu/$REAL_DOMAIN.pot $WORK_DIR/pot-tuu/$DOMAIN.pot
# Verify that patches don't introduce new strings.
msgcomm --uniq $REAL_DOMAIN.pot tuu/$REAL_DOMAIN.pot -o $WORK_DIR/pot-diff/$DOMAIN.pot
if ! test -f .gnome-patch-translation-implemented ; then
if test -f $WORK_DIR/pot-diff/$DOMAIN.pot ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: new pot file contains unique strings, please check gnome-patch-translation"
STATUS=OK_INCOMPLETE
fi
fi
else
# translation-update-upstream is implemented but fails:
get_pot_name
REAL_DOMAIN=${POT%.pot}
if test -f $REAL_DOMAIN.pot ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: pot file update error, continuing with original $DOMAIN.pot"
STATUS=OK_OUTDATED
cp -a $DOMAIN.pot $WORK_DIR/pot/
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: pot file update error, no way to update"
STATUS=POT_ERROR
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$REPODIR/${BRANCH:-__HEAD__}
# However we cannot do anything, build dir is successfully processed.
touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
continue
fi
fi
else
echo "$RPMPODIR: Missing or incorrect translation-update-upstream in the spec file."
# translation-update-upstream is implemented, try the default with the upstream domain:
if intltool-update --gettext-package=$DOMAIN --pot ; then
get_pot_name
REAL_DOMAIN=${POT%.pot}
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: packaging error, package does not call translation-update-upstream properly"
STATUS=TUU_NOT_CALLED
cp -a $DOMAIN.pot $WORK_DIR/pot/
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}: packaging error, package does not call translation-update-upstream properly and intltool-update fails, no way to update"
STATUS=TUU_BROKEN
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$REPODIR/${BRANCH:-__HEAD__}
# However we cannot do anything, build dir is successfully processed.
touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
continue
fi
fi
echo $PACKAGE $REAL_DOMAIN $STATUS >>$WORK_DIR/upstream-collect.domain-map.tmp
if $COLLECT_UPSTREAM ; then
cd $WORK_DIR/UPSTREAM
let SERIAL++ || :
mkdir $SERIAL
cd $SERIAL
echo >.info "package=$PACKAGE domain=$DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)}"
case "$METHOD" in
cvs )
cvs -z3 -d:pserver:anoncvs@$REPO co $REPODIR $BRANCH
cd $REPODIR
;;
svn )
if test -z "$BRANCH" ; then
svn co $REPO/${REPODIR%%/*}/trunk/${REPODIR#*/}
else
svn co $REPO/${REPODIR%%/*}/branches/$BRANCH/${REPODIR#*/}
fi
cd ${REPODIR##*/}
;;
git )
if ! test -d $WORK_DIR/GIT/${REPO//[\/:.]/_} ; then
mkdir -p $WORK_DIR/GIT/${REPO//[\/:.]/_}
cd $WORK_DIR/GIT/${REPO//[\/:.]/_}
git clone $REPO
cd $OLDPWD
fi
cp -a $WORK_DIR/GIT/${REPO//[\/:.]/_}/* .
if test -n "$BRANCH" ; then
cd *
git checkout remotes/origin/$BRANCH
cd $OLDPWD
fi
cd $REPODIR
;;
# Web-based Git repository viewer makes possible to download particular file.
cgit )
# Some tricks to be able to recycle git:// URI
CGIT_URI=$REPO
CGIT_URI=${CGIT_URI/git:\/\/anongit./http://cgit.}
CGIT_URI=${CGIT_URI/git:\/\//http://}
CGIT_BRANCH=${BRANCH:+?id=$BRANCH}
CGIT_SERVER=${CGIT_URI#http://}
CGIT_SERVER=${CGIT_SERVER%%/*}
curl $CGIT_URI/tree/${REPODIR#*/}$CGIT_BRANCH | sed -n 's:^.*class='\''ls-blob[^'\'']*'\'' href='\''\([^'\'']*\)'\''.*$:\1:p' |
while read ; do
wget -N http://$CGIT_SERVER${REPLY/\/tree\///plain/}
done
;;
# standard http directory with po files (BRANCH is not supported)
http )
wget -N -r --no-parent --level=1 http://$REPO/$REPODIR/$DIR
cd $REPO/$REPODIR/$DIR
;;
# GNOME Translation project l10n directory
gtp )
GTP_NAME_BASE=${REPODIR%%/*}
# Projects with multiple domains have custom handling in GTP.
case $DOMAIN in
gimp20-libgimp ) GTP_NAME_BASE=gimp-libgimp ;;
gimp20-python ) GTP_NAME_BASE=gimp-python ;;
gimp20-script-fu ) GTP_NAME_BASE=gimp-script-fu ;;
gimp20-std-plug-ins ) GTP_NAME_BASE=gimp-plug-ins ;;
gimp20-tags ) GTP_NAME_BASE=gimp-tags ;;
gimp20-tips ) GTP_NAME_BASE=gimp-tips ;;
gnumeric-functions ) GTP_NAME_BASE=gnumeric-functions ;;
gtk20-properties ) GTP_NAME_BASE=gtk+-properties ;;
libgweather-locations ) GTP_NAME_BASE=locations ;;
esac
curl https://$REPO/${REPODIR%%/*}.${BRANCH:-master}/ | sed -n 's:^.*href="\([^"]*\.po\)".*$:\1:p' |
while read ; do
case $REPLY in
*.reduced.po )
;;
$GTP_NAME_BASE.${BRANCH:-master}.*)
wget -N https://$REPO/${REPODIR%%/*}.${BRANCH:-master}/$REPLY
mv $REPLY ${REPLY#$GTP_NAME_BASE.${BRANCH:-master}.}
;;
esac
done
;;
tbz )
wget -N $REPO
tar -jxf ${REPO##*/}
cd $REPODIR
;;
tgz )
wget -N $REPO
tar -zxf ${REPO##*/}
cd $REPODIR
;;
txz )
wget -N $REPO
tar -Jxf ${REPO##*/}
cd $REPODIR
;;
static )
if ! test -d $WORK_DIR/translation-update-static ; then
cd $WORK_DIR
tar -jxf translation-update-static.tar.bz2
cd -
fi
cp -a $WORK_DIR/translation-update-static/$DOMAIN .
cd $DOMAIN
;;
lcn )
if ! test -d $WORK_DIR/LCN-${BRANCH:-trunk} ; then
mkdir $WORK_DIR/LCN-${BRANCH:-trunk}
cd $WORK_DIR/LCN-${BRANCH:-trunk}
if test "${BRANCH:-trunk}" = "trunk" ; then
BRANCH_PATH="${BRANCH:-trunk}"
else
BRANCH_PATH="branches/$BRANCH"
fi
svn co https://svn.opensuse.org/svn/opensuse-i18n/$BRANCH_PATH/lcn
for PO in lcn/*/po/*.po ; do
LCN_LANG=${PO%/*}
LCN_LANG=${LCN_LANG#lcn/}
LCN_LANG=${LCN_LANG%%/*}
LCN_DOMAIN=${PO##*/}
LCN_DOMAIN=${LCN_DOMAIN%.$LCN_LANG.po}
mkdir -p data/$LCN_DOMAIN
ln $PO data/$LCN_DOMAIN/$LCN_LANG.po
done
cd -
fi
cp -a $WORK_DIR/LCN-${BRANCH:-trunk}/data/${REPODIR%/po} .
cd ${REPODIR%/po}
;;
* )
echo "$PACKAGE: Unknown update method $METHOD"
exit 1
;;
esac
#
# Files created in this section:
#
# upstream dir:
# foo-backport.po: raw upstream translation backported to the downstream
# foo-uheader.po: header with the upstream date from the branch modified as last
# foo-upstream.po: backport + updates from all upstreams processed before (date from the branch modified as last)
# foo-allfz.po: downstream + joined updates. Strings that were modified create fuzzy multi-match.
# foo-all.po: downstream + joined updates. Strings that were modified use the last processed instance.
# foo-fixes.po: Strings that are different while comparing downstream and joined updates.
# foo-additions.po: Strings that are newly introduced in joined updates while comparing with downstream.
# foo-header.po: header with the lastest date, either upstream date from the branch modified as last or the downstream date
# foo-fixes-clean.po: fixes with a nice header.
#
# downstream dir:
# foo-downstream.po: cleaned and complete downstream po file
# foo-updatesraw.po: fixes + additions in a single file, raw form
# foo-updates.po: fixes + additions in a single file, clean form with an useful header (this file will be copied to UPDATE/)
#
for PO in *.po ; do
(
if $MANDATORY ; then
if ! validate_po $PO ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: mandatory validation error!"
exit
fi
# Mandatory sources: copy the whole source to the mandatory po directory. Strings must not be skipped.
mkdir -p $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN
if test -f $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO ; then
msgcat --use-first $PO $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO -o $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO~
mv $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO~ $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO
else
msgcat $PO -o $WORK_DIR/UPDATE/po-mandatory/$REAL_DOMAIN/$PO
fi
mkdir -p $WORK_DIR/po-mandatory-full/$REAL_DOMAIN
if test -f $RPMPODIR/$PO ; then
# FIXME: Downstream po file may be invalid and cause invalid po-full po file. It does not
# create any regression, but it would be nice to try harder to fix brokenness.
if ! msgmerge --no-fuzzy-matching $RPMPODIR/$PO $RPMPODIR/$REAL_DOMAIN.pot -o $RPMPODIR/${PO%.po}-downstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: package msgmerge error"
# Failed initial msgmerge is fatal. There is no way to update. Build may fail.
exit
fi
msgcat --use-first --force-po $PO $RPMPODIR/${PO%.po}-downstream.po -o $RPMPODIR/${PO%.po}-merged.po
msgattrib --no-obsolete $RPMPODIR/${PO%.po}-merged.po -o $WORK_DIR/po-mandatory-full/$REAL_DOMAIN/$PO
else
msgattrib --no-obsolete $PO -o $WORK_DIR/po-mandatory-full/$REAL_DOMAIN/$PO
fi
else
# step 0: Merge new po file into old project. Removes unused (too new) translations.
if ! msgmerge --no-fuzzy-matching $PO $RPMPODIR/$REAL_DOMAIN.pot -o ${PO%.po}-backport.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgmerge error"
exit
fi
if ! validate_po ${PO%.po}-backport.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: validation error (backported po file)"
exit
fi
if test -f $RPMPODIR/$PO ; then
# step 1: Clean the RPM po file to be safe.
if ! msgmerge --no-fuzzy-matching $RPMPODIR/$PO $RPMPODIR/$REAL_DOMAIN.pot -o $RPMPODIR/${PO%.po}-downstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: package msgmerge error"
# Failed initial msgmerge is fatal. There is no way to update. Build may fail.
exit
fi
# Do the magic:
# step 2: Merge new upstream po and previous upstream updates to RPM po (if any).
OLD_UPDATE=false
PLURAL_FAILURE=false
if test -f $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO ; then
if ! $WORK_DIR/msgheadermerge $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO $PO ${PO%.po}-uheader.po --mergemode --continue ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: old po file, skipping fixes"
OLD_UPDATE=true
fi
msgcat --force-po --use-first ${PO%.po}-uheader.po ${PO%.po}-backport.po~ $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO -o ${PO%.po}-upstream.po
if ! validate_po_with_plural_check ${PO%.po}-upstream.po ; then
if $PLURAL_FAILURE ; then
# Try to use downstream plural forms.
$WORK_DIR/msgheadermerge $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO $PO ${PO%.po}-uheader_opf.po --mergemode --continue --old-plural-forms
msgcat --force-po --use-first ${PO%.po}-uheader_opf.po ${PO%.po}-backport.po~ $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO -o ${PO%.po}-upstream.po
if ! validate_po ${PO%.po}-upstream.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: validation error (merged translation), possible plural forms clash"
exit
fi
else
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: validation error (merged translation)"
exit
fi
fi
else
cp -a ${PO%.po}-backport.po~ ${PO%.po}-upstream.po
fi
# step 3: Join both translations, without --use-first string changes will disappear as fuzzy.
msgcat --force-po ${PO%.po}-upstream.po $RPMPODIR/${PO%.po}-downstream.po -o ${PO%.po}-allfz.po
msgcat --use-first --force-po ${PO%.po}-upstream.po $RPMPODIR/${PO%.po}-downstream.po -o ${PO%.po}-all.po
# step 4: Find string fixes (existed before, now different).
msgcat --force-po --unique ${PO%.po}-all.po ${PO%.po}-allfz.po -o ${PO%.po}-fixes.po~
# step 5: Find newly translated strings (translation removal is not supported).
msgcat --force-po --unique $RPMPODIR/${PO%.po}-downstream.po ${PO%.po}-all.po -o ${PO%.po}-additions.po~
# step 6: Join both to collect all known fixes.
if $OLD_UPDATE ; then
# If the update has an old time stamp, don't include fixes. Use just additions.
msgcat ${PO%.po}-additions.po~ -o $RPMPODIR/${PO%.po}-updatesraw.po
# "updatesraw" can have inconsistent plural forms even if "upstream" has them consistent (bsc#894913).
# Other failures are not possible here (superset was already validated).
if ! validate_po_with_plural_check ${PO%.po}-updatesraw.po ; then
# => There is no chance that --old-plural-forms succeeds. Don't try it and fail.
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN repository=$REPO directory=$RPMPODIR branch=${BRANCH:(default)} po=$PO: validation error (merged translation additions), possible plural forms clash"
exit
fi
else
msgcat ${PO%.po}-fixes.po~ ${PO%.po}-additions.po~ -o $RPMPODIR/${PO%.po}-updatesraw.po
fi
# Are there any updates? If no, game over.
if test -f $RPMPODIR/${PO%.po}-updatesraw.po ; then
# step 7: Compose the best po file header.
if $PLURAL_FAILURE ; then
$WORK_DIR/msgheadermerge $RPMPODIR/$PO ${PO%.po}-upstream.po ${PO%.po}-header.po --newdate "" --old-plural-forms
else
$WORK_DIR/msgheadermerge $RPMPODIR/$PO ${PO%.po}-upstream.po ${PO%.po}-header.po --newdate
fi
# step 8: And yet another ugly game to get rid commented out garbage.
msgattrib --no-obsolete --force-po $RPMPODIR/${PO%.po}-updatesraw.po -o $RPMPODIR/${PO%.po}-updates.po~
# step 9: Merge correct header to the updates file.
msgcat --no-location --use-first ${PO%.po}-header.po $RPMPODIR/${PO%.po}-updates.po~ -o $RPMPODIR/${PO%.po}-updates.po
fi
# Prepare po-full file.
mkdir -p $WORK_DIR/po-full/$REAL_DOMAIN
if $OLD_UPDATE ; then
$WORK_DIR/msgheadermerge ${PO%.po}-upstream.po $RPMPODIR/$PO ${PO%.po}-dheader.po --mergemode --continue
msgcat --use-first --force-po ${PO%.po}-dheader.po $RPMPODIR/${PO%.po}-downstream.po ${PO%.po}-upstream.po -o ${PO%.po}-alldown.po
msgattrib --no-obsolete ${PO%.po}-alldown.po -o $WORK_DIR/po-full/$REAL_DOMAIN/$PO
else
msgattrib --no-obsolete ${PO%.po}-all.po -o $WORK_DIR/po-full/$REAL_DOMAIN/$PO
fi
# step 10: Prepare texts for review. We created them in previous steps, but files need cleanup.
if test -f ${PO%.po}-header.po ; then
if test -f ${PO%.po}-additions.po~ ; then
msgattrib --no-obsolete --force-po ${PO%.po}-additions.po~ -o ${PO%.po}-additions.po
mkdir -p $WORK_DIR/po-review/${PO%.po}/additions
msgcat --use-first ${PO%.po}-header.po ${PO%.po}-additions.po -o $WORK_DIR/po-review/${PO%.po}/additions/$REAL_DOMAIN.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/additions
fi
if test -f ${PO%.po}-fixes.po~ ; then
msgattrib --no-obsolete --force-po ${PO%.po}-fixes.po~ -o ${PO%.po}-fixes.po
msgcat --use-first ${PO%.po}-header.po ${PO%.po}-fixes.po -o ${PO%.po}-fixes-clean.po
fi
if test -f ${PO%.po}-fixes-clean.po ; then
msgmerge ${PO%.po}-allfz.po ${PO%.po}-fixes-clean.po -o ${PO%.po}-fixes-review.po~
msgattrib --no-obsolete --force-po ${PO%.po}-fixes-review.po~ -o ${PO%.po}-fixes-review.po~~
msgcat ${PO%.po}-fixes-review.po~~ -o ${PO%.po}-fixes-review.po
if $OLD_UPDATE ; then
mkdir -p $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR
cp -a ${PO%.po}-fixes-clean.po $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR/$REAL_DOMAIN.po
cp -a ${PO%.po}-fixes-review.po $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR/$REAL_DOMAIN-review.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/excluded-changes/${REPO//[\/:.]/_}/$REPODIR
if test -d $WORK_DIR/po-review/${PO%.po}/excluded-changes ; then
echo -e "Excluded changes contains changes introduced by upstream po files with\ntime stamp older than our package." >$WORK_DIR/po-review/${PO%.po}/excluded-changes/README
fi
else
mkdir -p $WORK_DIR/po-review/${PO%.po}/changes
cp -a ${PO%.po}-fixes-clean.po $WORK_DIR/po-review/${PO%.po}/changes/$REAL_DOMAIN.po
cp -a ${PO%.po}-fixes-review.po $WORK_DIR/po-review/${PO%.po}/changes/$REAL_DOMAIN-review.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/changes
fi
fi
fi
else
# Test is not needed in current msgmerge, file is generated even if there is nothing inside.
if test -f ${PO%.po}-backport.po~ ; then
# step 1: Merge new po and previous updates (if any).
msgattrib --no-obsolete --no-fuzzy --translated ${PO%.po}-backport.po~ -o ${PO%.po}-backport.po
if ! test -f ${PO%.po}-backport.po ; then
# backport file does not contain anything useful
exit
fi
if test -f $RPMPODIR/${PO%.po}-updates.po ; then
if ! msgcat --force-po --use-first ${PO%.po}-backport.po $RPMPODIR/${PO%.po}-updates.po -o $RPMPODIR/${PO%.po}-updates.po~ ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgcat error"
exit
fi
mv $RPMPODIR/${PO%.po}-updates.po~ $RPMPODIR/${PO%.po}-updates.po
else
# To get surely a valid po file, use msgcat instead of cp.
if ! msgcat ${PO%.po}-backport.po -o $RPMPODIR/${PO%.po}-updates.po ; then
echo >>$WORK_DIR/upstream-collect.log "package=$PACKAGE domain=$DOMAIN gettext-package=$REAL_DOMAIN method=$METHOD repository=$REPO directory=$DIR branch=${BRANCH:(default)} po=$PO: msgcat error"
exit
fi
fi
# step 2: Prepare texts for review.
mkdir -p $WORK_DIR/po-review/${PO%.po}/new-files
cp -a $RPMPODIR/${PO%.po}-updates.po $WORK_DIR/po-review/${PO%.po}/new-files/$REAL_DOMAIN.po
rmdir --ignore-fail-on-non-empty --parents $WORK_DIR/po-review/${PO%.po}/new-files
mkdir -p $WORK_DIR/po-full/$REAL_DOMAIN
cp -a $RPMPODIR/${PO%.po}-updates.po $WORK_DIR/po-full/$REAL_DOMAIN/$PO
fi
fi
fi
) &
if test $CPUS -le 1 ; then
wait
else
echo "JOBS: New job launched."
SHOWME=true
while test $(jobs -p | wc -l) -ge $CPUS ; do
if $SHOWME ; then
echo -n "JOBS: $CPUS jobs running. Will check later~"
SHOWME=false
else
echo -n "~"
fi
sleep 3
done
fi
if ! $SHOWME ; then
echo "JOBS: Releasing."
fi
done
if test $CPUS -gt 1 ; then
echo "JOBS: All tasks launched. Waiting for results."
wait
fi
mkdir -p $WORK_DIR/UPDATE/po/$REAL_DOMAIN
cd $RPMPODIR
for POX in *-updates.po ; do
PO=${POX/-updates/}
cp -a $POX $WORK_DIR/UPDATE/po/$REAL_DOMAIN/$PO
done
if ! $DEBUG ; then
rm -rf $WORK_DIR/UPSTREAM/$SERIAL
fi
fi
mkdir -p $WORK_DIR/STAMPS/$PACKAGE/$DOMAIN/$METHOD/${REPO//[\/:.]/_}/$DIR/${BRANCH:-__HEAD__}
touch $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
done
done
if ! $DEBUG ; then
if test -n "$RPMPKGDIR" ; then
rm -rf $RPMPKGDIR $WORK_DIR/STAMPS/$PACKAGE/.builddir_ok
fi
fi
if $COLLECT_UPSTREAM ; then
cd $WORK_DIR/UPDATE
if test -d po ; then
tar -j -c -f $WORK_DIR/translation-update-upstream-$SNAPSHOT.tar.bz2 po
fi
if test -d po-mandatory ; then
tar -j -c -f $WORK_DIR/translation-update-mandatory-$MSNAPSHOT.tar.bz2 po-mandatory
fi
fi
if $FULL_PROCESS ; then
# FIXME: Partial process should be able to update corresponding parts of domain-map.
echo >$WORK_DIR/upstream-collect.domain-map "# This file was generated $(LANG=C LC_ALL=C date) by upstream-collect.sh."
LC_ALL=C LANG=C sort -u <$WORK_DIR/upstream-collect.domain-map.tmp >>$WORK_DIR/upstream-collect.domain-map
rm $WORK_DIR/upstream-collect.domain-map.tmp
fi
cd $WORK_DIR
if ! $DEBUG ; then
rm -rf UPSTREAM OSC PACKAGES UPDATE UPDATE_OLD STAMPS BIN translation-update-static
fi
rm -rf ~/.upstream-collect.tmp
echo ""
echo "$(tput setf 2)Done. Please update version date in the spec file.$(tput init)"
