File 0009-imfile-proper-handling-of-multiline-messages-after-r.patch of Package rsyslog.6132
From dd546f82a51bf2e71a97383fce88fffaa3b2a611 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 24 Mar 2015 11:35:20 +0100
Subject: [PATCH] imfile: proper handling of multiline messages after rsyslog
restart
handling of persisted info (and polling loops) was not correct.
This needs some cleanup but I hope to find some folks for
independent testing before I declare victory.
see also https://github.com/rsyslog/rsyslog/issues/144
From ed8d132e3c7adaff715ebb12043a63c7b96a7436 Mon Sep 17 00:00:00 2001
From: Nirmoy Das <ndas@suse.de>
Date: Mon, 7 Aug 2017 17:31:09 +0200
Subject: [PATCH] remove debug prints from upstream patch
---
runtime/stream.c | 8 ++-
tests/imfile-readmode2-with-persists.sh | 65 ++++++++++++++++++++++
.../testsuites/imfile-readmode2-with-persists.conf | 21 +++++++
3 files changed, 92 insertions(+), 2 deletions(-)
create mode 100644 tests/imfile-readmode2-with-persists.sh
create mode 100644 tests/testsuites/imfile-readmode2-with-persists.conf
diff --git a/runtime/stream.c b/runtime/stream.c
index 1ee872c..1bc03c0 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -634,7 +634,7 @@ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC)
ASSERT(pThis != NULL);
ASSERT(pC != NULL);
- /* DEV debug only: DBGOPRINT((obj_t*) pThis, "strmRead index %d, max %d\n", pThis->iBufPtr, pThis->iBufPtrMax); */
+ /* DEV debug only: DBGOPRINT((obj_t*) pThis, "strmRead index %zd, max %zd\n", pThis->iBufPtr, pThis->iBufPtrMax); */
if(pThis->iUngetC != -1) { /* do we have an "unread" char that we need to provide? */
*pC = pThis->iUngetC;
++pThis->iCurrOffs; /* one more octet read */
@@ -790,7 +790,10 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF)
finalize_it:
if(iRet != RS_RET_OK && *ppCStr != NULL) {
+ if(cstrLen(*ppCStr) > 0) {
+ /* we may have an empty string in an unsuccsfull poll or after restart! */
rsCStrConstructFromCStr(&pThis->prevLineSegment, *ppCStr);
+ }
cstrDestruct(ppCStr);
}
@@ -1854,7 +1857,8 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm)
l = pThis->inode;
objSerializeSCALAR_VAR(pStrm, inode, INT64, l);
- objSerializePTR(pStrm, prevLineSegment, PSZ);
+ cstrFinalize(pThis->prevLineSegment);
+ objSerializePTR(pStrm, prevLineSegment, CSTR);
i = pThis->bPrevWasNL;
objSerializeSCALAR_VAR(pStrm, bPrevWasNL, INT, i);
diff --git a/tests/imfile-readmode2-with-persists.sh b/tests/imfile-readmode2-with-persists.sh
new file mode 100644
index 0000000..3063484
--- /dev/null
+++ b/tests/imfile-readmode2-with-persists.sh
@@ -0,0 +1,65 @@
+# This is part of the rsyslog testbench, licensed under ASL 2.0
+echo ======================================================================
+echo [imfile-readmode2-with-persists.sh]
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup imfile-readmode2-with-persists.conf
+
+# write the beginning of the file
+echo 'msgnum:0
+ msgnum:1' > rsyslog.input
+echo 'msgnum:2' >> rsyslog.input
+
+# sleep a little to give rsyslog a chance to begin processing
+sleep 1
+
+# now stop and restart rsyslog so that the file info must be
+# persisted and read again on startup. Results should still be
+# correct ;)
+echo stopping rsyslog
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished!
+echo restarting rsyslog
+source $srcdir/diag.sh startup imfile-readmode2-with-persists.conf
+echo restarted rsyslog, continuing with test
+
+# write some more lines (see https://github.com/rsyslog/rsyslog/issues/144)
+echo 'msgnum:3
+ msgnum:4' >> rsyslog.input
+echo 'msgnum:5' >> rsyslog.input # this one shouldn't be written to the output file because of ReadMode 2
+
+# give it time to finish
+sleep 1
+
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown # we need to wait until rsyslogd is finished!
+
+# give it time to write the output file
+sleep 1
+
+## check if we have the correct number of messages
+
+NUMLINES=$(grep -c HEADER $srcdir/rsyslog.out.log 2>/dev/null)
+
+if [ -z $NUMLINES ]; then
+ echo "ERROR: expecting at least a match for HEADER, maybe rsyslog.out.log wasn't even written?"
+ exit 1
+else
+ if [ ! $NUMLINES -eq 3 ]; then
+ echo "ERROR: expecting 3 headers, got $NUMLINES"
+ exit 1
+ fi
+fi
+
+## check if all the data we expect to get in the file is there
+
+for i in {1..4}; do
+ grep msgnum:$i $srcdir/rsyslog.out.log > /dev/null 2>&1
+ if [ ! $? -eq 0 ]; then
+ echo "ERROR: expecting the string 'msgnum:$i', it's not there"
+ exit 1
+ fi
+done
+
+## if we got here, all is good :)
+
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/imfile-readmode2-with-persists.conf b/tests/testsuites/imfile-readmode2-with-persists.conf
new file mode 100644
index 0000000..5d44641
--- /dev/null
+++ b/tests/testsuites/imfile-readmode2-with-persists.conf
@@ -0,0 +1,21 @@
+$IncludeConfig diag-common.conf
+global(workDirectory="test-spool")
+module(load="../plugins/imfile/.libs/imfile")
+
+input(type="imfile"
+ File="./rsyslog.input"
+ Tag="file:"
+ ReadMode="2")
+
+template(name="outfmt" type="list") {
+ constant(value="HEADER ")
+ property(name="msg" format="json")
+ constant(value="\n")
+}
+
+if $msg contains "msgnum:" then
+ action(
+ type="omfile"
+ file="rsyslog.out.log"
+ template="outfmt"
+ )
--
2.13.2
