File 0003-imfile-improve-truncation-detection.patch of Package rsyslog.15978

Overview Repositories Revisions Requests Users Attributes Meta

File 0003-imfile-improve-truncation-detection.patch of Package rsyslog.15978

From e84113816d69d428c96924647a0167be4f6a330a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 20 Sep 2018 10:39:47 +0200
Subject: [PATCH 3/7] imfile: improve truncation detection

previously, truncation was only detected at end of file. Especially with
busy files that could cause loss of data and possibly also stall imfile
reading. The new code now also checks during each read. Obviously, there
is some additional overhead associated with that, but this is unavoidable.

It still is highly recommended NOT to turn on "reopenOnTruncate" in imfile.
Note that there are also inherant reliability issues. There is no way to
"fix" these, as they are caused by races between the process(es) who truncate
and rsyslog reading the file. But with the new code, the "problem window"
should be much smaller and, more importantly, imfile should not stall.

see also https://github.com/rsyslog/rsyslog/issues/2659
see also https://github.com/rsyslog/rsyslog/issues/1605

(cherry picked from commit 2d15cbc8221e385c5aa821e4a851d7498ed81850)
---
 runtime/rsyslog.h |  6 +++---
 runtime/stream.c  | 13 +++++++++++++
 runtime/stream.h  |  1 +
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index f7107097f..a43149501 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -196,9 +196,9 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	/* begin regular error codes */
 	RS_RET_NOT_IMPLEMENTED = -7,	/**< implementation is missing (probably internal error or lazyness ;)) */
 	RS_RET_OUT_OF_MEMORY = -6,	/**< memory allocation failed */
-	RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50,
-/*< the caller provided a buffer, but the called function sees the size of this buffer is too small -
-operation not carried out */
+	RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50, /*< the caller provided a buffer, but the called function sees
+						  the size of this buffer is too small - operation not carried out */
+	RS_RET_FILE_TRUNCATED = -51,	/**< (input) file was truncated, not an error but a status */
 	RS_RET_TRUE = -3,		/**< to indicate a true state (can be used as TRUE, legacy) */
 	RS_RET_FALSE = -2,		/**< to indicate a false state (can be used as FALSE, legacy) */
 	RS_RET_NO_IRET = -8,	/**< This is a trick for the debuging system - it means no iRet is provided  */
diff --git a/runtime/stream.c b/runtime/stream.c
index 028a4652c..cbe618cdd 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -363,6 +363,7 @@ static rsRetVal strmOpenFile(strm_t *pThis)
 	CHKiRet(doPhysOpen(pThis));
 
 	pThis->iCurrOffs = 0;
+	pThis->iBufPtrMax = 0;
 	CHKiRet(getFileSize(pThis->pszCurrFName, &offset));
 	if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) {
 		pThis->iCurrOffs = offset;
@@ -655,6 +656,9 @@ checkTruncation(strm_t *const pThis)
 		iRet = rereadTruncated(pThis, "cannot seek backward to begin of last block");
 		FINALIZE;
 	}
+dbgprintf("checkTruncation seek backwrds: %d\n", ret);
+currpos = lseek64(pThis->fd, 0, SEEK_CUR);
+dbgprintf("checkTruncation in actual processing, NEW currpos %d, backseek is %d\n", (int)currpos, (int) backseek);
 
 	const ssize_t lenRead = read(pThis->fd, pThis->pIOBuf_truncation, pThis->iBufPtrMax);
 	dbgprintf("checkTruncation proof-read: %d bytes\n", (int) lenRead);
@@ -706,6 +710,13 @@ strmReadBuf(strm_t *pThis, int *padBytes)
 				toRead = (size_t) bytesLeft;
 			}
 		}
+		if(pThis->bReopenOnTruncate) {
+			rsRetVal localRet = checkTruncation(pThis);
+			if(localRet == RS_RET_FILE_TRUNCATED) {
+				continue;
+			}
+			CHKiRet(localRet);
+		}
 		iLenRead = read(pThis->fd, pThis->pIOBuf, toRead);
 		DBGOPRINT((obj_t*) pThis, "file %d read %ld bytes\n", pThis->fd, iLenRead);
 		/* end crypto */
@@ -1210,6 +1221,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis)
 	} else {
 		/* we work synchronously, so we need to alloc a fixed pIOBuf */
 		CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(pThis->sIOBufSize));
+		CHKmalloc(pThis->pIOBuf_truncation = (char*) MALLOC(pThis->sIOBufSize));
 	}
 
 finalize_it:
@@ -1257,6 +1269,7 @@ CODESTARTobjDestruct(strm)
 		}
 	} else {
 		free(pThis->pIOBuf);
+		free(pThis->pIOBuf_truncation);
 	}
 
 	/* Finally, we can free the resources.
diff --git a/runtime/stream.h b/runtime/stream.h
index 57d8c68b0..f395bc123 100644
--- a/runtime/stream.h
+++ b/runtime/stream.h
@@ -129,6 +129,7 @@ typedef struct strm_s {
 	ino_t inode;	/* current inode for files being monitored (undefined else) */
 	uchar *pszCurrFName; /* name of current file (if open) */
 	uchar *pIOBuf;	/* the iobuffer currently in use to gather data */
+	char *pIOBuf_truncation; /* iobuffer used during trucation detection block re-reads */
 	size_t iBufPtrMax;	/* current max Ptr in Buffer (if partial read!) */
 	size_t iBufPtr;	/* pointer into current buffer */
 	int iUngetC;	/* char set via UngetChar() call or -1 if none set */
-- 
2.16.4

Places

File 0003-imfile-improve-truncation-detection.patch of Package rsyslog.15978

Places