File 0007-mmkubernetes-fix-lnrules-add-defaults-add-test.patch of Package rsyslog.17135
From 2b7e62786a0771de3a82d6d477ff33f10b5b8816 Mon Sep 17 00:00:00 2001
From: Rich Megginson <rmeggins@redhat.com>
Date: Thu, 5 Apr 2018 21:57:17 -0600
Subject: [PATCH] mmkubernetes: fix lnrules, add defaults, add test
Fix lnrules for CONTAINER_NAME
Add pkg check for lognorm >= 2.0.3 so we can set the macro
to enable ln_loadSamplesFromString
Add some reasonable default values for parameters, such as
kubernetesurl https://kubernetes.default.svc.cluster.local:443
Clean up sample.conf configuration file
Add test for mmkubernetes, including mock kubernetes service
(cherry picked from commit 1d49aac5cb101704486bfb065fac362ca69f06bc)
---
configure.ac | 14 +++++
contrib/mmkubernetes/mmkubernetes.c | 69 ++++++++++++++++----
contrib/mmkubernetes/sample.conf | 22 ++-----
tests/Makefile.am | 13 +++-
tests/mmkubernetes-basic-vg.sh | 89 ++++++++++++++++++++++++++
tests/mmkubernetes-basic.out.json | 110 ++++++++++++++++++++++++++++++++
tests/mmkubernetes-basic.sh | 88 ++++++++++++++++++++++++++
tests/mmkubernetes_test_server.py | 121 ++++++++++++++++++++++++++++++++++++
8 files changed, 495 insertions(+), 31 deletions(-)
create mode 100755 tests/mmkubernetes-basic-vg.sh
create mode 100644 tests/mmkubernetes-basic.out.json
create mode 100755 tests/mmkubernetes-basic.sh
create mode 100644 tests/mmkubernetes_test_server.py
diff --git a/configure.ac b/configure.ac
index de679b3ef..17548b9fe 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2099,6 +2099,20 @@ AC_ARG_ENABLE(mmkubernetes,
)
if test "x$enable_mmkubernetes" = "xyes"; then
PKG_CHECK_MODULES([CURL], [libcurl])
+ PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 2.0.3)
+
+ save_CFLAGS="$CFLAGS"
+ save_LIBS="$LIBS"
+
+ CFLAGS="$CFLAGS $LIBLOGNORM_CFLAGS"
+ LIBS="$LIBS $LIBLOGNORM_LIBS"
+
+ AC_CHECK_FUNC([ln_loadSamplesFromString],
+ [AC_DEFINE([HAVE_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString exists.])],
+ [AC_DEFINE([NO_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString does not exist.])])
+
+ CFLAGS="$save_CFLAGS"
+ LIBS="$save_LIBS"
fi
AM_CONDITIONAL(ENABLE_MMKUBERNETES, test x$enable_mmkubernetes = xyes)
diff --git a/contrib/mmkubernetes/mmkubernetes.c b/contrib/mmkubernetes/mmkubernetes.c
index 5012c54f6..6aba930e3 100644
--- a/contrib/mmkubernetes/mmkubernetes.c
+++ b/contrib/mmkubernetes/mmkubernetes.c
@@ -75,10 +75,10 @@ DEFobjCurrIf(regexp)
* this is for _tag_ match, not actual filename match - in_tail turns filename
* into a fluentd tag
*/
-#define DFLT_FILENAME_LNRULES ":/var/log/containers/%pod_name:char-to:.%."\
+#define DFLT_FILENAME_LNRULES "rule=:/var/log/containers/%pod_name:char-to:.%."\
"%container_hash:char-to:_%_"\
"%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log\n"\
- ":/var/log/containers/%pod_name:char-to:_%_"\
+ "rule=:/var/log/containers/%pod_name:char-to:_%_"\
"%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log"
#define DFLT_FILENAME_RULEBASE "/etc/rsyslog.d/k8s_filename.rulebase"
/* original from fluentd plugin:
@@ -86,10 +86,10 @@ DEFobjCurrIf(regexp)
* (\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_\
* (?<namespace>[^_]+)_[^_]+_[^_]+$'
*/
-#define DFLT_CONTAINER_LNRULES ":%k8s_prefix:char-to:_%_%container_name:char-to:.%."\
- "%container_hash:char-to:_%_%"\
+#define DFLT_CONTAINER_LNRULES "rule=:%k8s_prefix:char-to:_%_%container_name:char-to:.%."\
+ "%container_hash:char-to:_%_"\
"%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%\n"\
- ":%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\
+ "rule=:%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\
"%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%"
#define DFLT_CONTAINER_RULEBASE "/etc/rsyslog.d/k8s_container_name.rulebase"
#define DFLT_SRCMD_PATH "$!metadata!filename"
@@ -98,6 +98,7 @@ DEFobjCurrIf(regexp)
#define DFLT_DE_DOT_SEPARATOR "_"
#define DFLT_CONTAINER_NAME "$!CONTAINER_NAME" /* name of variable holding CONTAINER_NAME value */
#define DFLT_CONTAINER_ID_FULL "$!CONTAINER_ID_FULL" /* name of variable holding CONTAINER_ID_FULL value */
+#define DFLT_KUBERNETES_URL "https://kubernetes.default.svc.cluster.local:443"
static struct cache_s {
const uchar *kbUrl;
@@ -953,9 +954,11 @@ CODESTARTnewActInst
loadModConf->contRules, loadModConf->contRulebase));
if(pData->kubernetesUrl == NULL) {
- if(loadModConf->kubernetesUrl == NULL)
- ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
- pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl);
+ if(loadModConf->kubernetesUrl == NULL) {
+ CHKmalloc(pData->kubernetesUrl = (uchar *) strdup(DFLT_KUBERNETES_URL));
+ } else {
+ CHKmalloc(pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl));
+ }
}
if(pData->srcMetadataDescr == NULL) {
CHKmalloc(pData->srcMetadataDescr = MALLOC(sizeof(msgPropDescr_t)));
@@ -1125,10 +1128,10 @@ extractMsgMetadata(smsg_t *pMsg, instanceData *pData, struct json_object **json)
/* extract metadata from the file name */
filename = MsgGetProp(pMsg, NULL, pData->srcMetadataDescr, &fnLen, &freeFn, NULL);
- dbgprintf("mmkubernetes: filename: '%s'.\n", filename);
- if(filename == NULL)
+ if((filename == NULL) || (fnLen == 0))
ABORT_FINALIZE(RS_RET_NOT_FOUND);
+ dbgprintf("mmkubernetes: filename: '%s' len %d.\n", filename, fnLen);
if ((lnret = ln_normalize(pData->fnCtxln, (char*)filename, fnLen, json))) {
ABORT_FINALIZE(RS_RET_ERR);
}
@@ -1173,18 +1176,58 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply)
CURLcode ccode;
struct json_tokener *jt = NULL;
struct json_object *jo;
+ long resp_code = 400;
/* query kubernetes for pod info */
ccode = curl_easy_setopt(pWrkrData->curlCtx, CURLOPT_URL, url);
if(ccode != CURLE_OK)
ABORT_FINALIZE(RS_RET_ERR);
- if (CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) {
+ if(CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) {
errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
"mmkubernetes: failed to connect to [%s] - %d:%s\n",
url, ccode, curl_easy_strerror(ccode));
ABORT_FINALIZE(RS_RET_ERR);
}
-
+ if(CURLE_OK != (ccode = curl_easy_getinfo(pWrkrData->curlCtx,
+ CURLINFO_RESPONSE_CODE, &resp_code))) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: could not get response code from query to [%s] - %d:%s\n",
+ url, ccode, curl_easy_strerror(ccode));
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ if(resp_code == 401) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: Unauthorized: not allowed to view url - "
+ "check token/auth credentials [%s]\n",
+ url);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ if(resp_code == 403) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: Forbidden: no access - "
+ "check permissions to view url [%s]\n",
+ url);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ if(resp_code == 404) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: Not Found: the resource does not exist at url [%s]\n",
+ url);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ if(resp_code == 429) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: Too Many Requests: the server is too heavily loaded "
+ "to provide the data for the requested url [%s]\n",
+ url);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
+ if(resp_code != 200) {
+ errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+ "mmkubernetes: server returned unexpected code [%ld] for url [%s]\n",
+ resp_code, url);
+ ABORT_FINALIZE(RS_RET_ERR);
+ }
/* parse retrieved data */
jt = json_tokener_new();
json_tokener_reset(jt);
@@ -1384,7 +1427,7 @@ CODESTARTdoAction
msgAddJSON(pMsg, (uchar *) pWrkrData->pData->dstMetadataPath + 1, jMetadataCopy, 0, 0);
finalize_it:
- json_object_put(jMsgMeta);
+ json_object_put(jMsgMeta);
free(mdKey);
ENDdoAction
diff --git a/contrib/mmkubernetes/sample.conf b/contrib/mmkubernetes/sample.conf
index 4c400ed51..55946003a 100644
--- a/contrib/mmkubernetes/sample.conf
+++ b/contrib/mmkubernetes/sample.conf
@@ -1,19 +1,7 @@
-module(load="imfile" mode="inotify")
-module(load="mmkubernetes" kubernetesurl="https://localhost:8443"
- tls.cacert="/etc/rsyslog.d/mmk8s.ca.crt"
- tokenfile="/etc/rsyslog.d/mmk8s.token" annotation_match=["."])
+module(load="mmkubernetes") # see docs for all module and action parameters
-template(name="tpl" type="list") {
- property(name="jsonmesg")
- constant(value="\n")
-}
+# $!metadata!filename added by imfile using addmetadata="on"
+# e.g. input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on")
+# $!CONTAINER_NAME and $!CONTAINER_ID_FULL added by imjournal
-ruleset(name="k8s") {
- action(type="mmkubernetes")
- action(type="omfile" file="/var/log/k8s.log" template="tpl")
-}
-
-input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on" ruleset="k8s")
-if ($!_SYSTEMD_UNIT == "docker.service") and (strlen($!CONTAINER_NAME) > 0) then {
- call k8s
-}
+action(type="mmkubernetes")
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4386c626d..bdee87134 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -810,6 +810,15 @@ TESTS += \
omtcl.sh
endif
+if ENABLE_MMKUBERNETES
+TESTS += \
+ mmkubernetes-basic.sh
+if HAVE_VALGRIND
+TESTS += \
+ mmkubernetes-basic-vg.sh
+endif
+endif
+
endif # if ENABLE_TESTBENCH
TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/
@@ -1841,7 +1850,9 @@ EXTRA_DIST= \
pgsql-basic-cnf6-vg.sh \
pgsql-template-cnf6-vg.sh \
pgsql-actq-mt-withpause-vg.sh \
- ../devtools/prep-mysql-db.sh
+ ../devtools/prep-mysql-db.sh \
+ mmkubernetes-basic.sh \
+ mmkubernetes-basic-vg.sh
ourtail_SOURCES = ourtail.c
msleep_SOURCES = msleep.c
diff --git a/tests/mmkubernetes-basic-vg.sh b/tests/mmkubernetes-basic-vg.sh
new file mode 100755
index 000000000..33f3d3588
--- /dev/null
+++ b/tests/mmkubernetes-basic-vg.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+# added 2018-04-06 by richm, released under ASL 2.0
+#export RSYSLOG_DEBUG="debug"
+. $srcdir/diag.sh init
+
+testsrv=mmk8s-test-server
+python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 &
+BGPROCESS=$!
+. $srcdir/diag.sh wait-startup $testsrv
+echo background mmkubernetes_test_server.py process id is $BGPROCESS
+
+pwd=$( pwd )
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imfile/.libs/imfile")
+module(load="../plugins/mmjsonparse/.libs/mmjsonparse")
+module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443"
+ filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log",
+ "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"]
+)
+
+template(name="mmk8s_template" type="list") {
+ property(name="$!all-json-plain")
+ constant(value="\n")
+}
+
+input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on")
+action(type="mmjsonparse" cookie="")
+action(type="mmkubernetes")
+action(type="omfile" file="rsyslog.out.log" template="mmk8s_template")
+'
+cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name3.log <<EOF
+{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"}
+EOF
+cat > pod-name4.log <<EOF
+{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"}
+EOF
+rm -f imfile-state\:*
+. $srcdir/diag.sh startup-vg-noleak
+sleep 10 || :
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+
+kill $BGPROCESS
+. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid
+cat mmk8s_srv.log
+
+# for each record in mmkubernetes-basic.out.json, see if the matching
+# record is found in rsyslog.out.log
+python -c 'import sys,json
+expected = {}
+for hsh in json.load(open(sys.argv[1])):
+ if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+ expected[hsh["kubernetes"]["pod_name"]] = hsh
+rc = 0
+actual = {}
+for line in open(sys.argv[2]):
+ hsh = json.loads(line)
+ if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+ actual[hsh["kubernetes"]["pod_name"]] = hsh
+for pod,hsh in expected.items():
+ if not pod in actual:
+ print("Error: record for pod {0} not found in output".format(pod))
+ rc = 1
+ else:
+ for kk,vv in hsh.items():
+ if not kk in actual[pod]:
+ print("Error: key {0} in record for pod {1} not found in output".format(kk, pod))
+ rc = 1
+ elif not vv == actual[pod][kk]:
+ print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv))
+ rc = 1
+sys.exit(rc)
+' mmkubernetes-basic.out.json rsyslog.out.log
+if [ $? -ne 0 ]; then
+ echo
+ echo "FAIL: expected data not found. rsyslog.out.log is:"
+ cat rsyslog.out.log
+ . $srcdir/diag.sh error-exit 1
+fi
+
+. $srcdir/diag.sh exit
diff --git a/tests/mmkubernetes-basic.out.json b/tests/mmkubernetes-basic.out.json
new file mode 100644
index 000000000..e5876ef21
--- /dev/null
+++ b/tests/mmkubernetes-basic.out.json
@@ -0,0 +1,110 @@
+[{
+ "kubernetes": {
+ "namespace_id": "namespace-name2-id",
+ "namespace_labels": {
+ "label_1_key": "label 1 value",
+ "label_with_empty_value": "",
+ "label_2_key": "label 2 value"
+ },
+ "creation_timestamp": "2018-04-09T21:56:39Z",
+ "pod_id": "pod-name2-id",
+ "labels": {
+ "custom_label": "pod-name2-label-value",
+ "deploymentconfig": "pod-name2-dc",
+ "component": "pod-name2-component",
+ "label_with_empty_value": "",
+ "deployment": "pod-name2-deployment"
+ },
+ "pod_name": "pod-name2",
+ "namespace_name": "namespace-name2",
+ "container_name": "container-name2",
+ "master_url": "http://localhost:18443"
+ },
+ "docker": {
+ "container_id": "id2"
+ }
+},
+{
+ "message": "a message from container 4",
+ "CONTAINER_NAME": "some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44",
+ "CONTAINER_ID_FULL": "id4",
+ "kubernetes": {
+ "namespace_id": "namespace-name4-id",
+ "namespace_labels": {
+ "label_1_key": "label 1 value",
+ "label_with_empty_value": "",
+ "label_2_key": "label 2 value"
+ },
+ "creation_timestamp": "2018-04-09T21:56:39Z",
+ "pod_id": "pod-name4-id",
+ "labels": {
+ "custom_label": "pod-name4-label-value",
+ "deploymentconfig": "pod-name4-dc",
+ "component": "pod-name4-component",
+ "label_with_empty_value": "",
+ "deployment": "pod-name4-deployment"
+ },
+ "pod_name": "pod-name4",
+ "namespace_name": "namespace-name4",
+ "container_name": "container-name4",
+ "master_url": "http://localhost:18443"
+ },
+ "docker": {
+ "container_id": "id4"
+ }
+},
+{
+ "kubernetes": {
+ "namespace_id": "namespace-name1-id",
+ "namespace_labels": {
+ "label_1_key": "label 1 value",
+ "label_with_empty_value": "",
+ "label_2_key": "label 2 value"
+ },
+ "creation_timestamp": "2018-04-09T21:56:39Z",
+ "pod_id": "pod-name1-id",
+ "labels": {
+ "custom_label": "pod-name1-label-value",
+ "deploymentconfig": "pod-name1-dc",
+ "component": "pod-name1-component",
+ "label_with_empty_value": "",
+ "deployment": "pod-name1-deployment"
+ },
+ "pod_name": "pod-name1",
+ "namespace_name": "namespace-name1",
+ "container_name": "container-name1",
+ "master_url": "http://localhost:18443"
+ },
+ "docker": {
+ "container_id": "id1"
+ }
+},
+{
+ "message": "a message from container 3",
+ "CONTAINER_NAME": "some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33",
+ "CONTAINER_ID_FULL": "id3",
+ "kubernetes": {
+ "namespace_id": "namespace-name3-id",
+ "namespace_labels": {
+ "label_1_key": "label 1 value",
+ "label_with_empty_value": "",
+ "label_2_key": "label 2 value"
+ },
+ "creation_timestamp": "2018-04-09T21:56:39Z",
+ "pod_id": "pod-name3-id",
+ "labels": {
+ "custom_label": "pod-name3-label-value",
+ "deploymentconfig": "pod-name3-dc",
+ "component": "pod-name3-component",
+ "label_with_empty_value": "",
+ "deployment": "pod-name3-deployment"
+ },
+ "pod_name": "pod-name3",
+ "namespace_name": "namespace-name3",
+ "container_name": "container-name3",
+ "master_url": "http://localhost:18443"
+ },
+ "docker": {
+ "container_id": "id3"
+ }
+}]
diff --git a/tests/mmkubernetes-basic.sh b/tests/mmkubernetes-basic.sh
new file mode 100755
index 000000000..0bbfd08ca
--- /dev/null
+++ b/tests/mmkubernetes-basic.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+# added 2018-04-06 by richm, released under ASL 2.0
+#export RSYSLOG_DEBUG="debug"
+. $srcdir/diag.sh init
+
+testsrv=mmk8s-test-server
+python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 &
+BGPROCESS=$!
+. $srcdir/diag.sh wait-startup $testsrv
+echo background mmkubernetes_test_server.py process id is $BGPROCESS
+
+pwd=$( pwd )
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imfile/.libs/imfile")
+module(load="../plugins/mmjsonparse/.libs/mmjsonparse")
+module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443"
+ filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log",
+ "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"]
+)
+
+template(name="mmk8s_template" type="list") {
+ property(name="$!all-json-plain")
+ constant(value="\n")
+}
+
+input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on")
+action(type="mmjsonparse" cookie="")
+action(type="mmkubernetes")
+action(type="omfile" file="rsyslog.out.log" template="mmk8s_template")
+'
+cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name3.log <<EOF
+{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"}
+EOF
+cat > pod-name4.log <<EOF
+{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"}
+EOF
+rm -f imfile-state\:*
+. $srcdir/diag.sh startup
+sleep 10 || :
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown
+
+kill $BGPROCESS
+. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid
+cat mmk8s_srv.log
+
+# for each record in mmkubernetes-basic.out.json, see if the matching
+# record is found in rsyslog.out.log
+python -c 'import sys,json
+expected = {}
+for hsh in json.load(open(sys.argv[1])):
+ if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+ expected[hsh["kubernetes"]["pod_name"]] = hsh
+rc = 0
+actual = {}
+for line in open(sys.argv[2]):
+ hsh = json.loads(line)
+ if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+ actual[hsh["kubernetes"]["pod_name"]] = hsh
+for pod,hsh in expected.items():
+ if not pod in actual:
+ print("Error: record for pod {0} not found in output".format(pod))
+ rc = 1
+ else:
+ for kk,vv in hsh.items():
+ if not kk in actual[pod]:
+ print("Error: key {0} in record for pod {1} not found in output".format(kk, pod))
+ rc = 1
+ elif not vv == actual[pod][kk]:
+ print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv))
+ rc = 1
+sys.exit(rc)
+' mmkubernetes-basic.out.json rsyslog.out.log
+if [ $? -ne 0 ]; then
+ echo
+ echo "FAIL: expected data not found. rsyslog.out.log is:"
+ cat rsyslog.out.log
+ . $srcdir/diag.sh error-exit 1
+fi
+
+. $srcdir/diag.sh exit
diff --git a/tests/mmkubernetes_test_server.py b/tests/mmkubernetes_test_server.py
new file mode 100644
index 000000000..0de215603
--- /dev/null
+++ b/tests/mmkubernetes_test_server.py
@@ -0,0 +1,121 @@
+# Used by the mmkubernetes tests
+# This is a simple http server which responds to kubernetes api requests
+# and responds with kubernetes api server responses
+# added 2018-04-06 by richm, released under ASL 2.0
+import os
+import json
+import sys
+
+try:
+ from http.server import HTTPServer, BaseHTTPRequestHandler
+except ImportError:
+ from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+
+ns_template = '''{{
+ "kind": "Namespace",
+ "apiVersion": "v1",
+ "metadata": {{
+ "name": "{namespace_name}",
+ "selfLink": "/api/v1/namespaces/{namespace_name}",
+ "uid": "{namespace_name}-id",
+ "resourceVersion": "2988",
+ "creationTimestamp": "2018-04-09T21:56:39Z",
+ "labels": {{
+ "label.1.key":"label 1 value",
+ "label.2.key":"label 2 value",
+ "label.with.empty.value":""
+ }},
+ "annotations": {{
+ "k8s.io/description": "",
+ "k8s.io/display-name": "",
+ "k8s.io/node-selector": "",
+ "k8s.io/sa.scc.mcs": "s0:c9,c4",
+ "k8s.io/sa.scc.supplemental-groups": "1000080000/10000",
+ "k8s.io/sa.scc.uid-range": "1000080000/10000",
+ "quota.k8s.io/cluster-resource-override-enabled": "false"
+ }}
+ }},
+ "spec": {{
+ "finalizers": [
+ "openshift.io/origin",
+ "kubernetes"
+ ]
+ }},
+ "status": {{
+ "phase": "Active"
+ }}
+}}'''
+
+pod_template = '''{{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {{
+ "name": "{pod_name}",
+ "generateName": "{pod_name}-prefix",
+ "namespace": "{namespace_name}",
+ "selfLink": "/api/v1/namespaces/{namespace_name}/pods/{pod_name}",
+ "uid": "{pod_name}-id",
+ "resourceVersion": "3486",
+ "creationTimestamp": "2018-04-09T21:56:39Z",
+ "labels": {{
+ "component": "{pod_name}-component",
+ "deployment": "{pod_name}-deployment",
+ "deploymentconfig": "{pod_name}-dc",
+ "custom.label": "{pod_name}-label-value",
+ "label.with.empty.value":""
+ }},
+ "annotations": {{
+ "k8s.io/deployment-config.latest-version": "1",
+ "k8s.io/deployment-config.name": "{pod_name}-dc",
+ "k8s.io/deployment.name": "{pod_name}-deployment",
+ "k8s.io/custom.name": "custom value",
+ "annotation.with.empty.value":""
+ }}
+ }},
+ "status": {{
+ "phase": "Running",
+ "hostIP": "172.18.4.32",
+ "podIP": "10.128.0.14",
+ "startTime": "2018-04-09T21:57:39Z"
+ }}
+}}'''
+
+class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+
+ def do_GET(self):
+ # "http://localhost:18443/api/v1/namespaces/namespace-name2"
+ # parse url - either /api/v1/namespaces/$ns_name
+ # or
+ # /api/v1/namespaces/$ns_name/pods/$pod_name
+ comps = self.path.split('/')
+ status = 400
+ if len(comps) >= 5 and comps[1] == 'api' and comps[2] == 'v1':
+ if len(comps) == 5 and comps[3] == 'namespaces': # namespace
+ resp = ns_template.format(namespace_name=comps[4])
+ status = 200
+ elif len(comps) == 7 and comps[3] == 'namespaces' and comps[5] == 'pods':
+ resp = pod_template.format(namespace_name=comps[4], pod_name=comps[6])
+ status = 200
+ else:
+ resp = '{{"error":"do not recognize {0}"}}'.format(self.path)
+ else:
+ resp = '{{"error":"do not recognize {0}"}}'.format(self.path)
+ if not status == 200:
+ self.log_error(resp)
+ self.send_response(status)
+ self.end_headers()
+ self.wfile.write(json.dumps(json.loads(resp), separators=(',',':')))
+
+port = int(sys.argv[1])
+
+httpd = HTTPServer(('localhost', port), SimpleHTTPRequestHandler)
+
+# write "started" to file named in argv[3]
+with open(sys.argv[3], "w") as ff:
+ ff.write("started\n")
+
+# write pid to file named in argv[2]
+with open(sys.argv[2], "w") as ff:
+ ff.write('{0}\n'.format(os.getpid()))
+
+httpd.serve_forever()
--
2.16.4
