File 0014-ServiceMonitor-and-PrometheusRule-API-detection-and-.patch of Package kubevirt.31335
From 8d4af84fd0674dd88564bd82e88fcd3c3eb2504e Mon Sep 17 00:00:00 2001
From: Chris Ho <chris.he@suse.com>
Date: Wed, 18 Jan 2023 21:16:57 +0800
Subject: [PATCH] ServiceMonitor and PrometheusRule API detection and uint test
Signed-off-by: Chris Ho <chris.he@suse.com>
---
pkg/testutils/mock_config.go | 34 +++++++++++
pkg/virt-config/configuration.go | 41 ++++++++++++-
pkg/virt-operator/BUILD.bazel | 1 +
pkg/virt-operator/application.go | 57 ++++++++++++++++--
pkg/virt-operator/application_test.go | 86 +++++++++++++++++++++++++++
5 files changed, 212 insertions(+), 7 deletions(-)
create mode 100644 pkg/virt-operator/application_test.go
diff --git a/pkg/testutils/mock_config.go b/pkg/testutils/mock_config.go
index f5c2cdd79..990e3ebde 100644
--- a/pkg/testutils/mock_config.go
+++ b/pkg/testutils/mock_config.go
@@ -93,3 +93,37 @@ func UpdateFakeKubeVirtClusterConfig(kubeVirtInformer cache.SharedIndexInformer,
kubeVirtInformer.GetStore().Update(clone)
}
+
+func AddServiceMonitorAPI(crdInformer cache.SharedIndexInformer) {
+ crdInformer.GetStore().Add(&extv1.CustomResourceDefinition{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "service-monitors.monitoring.coreos.com",
+ },
+ Spec: extv1.CustomResourceDefinitionSpec{
+ Names: extv1.CustomResourceDefinitionNames{
+ Kind: "ServiceMonitor",
+ },
+ },
+ })
+}
+
+func RemoveServiceMonitorAPI(crdInformer cache.SharedIndexInformer) {
+ crdInformer.GetStore().Replace(nil, "")
+}
+
+func AddPrometheusRuleAPI(crdInformer cache.SharedIndexInformer) {
+ crdInformer.GetStore().Add(&extv1.CustomResourceDefinition{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "prometheusrules.monitoring.coreos.com",
+ },
+ Spec: extv1.CustomResourceDefinitionSpec{
+ Names: extv1.CustomResourceDefinitionNames{
+ Kind: "PrometheusRule",
+ },
+ },
+ })
+}
+
+func RemovePrometheusRuleAPI(crdInformer cache.SharedIndexInformer) {
+ crdInformer.GetStore().Replace(nil, "")
+}
diff --git a/pkg/virt-config/configuration.go b/pkg/virt-config/configuration.go
index 06ddcd1a7..127c16172 100644
--- a/pkg/virt-config/configuration.go
+++ b/pkg/virt-config/configuration.go
@@ -119,10 +119,19 @@ func isDataSourceCrd(crd *extv1.CustomResourceDefinition) bool {
return crd.Spec.Names.Kind == "DataSource"
}
+func isServiceMonitor(crd *extv1.CustomResourceDefinition) bool {
+ return crd.Spec.Names.Kind == "ServiceMonitor"
+}
+
+func isPrometheusRules(crd *extv1.CustomResourceDefinition) bool {
+ return crd.Spec.Names.Kind == "PrometheusRule"
+}
+
func (c *ClusterConfig) crdAddedDeleted(obj interface{}) {
go c.GetConfig()
crd := obj.(*extv1.CustomResourceDefinition)
- if !isDataVolumeCrd(crd) && !isDataSourceCrd(crd) {
+ if !isDataVolumeCrd(crd) && !isDataSourceCrd(crd) &&
+ !isServiceMonitor(crd) && !isPrometheusRules(crd) {
return
}
@@ -379,6 +388,36 @@ func (c *ClusterConfig) HasDataVolumeAPI() bool {
return false
}
+func (c *ClusterConfig) HasServiceMonitorAPI() bool {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ objects := c.crdInformer.GetStore().List()
+ for _, obj := range objects {
+ if crd, ok := obj.(*extv1.CustomResourceDefinition); ok && crd.DeletionTimestamp == nil {
+ if isServiceMonitor(crd) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (c *ClusterConfig) HasPrometheusRuleAPI() bool {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ objects := c.crdInformer.GetStore().List()
+ for _, obj := range objects {
+ if crd, ok := obj.(*extv1.CustomResourceDefinition); ok && crd.DeletionTimestamp == nil {
+ if isPrometheusRules(crd) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
func parseNodeSelectors(str string) (map[string]string, error) {
nodeSelectors := make(map[string]string)
for _, s := range strings.Split(strings.TrimSpace(str), "\n") {
diff --git a/pkg/virt-operator/BUILD.bazel b/pkg/virt-operator/BUILD.bazel
index c6058e4a2..6b441c782 100644
--- a/pkg/virt-operator/BUILD.bazel
+++ b/pkg/virt-operator/BUILD.bazel
@@ -59,6 +59,7 @@ go_test(
name = "go_default_test",
timeout = "long",
srcs = [
+ "application_test.go",
"kubevirt_test.go",
"virt_operator_suite_test.go",
],
diff --git a/pkg/virt-operator/application.go b/pkg/virt-operator/application.go
index e9076921e..df2e42d54 100644
--- a/pkg/virt-operator/application.go
+++ b/pkg/virt-operator/application.go
@@ -91,6 +91,8 @@ type VirtOperatorApp struct {
kubeVirtInformer cache.SharedIndexInformer
kubeVirtCache cache.Store
+ crdInformer cache.SharedIndexInformer
+
stores util.Stores
informers util.Informers
@@ -99,6 +101,10 @@ type VirtOperatorApp struct {
operatorCertManager certificate.Manager
clusterConfig *virtconfig.ClusterConfig
+
+ ctx context.Context
+
+ reInitChan chan string
}
var (
@@ -223,6 +229,8 @@ func Execute() {
ConfigMapCache: app.informerFactory.OperatorConfigMap().GetStore(),
}
+ app.crdInformer = app.informerFactory.CRD()
+
onOpenShift, err := clusterutil.IsOnOpenShift(app.clientSet)
if err != nil {
golog.Fatalf("Error determining cluster type: %v", err)
@@ -284,7 +292,14 @@ func Execute() {
app.informerFactory.KubeVirt(),
app.operatorNamespace)
- app.Run()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ app.ctx = ctx
+
+ app.reInitChan = make(chan string, 0)
+
+ go app.Run()
+ <-app.reInitChan
}
func (app *VirtOperatorApp) Run() {
@@ -317,9 +332,6 @@ func (app *VirtOperatorApp) Run() {
}
}()
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
endpointName := VirtOperator
recorder := app.getNewRecorder(k8sv1.NamespaceAll, endpointName)
@@ -344,8 +356,13 @@ func (app *VirtOperatorApp) Run() {
apiAuthConfig := app.informerFactory.ApiAuthConfigMap()
- stop := ctx.Done()
+ stop := app.ctx.Done()
app.informerFactory.Start(stop)
+
+ stopChan := app.ctx.Done()
+ cache.WaitForCacheSync(stopChan, app.crdInformer.HasSynced, app.kubeVirtInformer.HasSynced)
+ app.clusterConfig.SetConfigModifiedCallback(app.configModificationCallback)
+
cache.WaitForCacheSync(stop, apiAuthConfig.HasSynced)
go app.operatorCertManager.Start()
@@ -404,11 +421,39 @@ func (app *VirtOperatorApp) Run() {
readyGauge.Set(1)
log.Log.Infof("Attempting to acquire leader status")
- leaderElector.Run(ctx)
+ leaderElector.Run(app.ctx)
+
panic("unreachable")
}
+// Detects if ServiceMonitor or PrometheusRule crd has been applied or deleted that
+// re-initializing virt-operator.
+func (app *VirtOperatorApp) configModificationCallback() {
+ msgf := "Reinitialize virt-operator, %s has been %s"
+
+ smEnabled := app.clusterConfig.HasServiceMonitorAPI()
+ if app.stores.ServiceMonitorEnabled != smEnabled {
+ if !app.stores.ServiceMonitorEnabled && smEnabled {
+ log.Log.Infof(msgf, "ServiceMonitor", "introduced")
+ } else {
+ log.Log.Infof(msgf, "ServiceMonitor", "removed")
+ }
+ app.reInitChan <- "reinit"
+ return
+ }
+
+ prEnabled := app.clusterConfig.HasPrometheusRuleAPI()
+ if app.stores.PrometheusRulesEnabled != prEnabled {
+ if !app.stores.PrometheusRulesEnabled && prEnabled {
+ log.Log.Infof(msgf, "PrometheusRule", "introduced")
+ } else {
+ log.Log.Infof(msgf, "PrometheusRule", "removed")
+ }
+ app.reInitChan <- "reinit"
+ }
+}
+
func (app *VirtOperatorApp) getNewRecorder(namespace string, componentName string) record.EventRecorder {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(&k8coresv1.EventSinkImpl{Interface: app.clientSet.CoreV1().Events(namespace)})
diff --git a/pkg/virt-operator/application_test.go b/pkg/virt-operator/application_test.go
new file mode 100644
index 000000000..e62ed9f77
--- /dev/null
+++ b/pkg/virt-operator/application_test.go
@@ -0,0 +1,86 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2017 Red Hat, Inc.
+ *
+ */
+
+package virt_operator
+
+import (
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ v1 "kubevirt.io/api/core/v1"
+
+ "kubevirt.io/kubevirt/pkg/testutils"
+)
+
+var _ = Describe("Reinitialization conditions", func() {
+ DescribeTable("Re-trigger initialization", func(
+ hasServiceMonitor bool, hasPrometheusRules bool,
+ addServiceMonitorCrd bool, removeServiceMonitorCrd bool,
+ addPrometheusRuleCrd bool, removePrometheusRuleCrd bool,
+ expectReInit bool) {
+ var reInitTriggered bool
+
+ app := VirtOperatorApp{}
+
+ clusterConfig, crdInformer, _ := testutils.NewFakeClusterConfigUsingKVConfig(&v1.KubeVirtConfiguration{})
+ app.clusterConfig = clusterConfig
+ app.reInitChan = make(chan string, 10)
+ app.stores.ServiceMonitorEnabled = hasServiceMonitor
+ app.stores.PrometheusRulesEnabled = hasPrometheusRules
+
+ if addServiceMonitorCrd {
+ testutils.AddServiceMonitorAPI(crdInformer)
+ } else if removeServiceMonitorCrd {
+ testutils.RemoveServiceMonitorAPI(crdInformer)
+ }
+
+ if addPrometheusRuleCrd {
+ testutils.AddPrometheusRuleAPI(crdInformer)
+ } else if removePrometheusRuleCrd {
+ testutils.RemovePrometheusRuleAPI(crdInformer)
+ }
+
+ app.clusterConfig.SetConfigModifiedCallback(app.configModificationCallback)
+
+ select {
+ case <-app.reInitChan:
+ reInitTriggered = true
+ case <-time.After(1 * time.Second):
+ reInitTriggered = false
+ }
+
+ Expect(reInitTriggered).To(Equal(expectReInit))
+ },
+ Entry("when ServiceMonitor is introduced", false, false, true, false, false, false, true),
+ Entry("when ServiceMonitor is removed", true, false, false, true, false, false, true),
+ Entry("when PrometheusRule is introduced", false, false, false, false, true, false, true),
+ Entry("when PrometheusRule is removed", false, true, false, false, false, true, true),
+
+ Entry("when ServiceMonitor and PrometheusRule are introduced", false, false, true, false, true, false, true),
+ Entry("when ServiceMonitor and PrometheusRule are removed", true, true, false, true, false, true, true),
+
+ Entry("not when nothing changed and ServiceMonitor and PrometheusRule exists", true, true, true, false, true, false, false),
+ Entry("not when nothing changed and ServiceMonitor and PrometheusRule does not exists", false, false, false, true, false, true, false),
+
+ Entry("when ServiceMonitor is introduced and PrometheusRule is removed", false, true, true, false, false, true, true),
+ Entry("when ServiceMonitor is removed and PrometheusRule is introduced", true, false, false, true, true, false, true),
+ )
+})
--
2.40.1
