File 0004-TSC-frequencies-add-250PPM-tolerance.patch of Package kubevirt
From f1e7f4d5bad44293b06d0f113c97174667688faf Mon Sep 17 00:00:00 2001
From: Jed Lejosne <jed@redhat.com>
Date: Thu, 20 Apr 2023 16:45:47 -0400
Subject: [PATCH] TSC frequencies: add 250PPM tolerance
It is possible to migrate VMs to nodes that have a TSC frequency within 250PPM of theirs.
This adds support for it by tagging nodes with VMI frequencies within the tolerance range.
Signed-off-by: Jed Lejosne <jed@redhat.com>
---
pkg/virt-controller/watch/topology/filter.go | 8 ++++++
pkg/virt-controller/watch/topology/tsc.go | 26 ++++++++++++++-----
.../watch/topology/tsc_test.go | 21 ++++++++++-----
3 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/pkg/virt-controller/watch/topology/filter.go b/pkg/virt-controller/watch/topology/filter.go
index 0108f5a41..ddf05489a 100644
--- a/pkg/virt-controller/watch/topology/filter.go
+++ b/pkg/virt-controller/watch/topology/filter.go
@@ -1,6 +1,8 @@
package topology
import (
+ "math"
+
v1 "k8s.io/api/core/v1"
virtv1 "kubevirt.io/api/core/v1"
@@ -10,6 +12,7 @@ import (
const TSCFrequencyLabel = virtv1.CPUTimerLabel + "tsc-frequency"
const TSCFrequencySchedulingLabel = "scheduling.node.kubevirt.io/tsc-frequency"
const TSCScalableLabel = virtv1.CPUTimerLabel + "tsc-scalable"
+const TSCTolerancePPM float64 = 250
type FilterPredicateFunc func(node *v1.Node) bool
@@ -89,3 +92,8 @@ func FilterNodesFromCache(objs []interface{}, predicates ...FilterPredicateFunc)
}
return match
}
+
+// ToleranceForFrequency returns TSCTolerancePPM parts per million of freq, rounded down to the nearest Hz
+func ToleranceForFrequency(freq int64) int64 {
+ return int64(math.Floor(float64(freq) * (TSCTolerancePPM / 1000000)))
+}
diff --git a/pkg/virt-controller/watch/topology/tsc.go b/pkg/virt-controller/watch/topology/tsc.go
index 376796eb1..6f03c91e4 100644
--- a/pkg/virt-controller/watch/topology/tsc.go
+++ b/pkg/virt-controller/watch/topology/tsc.go
@@ -85,14 +85,24 @@ func TSCFrequenciesOnNode(node *v1.Node) (frequencies []int64) {
return
}
-func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) {
- if scalable {
- frequenciesInUse = append(frequenciesInUse, nodeFrequency)
- } else {
- frequenciesInUse = []int64{nodeFrequency}
+func distance(freq1, freq2 int64) int64 {
+ if freq1 > freq2 {
+ return freq1 - freq2
}
+ return freq2 - freq1
+}
+
+func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool) (toAdd []int64, toRemove []int64) {
+ frequenciesInUse = append(frequenciesInUse, nodeFrequency)
+ tolerance := ToleranceForFrequency(nodeFrequency)
requiredMap := map[int64]struct{}{}
for _, freq := range frequenciesInUse {
+ if !scalable && distance(freq, nodeFrequency) > tolerance {
+ // A non-scalable node can only accept frequencies that are within Qemu's tolerance:
+ // nodeFrequency*(1-0.000250) < acceptableFrequency < nodeFrequency*(1+0.000250).
+ // Skip the frequencies that are outside that range
+ continue
+ }
requiredMap[freq] = struct{}{}
}
@@ -102,8 +112,10 @@ func CalculateTSCLabelDiff(frequenciesInUse []int64, frequenciesOnNode []int64,
}
}
- for _, freq := range frequenciesInUse {
- if freq <= nodeFrequency {
+ for freq := range requiredMap {
+ // For the non-scalable case, the map was already sanitized above.
+ // For the scalable case, a node can accept frequencies that are either lower than its own or within the tolerance range
+ if !scalable || freq <= nodeFrequency || distance(freq, nodeFrequency) <= tolerance {
toAdd = append(toAdd, freq)
}
}
diff --git a/pkg/virt-controller/watch/topology/tsc_test.go b/pkg/virt-controller/watch/topology/tsc_test.go
index 14af9cba5..b556bcb85 100644
--- a/pkg/virt-controller/watch/topology/tsc_test.go
+++ b/pkg/virt-controller/watch/topology/tsc_test.go
@@ -42,8 +42,8 @@ var _ = Describe("TSC", func() {
DescribeTable("should calculate the node label diff", func(frequenciesInUse []int64, frequenciesOnNode []int64, nodeFrequency int64, scalable bool, expectedToAdd []int64, expectedToRemove []int64) {
toAdd, toRemove := topology.CalculateTSCLabelDiff(frequenciesInUse, frequenciesOnNode, nodeFrequency, scalable)
- Expect(toAdd).To(Equal(expectedToAdd))
- Expect(toRemove).To(Equal(expectedToRemove))
+ Expect(toAdd).To(ConsistOf(expectedToAdd))
+ Expect(toRemove).To(ConsistOf(expectedToRemove))
},
Entry(
"on a scalable node",
@@ -56,15 +56,15 @@ var _ = Describe("TSC", func() {
),
Entry(
"on a scalable node where not all required frequencies are compatible",
- []int64{1, 2, 3, 200},
+ []int64{1, 2, 3, 123130, 200000}, // 123130 is above but within 250 PPM
[]int64{2, 4},
- int64(123),
+ int64(123123),
true,
- []int64{1, 2, 3, 123},
+ []int64{1, 2, 3, 123123, 123130},
[]int64{4},
),
Entry(
- "on a not scalable node where only the node frequency can be set",
+ "on a non-scalable node where only the node frequency can be set",
[]int64{1, 2, 3},
[]int64{2, 4},
int64(123),
@@ -72,6 +72,15 @@ var _ = Describe("TSC", func() {
[]int64{123},
[]int64{2, 4},
),
+ Entry(
+ "on a non-scalable node where other node frequencies are close-enough",
+ []int64{1, 2, 123120, 123130}, // 250 PPM of 123123 is 30
+ []int64{2, 4},
+ int64(123123),
+ false,
+ []int64{123123, 123120, 123130},
+ []int64{2, 4},
+ ),
)
Context("needs to be set when", func() {
--
2.40.0
