File 0006-topology-spread-constraints.patch of Package kubevirt.31631
From 8029bbf80b9a616d022a4811b1ffe68f3da6073f Mon Sep 17 00:00:00 2001
From: janeczku <jabruder@gmail.com>
Date: Wed, 1 Jun 2022 19:35:13 +0200
Subject: [PATCH 1/2] fix: generating crd validation code fails when API code
comments contain backticks
Signed-off-by: janeczku <jabruder@gmail.com>
---
.../validation-generator.go | 31 +++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/tools/crd-validation-generator/validation-generator.go b/tools/crd-validation-generator/validation-generator.go
index 2c016e928..67feb827a 100644
--- a/tools/crd-validation-generator/validation-generator.go
+++ b/tools/crd-validation-generator/validation-generator.go
@@ -68,6 +68,7 @@ func generateGoFile(outputDir string, validations map[string]*extv1.CustomResour
for _, crdname := range crds {
crd := validations[crdname]
+ crd.OpenAPIV3Schema = sanitizeSchema(crd.OpenAPIV3Schema)
b, _ := yaml.Marshal(crd)
file.WriteString(fmt.Sprintf(variable, crdname, string(b)))
}
@@ -89,3 +90,33 @@ func getValidation(filename string) (string, *extv1.CustomResourceValidation) {
}
return crd.Spec.Names.Singular, crd.Spec.Versions[0].Schema
}
+
+// sanitizeSchema traverses the given JSON-Schema object and replaces all occurrences of the
+// backtick (`) character in the (sub-)schema Description fields with single quote characters
+func sanitizeSchema(inSchema *extv1.JSONSchemaProps) *extv1.JSONSchemaProps {
+ schema := inSchema.DeepCopy()
+ if schema.Description != "" {
+ schema.Description = strings.ReplaceAll(schema.Description, "`", "'")
+ }
+
+ // Traverse Items
+ if schema.Items != nil {
+ if schema.Items.Schema != nil {
+ schema.Items.Schema = sanitizeSchema(schema.Items.Schema)
+ }
+ if len(schema.Items.JSONSchemas) > 0 {
+ sanitizedProps := make([]extv1.JSONSchemaProps, 0, len(schema.Items.JSONSchemas))
+ for _, schema := range schema.Items.JSONSchemas {
+ sanitizedProps = append(sanitizedProps, *sanitizeSchema(&schema))
+ }
+ schema.Items.JSONSchemas = sanitizedProps
+ }
+ }
+
+ // Traverse Properties
+ for name, prop := range schema.Properties {
+ schema.Properties[name] = *sanitizeSchema(&prop)
+ }
+
+ return schema
+}
--
2.37.1
From a35ea9c6654c196a52173a3d658e8bb801cc4ac3 Mon Sep 17 00:00:00 2001
From: janeczku <jabruder@gmail.com>
Date: Tue, 31 May 2022 22:17:05 +0200
Subject: [PATCH 2/2] support topology spread constraints for virtual machine
instances
Signed-off-by: janeczku <jabruder@gmail.com>
---
api/openapi-spec/swagger.json | 42 ++
pkg/virt-controller/services/template.go | 1 +
pkg/virt-controller/services/template_test.go | 30 +
.../components/validations_generated.go | 556 ++++++++++++++++++
.../api/core/v1/deepcopy_generated.go | 7 +
staging/src/kubevirt.io/api/core/v1/types.go | 10 +-
.../api/core/v1/types_swagger_generated.go | 1 +
.../client-go/api/openapi_generated.go | 26 +-
tests/vmi_configuration_test.go | 27 +
tests/vmi_lifecycle_test.go | 73 +++
10 files changed, 771 insertions(+), 2 deletions(-)
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
index d96bb60d7..d69c0fd57 100644
--- a/api/openapi-spec/swagger.json
+++ b/api/openapi-spec/swagger.json
@@ -12945,6 +12945,34 @@
}
}
},
+ "k8s.io.api.core.v1.TopologySpreadConstraint": {
+ "description": "TopologySpreadConstraint specifies how to spread matching pods among the given topology.",
+ "type": "object",
+ "required": [
+ "maxSkew",
+ "topologyKey",
+ "whenUnsatisfiable"
+ ],
+ "properties": {
+ "labelSelector": {
+ "description": "LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.",
+ "$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.LabelSelector"
+ },
+ "maxSkew": {
+ "description": "MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It's a required field. Default value is 1 and 0 is not allowed.",
+ "type": "integer",
+ "format": "int32"
+ },
+ "topologyKey": {
+ "description": "TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each \u003ckey, value\u003e as a \"bucket\", and try to put balanced number of pods into each bucket. It's a required field.",
+ "type": "string"
+ },
+ "whenUnsatisfiable": {
+ "description": "WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location,\n but giving higher precedence to topologies that would help reduce the\n skew.\nA constraint is considered \"Unsatisfiable\" for an incoming pod if and only if every possible node assignment for that pod would violate \"MaxSkew\" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won't make it *more* imbalanced. It's a required field.",
+ "type": "string"
+ }
+ }
+ },
"k8s.io.api.core.v1.TypedLocalObjectReference": {
"description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.",
"type": "object",
@@ -17371,6 +17399,20 @@
"$ref": "#/definitions/k8s.io.api.core.v1.Toleration"
}
},
+ "topologySpreadConstraints": {
+ "description": "TopologySpreadConstraints describes how a group of VMIs will be spread across a given topology domains. K8s scheduler will schedule VMI pods in a way which abides by the constraints.",
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/k8s.io.api.core.v1.TopologySpreadConstraint"
+ },
+ "x-kubernetes-list-map-keys": [
+ "topologyKey",
+ "whenUnsatisfiable"
+ ],
+ "x-kubernetes-list-type": "map",
+ "x-kubernetes-patch-merge-key": "topologyKey",
+ "x-kubernetes-patch-strategy": "merge"
+ },
"volumes": {
"description": "List of volumes that can be mounted by disks belonging to the vmi.",
"type": "array",
diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go
index 4d248756e..8c2ca44c1 100644
--- a/pkg/virt-controller/services/template.go
+++ b/pkg/virt-controller/services/template.go
@@ -1413,6 +1413,7 @@ func (t *templateService) renderLaunchManifest(vmi *v1.VirtualMachineInstance, i
DNSConfig: vmi.Spec.DNSConfig,
DNSPolicy: vmi.Spec.DNSPolicy,
ReadinessGates: readinessGates,
+ TopologySpreadConstraints: vmi.Spec.TopologySpreadConstraints,
},
}
diff --git a/pkg/virt-controller/services/template_test.go b/pkg/virt-controller/services/template_test.go
index 6b61c1b2c..4da8f6109 100644
--- a/pkg/virt-controller/services/template_test.go
+++ b/pkg/virt-controller/services/template_test.go
@@ -1518,6 +1518,36 @@ var _ = Describe("Template", func() {
Expect(pod.Spec.Tolerations).To(BeEquivalentTo([]kubev1.Toleration{{Key: podToleration.Key, TolerationSeconds: &tolerationSeconds}}))
})
+ It("should add topology spread constraints to pod", func() {
+ config, kvInformer, svc = configFactory(defaultArch)
+ topologySpreadConstraints := []kubev1.TopologySpreadConstraint{
+ {
+ MaxSkew: 1,
+ TopologyKey: "zone",
+ WhenUnsatisfiable: "DoNotSchedule",
+ LabelSelector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "foo": "bar",
+ },
+ },
+ },
+ }
+ vm := v1.VirtualMachineInstance{
+ ObjectMeta: metav1.ObjectMeta{Name: "testvm", Namespace: "default", UID: "1234"},
+ Spec: v1.VirtualMachineInstanceSpec{
+ TopologySpreadConstraints: topologySpreadConstraints,
+ Domain: v1.DomainSpec{
+ Devices: v1.Devices{
+ DisableHotplug: true,
+ },
+ },
+ },
+ }
+ pod, err := svc.RenderLaunchManifest(&vm)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(pod.Spec.TopologySpreadConstraints).To(Equal(topologySpreadConstraints))
+ })
+
It("should add the scheduler name to the pod", func() {
config, kvInformer, svc = configFactory(defaultArch)
vm := v1.VirtualMachineInstance{
diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go
index 90d350bcd..3f348bebf 100644
--- a/pkg/virt-operator/resource/generate/components/validations_generated.go
+++ b/pkg/virt-operator/resource/generate/components/validations_generated.go
@@ -6084,6 +6084,113 @@ var CRDsValidation map[string]string = map[string]string{
type: string
type: object
type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraints describes how a group of
+ VMIs will be spread across a given topology domains. K8s scheduler
+ will schedule VMI pods in a way which abides by the constraints.
+ items:
+ description: TopologySpreadConstraint specifies how to spread
+ matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching pods.
+ Pods that match this label selector are counted to determine
+ the number of pods in their corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector
+ requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector
+ that contains values, a key, and an operator that
+ relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector
+ applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship
+ to a set of values. Valid operators are In, NotIn,
+ Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values.
+ If the operator is In or NotIn, the values array
+ must be non-empty. If the operator is Exists or
+ DoesNotExist, the values array must be empty.
+ This array is replaced during a strategic merge
+ patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs.
+ A single {key,value} in the matchLabels map is equivalent
+ to an element of matchExpressions, whose key field is
+ "key", the operator is "In", and the values array contains
+ only "value". The requirements are ANDed.
+ type: object
+ type: object
+ maxSkew:
+ description: 'MaxSkew describes the degree to which pods may
+ be unevenly distributed. When ''whenUnsatisfiable=DoNotSchedule'',
+ it is the maximum permitted difference between the number
+ of matching pods in the target topology and the global minimum.
+ For example, in a 3-zone cluster, MaxSkew is set to 1, and
+ pods with the same labelSelector spread as 1/1/0: | zone1
+ | zone2 | zone3 | | P | P | | - if MaxSkew
+ is 1, incoming pod can only be scheduled to zone3 to become
+ 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0)
+ on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming
+ pod can be scheduled onto any zone. When ''whenUnsatisfiable=ScheduleAnyway'',
+ it is used to give higher precedence to topologies that
+ satisfy it. It''s a required field. Default value is 1 and
+ 0 is not allowed.'
+ format: int32
+ type: integer
+ topologyKey:
+ description: TopologyKey is the key of node labels. Nodes
+ that have a label with this key and identical values are
+ considered to be in the same topology. We consider each
+ <key, value> as a "bucket", and try to put balanced number
+ of pods into each bucket. It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal with
+ a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule
+ (default) tells the scheduler not to schedule it. - ScheduleAnyway
+ tells the scheduler to schedule the pod in any location, but
+ giving higher precedence to topologies that would help reduce
+ the skew. A constraint is considered "Unsatisfiable" for
+ an incoming pod if and only if every possible node assignment
+ for that pod would violate "MaxSkew" on some topology. For
+ example, in a 3-zone cluster, MaxSkew is set to 1, and pods
+ with the same labelSelector spread as 3/1/1: | zone1 | zone2
+ | zone3 | | P P P | P | P | If WhenUnsatisfiable
+ is set to DoNotSchedule, incoming pod can only be scheduled
+ to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1)
+ on zone2(zone3) satisfies MaxSkew(1). In other words, the
+ cluster can still be imbalanced, but scheduler won''t make
+ it *more* imbalanced. It''s a required field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - topologyKey
+ - whenUnsatisfiable
+ x-kubernetes-list-type: map
volumes:
description: List of volumes that can be mounted by disks belonging
to the vmi.
@@ -9896,6 +10003,109 @@ var CRDsValidation map[string]string = map[string]string{
type: string
type: object
type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraints describes how a group of VMIs will
+ be spread across a given topology domains. K8s scheduler will schedule
+ VMI pods in a way which abides by the constraints.
+ items:
+ description: TopologySpreadConstraint specifies how to spread matching
+ pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching pods. Pods that
+ match this label selector are counted to determine the number of
+ pods in their corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements.
+ The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that
+ contains values, a key, and an operator that relates the key
+ and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies
+ to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to
+ a set of values. Valid operators are In, NotIn, Exists
+ and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the
+ operator is In or NotIn, the values array must be non-empty.
+ If the operator is Exists or DoesNotExist, the values
+ array must be empty. This array is replaced during a strategic
+ merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single
+ {key,value} in the matchLabels map is equivalent to an element
+ of matchExpressions, whose key field is "key", the operator
+ is "In", and the values array contains only "value". The requirements
+ are ANDed.
+ type: object
+ type: object
+ maxSkew:
+ description: 'MaxSkew describes the degree to which pods may be unevenly
+ distributed. When ''whenUnsatisfiable=DoNotSchedule'', it is the
+ maximum permitted difference between the number of matching pods
+ in the target topology and the global minimum. For example, in a
+ 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector
+ spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | |
+ - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to
+ become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0)
+ on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming
+ pod can be scheduled onto any zone. When ''whenUnsatisfiable=ScheduleAnyway'',
+ it is used to give higher precedence to topologies that satisfy
+ it. It''s a required field. Default value is 1 and 0 is not allowed.'
+ format: int32
+ type: integer
+ topologyKey:
+ description: TopologyKey is the key of node labels. Nodes that have
+ a label with this key and identical values are considered to be
+ in the same topology. We consider each <key, value> as a "bucket",
+ and try to put balanced number of pods into each bucket. It's a
+ required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal with a pod if
+ it doesn''t satisfy the spread constraint. - DoNotSchedule (default)
+ tells the scheduler not to schedule it. - ScheduleAnyway tells the
+ scheduler to schedule the pod in any location, but giving higher
+ precedence to topologies that would help reduce the skew. A constraint
+ is considered "Unsatisfiable" for an incoming pod if and only if
+ every possible node assignment for that pod would violate "MaxSkew"
+ on some topology. For example, in a 3-zone cluster, MaxSkew is set
+ to 1, and pods with the same labelSelector spread as 3/1/1: | zone1
+ | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable
+ is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3)
+ to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies
+ MaxSkew(1). In other words, the cluster can still be imbalanced,
+ but scheduler won''t make it *more* imbalanced. It''s a required
+ field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - topologyKey
+ - whenUnsatisfiable
+ x-kubernetes-list-type: map
volumes:
description: List of volumes that can be mounted by disks belonging to the
vmi.
@@ -14199,6 +14409,113 @@ var CRDsValidation map[string]string = map[string]string{
type: string
type: object
type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraints describes how a group of
+ VMIs will be spread across a given topology domains. K8s scheduler
+ will schedule VMI pods in a way which abides by the constraints.
+ items:
+ description: TopologySpreadConstraint specifies how to spread
+ matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching pods.
+ Pods that match this label selector are counted to determine
+ the number of pods in their corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector
+ requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector
+ that contains values, a key, and an operator that
+ relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector
+ applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship
+ to a set of values. Valid operators are In, NotIn,
+ Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values.
+ If the operator is In or NotIn, the values array
+ must be non-empty. If the operator is Exists or
+ DoesNotExist, the values array must be empty.
+ This array is replaced during a strategic merge
+ patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs.
+ A single {key,value} in the matchLabels map is equivalent
+ to an element of matchExpressions, whose key field is
+ "key", the operator is "In", and the values array contains
+ only "value". The requirements are ANDed.
+ type: object
+ type: object
+ maxSkew:
+ description: 'MaxSkew describes the degree to which pods may
+ be unevenly distributed. When ''whenUnsatisfiable=DoNotSchedule'',
+ it is the maximum permitted difference between the number
+ of matching pods in the target topology and the global minimum.
+ For example, in a 3-zone cluster, MaxSkew is set to 1, and
+ pods with the same labelSelector spread as 1/1/0: | zone1
+ | zone2 | zone3 | | P | P | | - if MaxSkew
+ is 1, incoming pod can only be scheduled to zone3 to become
+ 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0)
+ on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming
+ pod can be scheduled onto any zone. When ''whenUnsatisfiable=ScheduleAnyway'',
+ it is used to give higher precedence to topologies that
+ satisfy it. It''s a required field. Default value is 1 and
+ 0 is not allowed.'
+ format: int32
+ type: integer
+ topologyKey:
+ description: TopologyKey is the key of node labels. Nodes
+ that have a label with this key and identical values are
+ considered to be in the same topology. We consider each
+ <key, value> as a "bucket", and try to put balanced number
+ of pods into each bucket. It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal with
+ a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule
+ (default) tells the scheduler not to schedule it. - ScheduleAnyway
+ tells the scheduler to schedule the pod in any location, but
+ giving higher precedence to topologies that would help reduce
+ the skew. A constraint is considered "Unsatisfiable" for
+ an incoming pod if and only if every possible node assignment
+ for that pod would violate "MaxSkew" on some topology. For
+ example, in a 3-zone cluster, MaxSkew is set to 1, and pods
+ with the same labelSelector spread as 3/1/1: | zone1 | zone2
+ | zone3 | | P P P | P | P | If WhenUnsatisfiable
+ is set to DoNotSchedule, incoming pod can only be scheduled
+ to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1)
+ on zone2(zone3) satisfies MaxSkew(1). In other words, the
+ cluster can still be imbalanced, but scheduler won''t make
+ it *more* imbalanced. It''s a required field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - topologyKey
+ - whenUnsatisfiable
+ x-kubernetes-list-type: map
volumes:
description: List of volumes that can be mounted by disks belonging
to the vmi.
@@ -17906,6 +18223,122 @@ var CRDsValidation map[string]string = map[string]string{
type: string
type: object
type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraints describes how a group
+ of VMIs will be spread across a given topology domains.
+ K8s scheduler will schedule VMI pods in a way which abides
+ by the constraints.
+ items:
+ description: TopologySpreadConstraint specifies how to
+ spread matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching
+ pods. Pods that match this label selector are counted
+ to determine the number of pods in their corresponding
+ topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label
+ selector requirements. The requirements are
+ ANDed.
+ items:
+ description: A label selector requirement is
+ a selector that contains values, a key, and
+ an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the
+ selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's
+ relationship to a set of values. Valid
+ operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string
+ values. If the operator is In or NotIn,
+ the values array must be non-empty. If
+ the operator is Exists or DoesNotExist,
+ the values array must be empty. This array
+ is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value}
+ pairs. A single {key,value} in the matchLabels
+ map is equivalent to an element of matchExpressions,
+ whose key field is "key", the operator is "In",
+ and the values array contains only "value".
+ The requirements are ANDed.
+ type: object
+ type: object
+ maxSkew:
+ description: 'MaxSkew describes the degree to which
+ pods may be unevenly distributed. When ''whenUnsatisfiable=DoNotSchedule'',
+ it is the maximum permitted difference between the
+ number of matching pods in the target topology and
+ the global minimum. For example, in a 3-zone cluster,
+ MaxSkew is set to 1, and pods with the same labelSelector
+ spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | |
+ - if MaxSkew is 1, incoming pod can only be scheduled
+ to zone3 to become 1/1/1; scheduling it onto zone1(zone2)
+ would make the ActualSkew(2-0) on zone1(zone2) violate
+ MaxSkew(1). - if MaxSkew is 2, incoming pod can
+ be scheduled onto any zone. When ''whenUnsatisfiable=ScheduleAnyway'',
+ it is used to give higher precedence to topologies
+ that satisfy it. It''s a required field. Default
+ value is 1 and 0 is not allowed.'
+ format: int32
+ type: integer
+ topologyKey:
+ description: TopologyKey is the key of node labels.
+ Nodes that have a label with this key and identical
+ values are considered to be in the same topology.
+ We consider each <key, value> as a "bucket", and
+ try to put balanced number of pods into each bucket.
+ It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal
+ with a pod if it doesn''t satisfy the spread constraint.
+ - DoNotSchedule (default) tells the scheduler not
+ to schedule it. - ScheduleAnyway tells the scheduler
+ to schedule the pod in any location, but giving
+ higher precedence to topologies that would help
+ reduce the skew. A constraint is considered "Unsatisfiable"
+ for an incoming pod if and only if every possible
+ node assignment for that pod would violate "MaxSkew"
+ on some topology. For example, in a 3-zone cluster,
+ MaxSkew is set to 1, and pods with the same labelSelector
+ spread as 3/1/1: | zone1 | zone2 | zone3 | | P P
+ P | P | P | If WhenUnsatisfiable is set
+ to DoNotSchedule, incoming pod can only be scheduled
+ to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1)
+ on zone2(zone3) satisfies MaxSkew(1). In other words,
+ the cluster can still be imbalanced, but scheduler
+ won''t make it *more* imbalanced. It''s a required
+ field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - topologyKey
+ - whenUnsatisfiable
+ x-kubernetes-list-type: map
volumes:
description: List of volumes that can be mounted by disks
belonging to the vmi.
@@ -22447,6 +22880,129 @@ var CRDsValidation map[string]string = map[string]string{
type: string
type: object
type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraints describes how
+ a group of VMIs will be spread across a given topology
+ domains. K8s scheduler will schedule VMI pods in a
+ way which abides by the constraints.
+ items:
+ description: TopologySpreadConstraint specifies how
+ to spread matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching
+ pods. Pods that match this label selector are
+ counted to determine the number of pods in their
+ corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of
+ label selector requirements. The requirements
+ are ANDed.
+ items:
+ description: A label selector requirement
+ is a selector that contains values, a
+ key, and an operator that relates the
+ key and values.
+ properties:
+ key:
+ description: key is the label key that
+ the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's
+ relationship to a set of values. Valid
+ operators are In, NotIn, Exists and
+ DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string
+ values. If the operator is In or NotIn,
+ the values array must be non-empty.
+ If the operator is Exists or DoesNotExist,
+ the values array must be empty. This
+ array is replaced during a strategic
+ merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value}
+ pairs. A single {key,value} in the matchLabels
+ map is equivalent to an element of matchExpressions,
+ whose key field is "key", the operator is
+ "In", and the values array contains only
+ "value". The requirements are ANDed.
+ type: object
+ type: object
+ maxSkew:
+ description: 'MaxSkew describes the degree to
+ which pods may be unevenly distributed. When
+ ''whenUnsatisfiable=DoNotSchedule'', it is the
+ maximum permitted difference between the number
+ of matching pods in the target topology and
+ the global minimum. For example, in a 3-zone
+ cluster, MaxSkew is set to 1, and pods with
+ the same labelSelector spread as 1/1/0: | zone1
+ | zone2 | zone3 | | P | P | |
+ - if MaxSkew is 1, incoming pod can only be
+ scheduled to zone3 to become 1/1/1; scheduling
+ it onto zone1(zone2) would make the ActualSkew(2-0)
+ on zone1(zone2) violate MaxSkew(1). - if MaxSkew
+ is 2, incoming pod can be scheduled onto any
+ zone. When ''whenUnsatisfiable=ScheduleAnyway'',
+ it is used to give higher precedence to topologies
+ that satisfy it. It''s a required field. Default
+ value is 1 and 0 is not allowed.'
+ format: int32
+ type: integer
+ topologyKey:
+ description: TopologyKey is the key of node labels.
+ Nodes that have a label with this key and identical
+ values are considered to be in the same topology.
+ We consider each <key, value> as a "bucket",
+ and try to put balanced number of pods into
+ each bucket. It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how
+ to deal with a pod if it doesn''t satisfy the
+ spread constraint. - DoNotSchedule (default)
+ tells the scheduler not to schedule it. - ScheduleAnyway
+ tells the scheduler to schedule the pod in any
+ location, but giving higher precedence to
+ topologies that would help reduce the skew.
+ A constraint is considered "Unsatisfiable" for
+ an incoming pod if and only if every possible
+ node assignment for that pod would violate "MaxSkew"
+ on some topology. For example, in a 3-zone cluster,
+ MaxSkew is set to 1, and pods with the same
+ labelSelector spread as 3/1/1: | zone1 | zone2
+ | zone3 | | P P P | P | P | If WhenUnsatisfiable
+ is set to DoNotSchedule, incoming pod can only
+ be scheduled to zone2(zone3) to become 3/2/1(3/1/2)
+ as ActualSkew(2-1) on zone2(zone3) satisfies
+ MaxSkew(1). In other words, the cluster can
+ still be imbalanced, but scheduler won''t make
+ it *more* imbalanced. It''s a required field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - topologyKey
+ - whenUnsatisfiable
+ x-kubernetes-list-type: map
volumes:
description: List of volumes that can be mounted by
disks belonging to the vmi.
diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
index 03446ec92..1ceaf2914 100644
--- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
+++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go
@@ -4498,6 +4498,13 @@ func (in *VirtualMachineInstanceSpec) DeepCopyInto(out *VirtualMachineInstanceSp
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
+ if in.TopologySpreadConstraints != nil {
+ in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints
+ *out = make([]corev1.TopologySpreadConstraint, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
if in.EvictionStrategy != nil {
in, out := &in.EvictionStrategy, &out.EvictionStrategy
*out = new(EvictionStrategy)
diff --git a/staging/src/kubevirt.io/api/core/v1/types.go b/staging/src/kubevirt.io/api/core/v1/types.go
index bc16c2624..69d5bacdb 100644
--- a/staging/src/kubevirt.io/api/core/v1/types.go
+++ b/staging/src/kubevirt.io/api/core/v1/types.go
@@ -100,7 +100,15 @@ type VirtualMachineInstanceSpec struct {
SchedulerName string `json:"schedulerName,omitempty"`
// If toleration is specified, obey all the toleration rules.
Tolerations []k8sv1.Toleration `json:"tolerations,omitempty"`
-
+ // TopologySpreadConstraints describes how a group of VMIs will be spread across a given topology
+ // domains. K8s scheduler will schedule VMI pods in a way which abides by the constraints.
+ // +optional
+ // +patchMergeKey=topologyKey
+ // +patchStrategy=merge
+ // +listType=map
+ // +listMapKey=topologyKey
+ // +listMapKey=whenUnsatisfiable
+ TopologySpreadConstraints []k8sv1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty" patchStrategy:"merge" patchMergeKey:"topologyKey"`
// EvictionStrategy can be set to "LiveMigrate" if the VirtualMachineInstance should be
// migrated instead of shut-off in case of a node drain.
//
diff --git a/staging/src/kubevirt.io/api/core/v1/types_swagger_generated.go b/staging/src/kubevirt.io/api/core/v1/types_swagger_generated.go
index 7049bc71f..9873c77eb 100644
--- a/staging/src/kubevirt.io/api/core/v1/types_swagger_generated.go
+++ b/staging/src/kubevirt.io/api/core/v1/types_swagger_generated.go
@@ -25,6 +25,7 @@ func (VirtualMachineInstanceSpec) SwaggerDoc() map[string]string {
"affinity": "If affinity is specifies, obey all the affinity rules",
"schedulerName": "If specified, the VMI will be dispatched by specified scheduler.\nIf not specified, the VMI will be dispatched by default scheduler.\n+optional",
"tolerations": "If toleration is specified, obey all the toleration rules.",
+ "topologySpreadConstraints": "TopologySpreadConstraints describes how a group of VMIs will be spread across a given topology\ndomains. K8s scheduler will schedule VMI pods in a way which abides by the constraints.\n+optional\n+patchMergeKey=topologyKey\n+patchStrategy=merge\n+listType=map\n+listMapKey=topologyKey\n+listMapKey=whenUnsatisfiable",
"evictionStrategy": "EvictionStrategy can be set to \"LiveMigrate\" if the VirtualMachineInstance should be\nmigrated instead of shut-off in case of a node drain.\n\n+optional",
"startStrategy": "StartStrategy can be set to \"Paused\" if Virtual Machine should be started in paused state.\n\n+optional",
"terminationGracePeriodSeconds": "Grace period observed after signalling a VirtualMachineInstance to stop after which the VirtualMachineInstance is force terminated.",
diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go
index 89fc5af24..b2bddce65 100644
--- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go
+++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go
@@ -21240,6 +21240,30 @@ func schema_kubevirtio_api_core_v1_VirtualMachineInstanceSpec(ref common.Referen
},
},
},
+ "topologySpreadConstraints": {
+ VendorExtensible: spec.VendorExtensible{
+ Extensions: spec.Extensions{
+ "x-kubernetes-list-map-keys": []interface{}{
+ "topologyKey",
+ "whenUnsatisfiable",
+ },
+ "x-kubernetes-list-type": "map",
+ "x-kubernetes-patch-merge-key": "topologyKey",
+ "x-kubernetes-patch-strategy": "merge",
+ },
+ },
+ SchemaProps: spec.SchemaProps{
+ Description: "TopologySpreadConstraints describes how a group of VMIs will be spread across a given topology domains. K8s scheduler will schedule VMI pods in a way which abides by the constraints.",
+ Type: []string{"array"},
+ Items: &spec.SchemaOrArray{
+ Schema: &spec.Schema{
+ SchemaProps: spec.SchemaProps{
+ Ref: ref("k8s.io/api/core/v1.TopologySpreadConstraint"),
+ },
+ },
+ },
+ },
+ },
"evictionStrategy": {
SchemaProps: spec.SchemaProps{
Description: "EvictionStrategy can be set to \"LiveMigrate\" if the VirtualMachineInstance should be migrated instead of shut-off in case of a node drain.",
@@ -21349,7 +21373,7 @@ func schema_kubevirtio_api_core_v1_VirtualMachineInstanceSpec(ref common.Referen
},
},
Dependencies: []string{
- "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.PodDNSConfig", "k8s.io/api/core/v1.Toleration", "kubevirt.io/api/core/v1.AccessCredential", "kubevirt.io/api/core/v1.DomainSpec", "kubevirt.io/api/core/v1.Network", "kubevirt.io/api/core/v1.Probe", "kubevirt.io/api/core/v1.Volume"},
+ "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.PodDNSConfig", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.TopologySpreadConstraint", "kubevirt.io/api/core/v1.AccessCredential", "kubevirt.io/api/core/v1.DomainSpec", "kubevirt.io/api/core/v1.Network", "kubevirt.io/api/core/v1.Probe", "kubevirt.io/api/core/v1.Volume"},
}
}
diff --git a/tests/vmi_configuration_test.go b/tests/vmi_configuration_test.go
index 1fdbb68a5..3e260e725 100644
--- a/tests/vmi_configuration_test.go
+++ b/tests/vmi_configuration_test.go
@@ -2989,4 +2989,31 @@ var _ = Describe("[sig-compute]Configurations", func() {
doesntExceedMemoryUsage(&processRss, "qemu-kvm", qemuExpected)
})
})
+
+ Context("When topology spread constraints are defined for the VMI", func() {
+ It("they should be applied to the launcher pod", func() {
+ vmi := libvmi.NewCirros()
+ tsc := []k8sv1.TopologySpreadConstraint{
+ {
+ MaxSkew: 1,
+ TopologyKey: "zone",
+ WhenUnsatisfiable: "DoNotSchedule",
+ LabelSelector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "foo": "bar",
+ },
+ },
+ },
+ }
+ vmi.Spec.TopologySpreadConstraints = tsc
+
+ By("Starting a VirtualMachineInstance")
+ vmi = tests.RunVMIAndExpectScheduling(vmi, 30)
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Ensuring that pod has expected topologySpreadConstraints")
+ pod := tests.GetPodByVirtualMachineInstance(vmi)
+ Expect(pod.Spec.TopologySpreadConstraints).To(Equal(tsc))
+ })
+ })
})
diff --git a/tests/vmi_lifecycle_test.go b/tests/vmi_lifecycle_test.go
index fe89ae67a..f214195c9 100644
--- a/tests/vmi_lifecycle_test.go
+++ b/tests/vmi_lifecycle_test.go
@@ -1717,6 +1717,79 @@ var _ = Describe("[rfe_id:273][crit:high][arm64][vendor:cnv-qe@redhat.com][level
Expect(event).To(BeNil(), "virt-handler tried to sync on a VirtualMachineInstance in final state")
})
})
+
+ Context("replicaset with topology spread constraints", func() {
+ It("Replicas should be spread across nodes", func() {
+ nodes := libnode.GetAllSchedulableNodes(virtClient)
+ Expect(nodes.Items).ToNot(BeEmpty(), "There should be some schedulable nodes")
+ numNodes := len(nodes.Items)
+ if numNodes < 2 {
+ Skip("Skipping spec if test environment has less than two schedulable nodes")
+ }
+ vmLabelKey := "test" + rand.String(5)
+ vmLabelValue := "test" + rand.String(5)
+ vmi := tests.NewRandomVMI()
+ vmi.Spec.TopologySpreadConstraints = []k8sv1.TopologySpreadConstraint{
+ {
+ MaxSkew: 1,
+ TopologyKey: "kubernetes.io/hostname",
+ WhenUnsatisfiable: "DoNotSchedule",
+ LabelSelector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ vmLabelKey: vmLabelValue,
+ },
+ },
+ },
+ }
+
+ By("Creating a VirtualMachineInstanceReplicaSet")
+ replicas := int32(numNodes)
+ // limit the number of replicas launched for this test
+ if replicas > 10 {
+ replicas = 10
+ }
+ rs := &v1.VirtualMachineInstanceReplicaSet{
+ ObjectMeta: metav1.ObjectMeta{Name: "replicaset" + rand.String(5)},
+ Spec: v1.VirtualMachineInstanceReplicaSetSpec{
+ Replicas: &replicas,
+ Selector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{vmLabelKey: vmLabelValue},
+ },
+ Template: &v1.VirtualMachineInstanceTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{vmLabelKey: vmLabelValue},
+ Name: vmi.ObjectMeta.Name,
+ },
+ Spec: vmi.Spec,
+ },
+ },
+ }
+ createdRs, err := virtClient.ReplicaSet(vmi.Namespace).Create(rs)
+ Expect(err).ToNot(HaveOccurred(), "Should create replicaset")
+
+ By("Ensuring that all VMIs are ready")
+ Eventually(func() int32 {
+ rs, err := virtClient.ReplicaSet(vmi.Namespace).Get(createdRs.ObjectMeta.Name, metav1.GetOptions{})
+ Expect(err).ToNot(HaveOccurred())
+ return rs.Status.ReadyReplicas
+ }, 120*time.Second, 1*time.Second).Should(Equal(replicas))
+
+ By("Ensuring that VMI replicas are scheduled to seperate nodes")
+ vmiSet, err := virtClient.VirtualMachineInstance(vmi.Namespace).List(&metav1.ListOptions{
+ LabelSelector: fmt.Sprintf("%s=%s", vmLabelKey, vmLabelValue),
+ })
+ Expect(err).ToNot(HaveOccurred(), "Should find VMIs by label")
+ Expect(vmiSet.Items).To(HaveLen(int(replicas)), "Should get expected number of VMIs")
+
+ nodeNames := make(map[string]bool)
+ for _, vmi := range vmiSet.Items {
+ nodeName := vmi.Status.NodeName
+ _, nodeHasReplica := nodeNames[nodeName]
+ Expect(nodeHasReplica).To(BeFalse(), "Multiple replicas should not be scheduled to the same node")
+ nodeNames[nodeName] = true
+ }
+ })
+ })
})
func renderPkillAllPod(processName string) *k8sv1.Pod {
--
2.37.1
