File xen.sr-min_remaining.patch of Package xen
From: Olaf Hering <olaf@aepfle.de>
Date: Thu, 7 Jan 2021 19:39:28 +0100
Subject: sr min_remaining
tools: add --min_remaining to libxl_domain_suspend
The decision to stop+move a domU to the new host must be based on two factors:
- the available network bandwidth for the migration stream
- the maximum time a workload within a domU can be savely suspended
Both values define how many dirty pages a workload may produce prior the
final stop+move.
The default value of 50 pages is much too low with todays network bandwidths.
On an idle 1GiB link these 200K will be transferred within ~2ms.
Give the admin a knob to adjust the point when the final stop+move will
be done, so he can base this decision on his own needs.
This patch adjusts xl(1) and the libxl API.
External users check LIBXL_HAVE_DOMAIN_SUSPEND_PROPS for the availibility
of the new .min_remaining property.
Signed-off-by: Olaf Hering <olaf@aepfle.de>
---
docs/man/xl.1.pod.in | 8 +++
tools/include/libxl.h | 1 +
tools/libs/light/libxl_dom_save.c | 2 +-
tools/libs/light/libxl_domain.c | 1 +
tools/libs/light/libxl_internal.h | 1 +
tools/xl/xl_cmdtable.c | 23 +++++----
tools/xl/xl_migrate.c | 9 +++-
7 files changed, 32 insertions(+), 13 deletions(-)
--- a/docs/man/xl.1.pod.in
+++ b/docs/man/xl.1.pod.in
@@ -490,24 +490,32 @@ Leave the domain on the receive side paused after migration.
=item B<-D>
Preserve the B<domain-id> in the domain coniguration that is transferred
such that it will be identical on the destination host, unless that
configuration is overridden using the B<-C> option. Note that it is not
possible to use this option for a 'localhost' migration.
=item B<--max_iters> I<iterations>
Number of copy iterations before final suspend+move (default: 5)
+=item B<--min_remaing> I<pages>
+
+Number of remaining dirty pages. If the number of dirty pages drops that
+low, the guest is suspended and the domU will finally be moved to I<host>.
+
+This allows the host admin to control for how long the domU will likely
+be suspended during transit.
+
=back
=item B<remus> [I<OPTIONS>] I<domain-id> I<host>
Enable Remus HA or COLO HA for domain. By default B<xl> relies on ssh as a
transport mechanism between the two hosts.
B<NOTES>
=over 4
Remus support in xl is still in experimental (proof-of-concept) phase.
--- a/tools/include/libxl.h
+++ b/tools/include/libxl.h
@@ -1787,24 +1787,25 @@ static inline int libxl_retrieve_domain_configuration_0x041200(
libxl_retrieve_domain_configuration_0x041200
#endif
/*
* LIBXL_HAVE_DOMAIN_SUSPEND_PROPS indicates that the
* libxl_domain_suspend_props() function takes a props struct.
*/
#define LIBXL_HAVE_DOMAIN_SUSPEND_PROPS 1
typedef struct {
uint32_t flags; /* LIBXL_SUSPEND_* */
uint32_t max_iters;
+ uint32_t min_remaining;
} libxl_domain_suspend_props;
#define LIBXL_SUSPEND_DEBUG 1
#define LIBXL_SUSPEND_LIVE 2
int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
libxl_domain_suspend_props *props,
const libxl_asyncop_how *ao_how)
LIBXL_EXTERNAL_CALLERS_ONLY;
#if defined(LIBXL_API_VERSION) && LIBXL_API_VERSION < 0x041600
static inline int libxl_domain_suspend_0x041500(libxl_ctx *ctx, uint32_t domid,
int fd, int flags, /* LIBXL_SUSPEND_* */
const libxl_asyncop_how *ao_how)
--- a/tools/libs/light/libxl_dom_save.c
+++ b/tools/libs/light/libxl_dom_save.c
@@ -372,25 +372,25 @@ int libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,
return rc;
}
static int libxl__domain_save_precopy_policy(struct precopy_stats stats, void *user)
{
libxl__save_helper_state *shs = user;
libxl__domain_save_state *dss = shs->caller_state;
STATE_AO_GC(dss->ao);
LOGD(DEBUG, shs->domid, "iteration %u dirty_count %ld total_written %lu",
stats.iteration, stats.dirty_count, stats.total_written);
- if (stats.dirty_count >= 0 && stats.dirty_count < LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT)
+ if (stats.dirty_count >= 0 && stats.dirty_count < dss->min_remaining)
goto stop_copy;
if (stats.iteration >= dss->max_iters)
goto stop_copy;
return XGS_POLICY_CONTINUE_PRECOPY;
stop_copy:
return XGS_POLICY_STOP_AND_COPY;
}
/*----- main code for saving, in order of execution -----*/
void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
--- a/tools/libs/light/libxl_domain.c
+++ b/tools/libs/light/libxl_domain.c
@@ -516,24 +516,25 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd,
}
libxl__domain_save_state *dss;
GCNEW(dss);
dss->ao = ao;
dss->callback = domain_suspend_cb;
dss->domid = domid;
dss->fd = fd;
dss->type = type;
dss->max_iters = props->max_iters ?: LIBXL_XGS_POLICY_MAX_ITERATIONS;
+ dss->min_remaining = props->min_remaining ?: LIBXL_XGS_POLICY_TARGET_DIRTY_COUNT;
dss->live = props->flags & LIBXL_SUSPEND_LIVE;
dss->debug = props->flags & LIBXL_SUSPEND_DEBUG;
dss->checkpointed_stream = LIBXL_CHECKPOINTED_STREAM_NONE;
rc = libxl__fd_flags_modify_save(gc, dss->fd,
~(O_NONBLOCK|O_NDELAY), 0,
&dss->fdfl);
if (rc < 0) goto out_err;
libxl__domain_save(egc, dss);
return AO_INPROGRESS;
--- a/tools/libs/light/libxl_internal.h
+++ b/tools/libs/light/libxl_internal.h
@@ -3649,24 +3649,25 @@ struct libxl__domain_save_state {
libxl__ao *ao;
libxl__domain_save_cb *callback;
uint32_t domid;
int fd;
int fdfl; /* original flags on fd */
int recv_fd;
libxl_domain_type type;
int live;
int debug;
int checkpointed_stream;
uint32_t max_iters;
+ uint32_t min_remaining;
const libxl_domain_remus_info *remus;
/* private */
int rc;
int xcflags;
libxl__domain_suspend_state dsps;
union {
/* for Remus */
libxl__remus_state rs;
/* for COLO */
libxl__colo_save_state css;
};
libxl__checkpoint_devices_state cds;
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -156,35 +156,36 @@ const struct cmd_spec cmd_table[] = {
&main_save, 0, 1,
"Save a domain state to restore later",
"[options] <Domain> <CheckpointFile> [<ConfigFile>]",
"-h Print this help.\n"
"-c Leave domain running after creating the snapshot.\n"
"-p Leave domain paused after creating the snapshot.\n"
"-D Store the domain id in the configuration."
},
{ "migrate",
&main_migrate, 0, 1,
"Migrate a domain to another host",
"[options] <Domain> <host>",
- "-h Print this help.\n"
- "-C <config> Send <config> instead of config file from creation.\n"
- "-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n"
- " to sh. If empty, run <host> instead of ssh <host> xl\n"
- " migrate-receive [-d -e]\n"
- "-e Do not wait in the background (on <host>) for the death\n"
- " of the domain.\n"
- "--debug Enable verification mode.\n"
- "-p Do not unpause domain after migrating it.\n"
- "-D Preserve the domain id\n"
- "--max_iters N Number of copy iterations before final stop+move"
+ "-h Print this help.\n"
+ "-C <config> Send <config> instead of config file from creation.\n"
+ "-s <sshcommand> Use <sshcommand> instead of ssh. String will be passed\n"
+ " to sh. If empty, run <host> instead of ssh <host> xl\n"
+ " migrate-receive [-d -e]\n"
+ "-e Do not wait in the background (on <host>) for the death\n"
+ " of the domain.\n"
+ "--debug Enable verification mode.\n"
+ "-p Do not unpause domain after migrating it.\n"
+ "-D Preserve the domain id\n"
+ "--max_iters N Number of copy iterations before final stop+move\n"
+ "--min_remaining N Number of remaining dirty pages before final stop+move"
},
{ "restore",
&main_restore, 0, 1,
"Restore a domain from a saved state",
"[options] [<ConfigFile>] <CheckpointFile>",
"-h Print this help.\n"
"-p Do not unpause domain after restoring it.\n"
"-e Do not wait in the background for the death of the domain.\n"
"-d Enable debug messages.\n"
"-V, --vncviewer Connect to the VNC display after the domain is created.\n"
"-A, --vncviewer-autopass Pass VNC password to viewer via stdin."
},
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -170,36 +170,38 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
migration_child_report(recv_fd);
exit(EXIT_FAILURE);
}
save_domain_core_writeconfig(send_fd, "migration stream",
config_data, config_len);
}
static void migrate_domain(uint32_t domid, int preserve_domid,
const char *rune, int debug,
uint32_t max_iters,
+ uint32_t min_remaining,
const char *override_config_file)
{
pid_t child = -1;
int rc;
int send_fd = -1, recv_fd = -1;
char *away_domname;
char rc_buf;
uint8_t *config_data;
int config_len;
libxl_domain_suspend_props props = {
.flags = LIBXL_SUSPEND_LIVE,
.max_iters = max_iters,
+ .min_remaining = min_remaining,
};
save_domain_core_begin(domid, preserve_domid, override_config_file,
&config_data, &config_len);
if (!config_len) {
fprintf(stderr, "No config file stored for running domain and "
"none supplied - cannot migrate.\n");
exit(EXIT_FAILURE);
}
child = create_migration_child(rune, &send_fd, &recv_fd);
@@ -536,27 +538,29 @@ int main_migrate_receive(int argc, char **argv)
}
int main_migrate(int argc, char **argv)
{
uint32_t domid;
const char *config_filename = NULL;
const char *ssh_command = "ssh";
char *rune = NULL;
char *host;
int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
int preserve_domid = 0;
uint32_t max_iters = 0;
+ uint32_t min_remaining = 0;
static struct option opts[] = {
{"debug", 0, 0, 0x100},
{"max_iters", 1, 0, 0x101},
+ {"min_remaining", 1, 0, 0x102},
{"live", 0, 0, 0x200},
COMMON_LONG_OPTS
};
SWITCH_FOREACH_OPT(opt, "FC:s:epD", opts, "migrate", 2) {
case 'C':
config_filename = optarg;
break;
case 's':
ssh_command = optarg;
break;
case 'F':
@@ -569,24 +573,27 @@ int main_migrate(int argc, char **argv)
case 'p':
pause_after_migration = 1;
break;
case 'D':
preserve_domid = 1;
break;
case 0x100: /* --debug */
debug = 1;
break;
case 0x101: /* --max_iters */
max_iters = atoi(optarg);
break;
+ case 0x102: /* --min_remaining */
+ min_remaining = atoi(optarg);
+ break;
case 0x200: /* --live */
/* ignored for compatibility with xm */
break;
}
domid = find_domain(argv[optind]);
host = argv[optind + 1];
bool pass_tty_arg = progress_use_cr || (isatty(2) > 0);
if (!ssh_command[0]) {
rune= host;
@@ -604,25 +611,25 @@ int main_migrate(int argc, char **argv)
}
xasprintf(&rune, "exec %s %s xl%s%s%.*s migrate-receive%s%s%s",
ssh_command, host,
pass_tty_arg ? " -t" : "",
timestamps ? " -T" : "",
verbose_len, verbose_buf,
daemonize ? "" : " -e",
debug ? " -d" : "",
pause_after_migration ? " -p" : "");
}
migrate_domain(domid, preserve_domid, rune, debug,
- max_iters, config_filename);
+ max_iters, min_remaining, config_filename);
return EXIT_SUCCESS;
}
int main_remus(int argc, char **argv)
{
uint32_t domid;
int opt, rc, daemonize = 1;
const char *ssh_command = "ssh";
char *host = NULL, *rune = NULL;
libxl_domain_remus_info r_info;
int send_fd = -1, recv_fd = -1;
pid_t child = -1;
