Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
filesystems
lustre_2_12
0010-LU-13344-ptlrpc-change-request-timeouts-to...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0010-LU-13344-ptlrpc-change-request-timeouts-to-s32.patch of Package lustre_2_12
From e778bf6f4e1246aa2094a7bcf0267e1553f47cdd Mon Sep 17 00:00:00 2001 From: James Simmons <jsimmons@infradead.org> Date: Fri, 20 Mar 2020 17:40:25 -0400 Subject: [PATCH 10/35] LU-13344 ptlrpc: change request timeouts to s32 In the latest kernels time_t has been removed since time_t has been a 64 bit value just like time64_t so no need for it anymore. This means all kernel time keeping are all 64 bits but Lustre sends 32 bit values, in seconds, for its timeout between nodes. To keep backwards compatibility as well as distinguish timeouts from timestamps change the Lustre fields representing timeouts to a new typedef timeout_t which is a s32. The reason for picking a signed number is that it makes it easier to detect if an incorrect timeout was calculated. In the past negative timeouts have happened which would look like an enormous timeout if its treated as an unsigned number. Change-Id: I7491bbac65023c544e8f4a0488b1d07315a739e5 Signed-off-by: James Simmons <jsimmons@infradead.org> Reviewed-on: https://review.whamcloud.com/37933 Tested-by: jenkins <devops@whamcloud.com> Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com> Tested-by: Maloo <maloo@whamcloud.com> Reviewed-by: Neil Brown <neilb@suse.de> Reviewed-by: Oleg Drokin <green@whamcloud.com> --- contrib/scripts/spelling.txt | 1 + libcfs/include/libcfs/libcfs.h | 2 ++ lustre/include/lustre_import.h | 8 +++---- lustre/include/lustre_net.h | 19 +++++++++++----- lustre/include/lustre_osc.h | 2 +- lustre/ldlm/ldlm_lib.c | 39 ++++++++++++++++++--------------- lustre/ldlm/ldlm_lockd.c | 6 ++--- lustre/ptlrpc/client.c | 17 +++++++------- lustre/ptlrpc/import.c | 16 ++++++++------ lustre/ptlrpc/niobuf.c | 11 ++++++---- lustre/ptlrpc/pack_generic.c | 17 +++++++++----- lustre/ptlrpc/ptlrpc_internal.h | 2 +- 12 files changed, 84 insertions(+), 56 deletions(-) diff --git a/contrib/scripts/spelling.txt b/contrib/scripts/spelling.txt index fd28a99ccf7c..0f2b121b945a 100644 --- a/contrib/scripts/spelling.txt +++ b/contrib/scripts/spelling.txt @@ -149,5 +149,6 @@ strcat||strncat strcpy||strncpy struct timeval||struct timespec64 tempnam||mkstemp +time_t||timeout_t timer_setup||cfs_timer_setup wait_queue_t||wait_queue_entry_t diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h index b372eeac3896..f68e3043c394 100644 --- a/libcfs/include/libcfs/libcfs.h +++ b/libcfs/include/libcfs/libcfs.h @@ -98,6 +98,8 @@ void lc_watchdog_delete(struct lc_watchdog *lcw); #endif #endif +typedef s32 timeout_t; + /* need both kernel and user-land acceptor */ #define LNET_ACCEPTOR_MIN_RESERVED_PORT 512 #define LNET_ACCEPTOR_MAX_RESERVED_PORT 1023 diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index 5421de41cf11..430fde2e9273 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -342,11 +342,11 @@ static inline unsigned int at_est2timeout(unsigned int val) return (val + (val >> 2) + 5); } -static inline unsigned int at_timeout2est(unsigned int val) +static inline timeout_t at_timeout2est(timeout_t timeout) { - /* restore estimate value from timeout: e=4/5(t-5) */ - LASSERT(val); - return (max((val << 2) / 5, 5U) - 4); + /* restore estimate value from timeout: e=4/5(t-5) */ + LASSERT(timeout > 0); + return max((timeout << 2) / 5, 5) - 4; } static inline void at_reset_nolock(struct adaptive_timeout *at, int val) diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index 0e986c344233..3a94a921e11d 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -1113,8 +1113,16 @@ struct ptlrpc_request { * service time estimate (secs) * If the request is not served by this time, it is marked as timed out. * Do not change to time64_t since this is transmitted over the wire. + * + * The linux kernel handles timestamps with time64_t and timeouts + * are normally done with jiffies. Lustre shares the rq_timeout between + * nodes. Since jiffies can vary from node to node Lustre instead + * will express the timeout value in seconds. To avoid confusion with + * timestamps (time64_t) and jiffy timeouts (long) Lustre timeouts + * are expressed in s32 (timeout_t). Also what is transmitted over + * the wire is 32 bits. */ - time_t rq_timeout; + timeout_t rq_timeout; /** * when request/reply sent (secs), or time when request should be sent */ @@ -2407,8 +2415,8 @@ void lustre_msg_set_limit(struct lustre_msg *msg, __u64 limit); int lustre_msg_get_status(struct lustre_msg *msg); __u32 lustre_msg_get_conn_cnt(struct lustre_msg *msg); __u32 lustre_msg_get_magic(struct lustre_msg *msg); -__u32 lustre_msg_get_timeout(struct lustre_msg *msg); -__u32 lustre_msg_get_service_time(struct lustre_msg *msg); +timeout_t lustre_msg_get_timeout(struct lustre_msg *msg); +timeout_t lustre_msg_get_service_timeout(struct lustre_msg *msg); char *lustre_msg_get_jobid(struct lustre_msg *msg); __u32 lustre_msg_get_cksum(struct lustre_msg *msg); __u64 lustre_msg_get_mbits(struct lustre_msg *msg); @@ -2425,8 +2433,9 @@ void lustre_msg_set_status(struct lustre_msg *msg, __u32 status); void lustre_msg_set_conn_cnt(struct lustre_msg *msg, __u32 conn_cnt); void ptlrpc_req_set_repsize(struct ptlrpc_request *req, int count, __u32 *sizes); void ptlrpc_request_set_replen(struct ptlrpc_request *req); -void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout); -void lustre_msg_set_service_time(struct lustre_msg *msg, __u32 service_time); +void lustre_msg_set_timeout(struct lustre_msg *msg, timeout_t timeout); +void lustre_msg_set_service_timeout(struct lustre_msg *msg, + timeout_t service_timeout); void lustre_msg_set_jobid(struct lustre_msg *msg, char *jobid); void lustre_msg_set_cksum(struct lustre_msg *msg, __u32 cksum); void lustre_msg_set_mbits(struct lustre_msg *msg, __u64 mbits); diff --git a/lustre/include/lustre_osc.h b/lustre/include/lustre_osc.h index c53cba81a5fa..f865036f897c 100644 --- a/lustre/include/lustre_osc.h +++ b/lustre/include/lustre_osc.h @@ -548,7 +548,7 @@ struct osc_brw_async_args { int aa_requested_nob; int aa_nio_count; u32 aa_page_count; - int aa_resends; + s32 aa_resends; struct brw_page **aa_ppga; struct client_obd *aa_cli; struct list_head aa_oaps; diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index a041d4128ebd..11bcd71f5853 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -1857,36 +1857,36 @@ check_and_start_recovery_timer(struct obd_device *obd, struct ptlrpc_request *req, int new_client) { - time_t service_time = lustre_msg_get_service_time(req->rq_reqmsg); + timeout_t service_timeout = lustre_msg_get_service_timeout(req->rq_reqmsg); struct obd_device_target *obt = &obd->u.obt; - if (!new_client && service_time) + if (!new_client && service_timeout) /* Teach server about old server's estimates, as first guess * at how long new requests will take. */ at_measured(&req->rq_rqbd->rqbd_svcpt->scp_at_estimate, - service_time); + service_timeout); target_start_recovery_timer(obd); /* Convert the service time to RPC timeout, - * and reuse service_time to limit stack usage. + * and reuse service_timeout to limit stack usage. */ - service_time = at_est2timeout(service_time); + service_timeout = at_est2timeout(service_timeout); if (OBD_FAIL_CHECK(OBD_FAIL_TGT_SLUGGISH_NET) && - service_time < at_extra) - service_time = at_extra; + service_timeout < at_extra) + service_timeout = at_extra; - /* We expect other clients to timeout within service_time, then try + /* We expect other clients to timeout within service_timeout, then try * to reconnect, then try the failover server. The max delay between * connect attempts is SWITCH_MAX + SWITCH_INC + INITIAL. */ - service_time += 2 * INITIAL_CONNECT_TIMEOUT; + service_timeout += 2 * INITIAL_CONNECT_TIMEOUT; LASSERT(obt->obt_magic == OBT_MAGIC); - service_time += 2 * (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC); - if (service_time > obd->obd_recovery_timeout && !new_client) - extend_recovery_timer(obd, service_time, false); + service_timeout += 2 * (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC); + if (service_timeout > obd->obd_recovery_timeout && !new_client) + extend_recovery_timer(obd, service_timeout, false); } /** Health checking routines */ @@ -2247,14 +2247,15 @@ static void handle_recovery_req(struct ptlrpc_thread *thread, /* don't reset timer for final stage */ if (!exp_finished(req->rq_export)) { - time_t to = obd_timeout; + timeout_t timeout = obd_timeout; /** - * Add request timeout to the recovery time so next request from + * Add request @timeout to the recovery time so next request from * this client may come in recovery time */ if (!AT_OFF) { struct ptlrpc_service_part *svcpt; + timeout_t est_timeout; svcpt = req->rq_rqbd->rqbd_svcpt; /* If the server sent early reply for this request, @@ -2263,15 +2264,17 @@ static void handle_recovery_req(struct ptlrpc_thread *thread, * use the maxium timeout here for waiting the client * sending the next req */ - to = max_t(time_t, at_est2timeout(at_get(&svcpt->scp_at_estimate)), - lustre_msg_get_timeout(req->rq_reqmsg)); + est_timeout = at_get(&svcpt->scp_at_estimate); + timeout = max_t(timeout_t, at_est2timeout(est_timeout), + lustre_msg_get_timeout(req->rq_reqmsg)); /* Add 2 net_latency, one for balance rq_deadline * (see ptl_send_rpc), one for resend the req to server, * Note: client will pack net_latency in replay req * (see ptlrpc_replay_req) */ - to += 2 * lustre_msg_get_service_time(req->rq_reqmsg); + timeout += 2 * lustre_msg_get_service_timeout(req->rq_reqmsg); } - extend_recovery_timer(class_exp2obd(req->rq_export), to, true); + extend_recovery_timer(class_exp2obd(req->rq_export), timeout, + true); } EXIT; } diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c index 137ba42e4f5c..ac7a9910e4d4 100644 --- a/lustre/ldlm/ldlm_lockd.c +++ b/lustre/ldlm/ldlm_lockd.c @@ -71,12 +71,12 @@ static struct ldlm_state *ldlm_state; /* timeout for initial callback (AST) reply (bz10399) * Due to having to send a 32 bit time value over the - * wire return it as time_t instead of time64_t + * wire return it as timeout_t instead of time64_t */ -static inline time_t ldlm_get_rq_timeout(void) +static inline timeout_t ldlm_get_rq_timeout(void) { /* Non-AT value */ - time_t timeout = min(ldlm_timeout, obd_timeout / 3); + timeout_t timeout = min(ldlm_timeout, obd_timeout / 3); return timeout < 1 ? 1 : timeout; } diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 995f4ddb9cf5..a9ce1544b322 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -377,7 +377,7 @@ int ptlrpc_at_get_net_latency(struct ptlrpc_request *req) /* Adjust expected network latency */ void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req, - unsigned int service_time) + timeout_t service_timeout) { unsigned int nl, oldnl; struct imp_at *at; @@ -385,7 +385,7 @@ void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req, LASSERT(req->rq_import); - if (service_time > now - req->rq_sent + 3) { + if (service_timeout > now - req->rq_sent + 3) { /* bz16408, however, this can also happen if early reply * is lost and client RPC is expired and resent, early reply * or reply of original RPC can still be fit in reply buffer @@ -396,13 +396,13 @@ void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req, CDEBUG((lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT) ? D_ADAPTTO : D_WARNING, "Reported service time %u > total measured time %lld\n", - service_time, now - req->rq_sent); + service_timeout, now - req->rq_sent); return; } /* Network latency is total time less server processing time */ nl = max_t(int, now - req->rq_sent - - service_time, 0) + 1; /* st rounding */ + service_timeout, 0) + 1; /* st rounding */ at = &req->rq_import->imp_at; oldnl = at_measured(&at->iat_net_latency, nl); @@ -443,6 +443,7 @@ static int ptlrpc_at_recv_early_reply(struct ptlrpc_request *req) __must_hold(&req->rq_lock) { struct ptlrpc_request *early_req; + timeout_t service_timeout; time64_t olddl; int rc; @@ -472,8 +473,8 @@ __must_hold(&req->rq_lock) lustre_msg_set_timeout(req->rq_reqmsg, req->rq_timeout); /* Network latency can be adjusted, it is pure network delays */ - ptlrpc_at_adj_net_latency(req, - lustre_msg_get_service_time(early_req->rq_repmsg)); + service_timeout = lustre_msg_get_service_timeout (early_req->rq_repmsg); + ptlrpc_at_adj_net_latency(req, service_timeout); sptlrpc_cli_finish_early_reply(early_req); @@ -1478,7 +1479,7 @@ static int after_reply(struct ptlrpc_request *req) CFS_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_PAUSE_REP, cfs_fail_val); ptlrpc_at_adj_service(req, lustre_msg_get_timeout(req->rq_repmsg)); ptlrpc_at_adj_net_latency(req, - lustre_msg_get_service_time(req->rq_repmsg)); + lustre_msg_get_service_timeout(req->rq_repmsg)); rc = ptlrpc_check_status(req); imp->imp_connect_error = rc; @@ -3144,7 +3145,7 @@ int ptlrpc_replay_req(struct ptlrpc_request *req) /* Tell server the net_latency, so the server can calculate how long * it should wait for next replay */ - lustre_msg_set_service_time(req->rq_reqmsg, + lustre_msg_set_service_timeout(req->rq_reqmsg, ptlrpc_at_get_net_latency(req)); DEBUG_REQ(D_HA, req, "REPLAY"); diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 2bb7d1d40ed4..46d92bf4ed2d 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -770,8 +770,8 @@ int ptlrpc_connect_import_locked(struct obd_import *imp) /* Report the rpc service time to the server so that it knows how long * to wait for clients to join recovery */ - lustre_msg_set_service_time(request->rq_reqmsg, - at_timeout2est(request->rq_timeout)); + lustre_msg_set_service_timeout(request->rq_reqmsg, + at_timeout2est(request->rq_timeout)); /* The amount of time we give the server to process the connect req. * import_select_connection will increase the net latency on @@ -808,7 +808,7 @@ int ptlrpc_connect_import_locked(struct obd_import *imp) lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_TRANSNO); - DEBUG_REQ(D_RPCTRACE, request, "(re)connect request (timeout %ld)", + DEBUG_REQ(D_RPCTRACE, request, "(re)connect request (timeout %d)", request->rq_timeout); ptlrpcd_add_req(request); rc = 0; @@ -1005,6 +1005,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env, struct obd_import *imp = request->rq_import; struct lustre_handle old_hdl; __u64 old_connect_flags; + timeout_t service_timeout; int msg_flags; struct obd_connect_data *ocd; struct obd_export *exp = NULL; @@ -1137,10 +1138,11 @@ static int ptlrpc_connect_interpret(const struct lu_env *env, imp->imp_obd->obd_self_export->exp_connect_data = *ocd; /* The net statistics after (re-)connect is not valid anymore, - * because may reflect other routing, etc. */ + * because may reflect other routing, etc. + */ + service_timeout = lustre_msg_get_service_timeout(request->rq_repmsg); at_reinit(&imp->imp_at.iat_net_latency, 0, 0); - ptlrpc_at_adj_net_latency(request, - lustre_msg_get_service_time(request->rq_repmsg)); + ptlrpc_at_adj_net_latency(request, service_timeout); /* Import flags should be updated before waking import at FULL state */ rc = ptlrpc_connect_set_flags(imp, ocd, old_connect_flags, exp, @@ -1680,7 +1682,7 @@ static struct ptlrpc_request *ptlrpc_disconnect_prep_req(struct obd_import *imp) /* We want client umounts to happen quickly, no matter the server state... */ - req->rq_timeout = min_t(int, req->rq_timeout, + req->rq_timeout = min_t(timeout_t, req->rq_timeout, INITIAL_CONNECT_TIMEOUT); import_set_state(imp, LUSTRE_IMP_CONNECTING); diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c index 52d0bc07c485..f6e0f57e2c78 100644 --- a/lustre/ptlrpc/niobuf.c +++ b/lustre/ptlrpc/niobuf.c @@ -498,9 +498,11 @@ static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags) { struct ptlrpc_service_part *svcpt = req->rq_rqbd->rqbd_svcpt; struct ptlrpc_service *svc = svcpt->scp_service; - int service_time = max_t(int, ktime_get_real_seconds() - - req->rq_arrival_time.tv_sec, 1); + timeout_t service_timeout; + service_timeout = clamp_t(timeout_t, ktime_get_real_seconds() - + req->rq_arrival_time.tv_sec, 1, + (AT_OFF ? obd_timeout * 3 / 2 : at_max)); if (!(flags & PTLRPC_REPLY_EARLY) && (req->rq_type != PTL_RPC_MSG_ERR) && (req->rq_reqmsg != NULL) && @@ -509,7 +511,8 @@ static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags) MSG_REQ_REPLAY_DONE | MSG_LOCK_REPLAY_DONE))) { /* early replies, errors and recovery requests don't count * toward our service time estimate */ - int oldse = at_measured(&svcpt->scp_at_estimate, service_time); + int oldse = at_measured(&svcpt->scp_at_estimate, + service_timeout); if (oldse != 0) { DEBUG_REQ(D_ADAPTTO, req, @@ -519,7 +522,7 @@ static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags) } } /* Report actual service time for client latency calc */ - lustre_msg_set_service_time(req->rq_repmsg, service_time); + lustre_msg_set_service_timeout(req->rq_repmsg, service_timeout); /* Report service time estimate for future client reqs, but report 0 * (to be ignored by client) if it's an error reply during recovery. * b=15815 diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 0d425804b1a1..5e2d384435fb 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -1235,11 +1235,12 @@ __u32 lustre_msg_get_magic(struct lustre_msg *msg) } } -__u32 lustre_msg_get_timeout(struct lustre_msg *msg) +timeout_t lustre_msg_get_timeout(struct lustre_msg *msg) { switch (msg->lm_magic) { case LUSTRE_MSG_MAGIC_V2: { struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg); + if (pb == NULL) { CERROR("invalid msg %p: no ptlrpc body!\n", msg); return 0; @@ -1252,11 +1253,12 @@ __u32 lustre_msg_get_timeout(struct lustre_msg *msg) } } -__u32 lustre_msg_get_service_time(struct lustre_msg *msg) +timeout_t lustre_msg_get_service_timeout(struct lustre_msg *msg) { switch (msg->lm_magic) { case LUSTRE_MSG_MAGIC_V2: { struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg); + if (pb == NULL) { CERROR("invalid msg %p: no ptlrpc body!\n", msg); return 0; @@ -1486,11 +1488,13 @@ void lustre_msg_set_conn_cnt(struct lustre_msg *msg, __u32 conn_cnt) } } -void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout) +void lustre_msg_set_timeout(struct lustre_msg *msg, timeout_t timeout) { switch (msg->lm_magic) { case LUSTRE_MSG_MAGIC_V2: { struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg); + + LASSERT(timeout >= 0); LASSERTF(pb != NULL, "invalid msg %p: no ptlrpc body!\n", msg); pb->pb_timeout = timeout; return; @@ -1500,13 +1504,16 @@ void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout) } } -void lustre_msg_set_service_time(struct lustre_msg *msg, __u32 service_time) +void lustre_msg_set_service_timeout(struct lustre_msg *msg, + timeout_t service_timeout) { switch (msg->lm_magic) { case LUSTRE_MSG_MAGIC_V2: { struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg); + + LASSERT(service_timeout >= 0); LASSERTF(pb, "invalid msg %p: no ptlrpc body!\n", msg); - pb->pb_service_time = service_time; + pb->pb_service_time = service_timeout; return; } default: diff --git a/lustre/ptlrpc/ptlrpc_internal.h b/lustre/ptlrpc/ptlrpc_internal.h index 19d0487a4641..41b9a268d52a 100644 --- a/lustre/ptlrpc/ptlrpc_internal.h +++ b/lustre/ptlrpc/ptlrpc_internal.h @@ -69,7 +69,7 @@ int ptlrpcd_start(struct ptlrpcd_ctl *pc); /* client.c */ void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req, - unsigned int service_time); + timeout_t service_timeout); struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned npages, unsigned max_brw, enum ptlrpc_bulk_op_type type, unsigned portal, -- 2.41.0
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
