File sprinkle-flushes.patch of Package ecmwfapi
When running requests in the background (cron, at), responsiveness is poor,
log messages are only seen at termination of the call. So sprinkle some flushes
to see the actual progress of the request.
--- ecmwfapi/api.py.orig 2022-01-21 13:22:00.000000000 +0100
+++ ecmwfapi/api.py 2022-01-28 13:44:02.578369003 +0100
@@ -169,6 +169,8 @@ def robust(func):
tries -= 1
# if all retries have been exhausted, raise the last exception caught
self.log("Could not contact the WebAPI after %d tries, failing !" % max_tries)
+ sys.stdout.flush()
+ sys.stderr.flush()
raise last_error
return wrapped
@@ -453,6 +455,8 @@ class APIRequest(object):
"Transfering %s into %s" % (self._bytename(size - existing_size), path)
)
self.log("From %s" % (url,))
+ sys.stdout.flush()
+ sys.stderr.flush()
bytes_transferred = 0
with open(path, mode) as f:
@@ -469,6 +473,8 @@ class APIRequest(object):
if end > start:
transfer_rate = bytes_transferred / (end - start)
self.log("Transfer rate %s/s" % self._bytename(transfer_rate))
+ sys.stdout.flush()
+ sys.stderr.flush()
return existing_size + bytes_transferred
@@ -478,19 +484,27 @@ class APIRequest(object):
self.connection.submit("%s/%s/requests" % (self.url, self.service), request)
self.log("Request submitted")
self.log("Request id: " + self.connection.last.get("name"))
+ sys.stdout.flush()
+ sys.stderr.flush()
if self.connection.status != status:
status = self.connection.status
self.log("Request is %s" % (status,))
+ sys.stdout.flush()
+ sys.stderr.flush()
while not self.connection.ready():
if self.connection.status != status:
status = self.connection.status
self.log("Request is %s" % (status,))
+ sys.stdout.flush()
+ sys.stderr.flush()
self.connection.wait()
if self.connection.status != status:
status = self.connection.status
self.log("Request is %s" % (status,))
+ sys.stdout.flush()
+ sys.stderr.flush()
result = self.connection.result()
if target:
@@ -509,6 +523,8 @@ class APIRequest(object):
if size != result["size"] and tries < 10:
tries += 1
self.log("Transfer interrupted, resuming in 60s...")
+ sys.stdout.flush()
+ sys.stderr.flush()
time.sleep(60)
else:
break
@@ -561,6 +577,8 @@ class ECMWFDataServer(object):
log=self.log,
verbose=self.verbose,
)
+ sys.stdout.flush()
+ sys.stderr.flush()
c.execute(req, target)
@@ -596,5 +614,9 @@ class ECMWFService(object):
verbose=self.verbose,
quiet=self.quiet,
)
+ sys.stdout.flush()
+ sys.stderr.flush()
c.execute(req, target)
self.log("Done")
+ sys.stdout.flush()
+ sys.stderr.flush()