File use_curl.patch of Package ecmwfapi
Use an external call to "curl" to retrieve data. This way we get
a robust and fast download procedure and the user can set additional retrieval
options in $CURL_HOME/.curlrc or ~/.curlrc such as proxies, rate limiting or timeouts.
--- ecmwfapi/api.py.old 2018-06-05 14:16:49.561786478 +0000
+++ ecmwfapi/api.py 2018-06-05 15:23:16.000972875 +0000
@@ -17,6 +17,7 @@ import time
import traceback
import textwrap
from contextlib import closing
+from subprocess import Popen, PIPE
# python 2 and 3 compatible urllib and httplib imports
try:
@@ -459,14 +460,9 @@ class APIRequest(object):
def _transfer(self, url, path, size):
start = time.time()
existing_size = 0
- req = Request(url)
if os.path.exists(path):
- mode = "ab"
existing_size = os.path.getsize(path)
- req.add_header("Range", "bytes=%s-" % existing_size)
- else:
- mode = "wb"
self.log(
"Transfering %s into %s" % (self._bytename(size - existing_size), path)
@@ -475,15 +471,13 @@ class APIRequest(object):
sys.stdout.flush()
sys.stderr.flush()
- bytes_transferred = 0
- with open(path, mode) as f:
- with closing(urlopen(req)) as http:
- while True:
- chunk = http.read(1048576) # 1MB chunks
- if not chunk:
- break
- f.write(chunk)
- bytes_transferred += len(chunk)
+ proc = Popen(["curl","-fsS","-C-","-w","%{size_download}",url,"-o",path], stdout=PIPE)
+ exitcode = proc.wait()
+ if exitcode != 0:
+ self.log("Transfer not successful, exit code %d" % exitcode)
+ sys.stdout.flush()
+ sys.stderr.flush()
+ bytes_transferred = int(proc.stdout.read())
end = time.time()