File reconnect-to-mongodb-on-failure.patch of Package openstack-ceilometer

From d8ba7bada3f952abefc97fb32604a5f35c178d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ionu=C8=9B=20Ar=C8=9B=C4=83ri=C8=99i?= <iartarisi@suse.cz>
Date: Tue, 29 Apr 2014 17:05:02 +0200
Subject: [PATCH] reconnect to mongodb on conection failure

Try to reconnect to mongodb when the client library raised a
ConnectionFailure error. This is especially useful when mongodb is in
the process of initializing a replica set and it does not respond to
connections even though the mongodb service is up.

Change-Id: I99f3938613436f09d9cff7deffd7565ab9484a16
---
 ceilometer/storage/impl_mongodb.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py
index aca3001..4853eb1 100644
--- a/ceilometer/storage/impl_mongodb.py
+++ b/ceilometer/storage/impl_mongodb.py
@@ -23,6 +23,7 @@
 import calendar
 import copy
 import operator
+import time
 import uuid
 import weakref
 
@@ -41,6 +42,10 @@ from ceilometer.openstack.common.gettextutils import _
 
 cfg.CONF.import_opt('time_to_live', 'ceilometer.storage',
                     group="database")
+cfg.CONF.import_opt('max_retries', 'ceilometer.storage',
+                    group="database")
+cfg.CONF.import_opt('retry_interval', 'ceilometer.storage',
+                    group="database")
 
 LOG = log.getLogger(__name__)
 
@@ -158,12 +163,32 @@ class ConnectionPool(object):
                 return client
         LOG.info(_('Connecting to MongoDB on %s'),
                  connection_options['nodelist'])
-        client = pymongo.MongoClient(
-            url,
-            safe=True)
+        client = self._mongodb_connect(url)
         self._pool[pool_key] = weakref.ref(client)
         return client
 
+    @staticmethod
+    def _mongodb_connect(url):
+        max_retries = cfg.CONF.database.max_retries
+        retry_interval = cfg.CONF.database.retry_interval
+        attempts = 0
+        while True:
+            try:
+                client = pymongo.MongoClient(url, safe=True)
+            except pymongo.errors.ConnectionFailure as e:
+                if max_retries >= 0 and attempts >= max_retries:
+                    LOG.error(_('Unable to connect to MongoDB after '
+                                '%(retries)d retries. Giving up.') %
+                              {'retries': max_retries})
+                    raise
+                LOG.error(_('Unable to connect to MongoDB server: %(errmsg)s.'
+                            ' Trying again in %(retry_interval)d seconds.') %
+                          {'errmsg': e, 'retry_interval': retry_interval})
+                attempts += 1
+                time.sleep(retry_interval)
+            else:
+                return client
+
 
 class Connection(base.Connection):
     """MongoDB connection.
-- 
1.8.4