[#1396] Implement CB connection recovery

src/bin/dhcp4/ctrl_dhcp4_srv.cc
    ControlledDhcpv4Srv::dbReconnect() - added logic to
    reopen CB backends (if any configured)

src/lib/database/database_connection.*
    Added DbConnectionUnusable exception
    DatabaseConnection - added:
        unusable_
        marUnusable()
        checkUnusable()

src/lib/mysql/mysql_connection.*
    MySqlConnection::checkError() - calls markUnusable()

    Added invocations of DatabaseConnection::checkUnusable()
    to functions using the connection

src/lib/pgsql/pgsql_connection.cc
    PgSqlConnection::checkStatementError() - calls markUnusable()

2 files changed, 13 insertions, 2 deletions

diff --git a/src/lib/mysql/mysql_connection.cc b/src/lib/mysql/mysql_connection.cc
index 5e559e36ba..b8577c97da 100644
--- a/src/lib/mysql/mysql_connection.cc
+++ b/src/lib/mysql/mysql_connection.cc
@@ -299,6 +299,7 @@ MySqlConnection::getVersion(const ParameterMap& parameters) {
 
 void
 MySqlConnection::prepareStatement(uint32_t index, const char* text) {
+    checkUnusable();
     // Validate that there is space for the statement in the statements array
     // and that nothing has been placed there before.
     if ((index >= statements_.size()) || (statements_[index] != NULL)) {
@@ -325,6 +326,7 @@ MySqlConnection::prepareStatement(uint32_t index, const char* text) {
 void
 MySqlConnection::prepareStatements(const TaggedStatement* start_statement,
                                    const TaggedStatement* end_statement) {
+    checkUnusable();
     // Created the MySQL prepared statements for each DML statement.
     for (const TaggedStatement* tagged_statement = start_statement;
          tagged_statement != end_statement; ++tagged_statement) {
@@ -390,6 +392,7 @@ MySqlConnection::convertFromDatabaseTime(const MYSQL_TIME& expire,
 void
 MySqlConnection::startTransaction() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_START_TRANSACTION);
+    checkUnusable();
     // We create prepared statements for all other queries, but MySQL
     // don't support prepared statements for START TRANSACTION.
     int status = mysql_query(mysql_, "START TRANSACTION");
@@ -402,6 +405,7 @@ MySqlConnection::startTransaction() {
 void
 MySqlConnection::commit() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_COMMIT);
+    checkUnusable();
     if (mysql_commit(mysql_) != 0) {
         isc_throw(DbOperationError, "commit failed: "
                   << mysql_error(mysql_));
@@ -411,6 +415,7 @@ MySqlConnection::commit() {
 void
 MySqlConnection::rollback() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_ROLLBACK);
+    checkUnusable();
     if (mysql_rollback(mysql_) != 0) {
         isc_throw(DbOperationError, "rollback failed: "
                   << mysql_error(mysql_));
diff --git a/src/lib/mysql/mysql_connection.h b/src/lib/mysql/mysql_connection.h
index 6fbfabef7e..c9808b803d 100644
--- a/src/lib/mysql/mysql_connection.h
+++ b/src/lib/mysql/mysql_connection.h
@@ -399,6 +399,7 @@ public:
                      const MySqlBindingCollection& in_bindings,
                      MySqlBindingCollection& out_bindings,
                      ConsumeResultFun process_result) {
+        checkUnusable();
         // Extract native input bindings.
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
@@ -476,6 +477,7 @@ public:
     template<typename StatementIndex>
     void insertQuery(const StatementIndex& index,
                      const MySqlBindingCollection& in_bindings) {
+        checkUnusable();
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
             in_bind_vec.push_back(in_binding->getMySqlBinding());
@@ -519,6 +521,7 @@ public:
     template<typename StatementIndex>
     uint64_t updateDeleteQuery(const StatementIndex& index,
                                const MySqlBindingCollection& in_bindings) {
+        checkUnusable();
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
             in_bind_vec.push_back(in_binding->getMySqlBinding());
@@ -603,7 +606,7 @@ public:
     ///        failed.
     template<typename StatementIndex>
     void checkError(const int status, const StatementIndex& index,
-                    const char* what) const {
+                    const char* what) {
         if (status != 0) {
             switch(mysql_errno(mysql_)) {
                 // These are the ones we consider fatal. Remember this method is
@@ -622,6 +625,9 @@ public:
                     .arg(mysql_error(mysql_))
                     .arg(mysql_errno(mysql_));
 
+                // Mark the connection as no longer usuable.
+                markUnusable();
+
                 // If there's no lost db callback or it returns false,
                 // then we're not attempting to recover so we're done.
                 if (!invokeDbLostCallback()) {
@@ -631,7 +637,7 @@ public:
 
                 // We still need to throw so caller can error out of the current
                 // processing.
-                isc_throw(db::DbOperationError,
+                isc_throw(db::DbConnectionUnusable,
                           "fatal database errror or connectivity lost");
             default:
                 // Connection is ok, so it must be an SQL error