AUTHORS

@@ -229,6 +229,7 @@ Yohan Jarosz <yohanjarosz@yahoo.fr>
 Zachary McGibbon <zachary.mcgibbon@gmail.com>
 Zoltan Nagy <abesto@abesto.net>
 akrus <akrus@flygroup.st>
+azthec <azthec@users.noreply.github.com>
 bascarsija <bascarsija.dev@gmail.com>
 chrostek <sebastian@chrostek.net>
 cstegm <cstegm@users.noreply.github.com>

CHANGELOG.md

@@ -7,6 +7,17 @@ documentation before upgrading to a new release.
 
 Released closed milestones can be found on [GitHub](https://github.com/Icinga/icinga2/milestones?state=closed).
 
+## 2.11.2 (2019-10-24)
+
+2.11.2 fixes a problem where the newly introduced config sync "check-change-then-reload" functionality
+could cause endless reload loops with agents. The most visible parts are failing command endpoint checks
+with "not connected" UNKNOWN state. **Only applies to HA enabled zones with 2 masters and/or 2 satellites.**
+
+### Bugfixes
+
+* Cluster Config Sync
+  * Config sync checksum change detection may not work within high load HA clusters #7565
+
 ## 2.11.1 (2019-10-17)
 
 This release fixes a hidden long lasting bug unveiled with 2.11 and distributed setups.

VERSION

-Version: 2.11.1
+Version: 2.11.2
 Revision: 1

debian/changelog

+icinga2 (2.11.2-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream release.
+
+ -- Bas Couwenberg <sebastic@debian.org>  Sat, 26 Oct 2019 16:53:23 +0200
+
 icinga2 (2.11.1-2) unstable; urgency=medium
 
   * Team upload.

doc/06-distributed-monitoring.md

@@ -2450,7 +2450,7 @@ object Zone "global-commands" {
 
 The default global zones generated by the setup wizards are called `global-templates` and `director-global`.
 
-While you can should `global-templates` for your global configuration, `director-global` is reserved for use
+While you can and should use `global-templates` for your global configuration, `director-global` is reserved for use
 by [Icinga Director](https://icinga.com/docs/director/latest/). Please don't
 place any configuration in it manually.
 

doc/16-upgrading-icinga-2.md

@@ -10,6 +10,29 @@ follow the instructions for v2.7 too.
 
 ## Upgrading to v2.11 <a id="upgrading-to-2-11"></a>
 
+### Bugfixes for 2.11 <a id="upgrading-to-2-11-bugfixes"></a>
+
+2.11.1 on agents/satellites fixes a problem where 2.10.x as config master would send out an unwanted config marker file,
+thus rendering the agent to think it is autoritative for the config, and never accepting any new
+config files for the zone(s). **If your config master is 2.11.x already, you are not affected by this problem.**
+
+In order to fix this, upgrade to at least 2.11.1, and purge away the local config sync storage once, then restart.
+
+```
+yum install icinga2
+
+rm -rf /var/lib/icinga2/api/zones/*
+rm -rf /var/lib/icinga2/api/zones-stage/*
+
+systemctl restart icinga2
+```
+
+2.11.2 fixes a problem where the newly introduced config sync "check-change-then-reload" functionality
+could cause endless reload loops with agents. The most visible parts are failing command endpoint checks
+with "not connected" UNKNOWN state. **Only applies to HA enabled zones with 2 masters and/or 2 satellites.**
+
+In order to fix this, upgrade all agents/satellites to at least 2.11.2 and restart them.
+
 ### Packages <a id="upgrading-to-2-11-packages"></a>
 
 EOL distributions where no packages are available with this release:

lib/remote/apilistener-filesync.cpp

@@ -229,6 +229,56 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient)
 	aclient->SendMessage(message);
 }
 
+static bool CompareTimestampsConfigChange(const Dictionary::Ptr& productionConfig, const Dictionary::Ptr& receivedConfig,
+	const String& stageConfigZoneDir)
+{
+	double productionTimestamp;
+	double receivedTimestamp;
+
+	// Missing production timestamp means that something really broke. Always trigger a config change then.
+	if (!productionConfig->Contains("/.timestamp"))
+		productionTimestamp = 0;
+	else
+		productionTimestamp = productionConfig->Get("/.timestamp");
+
+	// Missing received config timestamp means that something really broke. Always trigger a config change then.
+	if (!receivedConfig->Contains("/.timestamp"))
+		receivedTimestamp = Utility::GetTime() + 10;
+	else
+		receivedTimestamp = receivedConfig->Get("/.timestamp");
+
+	bool configChange;
+
+	// Skip update if our configuration files are more recent.
+	if (productionTimestamp >= receivedTimestamp) {
+
+		Log(LogInformation, "ApiListener")
+			<< "Our production configuration is more recent than the received configuration update."
+			<< " Ignoring configuration file update for path '" << stageConfigZoneDir << "'. Current timestamp '"
+			<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", productionTimestamp) << "' ("
+			<< std::fixed << std::setprecision(6) << productionTimestamp
+			<< ") >= received timestamp '"
+			<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", receivedTimestamp) << "' ("
+			<< receivedTimestamp << ").";
+
+		configChange = false;
+
+	} else {
+		configChange = true;
+	}
+
+	// Update the .timestamp file inside the staging directory.
+	String tsPath = stageConfigZoneDir + "/.timestamp";
+
+	if (!Utility::PathExists(tsPath)) {
+		std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc);
+		fp << std::fixed << receivedTimestamp;
+		fp.close();
+	}
+
+	return configChange;
+}
+
 /**
  * Registered handler when a new config::Update message is received.
  *
@@ -360,11 +410,13 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
 		if (checksums) {
 			Log(LogInformation, "ApiListener")
 				<< "Received configuration for zone '" << zoneName << "' from endpoint '"
-				<< fromEndpointName << "'. Comparing the checksums.";
+				<< fromEndpointName << "'. Comparing the timestamp and checksums.";
+
+			if (CompareTimestampsConfigChange(productionConfig, newConfig, stageConfigZoneDir)) {
 
-			// TODO: Do this earlier in hello-handshakes?
 				if (CheckConfigChange(productionConfigInfo, newConfigInfo))
 					configChange = true;
+			}
 
 		} else {
 			/* Fallback to timestamp handling when the parent endpoint didn't send checks.
@@ -377,33 +429,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
 				<< "Received configuration update without checksums from parent endpoint "
 				<< fromEndpointName << ". This behaviour is deprecated. Please upgrade the parent endpoint to 2.11+";
 
-			double productionTimestamp;
-
-			if (!productionConfig->Contains("/.timestamp"))
-				productionTimestamp = 0;
-			else
-				productionTimestamp = productionConfig->Get("/.timestamp");
-
-			double newTimestamp;
-
-			if (!newConfig->Contains("/.timestamp"))
-				newTimestamp = Utility::GetTime();
-			else
-				newTimestamp = newConfig->Get("/.timestamp");
-
-			// Skip update if our configuration files are more recent.
-			if (productionTimestamp >= newTimestamp) {
-
-				Log(LogInformation, "ApiListener")
-					<< "Our configuration is more recent than the received configuration update."
-					<< " Ignoring configuration file update for path '" << stageConfigZoneDir << "'. Current timestamp '"
-					<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", productionTimestamp) << "' ("
-					<< std::fixed << std::setprecision(6) << productionTimestamp
-					<< ") >= received timestamp '"
-					<< Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' ("
-					<< newTimestamp << ").";
-
-			} else {
+			if (CompareTimestampsConfigChange(productionConfig, newConfig, stageConfigZoneDir)) {
 				configChange = true;
 			}
 
@@ -420,14 +446,6 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
 					}
 				}
 			}
-
-			// Update the .timestamp file.
-			String tsPath = stageConfigZoneDir + "/.timestamp";
-			if (!Utility::PathExists(tsPath)) {
-				std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc);
-				fp << std::fixed << newTimestamp;
-				fp.close();
-			}
 		}
 
 		// Dump the received configuration for this zone into the stage directory.
@@ -507,7 +525,7 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
 	} else {
 		Log(LogInformation, "ApiListener")
 			<< "Received configuration updates (" << count << ") from endpoint '" << fromEndpointName
-			<< "' are equal to production, not triggering reload.";
+			<< "' do not qualify for production, not triggering reload.";
 	}
 
 	return Empty;