Track and prioritize routing according to application restart generations

CHANGELOG

@@ -1,5 +1,7 @@
 Release 6.0.24 (Not yet released)
 -------------
+ * Update the order Passenger routes requests to app processes. Processes are now chosen based on being in the latest generation (Enterprise), then by newest process, then by oldest, then by
+busyness. Closes GH-2551.
  * Fix a regression from 6.0.10 where running `passenger-config system-properties` would throw an error. Closes GH-2565.
  * [Enterprise] Fix a memory corruption-related crash that could occur during rolling restarting.
  * [Ubuntu] Add packages for Ubuntu 24.10 "oracular".

src/agent/Core/ApplicationPool/Group.h

@@ -223,9 +223,9 @@ class Group: public boost::enable_shared_from_this<Group> {
 	/****** Process list management ******/
 
 	Process *findProcessWithStickySessionId(unsigned int id) const;
-	Process *findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const;
-	Process *findProcessWithLowestBusyness(const ProcessList &processes) const;
-	Process *findEnabledProcessWithLowestBusyness() const;
+	Process *findBestProcessPreferringStickySessionId(unsigned int id) const;
+	Process *findBestProcess(const ProcessList &processes) const;
+	Process *findBestEnabledProcess() const;
 
 	void addProcessToList(const ProcessPtr &process, ProcessList &destination);
 	void removeProcessFromList(const ProcessPtr &process, ProcessList &source);

src/agent/Core/ApplicationPool/Group/InternalUtils.cpp

@@ -184,7 +184,7 @@ Group::createNullProcessObject() {
 	LockGuard l(context->memoryManagementSyncher);
 	Process *process = context->processObjectPool.malloc();
 	Guard guard(context, process);
-	process = new (process) Process(&info, args);
+	process = new (process) Process(&info, info.group->restartsInitiated, args);
 	process->shutdownNotRequired();
 	guard.clear();
 	return ProcessPtr(process, false);
@@ -221,7 +221,7 @@ Group::createProcessObject(const SpawningKit::Spawner &spawner,
 	LockGuard l(context->memoryManagementSyncher);
 	Process *process = context->processObjectPool.malloc();
 	Guard guard(context, process);
-	process = new (process) Process(&info, spawnResult, args);
+	process = new (process) Process(&info, info.group->restartsInitiated, spawnResult, args);
 	guard.clear();
 	return ProcessPtr(process, false);
 }

src/agent/Core/ApplicationPool/Group/ProcessListManagement.cpp

@@ -64,71 +64,126 @@ Group::findProcessWithStickySessionId(unsigned int id) const {
 }
 
 Process *
-Group::findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const {
-	int leastBusyProcessIndex = -1;
-	int lowestBusyness = 0;
-	unsigned int i, size = enabledProcessBusynessLevels.size();
-	const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
-
-	for (i = 0; i < size; i++) {
-		Process *process = enabledProcesses[i].get();
+Group::findBestProcessPreferringStickySessionId(unsigned int id) const {
+	Process *bestProcess = nullptr;
+	Process *fallbackProcess = nullptr;
+	ProcessList::const_iterator it;
+	ProcessList::const_iterator end = enabledProcesses.end();
+	for (it = enabledProcesses.begin(); it != end; it++) {
+		Process *process = (*it).get();
 		if (process->getStickySessionId() == id) {
 			return process;
-		} else if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
-			leastBusyProcessIndex = i;
-			lowestBusyness = enabledProcessBusynessLevels[i];
+		} else if (process->isTotallyBusy() && bestProcess == nullptr) {
+			if (fallbackProcess == nullptr ||
+				process->generation > fallbackProcess->generation ||
+				(process->generation == fallbackProcess->generation && process->spawnStartTime < fallbackProcess->spawnStartTime) ||
+				(process->generation == fallbackProcess->generation && process->spawnStartTime == fallbackProcess->spawnStartTime && process->busyness() < fallbackProcess->busyness())
+			) {
+				fallbackProcess = process;
+			}
+		} else if (!process->isTotallyBusy() && (bestProcess == nullptr ||
+				   process->generation > bestProcess->generation ||
+				   (process->generation == bestProcess->generation && process->spawnStartTime < bestProcess->spawnStartTime) ||
+				   (process->generation == bestProcess->generation && process->spawnStartTime == bestProcess->spawnStartTime && process->busyness() < bestProcess->busyness())
+		)) {
+			bestProcess = process;
 		}
 	}
-
-	if (leastBusyProcessIndex == -1) {
-		return NULL;
-	} else {
-		return enabledProcesses[leastBusyProcessIndex].get();
+	if (bestProcess == nullptr) {
+		return fallbackProcess;
 	}
+	return bestProcess;
 }
 
 Process *
-Group::findProcessWithLowestBusyness(const ProcessList &processes) const {
+Group::findBestProcess(const ProcessList &processes) const {
 	if (processes.empty()) {
-		return NULL;
+		return nullptr;
 	}
 
-	int lowestBusyness = -1;
-	Process *leastBusyProcess = NULL;
+	Process *bestProcess = nullptr;
+	Process *fallbackProcess = nullptr;
 	ProcessList::const_iterator it;
 	ProcessList::const_iterator end = processes.end();
 	for (it = processes.begin(); it != end; it++) {
 		Process *process = (*it).get();
-		int busyness = process->busyness();
-		if (lowestBusyness == -1 || lowestBusyness > busyness) {
-			lowestBusyness = busyness;
-			leastBusyProcess = process;
+
+	 if (process->isTotallyBusy() && bestProcess == nullptr) {
+			if (fallbackProcess == nullptr ||
+				process->generation > fallbackProcess->generation ||
+				(process->generation == fallbackProcess->generation && process->spawnStartTime < fallbackProcess->spawnStartTime) ||
+				(process->generation == fallbackProcess->generation && process->spawnStartTime == fallbackProcess->spawnStartTime && process->busyness() < fallbackProcess->busyness())
+			) {
+				fallbackProcess = process;
+			}
+	 } else if (!process->isTotallyBusy() && (bestProcess == nullptr ||
+			process->generation > bestProcess->generation ||
+			(process->generation == bestProcess->generation && process->spawnStartTime < bestProcess->spawnStartTime) ||
+			(process->generation == bestProcess->generation && process->spawnStartTime == bestProcess->spawnStartTime && process->busyness() < bestProcess->busyness())
+		)) {
+			bestProcess = process;
 		}
 	}
-	return leastBusyProcess;
+	if (bestProcess == nullptr) {
+		return fallbackProcess;
+	}
+	return bestProcess;
 }
 
 /**
- * Cache-optimized version of findProcessWithLowestBusyness() for the common case.
+ * Cache-optimized version of findBestProcess() for the common case.
  */
 Process *
-Group::findEnabledProcessWithLowestBusyness() const {
+Group::findBestEnabledProcess() const {
 	if (enabledProcesses.empty()) {
-		return NULL;
+		return nullptr;
 	}
 
-	int leastBusyProcessIndex = -1;
-	int lowestBusyness = 0;
-	unsigned int i, size = enabledProcessBusynessLevels.size();
+	Process *bestProcess = nullptr;
+	unsigned long long bestProcessStartTime = 0;
+	unsigned int bestProcessGen = 0;
+	int bestProcessBusyness = 0;
+
+	Process *fallbackProcess = nullptr;
+	unsigned long long fallbackProcessStartTime = 0;
+	unsigned int fallbackProcessGen = 0;
+	int fallbackProcessBusyness = 0;
+
+	unsigned int size = enabledProcessBusynessLevels.size();
 	const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
 
-	for (i = 0; i < size; i++) {
-		if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
-			leastBusyProcessIndex = i;
-			lowestBusyness = enabledProcessBusynessLevels[i];
+	for (unsigned int i = 0; i < size; i++) {
+		Process *process = enabledProcesses.at(i).get();
+		unsigned int gen = process->generation;
+		unsigned long long startTime = process->spawnStartTime;
+		int busyness = enabledProcessBusynessLevels[i];
+		bool totallyBusy = process->isTotallyBusy();
+		if (totallyBusy && bestProcess == nullptr) {
+			if (fallbackProcess == nullptr ||
+ 				gen > fallbackProcess->generation ||
+				(gen == fallbackProcessGen && startTime <  fallbackProcessStartTime) ||
+				(gen == fallbackProcessGen && startTime == fallbackProcessStartTime && busyness < fallbackProcessBusyness)
+			) {
+				fallbackProcess = process;
+				fallbackProcessGen = gen;
+				fallbackProcessBusyness = busyness;
+				fallbackProcessStartTime = startTime;
+			}
+		} else if (!totallyBusy && (bestProcess == nullptr ||
+ 			gen > bestProcess->generation ||
+			(gen == bestProcessGen && startTime <  bestProcessStartTime) ||
+			(gen == bestProcessGen && startTime == bestProcessStartTime && busyness < bestProcessBusyness)
+		)) {
+			bestProcess = process;
+			bestProcessGen = gen;
+			bestProcessBusyness = busyness;
+			bestProcessStartTime = startTime;
 		}
 	}
-	return enabledProcesses[leastBusyProcessIndex].get();
+	if (bestProcess == nullptr) {
+		return fallbackProcess;
+	}
+	return bestProcess;
 }
 
 /**

src/agent/Core/ApplicationPool/Group/SessionManagement.cpp

@@ -64,14 +64,14 @@ Group::RouteResult
 Group::route(const Options &options) const {
 	if (OXT_LIKELY(enabledCount > 0)) {
 		if (options.stickySessionId == 0) {
-			Process *process = findEnabledProcessWithLowestBusyness();
+			Process *process = findBestEnabledProcess();
 			if (process->canBeRoutedTo()) {
 				return RouteResult(process);
 			} else {
 				return RouteResult(NULL, true);
 			}
 		} else {
-			Process *process = findProcessWithStickySessionIdOrLowestBusyness(
+			Process *process = findBestProcessPreferringStickySessionId(
 				options.stickySessionId);
 			if (process != NULL) {
 				if (process->canBeRoutedTo()) {
@@ -84,7 +84,7 @@ Group::route(const Options &options) const {
 			}
 		}
 	} else {
-		Process *process = findProcessWithLowestBusyness(disablingProcesses);
+		Process *process = findBestProcess(disablingProcesses);
 		if (process->canBeRoutedTo()) {
 			return RouteResult(process);
 		} else {
@@ -310,7 +310,7 @@ Group::get(const Options &newOptions, const GetCallback &callback,
 		assert(m_spawning || restarting() || poolAtFullCapacity());
 
 		if (disablingCount > 0 && !restarting()) {
-			Process *process = findProcessWithLowestBusyness(disablingProcesses);
+			Process *process = findBestProcess(disablingProcesses);
 			assert(process != NULL);
 			if (!process->isTotallyBusy()) {
 				return newSession(process, newOptions.currentTime);

src/agent/Core/ApplicationPool/Pool.h

@@ -28,10 +28,8 @@
 
 #include <string>
 #include <vector>
-#include <algorithm>
 #include <utility>
 #include <sstream>
-#include <iomanip>
 #include <boost/thread.hpp>
 #include <boost/bind/bind.hpp>
 #include <boost/shared_ptr.hpp>

src/agent/Core/ApplicationPool/Process.h

@@ -99,6 +99,8 @@ typedef boost::container::vector<ProcessPtr> ProcessList;
  */
 class Process {
 public:
+	friend class Group;
+
 	static const unsigned int MAX_SOCKETS_ACCEPTING_HTTP_REQUESTS = 3;
 
 private:
@@ -383,6 +385,10 @@ class Process {
 
 	/** Last time when a session was opened for this Process. */
 	unsigned long long lastUsed;
+    /** Which generation of app processes this one belongs to,
+        inherited from the app group, incremented when a restart
+		is initiated*/
+	const unsigned int generation;
 	/** Number of sessions currently open.
 	 * @invariant session >= 0
 	 */
@@ -445,8 +451,7 @@ class Process {
 	/** Collected by Pool::collectAnalytics(). */
 	ProcessMetrics metrics;
 
-
-	Process(const BasicGroupInfo *groupInfo, const Json::Value &args)
+	Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const Json::Value &args)
 		: info(this, groupInfo, args),
 		  socketsAcceptingHttpRequestsCount(0),
 		  spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
@@ -457,6 +462,7 @@ class Process {
 		  refcount(1),
 		  index(-1),
 		  lastUsed(spawnEndTime),
+		  generation(gen),
 		  sessions(0),
 		  processed(0),
 		  lifeStatus(ALIVE),
@@ -470,7 +476,7 @@ class Process {
 		indexSocketsAcceptingHttpRequests();
 	}
 
-	Process(const BasicGroupInfo *groupInfo, const SpawningKit::Result &skResult,
+	Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const SpawningKit::Result &skResult,
 		const Json::Value &args)
 		: info(this, groupInfo, skResult),
 		  socketsAcceptingHttpRequestsCount(0),
@@ -482,6 +488,7 @@ class Process {
 		  refcount(1),
 		  index(-1),
 		  lastUsed(spawnEndTime),
+		  generation(gen),
 		  sessions(0),
 		  processed(0),
 		  lifeStatus(ALIVE),