Fix "load balancing" not working

ino-josh · gittiver · commit aef1471bd8a0 · 2025-06-11T14:15:40.000+02:00
Before this commit, the load balancing does not work.
The foundation is there, but in practice, the next connection is always picked.
For example, if there are 3 contexts available, it will always go: 0, 1, 2, 0, 1, 2, 0, 1, 2, etc.
If 0 is a long running connection, and 1 and 2 are short, it will pick 0 for the next connection and then block waiting for that long running connection to end even though 1 and 2 are free.
With this fix, the above example would result in this sequence: 0, 1, 2, 1, 2, 1, 2, 1, 2 (until 0 is done with its long running connection).
diff --git a/include/crow/http_connection.h b/include/crow/http_connection.h
@@ -69,6 +69,7 @@ namespace crow
           res_stream_threshold_(handler->stream_threshold()),
           queue_length_(queue_length)
         {
+            queue_length_++;
 #ifdef CROW_ENABLE_DEBUG
             connectionCount++;
             CROW_LOG_DEBUG << "Connection (" << this << ") allocated, total: " << connectionCount;
@@ -77,6 +78,7 @@ namespace crow
 
         ~Connection()
         {
+            queue_length_--;
 #ifdef CROW_ENABLE_DEBUG
             connectionCount--;
             CROW_LOG_DEBUG << "Connection (" << this << ") freed, total: " << connectionCount;
diff --git a/include/crow/http_server.h b/include/crow/http_server.h
@@ -49,14 +49,14 @@ namespace crow // NOTE: Already documented in "crow/app.h"
              uint16_t concurrency = 1,
              uint8_t timeout = 5,
              typename Adaptor::context* adaptor_ctx = nullptr):
+          concurrency_(concurrency),
+          task_queue_length_pool_(concurrency_ - 1),
           acceptor_(io_context_),
           signals_(io_context_),
           tick_timer_(io_context_),
           handler_(handler),
-          concurrency_(concurrency),
           timeout_(timeout),
           server_name_(server_name),
-          task_queue_length_pool_(concurrency_ - 1),
           middlewares_(middlewares),
           adaptor_ctx_(adaptor_ctx)
         {
@@ -300,12 +300,11 @@ namespace crow // NOTE: Already documented in "crow/app.h"
             {
                 uint16_t context_idx = pick_io_context_idx();
                 asio::io_context& ic = *io_context_pool_[context_idx];
-                task_queue_length_pool_[context_idx]++;
-                CROW_LOG_DEBUG << &ic << " {" << context_idx << "} queue length: " << task_queue_length_pool_[context_idx];
-
                 auto p = std::make_shared<Connection<Adaptor, Handler, Middlewares...>>(
-                  ic, handler_, server_name_, middlewares_,
-                  get_cached_date_str_pool_[context_idx], *task_timer_pool_[context_idx], adaptor_ctx_, task_queue_length_pool_[context_idx]);
+                    ic, handler_, server_name_, middlewares_,
+                    get_cached_date_str_pool_[context_idx], *task_timer_pool_[context_idx], adaptor_ctx_, task_queue_length_pool_[context_idx]);
+                    
+                CROW_LOG_DEBUG << &ic << " {" << context_idx << "} queue length: " << task_queue_length_pool_[context_idx];
 
                 acceptor_.async_accept(
                   p->socket(),
@@ -317,11 +316,6 @@ namespace crow // NOTE: Already documented in "crow/app.h"
                                 p->start();
                             });
                       }
-                      else
-                      {
-                          task_queue_length_pool_[context_idx]--;
-                          CROW_LOG_DEBUG << &ic << " {" << context_idx << "} queue length: " << task_queue_length_pool_[context_idx];
-                      }
                       do_accept();
                   });
             }
@@ -336,6 +330,8 @@ namespace crow // NOTE: Already documented in "crow/app.h"
         }
 
     private:
+        uint16_t concurrency_{2};
+        std::vector<std::atomic<unsigned int>> task_queue_length_pool_;
         std::vector<std::unique_ptr<asio::io_context>> io_context_pool_;
         asio::io_context io_context_;
         std::vector<detail::task_timer*> task_timer_pool_;
@@ -351,10 +347,8 @@ namespace crow // NOTE: Already documented in "crow/app.h"
         asio::basic_waitable_timer<std::chrono::high_resolution_clock> tick_timer_;
 
         Handler* handler_;
-        uint16_t concurrency_{2};
         std::uint8_t timeout_;
         std::string server_name_;
-        std::vector<std::atomic<unsigned int>> task_queue_length_pool_;
 
         std::chrono::milliseconds tick_interval_;
         std::function<void()> tick_function_;

Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,7 @@ namespace crow`
`69`	`69`	`res_stream_threshold_(handler->stream_threshold()),`
`70`	`70`	`queue_length_(queue_length)`
`71`	`71`	`{`
	`72`	`+ queue_length_++;`
`72`	`73`	`#ifdef CROW_ENABLE_DEBUG`
`73`	`74`	`connectionCount++;`
`74`	`75`	`CROW_LOG_DEBUG << "Connection (" << this << ") allocated, total: " << connectionCount;`
`@@ -77,6 +78,7 @@ namespace crow`
`77`	`78`
`78`	`79`	`~Connection()`
`79`	`80`	`{`
	`81`	`+ queue_length_--;`
`80`	`82`	`#ifdef CROW_ENABLE_DEBUG`
`81`	`83`	`connectionCount--;`
`82`	`84`	`CROW_LOG_DEBUG << "Connection (" << this << ") freed, total: " << connectionCount;`