redesign announce jitter to better spread the load after startup

CHANGES

@@ -1,3 +1,21 @@
+-- 2.1.0 (2024-02-11)
+FEATURES
+* announce jitter is now a uniform distribution and can be modified on the fly
+
+-- 2.0.1 (2024-02-10)
+BREAKING
+* API torrent endpoint output changed to JSON
+* API user endpoint shows more information about a user in JSON format
+FEATURES
+* Mysql port may be specified in configuration file
+* API stats report contains Ocelot version
+* Queue sizes are reported in stats
+* Sleep if Gazelle API is down
+* Handle DB connection problems gracefully
+* Add Prometheus and Grafana support
+BUILD
+* Docker image based on Bookworm
+
 -- 2.0.0 (2019-10-22)
 BREAKING
 * Write leech stats into their own dedicated tables (users_leech_stats, torrents_leech_stats)
@@ -15,7 +33,7 @@ BUILD
 Move to spdlog (https://github.com/gabime/spdlog)to handle logging instead of cout
     spdlog is vendored under src/, currently using v0.13.0
 Add logging directive to config to configure how Ocelot logs and where it logs
-Add deamon (--daemon, -d) to run Ocelot as a daemon
+Add daemon switch (--daemon, -d) to run Ocelot as a daemon
 Move source code into src/ directory
 Add Vagrantfile and Dockerfile for running Ocelot
 

Dockerfile

@@ -6,7 +6,6 @@ RUN apt-get update \
     && apt-get install --no-install-recommends -y \
         build-essential \
         cmake \
-        curl \
         default-libmysqlclient-dev \
         libboost-iostreams-dev \
         libboost-system-dev \
@@ -20,10 +19,8 @@ RUN apt-get update \
 COPY . /srv
 WORKDIR /srv
 
-RUN mkdir build \
-    && cd build \
-    && cmake .. \
-    && make \
+RUN cmake -Wno-dev -S /srv -B /srv/build \
+    && make -C build \
     && apt-get purge -y \
         build-essential \
         cmake \

README.md

@@ -5,17 +5,16 @@ It supports requests over TCP and can only track IPv4 peers.
 
 ## Ocelot Compile-time Dependencies
 
-* [GCC/G++](http://gcc.gnu.org/) (4.7+ required; 4.8.1+ recommended)
-* [Boost](http://www.boost.org/) (1.55.0+ required)
+* [GCC/G++](http://gcc.gnu.org/) (11+ required; 11.4.0+ recommended)
+* [Boost](http://www.boost.org/) (1.74.0+ required)
 * [libev](http://software.schmorp.de/pkg/libev.html) (required)
-* [spdlog](https://github.com/gabime/spdlog) (required)
 * [MySQL++](http://tangentsoft.net/mysql++/) (3.2.0+ required)
 * [jemalloc](http://jemalloc.net/) (optional, but strongly recommended)
 * [TCMalloc](http://goog-perftools.sourceforge.net/doc/tcmalloc.html) (optional)
 
 ## Installation
 
-### Debian Stretch
+### Debian Bookworm
 ```bash
 sudo apt-get install \
     build-essential \
@@ -26,11 +25,10 @@ sudo apt-get install \
     libev-dev \
     libjemalloc-dev \
     libmysql++-dev \
+	netcat-traditional \
     pkg-config
-mkdir build
-cd build
-cmake ..
-make
+cmake -Wno-dev . -B build 
+make -C build
 ```
 
 The [Gazelle installation guides](https://github.com/WhatCD/Gazelle/wiki/Gazelle-installation) include instructions for installing Ocelot as a part of the Gazelle project.
@@ -44,6 +42,10 @@ docker run -v $(pwd)/ocelot.conf:/srv/ocelot.conf ocelot
 
 ### Standalone Installation
 
+* Prepare the build environment. (This must be re-executed when new source files are added).
+
+        cmake -Wno-dev . -B build
+
 * Create the following tables according to the [Gazelle database schema](https://raw.githubusercontent.com/WhatCD/Gazelle/master/gazelle.sql):
  - `torrents`
  - `torrents_leech_stats`
@@ -58,10 +60,7 @@ docker run -v $(pwd)/ocelot.conf:/srv/ocelot.conf ocelot
 
 * Build Ocelot:
 
-        mkdir build
-        cd build
-        cmake ..
-        make
+        make -C build
 
 ## Running Ocelot
 

ocelot.conf.dist

@@ -11,15 +11,16 @@ connection_timeout  = 10
 keepalive_timeout   = 0
 
 announce_interval   = 1800
+announce_jitter     = 240
 max_request_size    = 4096
 numwant_limit       = 50
 request_log_size    = 500
 
-mysql_host          = 
-mysql_username      = 
-mysql_password      = 
-mysql_db            = 
-mysql_port          = 3306
+mysql_host          = mysql
+mysql_port          = 36000
+mysql_username      = gazelle
+mysql_password      = password
+mysql_db            = gazelle
 
 # The passwords must be 32 characters and match the Gazelle config
 report_password     = 00000000000000000000000000000000

src/config.cpp

@@ -66,6 +66,7 @@ void config::init() {
 
 	// Tracker requests
 	add("announce_interval", 1800u);
+	add("announce_jitter", 240u);
 	add("max_request_size", 4096u);
 	add("numwant_limit", 50u);
 	add("request_log_size", 500u);
@@ -79,6 +80,7 @@ void config::init() {
 	// MySQL
 	add("mysql_db", "gazelle");
 	add("mysql_host", "localhost");
+	add("mysql_port", 3306u);
 	add("mysql_username", "");
 	add("mysql_password", "");
 	add("mysql_port", 3306u);
@@ -101,7 +103,7 @@ void config::init() {
 confval * config::get(const std::string &setting_name) {
 	const auto setting = settings.find(setting_name);
 	if (setting == settings.end()) {
-		 spdlog::get("logger")->info("WARNING: Unrecognized setting '" + setting_name + "'");
+		spdlog::get("logger")->info("WARNING: Unrecognized setting '" + setting_name + "'");
 		return dummy_setting;
 	}
 	return &setting->second;

src/ocelot.cpp

@@ -24,7 +24,7 @@ struct stats_t stats;
 
 const char * version()
 {
-	return "2.0.1";
+	return "2.1.0";
 }
 
 static void create_daemon()
@@ -97,7 +97,9 @@ int main(int argc, char **argv) {
 
 	conf = new config();
 
-	bool verbose = false, conf_arg = false, daemonize = false;
+	bool conf_arg  = false;
+	bool daemonize = false;
+	bool verbose   = false;
 	std::string conf_file_path("./ocelot.conf");
 
 	for (int i = 1; i < argc; i++) {
@@ -116,7 +118,7 @@ int main(int argc, char **argv) {
 			return 0;
 		}
 		else {
-			std::cout << "Usage: " << argv[0] << " [-v] [-c configfile]" << std::endl;
+			std::cout << "Usage: " << argv[0] << " [-v] [-c configfile] [--daemonize]" << std::endl;
 			return 0;
 		}
 	}

src/report.cpp

@@ -7,12 +7,10 @@
 #include "response.h"
 #include "user.h"
 
-std::string report(params_type &params, user_list &users_list, client_opts_t &client_opts) {
+std::string report(params_type &params, user_list &users_list, client_opts_t &client_opts, unsigned int announce_interval, unsigned int announce_jitter) {
 	std::stringstream output;
 	std::string action = params["get"];
-	if (action.empty()) {
-		output << "Invalid action\n";
-	} else if (action == "stats") {
+	if (action == "stats") {
 		time_t uptime = time(NULL) - stats.start_time;
 		int up_d = uptime / 86400;
 		uptime -= up_d * 86400;
@@ -20,11 +18,11 @@ std::string report(params_type &params, user_list &users_list, client_opts_t &cl
 		uptime -= up_h * 3600;
 		int up_m = uptime / 60;
 		int up_s = uptime - up_m * 60;
-		std::string up_ht = up_h <= 9 ? '0' + inttostr(up_h) : inttostr(up_h);
-		std::string up_mt = up_m <= 9 ? '0' + inttostr(up_m) : inttostr(up_m);
-		std::string up_st = up_s <= 9 ? '0' + inttostr(up_s) : inttostr(up_s);
 
-		output << "Uptime: " << up_d << " days, " << up_ht << ':' << up_mt << ':' << up_st << "\n"
+		output << "Uptime: " << up_d << " days, "
+				<< (up_h < 10 ? "0" : "") << inttostr(up_h) << ':'
+				<< (up_m < 10 ? "0" : "") << inttostr(up_m) << ':'
+				<< (up_s < 10 ? "0" : "") << inttostr(up_s) << "\n"
 			<< "version: " << version() << "\n"
 			<< stats.opened_connections << " connections opened\n"
 			<< stats.open_connections << " open connections\n"
@@ -42,7 +40,10 @@ std::string report(params_type &params, user_list &users_list, client_opts_t &cl
 			<< stats.snatch_queue_size << " items in snatch queue\n"
 			<< stats.token_queue_size << " items in token queue\n"
 			<< stats.bytes_read << " bytes read\n"
-			<< stats.bytes_written << " bytes written\n";
+			<< stats.bytes_written << " bytes written\n"
+			<< announce_interval << " announce interval\n"
+			<< announce_jitter << " announce jitter\n"
+			;
 	} else if (action == "prom_stats") {
 		time_t uptime = time(NULL) - stats.start_time;
 		output << "ocelot_uptime " << uptime << "\n"

src/report.h

@@ -1,4 +1,9 @@
-#include <string>
 #include "ocelot.h"
 
-std::string report(params_type &params, user_list &users_list, client_opts_t &client_opts);
+std::string report(
+	params_type &params,
+	user_list &users_list,
+	client_opts_t &client_opts,
+	unsigned int announce_interval,
+	unsigned int announce_jitter 
+);

src/worker.cpp

@@ -22,20 +22,21 @@
 
 //---------- Worker - does stuff with input
 worker::worker(config * conf_obj, torrent_list &torrents, user_list &users, std::vector<std::string> &_whitelist, mysql * db_obj, site_comm * sc) :
-	conf(conf_obj), db(db_obj), s_comm(sc), torrents_list(torrents), users_list(users), whitelist(_whitelist), status(OPEN), reaper_active(false)
+	conf(conf_obj), db(db_obj), s_comm(sc), torrents_list(torrents), users_list(users), whitelist(_whitelist), status(OPEN), reaper_active(false), randgen((std::random_device())())
 {
 	logger = spdlog::get("logger");
 	load_config(conf);
 }
 
 void worker::load_config(config * conf) {
-	announce_interval = conf->get_uint("announce_interval");
+	announce_interval   = conf->get_uint("announce_interval");
 	del_reason_lifetime = conf->get_uint("del_reason_lifetime");
-	peers_timeout = conf->get_uint("peers_timeout");
-	numwant_limit = conf->get_uint("numwant_limit");
-	keepalive_enabled = conf->get_uint("keepalive_timeout") != 0;
-	site_password = conf->get_str("site_password");
-	report_password = conf->get_str("report_password");
+	peers_timeout       = conf->get_uint("peers_timeout");
+	numwant_limit       = conf->get_uint("numwant_limit");
+	keepalive_enabled   = conf->get_uint("keepalive_timeout") != 0;
+	site_password       = conf->get_str("site_password");
+	report_password     = conf->get_str("report_password");
+	jitter              = std::uniform_int_distribution<int>(0, conf->get_uint("announce_jitter"));
 }
 
 void worker::reload_config(config * conf) {
@@ -226,7 +227,7 @@ std::string worker::work(const std::string &input, std::string &ip, client_opts_
 	if (action == REPORT) {
 		if (passkey == report_password) {
 			std::lock_guard<std::mutex> ul_lock(db->user_list_mutex);
-			return report(params, users_list, client_opts);
+			return report(params, users_list, client_opts, announce_interval, conf->get_uint("announce_jitter"));
 		} else {
 			return error("Authentication failure", client_opts);
 		}
@@ -712,7 +713,7 @@ std::string worker::announce(const std::string &input, torrent &tor, user_ptr &u
 	output += "e10:incompletei";
 	output += inttostr(tor.leechers.size());
 	output += "e8:intervali";
-	output += inttostr(announce_interval + std::min((size_t)600, tor.seeders.size())); // ensure a more even distribution of announces/second
+	output += inttostr(announce_interval + jitter(randgen));
 	output += "e12:min intervali";
 	output += inttostr(announce_interval);
 	output += "e5:peers";
@@ -983,6 +984,11 @@ std::string worker::update(params_type &params, client_opts_t &client_opts) {
 		conf->set("announce_interval", interval);
 		announce_interval = conf->get_uint("announce_interval");
 		logger->info("Edited announce interval to " + std::to_string(announce_interval));
+	} else if (params["action"] == "update_announce_jitter") {
+		const std::string new_jitter = params["new_announce_jitter"];
+		conf->set("announce_jitter", new_jitter);
+		jitter = std::uniform_int_distribution<int>(0, conf->get_uint("announce_jitter"));
+		logger->info("Edited announce jitter to " + std::to_string(conf->get_uint("announce_jitter")));
 	} else if (params["action"] == "info_torrent") {
 		std::stringstream output;
 		std::string info_hash_hex = params["info_hash"];

src/worker.h

@@ -9,6 +9,8 @@
 #include <mutex>
 #include <ctime>
 #include <spdlog/spdlog.h>
+#include <random>
+
 #include "site_comm.h"
 #include "ocelot.h"
 
@@ -27,6 +29,8 @@ class worker {
 		bool reaper_active;
 		time_t cur_time;
 		std::shared_ptr<spdlog::logger> logger;
+		std::mt19937 randgen;
+		std::uniform_int_distribution<int> jitter;
 
 		unsigned int announce_interval;
 		unsigned int del_reason_lifetime;