22 Commits
Author SHA1 Message Date
Spine 54370ad892 lock entire reap operations in a mutex 2026-02-14 12:31:19 +00:00
Spine d4aaff7bb5 refactor tracker report functionality 2024-03-10 21:13:13 +01:00
Spine 37685bd8da add resource usage to grafana and report endpoints 2024-03-10 21:12:45 +01:00
Spine ce2192c018 detabbed source code, no code changes 2024-02-15 16:05:57 +01:00
Spine 2a8a7eb24b Merge branch 'announce-jitter' into 'master'
redesign announce jitter to better spread the load after startup

See merge request tracker/ocelot!8
2024-02-11 15:04:57 +00:00
Spine 4a5687b607 redesign announce jitter to better spread the load after startup 2024-02-11 15:39:22 +01:00
Spine f80f37c025 Merge branch 'torrent-info' into 'master'
output user report and info_torrent as JSON

See merge request tracker/ocelot!7
2024-02-10 15:26:59 +00:00
Spine 51c5f64a8e output user report and info_torrent as JSON 2024-01-28 12:38:25 +01:00
Spider 59a91200fa Update docker base image 2023-10-14 20:57:34 +02:00
sheepish 665e8b7a24 track queue sizes in stats 2023-10-14 20:25:15 +02:00
sheepish 7efb98654a sleep if api returns error 2023-10-14 20:25:15 +02:00
sheepish c4153d7585 fix queue worker threads getting stuck if connection fails 2023-10-14 20:25:15 +02:00
sheepish f42c7cdf47 add grafana dashboard 2023-10-14 20:19:50 +02:00
sheepish e5d20d7851 add prometheus-compatible stats reporting 2023-10-14 20:19:46 +02:00
itismadness cd1b031d4f Add curl to Dockerfile for downstream health checks
(cherry picked from commit fac0fd1686)
2023-10-14 20:19:32 +02:00
itismadness d2bae2622a Ignore build output files for docker
(cherry picked from commit 23e5b92b2a)
2023-10-14 20:19:29 +02:00
itismadness ad9fb66efa add image.source label to Dockerfile
(cherry picked from commit fc56c7a35a)
2023-10-14 20:19:25 +02:00
itismadness b248a4c766 improve dev workflow
(cherry picked from commit 50a08a2b66)
2023-10-14 20:19:14 +02:00
undertheironbridge e04b30366e Small CMakeLists.txt cleanup (#19)
Co-authored-by: strangeways <[email protected]>
(cherry picked from commit 5412bee85a)
2023-10-14 20:18:46 +02:00
itismadness 38ac1af7a2 use chrono to sleep in threads
(cherry picked from commit 3a430a9a1c)
2023-10-14 20:16:58 +02:00
Spine 9c2051ef71 change xbt_snatched insert to allow a primary key to be added 2020-04-22 01:24:02 +01:00
itismadness febde1daf7 use string empty method for checking empty strings 2020-04-22 01:23:52 +01:00
32 changed files with 4965 additions and 2675 deletions

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
.vscode/
build/
/ocelot
+1
View File
@@ -1,4 +1,5 @@
.idea/
.vscode/
.vagrant/
*.tar.*
MakeFile.in
+50 -1
View File
@@ -1,3 +1,52 @@
-- 2.1.5 (2026-02-12)
FEATURES
* connection_mother() does not need db access
* validate client whitelist before acquiring torrent lock
* add timers to measure lock acquisition, announce() and reap_peers() duration
BUGFIXES
* Wrap peer reaper code in a mutex to avoid stats desynchronization.
* Prevent upspeed and downspeed calculation from exceeding 2^31-1.
* Fix error when seeders reannounce with left > 0.
NOTE
* 2.1.4 was not officially released.
-- 2.1.3 (2024-03-09)
BREAKING
* API user endpoint output changed to JSON
FEATURES
* Peak number of simultaneous open connections is now tracked.
* Maximum size of client http request is now tracked.
* Number of errors (bad tracker secrets, bad client requests) is now tracked.
* Show jemalloc stats in Gazelle and Grafana report endpoints.
BUILD
* refactor report functions to produce only report payload
-- 2.1.2 (2024-02-17)
FEATURES
* show resource usage in Gazelle and Grafana report endpoints
-- 2.1.1 (2024-02-15)
BUILD
* Detabbed source code in first step towards having the code lint cleanly, no code changes
-- 2.1.0 (2024-02-11)
FEATURES
* announce jitter is now a uniform distribution and can be modified on the fly
-- 2.0.1 (2024-02-10)
BREAKING
* API torrent endpoint output changed to JSON
* API user endpoint shows more information about a user in JSON format
FEATURES
* Mysql port may be specified in configuration file
* API stats report contains Ocelot version
* Queue sizes are reported in stats
* Sleep if Gazelle API is down
* Handle DB connection problems gracefully
* Add Prometheus and Grafana support
BUILD
* Docker image based on Bookworm
-- 2.0.0 (2019-10-22)
BREAKING
* Write leech stats into their own dedicated tables (users_leech_stats, torrents_leech_stats)
@@ -15,7 +64,7 @@ BUILD
Move to spdlog (https://github.com/gabime/spdlog)to handle logging instead of cout
spdlog is vendored under src/, currently using v0.13.0
Add logging directive to config to configure how Ocelot logs and where it logs
Add deamon (--daemon, -d) to run Ocelot as a daemon
Add daemon switch (--daemon, -d) to run Ocelot as a daemon
Move source code into src/ directory
Add Vagrantfile and Dockerfile for running Ocelot
+3 -5
View File
@@ -12,7 +12,7 @@ endif()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMake")
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -march=native -O2 -fvisibility=hidden -fvisibility-inlines-hidden -fomit-frame-pointer -fno-ident -Wall -Wfatal-errors")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -fvisibility-inlines-hidden -Wall -Wfatal-errors")
message(STATUS "CMAKE_MODULE_PATH='${CMAKE_MODULE_PATH}'")
find_package(Jemalloc)
@@ -66,12 +66,10 @@ SET(LINK_LIBRARIES
if (USE_JEMALLOC)
message(STATUS "Build with jemalloc")
list(APPEND ${LINK_LIBRARIES} ${JEMALLOC_LIBRARY})
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ljemalloc")
list(APPEND LINK_LIBRARIES ${JEMALLOC_LIBRARY})
elseif (USE_TCMALLOC)
message(STATUS "Build with tcmalloc")
list(APPEND ${LINK_LIBRARIES} ${TCMALLOC_LIBRARY})
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ltcmalloc")
list(APPEND LINK_LIBRARIES ${TCMALLOC_LIBRARY})
endif()
target_link_libraries(
+18 -11
View File
@@ -1,7 +1,6 @@
FROM debian:stretch-slim
FROM debian:bookworm-slim
COPY . /srv
WORKDIR /srv
LABEL org.opencontainers.image.source=https://github.com/OPSnet/Ocelot
RUN apt-get update \
&& apt-get install --no-install-recommends -y \
@@ -13,19 +12,27 @@ RUN apt-get update \
libev-dev \
libjemalloc-dev \
libmysql++-dev \
netcat-traditional \
pkg-config \
&& mkdir build \
&& cd build \
&& cmake .. \
&& make \
&& rm -rf /var/lib/apt/lists/*
COPY . /srv
WORKDIR /srv
RUN cmake -Wno-dev -S /srv -B /srv/build \
&& make -C build \
&& apt-get purge -y \
build-essential \
cmake \
cmake \
pkg-config \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* \
&& mv /srv/build/ocelot /srv/ocelot \
&& mv /srv/ocelot.conf.dist /srv/ocelot.conf
&& mv /srv/ocelot.conf.dist /srv/ocelot.conf \
&& rm -rf /srv/build \
&& mkdir -p /tmp/ocelot
CMD ["/srv/ocelot"]
# default listen_port value in ocelot.conf
EXPOSE 34000
CMD ["/srv/ocelot"]
+22 -12
View File
@@ -5,17 +5,16 @@ It supports requests over TCP and can only track IPv4 peers.
## Ocelot Compile-time Dependencies
* [GCC/G++](http://gcc.gnu.org/) (4.7+ required; 4.8.1+ recommended)
* [Boost](http://www.boost.org/) (1.55.0+ required)
* [GCC/G++](http://gcc.gnu.org/) (11+ required; 11.4.0+ recommended)
* [Boost](http://www.boost.org/) (1.74.0+ required)
* [libev](http://software.schmorp.de/pkg/libev.html) (required)
* [spdlog](https://github.com/gabime/spdlog) (required)
* [MySQL++](http://tangentsoft.net/mysql++/) (3.2.0+ required)
* [jemalloc](http://jemalloc.net/) (optional, but strongly recommended)
* [TCMalloc](http://goog-perftools.sourceforge.net/doc/tcmalloc.html) (optional)
## Installation
### Debian Stretch
### Debian Bookworm
```bash
sudo apt-get install \
build-essential \
@@ -26,17 +25,27 @@ sudo apt-get install \
libev-dev \
libjemalloc-dev \
libmysql++-dev \
netcat-traditional \
pkg-config
mkdir build
cd build
cmake ..
make
cmake -Wno-dev . -B build
make -C build
```
The [Gazelle installation guides](https://github.com/WhatCD/Gazelle/wiki/Gazelle-installation) include instructions for installing Ocelot as a part of the Gazelle project.
### Docker
```bash
docker build . -t ocelot
docker run -v $(pwd)/ocelot.conf:/srv/ocelot.conf ocelot
```
### Standalone Installation
* Prepare the build environment. (This must be re-executed when new source files are added).
cmake -Wno-dev . -B build
* Create the following tables according to the [Gazelle database schema](https://raw.githubusercontent.com/WhatCD/Gazelle/master/gazelle.sql):
- `torrents`
- `torrents_leech_stats`
@@ -51,10 +60,7 @@ The [Gazelle installation guides](https://github.com/WhatCD/Gazelle/wiki/Gazelle
* Build Ocelot:
mkdir build
cd build
cmake ..
make
make -C build
## Running Ocelot
@@ -63,6 +69,10 @@ The [Gazelle installation guides](https://github.com/WhatCD/Gazelle/wiki/Gazelle
* `-c <path/to/ocelot.conf>` - Path to config file. If unspecified, the current working directory is used.
* `-v` - Print queue status every time a flush is initiated.
You can run a test ocelot daemon alongside a production daemon by specifying
a separate configuration file, and setting `readonly = true`. This will
prevent the database peer tables from being reset.
### Signals
* `SIGHUP` - Reload config
File diff suppressed because it is too large. Load diff
+11 -4
View File
@@ -11,14 +11,16 @@ connection_timeout = 10
keepalive_timeout = 0
announce_interval = 1800
announce_jitter = 240
max_request_size = 4096
numwant_limit = 50
request_log_size = 500
mysql_host =
mysql_username =
mysql_password =
mysql_db =
mysql_host = mysql
mysql_port = 36000
mysql_username = gazelle
mysql_password = password
mysql_db = gazelle
# The passwords must be 32 characters and match the Gazelle config
report_password = 00000000000000000000000000000000
@@ -32,4 +34,9 @@ schedule_interval = 3
log = false
log_path = /tmp/ocelot
# the report path should be placed on a tmpfs partition in production
report_path = /tmp/ocelot
# set to true if prevent peer data from being zeroed out on startup
# useful to test a new version alongside an instance running in production
readonly = false
+104 -95
View File
@@ -1,163 +1,172 @@
// Copyright [2017-2024] Orpheus
#include <spdlog/spdlog.h>
#include <iostream>
#include <fstream>
#include <string>
#include <spdlog/spdlog.h>
#include "config.h"
#include "misc_functions.h"
confval::confval() {
bool_val = 0;
uint_val = 0;
str_val = "";
val_type = CONF_NONEXISTENT;
bool_val = 0;
uint_val = 0;
str_val = "";
val_type = CONF_NONEXISTENT;
}
confval::confval(bool value) {
bool_val = value;
val_type = CONF_BOOL;
bool_val = value;
val_type = CONF_BOOL;
}
confval::confval(unsigned int value) {
uint_val = value;
val_type = CONF_UINT;
uint_val = value;
val_type = CONF_UINT;
}
confval::confval(const char * value) {
str_val = value;
val_type = CONF_STR;
str_val = value;
val_type = CONF_STR;
}
bool confval::get_bool() {
return bool_val;
return bool_val;
}
unsigned int confval::get_uint() {
return uint_val;
return uint_val;
}
std::string confval::get_str() {
return str_val;
return str_val;
}
void confval::set(const std::string &value) {
if (val_type == CONF_BOOL) {
bool_val = value == "1" || value == "true" || value == "yes";
} else if (val_type == CONF_UINT) {
uint_val = strtoint32(value);
} else if (val_type == CONF_STR) {
str_val = value;
}
if (val_type == CONF_BOOL) {
bool_val = value == "1" || value == "true" || value == "yes";
} else if (val_type == CONF_UINT) {
uint_val = strtoint32(value);
} else if (val_type == CONF_STR) {
str_val = value;
}
}
config::config() {
init();
dummy_setting = new confval(); // Safety value to use if we're accessing nonexistent settings
init();
dummy_setting = new confval(); // Safety value to use if we're accessing nonexistent settings
}
void config::init() {
// Internal stuff
add("listen_port", 34000u);
add("max_connections", 1024u);
add("max_middlemen", 20000u);
add("max_read_buffer", 4096u);
add("connection_timeout", 10u);
add("keepalive_timeout", 0u);
add("daemonize", false);
// Internal stuff
add("listen_port", 34000u);
add("max_connections", 1024u);
add("max_middlemen", 20000u);
add("max_read_buffer", 4096u);
add("connection_timeout", 10u);
add("keepalive_timeout", 0u);
add("daemonize", false);
// Tracker requests
add("announce_interval", 1800u);
add("max_request_size", 4096u);
add("numwant_limit", 50u);
add("request_log_size", 500u);
// Tracker requests
add("announce_interval", 1800u);
add("announce_jitter", 240u);
add("max_request_size", 4096u);
add("numwant_limit", 50u);
add("request_log_size", 500u);
// Timers
add("del_reason_lifetime", 86400u);
add("peers_timeout", 7200u);
add("reap_peers_interval", 1800u);
add("schedule_interval", 3u);
// Timers
add("del_reason_lifetime", 86400u);
add("peers_timeout", 7200u);
add("reap_peers_interval", 1800u);
add("schedule_interval", 3u);
// MySQL
add("mysql_db", "gazelle");
add("mysql_host", "localhost");
add("mysql_username", "");
add("mysql_password", "");
// MySQL
add("mysql_db", "gazelle");
add("mysql_host", "localhost");
add("mysql_port", 3306u);
add("mysql_username", "");
add("mysql_password", "");
add("mysql_port", 3306u);
// Site communication
add("site_host", "127.0.0.1");
add("site_service", "http");
add("site_path", "");
add("site_password", "00000000000000000000000000000000");
add("report_password", "00000000000000000000000000000000");
// Site communication
add("site_host", "127.0.0.1");
add("site_service", "http");
add("site_path", "");
add("site_password", "00000000000000000000000000000000");
add("report_password", "00000000000000000000000000000000");
// Logging
add("log", false);
add("log_path", "ocelot"); // path to where to write log + name of log (don't need to put file extension)
// Logging
add("log", false);
add("log_path", "ocelot"); // path to where to write log + name of log (don't need to put file extension)
// Debugging
add("readonly", false);
add("report_path", "/tmp"); // path for transfer to website (should be a tmpfs in production)
// Debugging
add("readonly", false);
}
confval * config::get(const std::string &setting_name) {
const auto setting = settings.find(setting_name);
if (setting == settings.end()) {
spdlog::get("logger")->info("WARNING: Unrecognized setting '" + setting_name + "'");
return dummy_setting;
}
return &setting->second;
const auto setting = settings.find(setting_name);
if (setting == settings.end()) {
spdlog::get("logger")->info("WARNING: Unrecognized setting '" + setting_name + "'");
return dummy_setting;
}
return &setting->second;
}
bool config::get_bool(const std::string &setting_name) {
return get(setting_name)->get_bool();
return get(setting_name)->get_bool();
}
unsigned int config::get_uint(const std::string &setting_name) {
return get(setting_name)->get_uint();
return get(setting_name)->get_uint();
}
std::string config::get_str(const std::string &setting_name) {
return get(setting_name)->get_str();
return get(setting_name)->get_str();
}
void config::set(const std::string &setting_name, const std::string &value) {
get(setting_name)->set(value);
get(setting_name)->set(value);
}
void config::load(const std::string &conf_file_path, std::istream &conf_file) {
load(conf_file);
add("conf_file_path", conf_file_path.c_str());
load(conf_file);
add("conf_file_path", conf_file_path.c_str());
}
void config::load(std::istream &conf_file) {
std::string line;
while (getline(conf_file, line)) {
size_t pos;
if (line[0] != '#' && (pos = line.find('=')) != std::string::npos) {
std::string key(trim(line.substr(0, pos)));
std::string value(trim(line.substr(pos + 1)));
set(key, value);
}
}
std::string line;
while (getline(conf_file, line)) {
size_t pos;
if (line[0] != '#' && (pos = line.find('=')) != std::string::npos) {
std::string key(trim(line.substr(0, pos)));
std::string value(trim(line.substr(pos + 1)));
set(key, value);
}
}
}
void config::reload() {
const std::string conf_file_path(get_str("conf_file_path"));
std::ifstream conf_file(conf_file_path);
if (conf_file.fail()) {
spdlog::get("logger")->error("Config file '" + conf_file_path + "' couldn't be opened");
} else {
init();
load(conf_file);
}
const std::string conf_file_path(get_str("conf_file_path"));
std::ifstream conf_file(conf_file_path);
if (conf_file.fail()) {
spdlog::get("logger")->error("Config file '" + conf_file_path + "' couldn't be opened");
} else {
init();
load(conf_file);
}
}
std::string config::trim(const std::string str) {
size_t ltrim = str.find_first_not_of(" \t");
if (ltrim == std::string::npos) {
ltrim = 0;
}
size_t rtrim = str.find_last_not_of(" \t");
if (ltrim != 0 || rtrim != str.length() - 1) {
return str.substr(ltrim, rtrim - ltrim + 1);
}
return str;
size_t ltrim = str.find_first_not_of(" \t");
if (ltrim == std::string::npos) {
ltrim = 0;
}
size_t rtrim = str.find_last_not_of(" \t");
if (ltrim != 0 || rtrim != str.length() - 1) {
return str.substr(ltrim, rtrim - ltrim + 1);
}
return str;
}
+43 -40
View File
@@ -1,52 +1,55 @@
#ifndef OCELOT_CONFIG_H
#define OCELOT_CONFIG_H
#ifndef SRC_CONFIG_H_
#define SRC_CONFIG_H_
// Copyright [2017-2024] Orpheus
#include <string>
#include <map>
class confval {
private:
bool bool_val;
uint32_t uint_val;
std::string str_val;
enum {
CONF_NONEXISTENT,
CONF_BOOL,
CONF_UINT,
CONF_STR,
} val_type;
public:
confval();
confval(bool value);
confval(uint32_t value);
confval(const char * value);
uint32_t get_uint();
bool get_bool();
std::string get_str();
void set(const std::string &val);
private:
bool bool_val;
uint32_t uint_val;
std::string str_val;
enum {
CONF_NONEXISTENT,
CONF_BOOL,
CONF_UINT,
CONF_STR,
} val_type;
public:
confval();
confval(bool value);
confval(uint32_t value);
confval(const char * value);
uint32_t get_uint();
bool get_bool();
std::string get_str();
void set(const std::string &val);
};
class config {
private:
template <typename T> void add(const std::string &setting_name, T value);
std::string trim(const std::string str);
void init();
confval * get(const std::string &setting_name);
std::map<std::string, confval> settings;
confval * dummy_setting;
public:
config();
void load(std::istream &conf_file);
void load(const std::string &conf_file_path, std::istream &conf_file);
void reload();
bool get_bool(const std::string &setting_name);
uint32_t get_uint(const std::string &setting_name);
std::string get_str(const std::string &setting_name);
void set(const std::string &setting_name, const std::string &value);
private:
template <typename T> void add(const std::string &setting_name, T value);
std::string trim(const std::string str);
void init();
confval * get(const std::string &setting_name);
std::map<std::string, confval> settings;
confval * dummy_setting;
public:
config();
void load(std::istream &conf_file);
void load(const std::string &conf_file_path, std::istream &conf_file);
void reload();
bool get_bool(const std::string &setting_name);
uint32_t get_uint(const std::string &setting_name);
std::string get_str(const std::string &setting_name);
void set(const std::string &setting_name, const std::string &value);
};
template <typename T> void config::add(const std::string &setting_name, T value) {
confval setting(value);
settings[setting_name] = setting;
confval setting(value);
settings[setting_name] = setting;
}
#endif
#endif // SRC_CONFIG_H_
+561 -532
View File
File diff suppressed because it is too large. Load diff
+77 -61
View File
@@ -1,86 +1,102 @@
#ifndef OCELOT_DB_H
#define OCELOT_DB_H
#ifndef SRC_DB_H_
#define SRC_DB_H_
// Copyright [2017-2024] Orpheus
#pragma GCC visibility push(default)
#include <mysql++/mysql++.h>
#include <spdlog/spdlog.h>
#include <string>
#include <unordered_map>
#include <queue>
#include <mutex>
#include <memory>
#include <vector>
#include "config.h"
class mysql {
private:
mysqlpp::Connection conn;
std::string update_user_buffer;
std::string update_torrent_buffer;
std::string update_heavy_peer_buffer;
std::string update_light_peer_buffer;
std::string update_snatch_buffer;
std::string update_token_buffer;
private:
mysqlpp::Connection conn;
std::string update_user_buffer;
std::string update_torrent_buffer;
std::string update_heavy_peer_buffer;
std::string update_light_peer_buffer;
std::string update_snatch_buffer;
std::string update_token_buffer;
std::queue<std::string> user_queue;
std::queue<std::string> torrent_queue;
std::queue<std::string> peer_queue;
std::queue<std::string> snatch_queue;
std::queue<std::string> token_queue;
std::queue<std::string> user_queue;
std::queue<std::string> torrent_queue;
std::queue<std::string> peer_queue;
std::queue<std::string> snatch_queue;
std::queue<std::string> token_queue;
std::string mysql_db, mysql_host, mysql_username, mysql_password;
bool u_active, t_active, p_active, s_active, tok_active;
bool readonly;
std::string mysql_db, mysql_host, mysql_username, mysql_password;
unsigned int mysql_port;
bool u_active, t_active, p_active, s_active, tok_active;
bool readonly;
// These locks prevent more than one thread from reading/writing the buffers.
// These should be held for the minimum time possible.
std::mutex user_queue_lock;
std::mutex torrent_buffer_lock;
std::mutex torrent_queue_lock;
std::mutex peer_queue_lock;
std::mutex snatch_queue_lock;
std::mutex token_queue_lock;
// These locks prevent more than one thread from reading/writing the buffers.
// These should be held for the minimum time possible.
std::mutex user_queue_lock;
std::mutex torrent_buffer_lock;
std::mutex torrent_queue_lock;
std::mutex peer_queue_lock;
std::mutex snatch_queue_lock;
std::mutex token_queue_lock;
std::shared_ptr<spdlog::logger> logger;
std::shared_ptr<spdlog::logger> logger;
void load_config(config * conf);
void load_tokens(torrent_list &torrents);
void load_config(config * conf);
void load_tokens(torrent_list &torrents);
mysqlpp::Connection create_connection();
void do_flush_users();
void do_flush_torrents();
void do_flush_snatches();
void do_flush_peers();
void do_flush_tokens();
void do_flush_users();
void do_flush_torrents();
void do_flush_snatches();
void do_flush_peers();
void do_flush_tokens();
void flush_users();
void flush_torrents();
void flush_snatches();
void flush_peers();
void flush_tokens();
void clear_peer_data();
void flush_users();
void flush_torrents();
void flush_snatches();
void flush_peers();
void flush_tokens();
void clear_peer_data();
public:
bool verbose_flush;
public:
bool verbose_flush;
mysql(config * conf);
void reload_config(config * conf);
bool connected();
void load_torrents(torrent_list &torrents);
void load_users(user_list &users);
void load_whitelist(std::vector<std::string> &whitelist);
mysql(config * conf);
void reload_config(config * conf);
void flush();
bool connected();
bool all_clear();
void load_torrents(torrent_list &torrents);
void load_users(user_list &users);
void load_whitelist(std::vector<std::string> &whitelist);
void record_user(const std::string &record); // (id,uploaded_change,downloaded_change)
void record_torrent(const std::string &record); // (id,seeders,leechers,snatched_change,balance)
void record_snatch(const std::string &record, const std::string &ip); // (uid,fid,tstamp)
void record_peer(const std::string &record, const std::string &ip, const std::string &peer_id, const std::string &useragent); // (uid,fid,active,peerid,useragent,ip,uploaded,downloaded,upspeed,downspeed,left,timespent,announces,tstamp)
void record_peer(const std::string &record, const std::string &peer_id); // (fid,peerid,timespent,announces,tstamp)
void record_token(const std::string &record);
// (id,uploaded_change,downloaded_change)
void record_user(const std::string &record);
void flush();
// (id,seeders,leechers,snatched_change,balance)
void record_torrent(const std::string &record);
bool all_clear();
// (uid,fid,tstamp)
void record_snatch(const std::string &record, const std::string &ip);
std::mutex torrent_list_mutex;
std::mutex user_list_mutex;
std::mutex whitelist_mutex;
// (uid,fid,active,peerid,useragent,ip,uploaded,downloaded,upspeed,downspeed,left,timespent,announces,tstamp)
void record_peer(const std::string &record, const std::string &ip, const std::string &peer_id, const std::string &useragent);
// (fid,peerid,timespent,announces,tstamp)
void record_peer(const std::string &record, const std::string &peer_id);
void record_token(const std::string &record);
std::mutex torrent_list_mutex;
std::mutex user_list_mutex;
std::mutex whitelist_mutex;
};
#pragma GCC visibility pop
#endif
#endif // SRC_DB_H_
+180 -171
View File
@@ -1,5 +1,9 @@
#include <cerrno>
// Copyright [2017-2026] Orpheus
#include <spdlog/spdlog.h>
#include <cerrno>
#include "ocelot.h"
#include "config.h"
#include "db.h"
@@ -8,237 +12,242 @@
#include "response.h"
#include "events.h"
// Define the connection mother (first half) and connection middlemen (second half)
std::mutex peak_open_mutex;
//TODO Better errors
// Define the connection mother (first half) and connection middlemen (second half)
//---------- Connection mother - spawns middlemen and lets them deal with the connection
connection_mother::connection_mother(config * conf, worker * worker_obj, mysql * db_obj, site_comm * sc_obj, schedule * sched) : work(worker_obj), db(db_obj) {
logger = spdlog::get("logger");
connection_mother::connection_mother(config * conf, worker * worker_obj, site_comm * sc_obj, schedule * sched) : work(worker_obj) {
logger = spdlog::get("logger");
// Handle config stuff first
load_config(conf);
// Handle config stuff first
load_config(conf);
listen_socket = create_listen_socket();
listen_socket = create_listen_socket();
// We failed to create the socket, so no point in running Ocelot
if (listen_socket == 0) {
logger->critical("Failed to create socket for Ocelot. Exiting.");
exit(1);
}
// We failed to create the socket, so no point in running Ocelot
if (listen_socket == 0) {
logger->critical("Failed to create socket for Ocelot. Exiting.");
exit(1);
}
listen_event.set<connection_mother, &connection_mother::handle_connect>(this);
listen_event.start(listen_socket, ev::READ);
// Create libev timer
schedule_event.set<schedule, &schedule::handle>(sched);
schedule_event.start(sched->schedule_interval, sched->schedule_interval); // After interval, every interval
listen_event.set<connection_mother, &connection_mother::handle_connect>(this);
listen_event.start(listen_socket, ev::READ);
// Create libev timer
schedule_event.set<schedule, &schedule::handle>(sched);
schedule_event.start(sched->schedule_interval, sched->schedule_interval); // After interval, every interval
}
void connection_mother::load_config(config * conf) {
listen_port = conf->get_uint("listen_port");
max_connections = conf->get_uint("max_connections");
max_middlemen = conf->get_uint("max_middlemen");
connection_timeout = conf->get_uint("connection_timeout");
keepalive_timeout = conf->get_uint("keepalive_timeout");
max_read_buffer = conf->get_uint("max_read_buffer");
max_request_size = conf->get_uint("max_request_size");
listen_port = conf->get_uint("listen_port");
max_connections = conf->get_uint("max_connections");
max_middlemen = conf->get_uint("max_middlemen");
connection_timeout = conf->get_uint("connection_timeout");
keepalive_timeout = conf->get_uint("keepalive_timeout");
max_read_buffer = conf->get_uint("max_read_buffer");
max_request_size = conf->get_uint("max_request_size");
}
void connection_mother::reload_config(config * conf) {
unsigned int old_listen_port = listen_port;
unsigned int old_max_connections = max_connections;
load_config(conf);
if (old_listen_port != listen_port) {
logger->info("Changing listen port from " + std::to_string(old_listen_port) +" to " + std::to_string(listen_port));
int new_listen_socket = create_listen_socket();
if (new_listen_socket != 0) {
listen_event.stop();
listen_event.start(new_listen_socket, ev::READ);
close(listen_socket);
listen_socket = new_listen_socket;
} else {
logger->error("Couldn't create new listen socket when reloading config");
}
} else if (old_max_connections != max_connections) {
listen(listen_socket, max_connections);
}
unsigned int old_listen_port = listen_port;
unsigned int old_max_connections = max_connections;
load_config(conf);
if (old_listen_port != listen_port) {
logger->info("Changing listen port from " + std::to_string(old_listen_port) +" to " + std::to_string(listen_port));
int new_listen_socket = create_listen_socket();
if (new_listen_socket != 0) {
listen_event.stop();
listen_event.start(new_listen_socket, ev::READ);
close(listen_socket);
listen_socket = new_listen_socket;
} else {
logger->error("Couldn't create new listen socket when reloading config");
}
} else if (old_max_connections != max_connections) {
listen(listen_socket, max_connections);
}
}
int connection_mother::create_listen_socket() {
sockaddr_in address;
memset(&address, 0, sizeof(address));
int new_listen_socket = socket(AF_INET, SOCK_STREAM, 0);
sockaddr_in address;
memset(&address, 0, sizeof(address));
int new_listen_socket = socket(AF_INET, SOCK_STREAM, 0);
// Stop old sockets from hogging the port
int yes = 1;
if (setsockopt(new_listen_socket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) {
logger->error("Could not reuse socket: " + std::string(strerror(errno)));
return 0;
}
// Stop old sockets from hogging the port
int yes = 1;
if (setsockopt(new_listen_socket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) {
logger->error("Could not reuse socket: " + std::string(strerror(errno)));
return 0;
}
// Get ready to bind
address.sin_family = AF_INET;
//address.sin_addr.s_addr = inet_addr(conf->host.c_str()); // htonl(INADDR_ANY)
address.sin_addr.s_addr = htonl(INADDR_ANY);
address.sin_port = htons(listen_port);
// Get ready to bind
address.sin_family = AF_INET;
address.sin_addr.s_addr = htonl(INADDR_ANY);
address.sin_port = htons(listen_port);
// Bind
if (bind(new_listen_socket, (sockaddr *) &address, sizeof(address)) == -1) {
logger->error("Bind failed: " + std::string(strerror(errno)));
return 0;
}
// Bind
if (bind(new_listen_socket, (sockaddr *) &address, sizeof(address)) == -1) {
logger->error("Bind failed: " + std::string(strerror(errno)));
return 0;
}
// Listen
if (listen(new_listen_socket, max_connections) == -1) {
logger->error("Listen failed: " + std::string(strerror(errno)));
return 0;
}
// Listen
if (listen(new_listen_socket, max_connections) == -1) {
logger->error("Listen failed: " + std::string(strerror(errno)));
return 0;
}
// Set non-blocking
int flags = fcntl(new_listen_socket, F_GETFL);
if (flags == -1) {
logger->error("Could not get socket flags: " + std::string(strerror(errno)));
return 0;
}
if (fcntl(new_listen_socket, F_SETFL, flags | O_NONBLOCK) == -1) {
logger->error("Could not set non-blocking: " + std::string(strerror(errno)));
return 0;
}
// Set non-blocking
int flags = fcntl(new_listen_socket, F_GETFL);
if (flags == -1) {
logger->error("Could not get socket flags: " + std::string(strerror(errno)));
return 0;
}
if (fcntl(new_listen_socket, F_SETFL, flags | O_NONBLOCK) == -1) {
logger->error("Could not set non-blocking: " + std::string(strerror(errno)));
return 0;
}
return new_listen_socket;
return new_listen_socket;
}
void connection_mother::run() {
logger->info("Sockets up on port " + std::to_string(listen_port) + ", starting event loop!");
ev_loop(ev_default_loop(0), 0);
logger->info("Sockets up on port " + std::to_string(listen_port) + ", starting event loop!");
ev_loop(ev_default_loop(0), 0);
}
void connection_mother::handle_connect(ev::io &watcher, int events_flags) {
// Spawn a new middleman
if (stats.open_connections < max_middlemen) {
stats.opened_connections++;
stats.open_connections++;
new connection_middleman(listen_socket, work, this);
}
// Spawn a new middleman
if (stats.open_connections < max_middlemen) {
stats.opened_connections++;
{
const std::lock_guard<std::mutex> lock(peak_open_mutex);
unsigned int current_open = stats.open_connections++;
if (stats.peak_connections < current_open) {
stats.peak_connections = current_open;
}
}
new connection_middleman(listen_socket, work, this);
}
}
connection_mother::~connection_mother()
{
close(listen_socket);
close(listen_socket);
}
//---------- Connection middlemen - these little guys live until their connection is closed
connection_middleman::connection_middleman(int &listen_socket, worker * new_work, connection_mother * mother_arg) :
written(0), mother(mother_arg), work(new_work)
written(0), mother(mother_arg), work(new_work)
{
auto logger = spdlog::get("logger");
connect_sock = accept(listen_socket, NULL, NULL);
if (connect_sock == -1) {
logger->error("Accept failed, errno " + std::to_string(errno) + ": " + std::string(strerror(errno)));
delete this;
return;
}
auto logger = spdlog::get("logger");
connect_sock = accept(listen_socket, NULL, NULL);
if (connect_sock == -1) {
logger->error("Accept failed, errno " + std::to_string(errno) + ": " + std::string(strerror(errno)));
delete this;
return;
}
// Set non-blocking
int flags = fcntl(connect_sock, F_GETFL);
if (flags == -1) {
logger->warn("Could not get connect socket flags");
}
if (fcntl(connect_sock, F_SETFL, flags | O_NONBLOCK) == -1) {
logger->warn("Could not set non-blocking");
}
// Set non-blocking
int flags = fcntl(connect_sock, F_GETFL);
if (flags == -1) {
logger->warn("Could not get connect socket flags");
}
if (fcntl(connect_sock, F_SETFL, flags | O_NONBLOCK) == -1) {
logger->warn("Could not set non-blocking");
}
// Get their info
request.reserve(mother->max_read_buffer);
written = 0;
// Get their info
request.reserve(mother->max_read_buffer);
written = 0;
read_event.set<connection_middleman, &connection_middleman::handle_read>(this);
read_event.start(connect_sock, ev::READ);
read_event.set<connection_middleman, &connection_middleman::handle_read>(this);
read_event.start(connect_sock, ev::READ);
// Let the socket timeout in timeout_interval seconds
timeout_event.set<connection_middleman, &connection_middleman::handle_timeout>(this);
timeout_event.set(mother->connection_timeout, mother->keepalive_timeout);
timeout_event.start();
// Let the socket timeout in timeout_interval seconds
timeout_event.set<connection_middleman, &connection_middleman::handle_timeout>(this);
timeout_event.set(mother->connection_timeout, mother->keepalive_timeout);
timeout_event.start();
}
connection_middleman::~connection_middleman() {
close(connect_sock);
stats.open_connections--;
close(connect_sock);
stats.open_connections--;
}
// Handler to read data from the socket, called by event loop when socket is readable
void connection_middleman::handle_read(ev::io &watcher, int events_flags) {
char buffer[mother->max_read_buffer + 1];
memset(buffer, 0, mother->max_read_buffer + 1);
int ret = recv(connect_sock, &buffer, mother->max_read_buffer, 0);
char buffer[mother->max_read_buffer + 1];
memset(buffer, 0, mother->max_read_buffer + 1);
int ret = recv(connect_sock, &buffer, mother->max_read_buffer, 0);
if (ret <= 0) {
delete this;
return;
}
stats.bytes_read += ret;
request.append(buffer, ret);
size_t request_size = request.size();
if (request_size > mother->max_request_size || (request_size >= 4 && request.compare(request_size - 4, std::string::npos, "\r\n\r\n") == 0)) {
stats.requests++;
read_event.stop();
client_opts.gzip = false;
client_opts.html = false;
client_opts.http_close = true;
if (ret <= 0) {
delete this;
return;
}
stats.bytes_read += ret;
request.append(buffer, ret);
size_t request_size = request.size();
if (request_size > mother->max_request_size || (request_size >= 4 && request.compare(request_size - 4, std::string::npos, "\r\n\r\n") == 0)) {
stats.requests++;
read_event.stop();
client_opts.gzip = false;
client_opts.html = false;
client_opts.http_close = true;
if (request_size > mother->max_request_size) {
shutdown(connect_sock, SHUT_RD);
response = error("GET string too long", client_opts);
} else {
char ip[INET_ADDRSTRLEN];
sockaddr_in client_addr;
socklen_t addr_len = sizeof(client_addr);
getpeername(connect_sock, (sockaddr *) &client_addr, &addr_len);
inet_ntop(AF_INET, &(client_addr.sin_addr), ip, INET_ADDRSTRLEN);
std::string ip_str = ip;
if (request_size > mother->max_request_size) {
shutdown(connect_sock, SHUT_RD);
response = error("GET string too long", client_opts);
} else {
char ip[INET_ADDRSTRLEN];
sockaddr_in client_addr;
socklen_t addr_len = sizeof(client_addr);
getpeername(connect_sock, (sockaddr *) &client_addr, &addr_len);
inet_ntop(AF_INET, &(client_addr.sin_addr), ip, INET_ADDRSTRLEN);
std::string ip_str = ip;
//--- CALL WORKER
response = work->work(request, ip_str, client_opts);
request.clear();
request_size = 0;
}
//--- CALL WORKER
response = work->work(request, ip_str, client_opts);
request.clear();
request_size = 0;
}
// Find out when the socket is writeable.
// The loop in connection_mother will call handle_write when it is.
write_event.set<connection_middleman, &connection_middleman::handle_write>(this);
write_event.start(connect_sock, ev::WRITE);
}
// Find out when the socket is writeable.
// The loop in connection_mother will call handle_write when it is.
write_event.set<connection_middleman, &connection_middleman::handle_write>(this);
write_event.start(connect_sock, ev::WRITE);
}
}
// Handler to write data to the socket, called by event loop when socket is writeable
void connection_middleman::handle_write(ev::io &watcher, int events_flags) {
int ret = send(connect_sock, response.c_str()+written, response.size()-written, MSG_NOSIGNAL);
if (ret == -1) {
return;
}
stats.bytes_written += ret;
written += ret;
if (written == response.size()) {
write_event.stop();
if (client_opts.http_close) {
timeout_event.stop();
delete this;
return;
}
timeout_event.again();
read_event.start();
response.clear();
written = 0;
}
int ret = send(connect_sock, response.c_str()+written, response.size()-written, MSG_NOSIGNAL);
if (ret == -1) {
return;
}
stats.bytes_written += ret;
written += ret;
if (written == response.size()) {
write_event.stop();
if (client_opts.http_close) {
timeout_event.stop();
delete this;
return;
}
timeout_event.again();
read_event.start();
response.clear();
written = 0;
}
}
// After a middleman has been alive for timout_interval seconds, this is called
void connection_middleman::handle_timeout(ev::timer &watcher, int events_flags) {
timeout_event.stop();
read_event.stop();
write_event.stop();
delete this;
timeout_event.stop();
read_event.stop();
write_event.stop();
delete this;
}
+63 -71
View File
@@ -1,3 +1,8 @@
#ifndef SRC_EVENTS_H_
#define SRC_EVENTS_H_
// Copyright [2017-2026] Orpheus
#ifdef EV_ERROR
#undef EV_ERROR
#endif
@@ -6,108 +11,95 @@
#define MSG_NOSIGNAL 0x2000 /* don't raise SIGPIPE */
#endif
#include <iostream>
#include <string>
#include <cstring>
// Sockets
#include <arpa/inet.h>
#include <fcntl.h>
#include <spdlog/spdlog.h>
#include <sys/socket.h>
#include <unistd.h>
// libev
#include <ev++.h>
// Sockets
#include <sys/socket.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <iostream>
#include <cstring>
#include <memory>
#include <string>
#include "config.h"
#include "worker.h"
#include "schedule.h"
#include "db.h"
#include "site_comm.h"
/*
TODO find out what these do
#include <netinet/in.h>
#include <netdb.h>
#include <sys/types.h>
*/
/*
We have three classes - the mother, the middlemen, and the worker
THE MOTHER
The mother is called when a client opens a connection to the server.
It creates a middleman for every new connection, which will be called
when its socket is ready for reading.
The mother is called when a client opens a connection to the server.
It creates a middleman for every new connection, which will be called
when its socket is ready for reading.
THE MIDDLEMEN
Each middleman hang around until data is written to its socket. It then
reads the data and sends it to the worker. When it gets the response, it
gets called to write its data back to the client.
Each middleman hang around until data is written to its socket. It then
reads the data and sends it to the worker. When it gets the response, it
gets called to write its data back to the client.
THE WORKER
The worker gets data from the middleman, and returns the response. It
doesn't concern itself with silly things like sockets.
The worker gets data from the middleman, and returns the response. It
doesn't concern itself with silly things like sockets.
see worker.h for the worker.
see worker.h for the worker.
*/
// THE MOTHER - Spawns connection middlemen
class connection_mother {
private:
void load_config(config * conf);
unsigned int listen_port;
unsigned int max_connections;
private:
void load_config(config * conf);
unsigned int listen_port;
unsigned int max_connections;
int listen_socket;
worker * work;
mysql * db;
ev::io listen_event;
ev::timer schedule_event;
std::shared_ptr<spdlog::logger> logger;
int listen_socket;
worker * work;
ev::io listen_event;
ev::timer schedule_event;
std::shared_ptr<spdlog::logger> logger;
public:
connection_mother(config * conf, worker * worker_obj, mysql * db_obj, site_comm * sc_obj, schedule * sched_obj);
~connection_mother();
void reload_config(config * conf);
int create_listen_socket();
void run();
void handle_connect(ev::io &watcher, int events_flags);
public:
connection_mother(config * conf, worker * worker_obj, site_comm * sc_obj, schedule * sched_obj);
~connection_mother();
void reload_config(config * conf);
int create_listen_socket();
void run();
void handle_connect(ev::io &watcher, int events_flags);
unsigned int max_middlemen;
unsigned int connection_timeout;
unsigned int keepalive_timeout;
unsigned int max_read_buffer;
unsigned int max_request_size;
unsigned int max_middlemen;
unsigned int connection_timeout;
unsigned int keepalive_timeout;
unsigned int max_read_buffer;
unsigned int max_request_size;
};
// THE MIDDLEMAN
// Created by connection_mother
// Add their own watchers to see when sockets become readable
class connection_middleman {
private:
int connect_sock;
client_opts_t client_opts;
unsigned int written;
ev::io read_event;
ev::io write_event;
ev::timer timeout_event;
std::string request;
std::string response;
private:
int connect_sock;
client_opts_t client_opts;
unsigned int written;
ev::io read_event;
ev::io write_event;
ev::timer timeout_event;
std::string request;
std::string response;
connection_mother * mother;
worker * work;
connection_mother * mother;
worker * work;
public:
connection_middleman(int &listen_socket, worker* work, connection_mother * mother_arg);
~connection_middleman();
public:
connection_middleman(int &listen_socket, worker* work, connection_mother * mother_arg);
~connection_middleman();
void handle_read(ev::io &watcher, int events_flags);
void handle_write(ev::io &watcher, int events_flags);
void handle_timeout(ev::timer &watcher, int events_flags);
void handle_read(ev::io &watcher, int events_flags);
void handle_write(ev::io &watcher, int events_flags);
void handle_timeout(ev::timer &watcher, int events_flags);
};
#endif // SRC_EVENTS_H_
+200
View File
@@ -0,0 +1,200 @@
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "jemalloc_parse.h"
/* parse a size_t decimal text representation */
const char *parse_sz(const char *in, const char *find, size_t *target) {
const char *p = strstr(in, find);
if (!p) {
// didn't find what we were looking for
return NULL;
}
p += strlen(find);
// copy the digits to a null-terminated string in order to pass it to strtoul()
// the largest unsigned long decimal representation is 4294967295 (10 bytes)
char convert[11];
char *c = convert;
while (isspace(*p)) {
p++;
}
while (isdigit(*p) && c < convert + sizeof(convert) - 1) {
*c++ = *p++;
}
*c = '\0';
*target = strtoul(convert, &c, 10);
return p + 1; // one past the last digit
}
/* parse an unsigned long long decimal text representation */
const char *parse_ull(const char *in, const char *find, unsigned long long *target) {
const char *p = strstr(in, find);
if (!p) {
// didn't find what we were looking for
return NULL;
}
p += strlen(find);
// copy the digits to a null-terminated string in order to pass it to strtoull()
// the largest unsigned long long decimal representation is 18446744073709551615 (21 bytes)
char convert[22];
char *c = convert;
while (isspace(*p)) {
p++;
}
while(isdigit(*p) && c < convert + sizeof(convert) - 1) {
*c++ = *p++;
}
*c = '\0';
*target = strtoull(convert, &c, 10);
return p + 1; // one past the last digit
}
int jemalloc_parse(const char *in, struct ocelot_alloc_info *out) {
/* Parse the output of jemalloc_status() plain output
* This is an ugly hack; it would be faster to build directly what we want
* rather than walk down a formatted string that has generated lots of
* information that is discarded.
* If the input breaks, the error code indicates where the parsing failed.
*/
// set everything to zero
memset (out, 0, sizeof(struct ocelot_alloc_info));
/*
___ Begin jemalloc statistics ___
Version: "5.3.0-0-g54eaed1d8b56b1aa528be3bdd1877e59c56fa90c"
Build-time option settings
...
Run-time option settings
...
Profiling settings
...
Arenas: 17
Quantum size: 16
Page size: 4096
Maximum thread-cached size class: 32768
Number of bin size classes: 36
Number of thread-cache bin size classes: 41
Number of large size classes: 196
*/
if (!(in = strstr(in, "___ Begin jemalloc statistics ___"))) {
return 1;
}
if (!(in = parse_sz(in, "Arenas: ", &out->nr_arena))) {
return 10;
}
if (!(in = parse_sz(in, "Number of bin size classes: ", &out->nr_bin_small))) {
return 11;
}
if (!(in = parse_sz(in, "Number of thread-cache bin size classes: ", &out->nr_bin_tc))) {
return 12;
}
if (!(in = parse_sz(in, "Number of large size classes: ", &out->nr_bin_large))) {
return 13;
}
// Allocated: 895744, active: 1024000, metadata: 3047664 (n_thp 0), resident: 3985408, mapped: 7315456, retained: 1073152
if (!(in = parse_ull(in, "Allocated: ", &out->mem_allocated))) {
return 20;
}
if (!(in = parse_ull(in, "active: ", &out->mem_active))) {
return 21;
}
if (!(in = parse_ull(in, "metadata: ", &out->mem_metadata))) {
return 22;
}
if (!(in = parse_ull(in, "resident: ", &out->mem_resident))) {
return 23;
}
if (!(in = parse_ull(in, "mapped: ", &out->mem_mapped))) {
return 24;
}
if (!(in = parse_ull(in, "retained: ", &out->mem_retained))) {
return 25;
}
/*
allocated nmalloc (#/sec) ndalloc (#/sec) nrequests (#/sec) nfill (#/sec) nflush (#/sec)
small: 247840 131452 0 129036 0 24219 0 2402 0 8543 0
large: 290816 815 0 808 0 971 0 815 0 2351 0
total: 538656 132267 0 129844 0
*/
if (!(in = strstr(in, "(#/sec)\nsmall:"))) {
return 30;
}
if (!(in = parse_sz(in, " ", &out->small.allocated))) {
return 40;
}
if (!(in = parse_sz(in, " ", &out->small.nmalloc_total))) {
return 41;
}
if (!(in = parse_sz(in, " ", &out->small.ndalloc_rate))) {
return 42;
}
if (!(in = parse_sz(in, " ", &out->small.ndalloc_total))) {
return 43;
}
if (!(in = parse_sz(in, " ", &out->small.nrequests_rate))) {
return 44;
}
if (!(in = parse_sz(in, " ", &out->small.nrequests_total))) {
return 45;
}
if (!(in = parse_sz(in, " ", &out->small.nfill_rate))) {
return 46;
}
if (!(in = parse_sz(in, " ", &out->small.nfill_total))) {
return 47;
}
if (!(in = parse_sz(in, " ", &out->small.nflush_rate))) {
return 48;
}
if (!(in = parse_sz(in, " ", &out->small.nflush_total))) {
return 49;
}
if (!(in = strstr(in, "\nlarge:"))) {
return 60;
}
if (!(in = parse_sz(in, " ", &out->large.allocated))) {
return 70;
}
if (!(in = parse_sz(in, " ", &out->large.nmalloc_total))) {
return 71;
}
if (!(in = parse_sz(in, " ", &out->large.ndalloc_rate))) {
return 72;
}
if (!(in = parse_sz(in, " ", &out->large.ndalloc_total))) {
return 73;
}
if (!(in = parse_sz(in, " ", &out->large.nrequests_rate))) {
return 74;
}
if (!(in = parse_sz(in, " ", &out->large.nrequests_total))) {
return 75;
}
if (!(in = parse_sz(in, " ", &out->large.nfill_rate))) {
return 76;
}
if (!(in = parse_sz(in, " ", &out->large.nfill_total))) {
return 77;
}
if (!(in = parse_sz(in, " ", &out->large.nflush_rate))) {
return 78;
}
if (!(in = parse_sz(in, " ", &out->large.nflush_total))) {
return 79;
}
return 0;
}
+37
View File
@@ -0,0 +1,37 @@
#ifndef SRC_PARSE_JEMALLOC_H_
#define SRC_PARSE_JEMALLOC_H_
struct ocelot_alloc_stat {
size_t allocated;
size_t nmalloc_total;
size_t nmalloc_rate;
size_t ndalloc_total;
size_t ndalloc_rate;
size_t nrequests_total;
size_t nrequests_rate;
size_t nfill_total;
size_t nfill_rate;
size_t nflush_total;
size_t nflush_rate;
};
struct ocelot_alloc_info {
size_t nr_arena;
size_t nr_bin_small;
size_t nr_bin_tc;
size_t nr_bin_large;
unsigned long long mem_allocated;
unsigned long long mem_active;
unsigned long long mem_metadata;
unsigned long long mem_resident;
unsigned long long mem_mapped;
unsigned long long mem_retained;
struct ocelot_alloc_stat small;
struct ocelot_alloc_stat large;
};
int jemalloc_parse(const char *in, struct ocelot_alloc_info *out);
#endif // SRC_PARSE_JEMALLOC_H_
+66 -62
View File
@@ -1,81 +1,85 @@
#include <string>
// Copyright [2017-2024] Orpheus
#include <iostream>
#include <sstream>
#include <string>
#include "misc_functions.h"
int32_t strtoint32(const std::string& str) {
std::istringstream stream(str);
int32_t i = 0;
stream >> i;
return i;
std::istringstream stream(str);
int32_t i = 0;
stream >> i;
return i;
}
int64_t strtoint64(const std::string& str) {
std::istringstream stream(str);
int64_t i = 0;
stream >> i;
return i;
std::istringstream stream(str);
int64_t i = 0;
stream >> i;
return i;
}
std::string inttostr(const int i) {
std::string str;
std::stringstream out;
out << i;
str = out.str();
return str;
std::string str;
std::stringstream out;
out << i;
str = out.str();
return str;
}
std::string hex_decode(const std::string &in) {
std::string out;
out.reserve(20);
unsigned int in_length = in.length();
for (unsigned int i = 0; i < in_length; i++) {
unsigned char x = '0';
if (in[i] == '%' && (i + 2) < in_length) {
i++;
if (in[i] >= 'a' && in[i] <= 'f') {
x = static_cast<unsigned char>((in[i]-87) << 4);
} else if (in[i] >= 'A' && in[i] <= 'F') {
x = static_cast<unsigned char>((in[i]-55) << 4);
} else if (in[i] >= '0' && in[i] <= '9') {
x = static_cast<unsigned char>((in[i]-48) << 4);
}
std::string out;
out.reserve(20);
unsigned int in_length = in.length();
for (unsigned int i = 0; i < in_length; i++) {
unsigned char x = '0';
if (in[i] == '%' && (i + 2) < in_length) {
i++;
if (in[i] >= 'a' && in[i] <= 'f') {
x = static_cast<unsigned char>((in[i]-87) << 4);
} else if (in[i] >= 'A' && in[i] <= 'F') {
x = static_cast<unsigned char>((in[i]-55) << 4);
} else if (in[i] >= '0' && in[i] <= '9') {
x = static_cast<unsigned char>((in[i]-48) << 4);
}
i++;
if (in[i] >= 'a' && in[i] <= 'f') {
x += static_cast<unsigned char>(in[i]-87);
} else if (in[i] >= 'A' && in[i] <= 'F') {
x += static_cast<unsigned char>(in[i]-55);
} else if (in[i] >= '0' && in[i] <= '9') {
x += static_cast<unsigned char>(in[i]-48);
}
} else {
x = in[i];
}
out.push_back(x);
}
return out;
i++;
if (in[i] >= 'a' && in[i] <= 'f') {
x += static_cast<unsigned char>(in[i]-87);
} else if (in[i] >= 'A' && in[i] <= 'F') {
x += static_cast<unsigned char>(in[i]-55);
} else if (in[i] >= '0' && in[i] <= '9') {
x += static_cast<unsigned char>(in[i]-48);
}
} else {
x = in[i];
}
out.push_back(x);
}
return out;
}
std::string bintohex(const std::string &in) {
std::string out;
size_t length = in.length();
out.reserve(2*length);
for (unsigned int i = 0; i < length; i++) {
unsigned char x = static_cast<unsigned char>((in[i] & 0xF0) >> 4);
if (x > 9) {
x += 'a' - 10;
} else {
x += '0';
}
out.push_back(x);
x = in[i] & 0x0F;
if (x > 9) {
x += 'a' - 10;
} else {
x += '0';
}
out.push_back(x);
}
return out;
std::string out;
size_t length = in.length();
out.reserve(2*length);
for (unsigned int i = 0; i < length; i++) {
unsigned char x = static_cast<unsigned char>((in[i] & 0xF0) >> 4);
if (x > 9) {
x += 'a' - 10;
} else {
x += '0';
}
out.push_back(x);
x = in[i] & 0x0F;
if (x > 9) {
x += 'a' - 10;
} else {
x += '0';
}
out.push_back(x);
}
return out;
}
+6 -3
View File
@@ -1,5 +1,8 @@
#ifndef MISC_FUNCTIONS__H
#define MISC_FUNCTIONS__H
#ifndef SRC_MISC_FUNCTIONS_H_
#define SRC_MISC_FUNCTIONS_H_
// Copyright [2017-2024] Orpheus
#include <string>
int32_t strtoint32(const std::string& str);
@@ -8,4 +11,4 @@ std::string inttostr(int i);
std::string hex_decode(const std::string &in);
std::string bintohex(const std::string &in);
#endif
#endif // SRC_MISC_FUNCTIONS_H_
+172 -150
View File
@@ -1,12 +1,16 @@
#include <fstream>
#include <iostream>
#include <csignal>
#include <thread>
// Copyright [2017-2026] Orpheus
#include <sys/stat.h>
#include <syslog.h>
#include <spdlog/spdlog.h>
#include <spdlog/sinks/daily_file_sink.h>
#include <spdlog/sinks/stdout_sinks.h>
#include <fstream>
#include <iostream>
#include <csignal>
#include <thread>
#include "ocelot.h"
#include "config.h"
#include "db.h"
@@ -22,183 +26,201 @@ static schedule *sched;
struct stats_t stats;
static void create_daemon()
{
pid_t pid;
static void create_daemon() {
pid_t pid;
/* Fork off the parent process */
pid = fork();
/* Fork off the parent process */
pid = fork();
/* An error occurred */
if (pid < 0) {
exit(EXIT_FAILURE);
}
/* An error occurred */
if (pid < 0) {
exit(EXIT_FAILURE);
}
/* Success: Let the parent terminate */
if (pid > 0) {
exit(EXIT_SUCCESS);
}
/* Success: Let the parent terminate */
if (pid > 0) {
exit(EXIT_SUCCESS);
}
/* On success: The child process becomes session leader */
if (setsid() < 0)
exit(EXIT_FAILURE);
/* On success: The child process becomes session leader */
if (setsid() < 0)
exit(EXIT_FAILURE);
/* Set new file permissions */
umask(0);
/* Set new file permissions */
umask(0);
/* Change the working directory to the root directory */
/* or another appropriated directory */
if (chdir("/") < 0) {
exit(EXIT_FAILURE);
}
/* Change the working directory to the root directory */
/* or another appropriated directory */
if (chdir("/") < 0) {
exit(EXIT_FAILURE);
}
/* Close all open file descriptors */
int x;
for (x = static_cast<int>(sysconf(_SC_OPEN_MAX)); x >= 0; x--) {
close (x);
}
/* Close all open file descriptors */
int x;
for (x = static_cast<int>(sysconf(_SC_OPEN_MAX)); x >= 0; x--) {
close(x);
}
/* Open the log file */
openlog("ocelot", LOG_PID, LOG_DAEMON);
/* Open the log file */
openlog("ocelot", LOG_PID, LOG_DAEMON);
}
static void sig_handler(int sig) {
std::shared_ptr<spdlog::logger> logger = spdlog::get("logger");
if (sig == SIGINT || sig == SIGTERM) {
logger->info("Caught SIGINT/SIGTERM");
if (work->shutdown()) {
exit(0);
}
} else if (sig == SIGHUP) {
logger->info("Reloading config");
logger->flush();
conf->reload();
db->reload_config(conf);
mother->reload_config(conf);
sc->reload_config(conf);
sched->reload_config(conf);
work->reload_config(conf);
logger->info("Done reloading config");
} else if (sig == SIGUSR1) {
logger->info("Reloading from database");
std::thread w_thread(&worker::reload_lists, work);
w_thread.detach();
}
std::shared_ptr<spdlog::logger> logger = spdlog::get("logger");
if (sig == SIGINT || sig == SIGTERM) {
logger->info("Caught SIGINT/SIGTERM");
if (work->shutdown()) {
exit(0);
}
} else if (sig == SIGHUP) {
logger->info("Reloading config");
logger->flush();
conf->reload();
db->reload_config(conf);
mother->reload_config(conf);
sc->reload_config(conf);
sched->reload_config(conf);
work->reload_config(conf);
logger->info("Done reloading config");
} else if (sig == SIGUSR1) {
logger->info("Reloading from database");
std::thread w_thread(&worker::reload_lists, work);
w_thread.detach();
}
}
int main(int argc, char **argv) {
// we don't use printf so make cout/cerr a little bit faster
std::ios_base::sync_with_stdio(false);
// we don't use printf so make cout/cerr a little bit faster
std::ios_base::sync_with_stdio(false);
conf = new config();
conf = new config();
bool verbose = false, conf_arg = false, daemonize = false;
std::string conf_file_path("./ocelot.conf");
bool conf_arg = false;
bool daemonize = false;
bool verbose = false;
std::string conf_file_path("./ocelot.conf");
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0) {
verbose = true;
}
else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--daemonize") == 0) {
daemonize = true;
}
else if (strcmp(argv[i], "-c") == 0 && i < argc - 1) {
conf_arg = true;
conf_file_path = argv[++i];
}
else if(strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) {
std::cout << "Ocelot, version 2.0.0" << std::endl;
return 0;
}
else {
std::cout << "Usage: " << argv[0] << " [-v] [-c configfile]" << std::endl;
return 0;
}
}
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0) {
verbose = true;
} else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--daemonize") == 0) {
daemonize = true;
} else if (strcmp(argv[i], "-c") == 0 && i < argc - 1) {
conf_arg = true;
conf_file_path = argv[++i];
} else if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) {
// preprocessor concatenation
std::cout << "Ocelot version " OCELOT_VERSION ", compiled " __DATE__ " " __TIME__ << std::endl;
return 0;
} else {
std::cout << "Usage: " << argv[0] << " [-v] [-c configfile] [--daemonize]" << std::endl;
return 0;
}
}
std::ifstream conf_file(conf_file_path);
if (conf_file.fail()) {
std::cout << "Using default config because '" << conf_file_path << "' couldn't be opened" << std::endl;
if (!conf_arg) {
std::cout << "Start Ocelot with -c <path> to specify config file if necessary" << std::endl;
}
}
else {
conf->load(conf_file_path, conf_file);
}
std::ifstream conf_file(conf_file_path);
if (conf_file.fail()) {
std::cout << "Using default config because '" << conf_file_path << "' couldn't be opened" << std::endl;
if (!conf_arg) {
std::cout << "Start Ocelot with -c <path> to specify config file if necessary" << std::endl;
}
} else {
conf->load(conf_file_path, conf_file);
}
if (conf->get_bool("daemonize") || daemonize) {
create_daemon();
}
if (conf->get_bool("daemonize") || daemonize) {
create_daemon();
}
std::vector<spdlog::sink_ptr> sinks;
if (!conf->get_bool("daemonize") && !daemonize) {
sinks.push_back(std::make_shared<spdlog::sinks::stdout_sink_mt>());
}
if (conf->get_bool("log") && !conf->get_str("log_path").empty()) {
sinks.push_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(conf->get_str("log_path"), 23, 59));
}
std::vector<spdlog::sink_ptr> sinks;
if (!conf->get_bool("daemonize") && !daemonize) {
sinks.push_back(std::make_shared<spdlog::sinks::stdout_sink_mt>());
}
if (conf->get_bool("log") && !conf->get_str("log_path").empty()) {
sinks.push_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(conf->get_str("log_path"), 23, 59));
}
auto combined_logger = std::make_shared<spdlog::logger>("logger", begin(sinks), end(sinks));
// If we don't set flush on info, the file log takes a long while to actually flush
combined_logger->flush_on(spdlog::level::info);
spdlog::register_logger(combined_logger);
auto combined_logger = std::make_shared<spdlog::logger>("logger", begin(sinks), end(sinks));
// If we don't set flush on info, the file log takes a long while to actually flush
combined_logger->flush_on(spdlog::level::info);
combined_logger->info(
"Ocelot version " OCELOT_VERSION ", compiled " __DATE__ " " __TIME__
);
spdlog::register_logger(combined_logger);
db = new mysql(conf);
db = new mysql(conf);
if (!db->connected()) {
combined_logger->critical("Could not connect to DB. Exiting!");
return 0;
}
db->verbose_flush = verbose;
if (!db->connected()) {
combined_logger->critical("Could not connect to DB. Exiting!");
return 0;
}
db->verbose_flush = verbose;
sc = new site_comm(conf);
sc->verbose_flush = verbose;
sc = new site_comm(conf);
sc->verbose_flush = verbose;
user_list users_list;
torrent_list torrents_list;
std::vector<std::string> whitelist;
db->load_users(users_list);
db->load_torrents(torrents_list);
db->load_whitelist(whitelist);
user_list users_list;
torrent_list torrents_list;
std::vector<std::string> whitelist;
db->load_users(users_list);
db->load_torrents(torrents_list);
db->load_whitelist(whitelist);
stats.open_connections = 0;
stats.opened_connections = 0;
stats.connection_rate = 0;
stats.requests = 0;
stats.request_rate = 0;
stats.leechers = 0;
stats.seeders = 0;
stats.announcements = 0;
stats.succ_announcements = 0;
stats.scrapes = 0;
stats.bytes_read = 0;
stats.bytes_written = 0;
stats.start_time = time(NULL);
stats.open_connections = 0;
stats.peak_connections = 0;
stats.opened_connections = 0;
stats.connection_rate = 0;
stats.requests = 0;
stats.request_rate = 0;
stats.leechers = 0;
stats.seeders = 0;
stats.user_queue_size = 0;
stats.torrent_queue_size = 0;
stats.peer_queue_size = 0;
stats.snatch_queue_size = 0;
stats.token_queue_size = 0;
stats.max_client_request_len = 0;
stats.announcements = 0;
stats.succ_announcements = 0;
stats.scrapes = 0;
stats.announce_duration = 0;
stats.torrent_lock_total = 0;
stats.torrent_lock_duration = 0;
stats.reap_total = 0;
stats.reap_lock_duration = 0;
stats.reap_duration = 0;
stats.bytes_read = 0;
stats.bytes_written = 0;
stats.auth_error_secret = 0;
stats.auth_error_report = 0;
stats.auth_error_announce_key = 0;
stats.client_error = 0;
stats.http_error = 0;
stats.start_time = time(NULL);
// Create worker object, which handles announces and scrapes and all that jazz
work = new worker(conf, torrents_list, users_list, whitelist, db, sc);
// Create worker object, which handles announces and scrapes and all that jazz
work = new worker(conf, torrents_list, users_list, whitelist, db, sc);
// Create schedule object
sched = new schedule(conf, work, db, sc);
// Create schedule object
sched = new schedule(conf, work, db, sc);
// Create connection mother, which binds to its socket and handles the event stuff
mother = new connection_mother(conf, work, db, sc, sched);
// Create connection mother, which binds to its socket and handles the event stuff
mother = new connection_mother(conf, work, sc, sched);
// Add signal handlers now that all objects have been created
struct sigaction handler{}, ignore{};
ignore.sa_handler = SIG_IGN;
handler.sa_handler = sig_handler;
sigemptyset(&handler.sa_mask);
handler.sa_flags = 0;
// Add signal handlers now that all objects have been created
struct sigaction handler{}, ignore{};
ignore.sa_handler = SIG_IGN;
handler.sa_handler = sig_handler;
sigemptyset(&handler.sa_mask);
handler.sa_flags = 0;
sigaction(SIGINT, &handler, NULL);
sigaction(SIGTERM, &handler, NULL);
sigaction(SIGHUP, &handler, NULL);
sigaction(SIGUSR1, &handler, NULL);
sigaction(SIGUSR2, &ignore, NULL);
sigaction(SIGINT, &handler, NULL);
sigaction(SIGTERM, &handler, NULL);
sigaction(SIGHUP, &handler, NULL);
sigaction(SIGUSR1, &handler, NULL);
sigaction(SIGUSR2, &ignore, NULL);
mother->run();
mother->run();
return 0;
return 0;
}
+96 -67
View File
@@ -1,5 +1,15 @@
#ifndef OCELOT_H
#define OCELOT_H
#ifndef SRC_OCELOT_H_
#define SRC_OCELOT_H_
// Copyright [2017-2026] Orpheus
#define OCELOT_VERSION_MAJOR 2
#define OCELOT_VERSION_MINOR 1
#define OCELOT_VERSION_BUGFIX 5
#define OCELOT_VERSION_NREV 0
#define OCELOT_VERSION "2.1.5"
#include <time.h>
#include <string>
#include <map>
@@ -8,7 +18,6 @@
#include <set>
#include <memory>
#include <atomic>
#include <time.h>
typedef uint32_t torid_t;
typedef uint32_t userid_t;
@@ -17,19 +26,19 @@ class user;
typedef std::shared_ptr<user> user_ptr;
typedef struct {
int64_t uploaded;
int64_t downloaded;
int64_t corrupt;
int64_t left;
time_t last_announced;
time_t first_announced;
uint32_t announces;
uint16_t port;
bool visible;
bool invalid_ip;
user_ptr user;
std::string ip_port;
std::string ip;
int64_t uploaded;
int64_t downloaded;
int64_t corrupt;
int64_t left;
time_t last_announced;
time_t first_announced;
uint32_t announces;
uint16_t port;
bool visible;
bool invalid_ip;
user_ptr user;
std::string ip_port;
std::string ip;
} peer;
typedef std::map<std::string, peer> peer_list;
@@ -37,52 +46,52 @@ typedef std::map<std::string, peer> peer_list;
enum freetype { NORMAL, FREE, NEUTRAL };
typedef struct {
torid_t id;
uint32_t completed;
int64_t balance;
freetype free_torrent;
time_t last_flushed;
peer_list seeders;
peer_list leechers;
std::string last_selected_seeder;
std::set<userid_t> tokened_users;
torid_t id;
uint32_t completed;
int64_t balance;
freetype free_torrent;
time_t last_flushed;
peer_list seeders;
peer_list leechers;
std::string last_selected_seeder;
std::set<userid_t> tokened_users;
} torrent;
enum {
DUPE, // 0
TRUMP, // 1
BAD_FILE_NAMES, // 2
BAD_FOLDER_NAMES, // 3
BAD_TAGS, // 4
BAD_FORMAT, // 5
DISCS_MISSING, // 6
DISCOGRAPHY,// 7
EDITED_LOG,// 8
INACCURATE_BITRATE, // 9
LOW_BITRATE, // 10
MUTT_RIP,// 11
BAD_SOURCE,// 12
ENCODE_ERRORS,// 13
BANNED, // 14
TRACKS_MISSING,// 15
TRANSCODE, // 16
CASSETTE, // 17
UNSPLIT_ALBUM, // 18
USER_COMPILATION, // 19
WRONG_FORMAT, // 20
WRONG_MEDIA, // 21
AUDIENCE // 22
DUPE, // 0
TRUMP, // 1
BAD_FILE_NAMES, // 2
BAD_FOLDER_NAMES, // 3
BAD_TAGS, // 4
BAD_FORMAT, // 5
DISCS_MISSING, // 6
DISCOGRAPHY, // 7
EDITED_LOG, // 8
INACCURATE_BITRATE, // 9
LOW_BITRATE, // 10
MUTT_RIP, // 11
BAD_SOURCE, // 12
ENCODE_ERRORS, // 13
BANNED, // 14
TRACKS_MISSING, // 15
TRANSCODE, // 16
CASSETTE, // 17
UNSPLIT_ALBUM, // 18
USER_COMPILATION, // 19
WRONG_FORMAT, // 20
WRONG_MEDIA, // 21
AUDIENCE // 22
};
typedef struct {
int reason;
time_t time;
int reason;
time_t time;
} del_message;
typedef struct {
bool gzip;
bool html;
bool http_close;
bool gzip;
bool html;
bool http_close;
} client_opts_t;
typedef std::unordered_map<std::string, torrent> torrent_list;
@@ -90,19 +99,39 @@ typedef std::unordered_map<std::string, user_ptr> user_list;
typedef std::unordered_map<std::string, std::string> params_type;
struct stats_t {
std::atomic<uint32_t> open_connections;
std::atomic<uint64_t> opened_connections;
std::atomic<uint64_t> connection_rate;
std::atomic<uint32_t> leechers;
std::atomic<uint32_t> seeders;
std::atomic<uint64_t> requests;
std::atomic<uint64_t> request_rate;
std::atomic<uint64_t> announcements;
std::atomic<uint64_t> succ_announcements;
std::atomic<uint64_t> scrapes;
std::atomic<uint64_t> bytes_read;
std::atomic<uint64_t> bytes_written;
time_t start_time;
std::atomic<uint32_t> open_connections;
std::atomic<uint32_t> peak_connections;
std::atomic<uint64_t> opened_connections;
std::atomic<uint64_t> connection_rate;
std::atomic<uint32_t> leechers;
std::atomic<uint32_t> seeders;
std::atomic<uint32_t> user_queue_size;
std::atomic<uint32_t> torrent_queue_size;
std::atomic<uint32_t> peer_queue_size;
std::atomic<uint32_t> snatch_queue_size;
std::atomic<uint32_t> token_queue_size;
std::atomic<uint32_t> max_client_request_len;
std::atomic<uint64_t> requests;
std::atomic<uint64_t> request_rate;
std::atomic<uint64_t> announcements;
std::atomic<uint64_t> succ_announcements;
std::atomic<uint64_t> scrapes;
std::atomic<uint64_t> announce_duration; // all durations measure microseconds
std::atomic<uint64_t> torrent_lock_total;
std::atomic<uint64_t> torrent_lock_duration;
std::atomic<uint64_t> reap_total;
std::atomic<uint64_t> reap_lock_duration;
std::atomic<uint64_t> reap_duration;
std::atomic<uint64_t> bytes_read;
std::atomic<uint64_t> bytes_written;
std::atomic<uint64_t> auth_error_secret;
std::atomic<uint64_t> auth_error_report;
std::atomic<uint64_t> auth_error_announce_key;
std::atomic<uint64_t> client_error;
std::atomic<uint64_t> http_error;
time_t start_time;
};
extern struct stats_t stats;
#endif
extern const char *version();
#endif // SRC_OCELOT_H_
+282 -47
View File
@@ -1,56 +1,291 @@
#include <iostream>
#include <map>
// Copyright [2017-2026] Orpheus
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/resource.h>
#include <jemalloc/jemalloc.h>
#include <spdlog/spdlog.h> // debug
#include <fstream>
#include <sstream>
#include <iostream>
#include <iomanip> // for std::setfill() and std::setw()
#include <map>
#include <mutex>
#include <thread>
#include "ocelot.h"
#include "misc_functions.h"
#include "jemalloc_parse.h"
#include "report.h"
#include "response.h"
#include "user.h"
std::string report(params_type &params, user_list &users_list, client_opts_t &client_opts) {
std::stringstream output;
std::string action = params["get"];
if (action == "") {
output << "Invalid action\n";
} else if (action == "stats") {
time_t uptime = time(NULL) - stats.start_time;
int up_d = uptime / 86400;
uptime -= up_d * 86400;
int up_h = uptime / 3600;
uptime -= up_h * 3600;
int up_m = uptime / 60;
int up_s = uptime - up_m * 60;
std::string up_ht = up_h <= 9 ? '0' + inttostr(up_h) : inttostr(up_h);
std::string up_mt = up_m <= 9 ? '0' + inttostr(up_m) : inttostr(up_m);
std::string up_st = up_s <= 9 ? '0' + inttostr(up_s) : inttostr(up_s);
int jemalloc_fd;
std::mutex jemalloc_fd_mutex;
output << "Uptime: " << up_d << " days, " << up_ht << ':' << up_mt << ':' << up_st << '\n'
<< stats.opened_connections << " connections opened\n"
<< stats.open_connections << " open connections\n"
<< stats.connection_rate << " connections/s\n"
<< stats.requests << " requests handled\n"
<< stats.request_rate << " requests/s\n"
<< stats.succ_announcements << " successful announcements\n"
<< (stats.announcements - stats.succ_announcements) << " failed announcements\n"
<< stats.scrapes << " scrapes\n"
<< stats.leechers << " leechers tracked\n"
<< stats.seeders << " seeders tracked\n"
<< stats.bytes_read << " bytes read\n"
<< stats.bytes_written << " bytes written\n";
} else if (action == "user") {
std::string key = params["key"];
if (key.empty()) {
output << "Invalid action\n";
} else {
user_list::const_iterator u = users_list.find(key);
if (u != users_list.end()) {
output << u->second->get_leeching() << " leeching" << std::endl
<< u->second->get_seeding() << " seeding" << std::endl;
}
}
} else {
output << "Invalid action\n";
}
output << "success";
return response(output.str(), client_opts);
/* Getting jemalloc stats back to Gazelle is a bit cumbersome
* because all jemalloc can do is write to a file descriptor.
* So we let it write to a file (which is why using a ramdisk
* is important) and then read it back to return as an HTTP
* response.
*/
// callback function for malloc_stats_print()
void jemalloc_stats_export(void *opaque, const char *buf) {
if (jemalloc_fd > 0) {
write(jemalloc_fd, buf, strlen(buf));
}
}
int dump_jemalloc(const char *filename, const char *opts) {
// open the file descriptor
std::lock_guard<std::mutex> guard(jemalloc_fd_mutex);
jemalloc_fd = open(filename, O_CREAT|O_WRONLY, 0600);
if (jemalloc_fd == -1) {
return jemalloc_fd;
}
// dump stats
malloc_stats_print(jemalloc_stats_export, NULL, opts);
// tidy up
int result = close(jemalloc_fd);
jemalloc_fd = 0;
return result;
}
std::string report_jemalloc_plain(const char *opts, const std::string path) {
std::string filename(path + "/jemalloc.json." + std::to_string(pthread_self()));
if (dump_jemalloc(filename.c_str(), opts) != 0) {
// return an empty string if something went bananas
unlink(filename.c_str());
return std::string("");
}
// slurp the contents into a string
std::ifstream input(filename);
std::ostringstream output;
while(input >> output.rdbuf())
;
unlink(filename.c_str());
output << "\n";
std::shared_ptr<spdlog::logger> logger(spdlog::get("logger"));
logger->debug("jemalloc stats written to " + filename + ", size=" + std::to_string(output.str().size()));
return output.str();
}
#define ITEM_BYTE(LABEL, VALUE) "\"" LABEL "\":{\"type\":\"byte\",\"value\":" << VALUE << "}"
#define ITEM_ELAPSED(LABEL, VALUE) "\"" LABEL "\":{\"type\":\"elapsed\",\"value\":" << VALUE << "}"
#define ITEM_NUM(LABEL, VALUE) "\"" LABEL "\":{\"type\":\"number\",\"value\":" << VALUE << "}"
#define ITEM_STR(LABEL, VALUE) "\"" LABEL "\":{\"type\":\"string\",\"value\":\"" VALUE "\"}"
#define ITEM_VERSION(LABEL, MAJ, MIN, BUGFIX) "\"" LABEL "\":{\"type\":\"string\",\"value\":\"" << MAJ << "." << MIN << "." << BUGFIX << "\"}"
std::string report(const uint32_t announce_interval, const uint32_t announce_jitter) {
std::ostringstream output;
output << "{"
<< ITEM_STR("version", OCELOT_VERSION) << ','
<< ITEM_VERSION("jemalloc version", JEMALLOC_VERSION_MAJOR, JEMALLOC_VERSION_MINOR, JEMALLOC_VERSION_BUGFIX) << ','
<< ITEM_NUM("connections opened", stats.opened_connections) << ','
<< ITEM_NUM("open connections", stats.open_connections) << ','
<< ITEM_NUM("peak connections", stats.peak_connections) << ','
<< ITEM_NUM("connections/s", stats.connection_rate) << ','
<< ITEM_NUM("requests handled", stats.requests) << ','
<< ITEM_NUM("requests/s", stats.request_rate) << ','
<< ITEM_NUM("successful announcements", stats.succ_announcements) << ','
<< ITEM_NUM("failed announcements", (stats.announcements - stats.succ_announcements)) << ','
<< ITEM_NUM("scrapes", stats.scrapes) << ','
<< ITEM_NUM("leechers tracked", stats.leechers) << ','
<< ITEM_NUM("seeders tracked", stats.seeders) << ','
<< ITEM_NUM("announce duration", stats.announce_duration) << ','
<< ITEM_NUM("torrent lock total", stats.torrent_lock_total) << ','
<< ITEM_NUM("torrent lock duration", stats.torrent_lock_duration) << ','
<< ITEM_NUM("reap lock total", stats.reap_total) << ','
<< ITEM_NUM("reap lock duration", stats.reap_lock_duration) << ','
<< ITEM_NUM("reap duration", stats.reap_duration) << ','
<< ITEM_NUM("items in user queue", stats.user_queue_size) << ','
<< ITEM_NUM("items in torrent queue", stats.torrent_queue_size) << ','
<< ITEM_NUM("items in peer queue", stats.peer_queue_size) << ','
<< ITEM_NUM("items in snatch queue", stats.snatch_queue_size) << ','
<< ITEM_NUM("items in token queue", stats.token_queue_size) << ','
<< ITEM_BYTE("max client request length", stats.max_client_request_len) << ','
<< ITEM_BYTE("bytes read", stats.bytes_read) << ','
<< ITEM_BYTE("bytes written", stats.bytes_written) << ','
<< ITEM_NUM("bad tracker secret received", stats.auth_error_secret) << ','
<< ITEM_NUM("bad report secret received", stats.auth_error_report) << ','
<< ITEM_NUM("bad announce key received", stats.auth_error_announce_key) << ','
<< ITEM_NUM("bad client configuration", stats.client_error) << ','
<< ITEM_NUM("bad http request", stats.http_error) << ','
<< ITEM_NUM("announce interval", announce_interval) << ','
<< ITEM_NUM("announce jitter", announce_jitter) << ','
<< ITEM_ELAPSED("uptime", (time(NULL) - stats.start_time))
;
struct rusage r;
if (getrusage(RUSAGE_SELF, &r) == 0) {
output << ','
<< ITEM_ELAPSED("user time", r.ru_utime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_utime.tv_usec) << ','
<< ITEM_ELAPSED("system time", r.ru_stime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_stime.tv_usec) << ','
<< ITEM_BYTE("maximum resident set size", r.ru_maxrss * 1024) << ','
<< ITEM_NUM("minor page faults", r.ru_minflt) << ','
<< ITEM_NUM("major page faults", r.ru_majflt) << ','
<< ITEM_NUM("blocks in", r.ru_inblock) << ','
<< ITEM_NUM("blocks out", r.ru_oublock) << ','
<< ITEM_NUM("voluntary context switches", r.ru_nvcsw ) << ','
<< ITEM_NUM("involuntary context switches", r.ru_nivcsw )
;
}
output << "}\n";
return output.str();
}
std::string report_prom_stats(const char *jemalloc) {
std::ostringstream output;
struct ocelot_alloc_info ji;
int result = jemalloc_parse(jemalloc, &ji);
output << "ocelot_uptime " << time(NULL) - stats.start_time << "\n"
"ocelot_version " << OCELOT_VERSION_MAJOR
<< std::setfill('0') << std::setw(3) << OCELOT_VERSION_MINOR
<< std::setfill('0') << std::setw(3) << OCELOT_VERSION_BUGFIX
<< std::setfill('0') << std::setw(3) << OCELOT_VERSION_NREV << "\n"
"jemalloc_version " << JEMALLOC_VERSION_MAJOR
<< std::setfill('0') << std::setw(3) << JEMALLOC_VERSION_MINOR
<< std::setfill('0') << std::setw(3) << JEMALLOC_VERSION_BUGFIX << "\n"
"ocelot_open_connections " << stats.open_connections << "\n"
"#TYPE ocelot_peak_connections counter\n"
"ocelot_peak_connections " << stats.peak_connections << "\n"
"ocelot_connection_rate " << stats.connection_rate << "\n"
"ocelot_requests " << stats.requests << "\n"
"ocelot_request_rate " << stats.request_rate << "\n"
"ocelot_succ_announcements " << stats.succ_announcements << "\n"
"ocelot_total_announcements " << stats.announcements << "\n"
"ocelot_scrapes " << stats.scrapes << "\n"
"ocelot_leechers " << stats.leechers << "\n"
"ocelot_seeders " << stats.seeders << "\n"
"ocelot_user_queue " << stats.user_queue_size << "\n"
"ocelot_torrent_queue " << stats.torrent_queue_size << "\n"
"ocelot_peer_queue " << stats.peer_queue_size << "\n"
"ocelot_snatch_queue " << stats.snatch_queue_size << "\n"
"ocelot_token_queue " << stats.token_queue_size << "\n"
"ocelot_bytes_read " << stats.bytes_read << "\n"
"ocelot_bytes_written " << stats.bytes_written << "\n"
"#TYPE ocelot_max_client_request_len counter\n"
"ocelot_max_client_request_len " << stats.max_client_request_len << "\n"
"#TYPE ocelot_error counter\n"
"ocelot_error{kind=\"secret\"} " << stats.auth_error_secret << "\n"
"ocelot_error{kind=\"report\"} " << stats.auth_error_report << "\n"
"ocelot_error{kind=\"announce\"} " << stats.auth_error_announce_key << "\n"
"ocelot_error{kind=\"client\"} " << stats.client_error << "\n"
"ocelot_error{kind=\"http\"} " << stats.http_error << "\n"
"ocelot_error{kind=\"jemalloc_parse\"} " << result << "\n"
"#TYPE ocelot_timing counter\n"
"ocelot_timing{kind=\"announce_duration\"} " << stats.announce_duration << "\n"
"ocelot_timing{kind=\"torrent_lock_total\"} " << stats.torrent_lock_total << "\n"
"ocelot_timing{kind=\"torrent_lock_duration\"} " << stats.torrent_lock_duration << "\n"
"ocelot_timing{kind=\"reap_total\"} " << stats.reap_total << "\n"
"ocelot_timing{kind=\"reap_lock_duration\"} " << stats.reap_lock_duration << "\n"
"ocelot_timing{kind=\"reap_duration\"} " << stats.reap_duration << "\n"
"#TYPE jemalloc_arena_total counter\n"
"jemalloc_arena_total " << ji.nr_arena << "\n"
"#TYPE jemalloc_bin_total counter\n"
"jemalloc_bin_total{bin=\"small\"} " << ji.nr_bin_small << "\n"
"jemalloc_bin_total{bin=\"thread_cache\"} " << ji.nr_bin_tc << "\n"
"jemalloc_bin_total{bin=\"large\"} " << ji.nr_bin_large << "\n"
"#TYPE jemalloc_allocated counter\n"
"jemalloc_allocated{bucket=\"small\"} " << ji.small.allocated << "\n"
"jemalloc_allocated{bucket=\"large\"} " << ji.large.allocated << "\n"
"#TYPE jemalloc_nmalloc_total counter\n"
"jemalloc_nmalloc_total{bucket=\"small\"} " << ji.small.nmalloc_total << "\n"
"jemalloc_nmalloc_total{bucket=\"large\"} " << ji.large.nmalloc_total << "\n"
"#TYPE jemalloc_ndalloc_total counter\n"
"jemalloc_ndalloc_total{bucket=\"small\"} " << ji.small.ndalloc_total << "\n"
"jemalloc_ndalloc_total{bucket=\"large\"} " << ji.large.ndalloc_total << "\n"
"#TYPE jemalloc_nrequests_total counter\n"
"jemalloc_nrequests_total{bucket=\"small\"} " << ji.small.nrequests_total << "\n"
"jemalloc_nrequests_total{bucket=\"large\"} " << ji.large.nrequests_total << "\n"
"#TYPE jemalloc_nfill_total counter\n"
"jemalloc_nfill_total{bucket=\"small\"} " << ji.small.nfill_total << "\n"
"jemalloc_nfill_total{bucket=\"large\"} " << ji.large.nfill_total << "\n"
"#TYPE jemalloc_nflush_total counter\n"
"jemalloc_nflush_total{bucket=\"small\"} " << ji.small.nflush_total << "\n"
"jemalloc_nflush_total{bucket=\"large\"} " << ji.large.nflush_total << "\n"
"#TYPE jemalloc_nflush_total gauge\n"
"jemalloc_memory{global=\"allocated\"} " << ji.mem_allocated << "\n"
"jemalloc_memory{global=\"active\"} " << ji.mem_active << "\n"
"jemalloc_memory{global=\"metadata\"} " << ji.mem_metadata << "\n"
"jemalloc_memory{global=\"resident\"} " << ji.mem_resident << "\n"
"jemalloc_memory{global=\"mapped\"} " << ji.mem_mapped << "\n"
"jemalloc_memory{global=\"retained\"} " << ji.mem_retained << "\n"
"#TYPE jemalloc_nflush_rate gauge\n"
"jemalloc_allocated{bucket=\"small\"} " << ji.small.allocated << "\n"
"jemalloc_allocated{bucket=\"large\"} " << ji.large.allocated << "\n"
"#TYPE jemalloc_nflush_rate gauge\n"
"jemalloc_nmalloc_rate{bucket=\"small\"} " << ji.small.nmalloc_rate << "\n"
"jemalloc_nmalloc_rate{bucket=\"large\"} " << ji.large.nmalloc_rate << "\n"
"#TYPE jemalloc_nflush_total gauge\n"
"jemalloc_ndalloc_rate{bucket=\"small\"} " << ji.small.ndalloc_rate << "\n"
"jemalloc_ndalloc_rate{bucket=\"large\"} " << ji.large.ndalloc_rate << "\n"
"#TYPE jemalloc_nflush_rate gauge\n"
"jemalloc_nrequests_rate{bucket=\"small\"} " << ji.small.nrequests_rate << "\n"
"jemalloc_nrequests_rate{bucket=\"large\"} " << ji.large.nrequests_rate << "\n"
"#TYPE jemalloc_nflush_rate gauge\n"
"jemalloc_fill_rate{bucket=\"small\"} " << ji.small.nfill_rate << "\n"
"jemalloc_fill_rate{bucket=\"large\"} " << ji.large.nfill_rate << "\n"
"#TYPE jemalloc_nflush_rate gauge\n"
"jemalloc_nflush_rate{bucket=\"small\"} " << ji.small.nflush_rate << "\n"
"jemalloc_nflush_rate{bucket=\"large\"} " << ji.large.nflush_rate << "\n"
;
struct rusage r;
if (getrusage(RUSAGE_SELF, &r) == 0) {
output << "ocelot_time_user " << r.ru_utime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_utime.tv_usec << "\n"
"ocelot_time_system " << r.ru_stime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_stime.tv_usec << "\n"
"ocelot_max_rss " << r.ru_maxrss << "\n"
"ocelot_minor_fault " << r.ru_minflt << "\n"
"ocelot_major_fault " << r.ru_majflt << "\n"
"#TYPE ocelot_block counter\n"
"ocelot_block{op=\"in\"} " << r.ru_inblock << "\n"
"ocelot_block{op=\"out\"} " << r.ru_oublock << "\n"
"ocelot_nvcsw " << r.ru_nvcsw << "\n"
"ocelot_nivcsw " << r.ru_nivcsw << "\n"
;
}
output << '#';
return output.str();
}
std::string report_user(const user_ptr u) {
std::ostringstream output;
output << "{\"id\":" << u->get_id()
<< ",\"leeching\":" << u->get_leeching()
<< ",\"seeding\":" << u->get_seeding()
<< ",\"deleted\":" << u->is_deleted()
<< ",\"protected\":" << u->is_protected()
<< ",\"can_leech\":" << u->can_leech()
<< "}\n";
return output.str();
}
+22 -3
View File
@@ -1,4 +1,23 @@
#include <string>
#include "ocelot.h"
#ifndef SRC_REPORT_H_
#define SRC_REPORT_H_
std::string report(params_type &params, user_list &users_list, client_opts_t &client_opts);
// Copyright [2017-2026] Orpheus
#include <string>
// take a snapshot of the current stats
void copy_stats(const struct stats_t *in, struct stats_t *out);
// status report
std::string report(const uint32_t announce_interval, const uint32_t announce_jitter);
// a snapshot of jemalloc statistics
std::string report_jemalloc_plain(const char *opts, const std::string path);
// a prometheus scrape
std::string report_prom_stats(const char *jemalloc_stats);
// user report
std::string report_user(const user_ptr u);
#endif // SRC_REPORT_H_
+47 -40
View File
@@ -1,51 +1,58 @@
// Copyright [2017-2024] Orpheus
#include <algorithm>
#include <sstream>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include "response.h"
#include "misc_functions.h"
#include "response.h"
const std::string response(const std::string &body, client_opts_t &client_opts) {
std::string out;
bool processed = false;
if (client_opts.html) {
out = "<html><head><meta name=\"robots\" content=\"noindex, nofollow\" /></head><body>" + body + "</body></html>";
processed = true;
}
if (client_opts.gzip) {
std::stringstream ss, zss;
ss << body;
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::gzip_compressor());
in.push(ss);
boost::iostreams::copy(in, zss);
out = zss.str();
processed = true;
}
if (processed) {
return response_head(out.length(), client_opts) + out;
}
return response_head(body.length(), client_opts) + body;
const std::string http_head(size_t content_length, client_opts_t &client_opts) {
const std::string content_type = client_opts.html ? "text/html" : "text/plain";
// rely on preprocessor string concatenation
std::string head = "HTTP/1.1 200 OK\r\nServer: Ocelot " OCELOT_VERSION "\r\n"
"Content-Type: " + content_type + "\r\n"
"Content-Length: " + inttostr(content_length) + "\r\n";
if (client_opts.gzip) {
head += "Content-Encoding: gzip\r\n";
}
if (client_opts.http_close) {
head += "Connection: Close\r\n";
}
return head + "\r\n";
}
const std::string response_head(size_t content_length, client_opts_t &client_opts) {
const std::string content_type = client_opts.html ? "text/html" : "text/plain";
std::string head = "HTTP/1.1 200 OK\r\nServer: Ocelot 1.0";
head += "\r\nContent-Type: " + content_type;
if (client_opts.gzip) {
head += "\r\nContent-Encoding: gzip";
}
if (client_opts.http_close) {
head += "\r\nConnection: Close";
}
head += "\r\nContent-Length: " + inttostr(content_length) + "\r\n\r\n";
return head;
const std::string http_response(const std::string &body, client_opts_t &client_opts) {
std::string out;
bool processed = false;
if (client_opts.html) {
out = "<html><head><meta name=\"robots\" content=\"noindex, nofollow\" /></head><body>" + body + "</body></html>";
processed = true;
}
if (client_opts.gzip) {
std::stringstream ss, zss;
ss << body;
boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
in.push(boost::iostreams::gzip_compressor());
in.push(ss);
boost::iostreams::copy(in, zss);
out = zss.str();
processed = true;
}
if (processed) {
return http_head(out.length(), client_opts) + out;
}
return http_head(body.length(), client_opts) + body;
}
const std::string error(const std::string &err, client_opts_t &client_opts) {
return response("d14:failure reason" + inttostr(err.length()) + ':' + err + "12:min intervali5400e8:intervali5400ee", client_opts);
}
const std::string warning(const std::string &msg) {
return "15:warning message" + inttostr(msg.length()) + ':' + msg;
const std::string error(const std::string &message, client_opts_t &client_opts) {
return http_response(
"d14:failure reason" + inttostr(message.length()) + ':' + message
+ "12:min intervali5400e8:intervali5400ee",
client_opts
);
}
+8 -6
View File
@@ -1,11 +1,13 @@
#ifndef RESPONSE_H
#define RESPONSE_H
#ifndef SRC_RESPONSE_H_
#define SRC_RESPONSE_H_
// Copyright [2017-2024] Orpheus
#include <string>
#include "ocelot.h"
const std::string response(const std::string &body, client_opts_t &client_opts);
const std::string response_head(size_t content_length, client_opts_t &client_opts);
const std::string http_response(const std::string &body, client_opts_t &client_opts);
const std::string error(const std::string &err, client_opts_t &client_opts);
const std::string warning(const std::string &msg);
#endif
#endif // SRC_RESPONSE_H_
+38 -35
View File
@@ -1,6 +1,9 @@
#include <iostream>
// Copyright [2017-2024] Orpheus
#include <spdlog/spdlog.h>
#include <iostream>
#include "ocelot.h"
#include "config.h"
#include "db.h"
@@ -8,53 +11,53 @@
#include "schedule.h"
schedule::schedule(config * conf, worker * worker_obj, mysql * db_obj, site_comm * sc_obj) : work(worker_obj), db(db_obj), sc(sc_obj) {
load_config(conf);
counter = 0;
last_opened_connections = 0;
last_request_count = 0;
next_reap_peers = reap_peers_interval;
logger = spdlog::get("logger");
load_config(conf);
counter = 0;
last_opened_connections = 0;
last_request_count = 0;
next_reap_peers = reap_peers_interval;
logger = spdlog::get("logger");
}
void schedule::load_config(config * conf) {
reap_peers_interval = conf->get_uint("reap_peers_interval");
schedule_interval = conf->get_uint("schedule_interval");
reap_peers_interval = conf->get_uint("reap_peers_interval");
schedule_interval = conf->get_uint("schedule_interval");
}
void schedule::reload_config(config * conf) {
load_config(conf);
load_config(conf);
}
//---------- Schedule - gets called every schedule_interval seconds
void schedule::handle(ev::timer &watcher, int events_flags) {
unsigned int cur_schedule_interval = watcher.repeat;
stats.connection_rate = (stats.opened_connections - last_opened_connections) / cur_schedule_interval;
stats.request_rate = (stats.requests - last_request_count) / cur_schedule_interval;
if (counter % 20 == 0) {
logger->info(std::to_string(stats.open_connections) + " open, "
+ std::to_string(stats.opened_connections) + " connections (" + std::to_string(stats.connection_rate) + "/s), "
+ std::to_string(stats.requests) + " requests (" + std::to_string(stats.request_rate) + "/s)");
}
unsigned int cur_schedule_interval = watcher.repeat;
stats.connection_rate = (stats.opened_connections - last_opened_connections) / cur_schedule_interval;
stats.request_rate = (stats.requests - last_request_count) / cur_schedule_interval;
if (counter % 20 == 0) {
logger->info(std::to_string(stats.open_connections) + " open, "
+ std::to_string(stats.opened_connections) + " connections (" + std::to_string(stats.connection_rate) + "/s), "
+ std::to_string(stats.requests) + " requests (" + std::to_string(stats.request_rate) + "/s)");
}
if (work->get_status() == CLOSING && db->all_clear() && sc->all_clear()) {
logger->info("all clear, shutting down");
exit(0);
}
if (work->get_status() == CLOSING && db->all_clear() && sc->all_clear()) {
logger->info("all clear, shutting down");
exit(0);
}
last_opened_connections = stats.opened_connections;
last_request_count = stats.requests;
last_opened_connections = stats.opened_connections;
last_request_count = stats.requests;
db->flush();
sc->flush_tokens();
db->flush();
sc->flush_tokens();
next_reap_peers -= cur_schedule_interval;
if (next_reap_peers <= 0) {
work->start_reaper();
next_reap_peers = reap_peers_interval;
}
next_reap_peers -= cur_schedule_interval;
if (next_reap_peers <= 0) {
work->start_reaper();
next_reap_peers = reap_peers_interval;
}
counter++;
if (schedule_interval != cur_schedule_interval) {
watcher.set(schedule_interval, schedule_interval);
}
counter++;
if (schedule_interval != cur_schedule_interval) {
watcher.set(schedule_interval, schedule_interval);
}
}
+24 -19
View File
@@ -1,29 +1,34 @@
#ifndef SCHEDULE_H
#define SCHEDULE_H
#ifndef SRC_SCHEDULE_H_
#define SRC_SCHEDULE_H_
// Copyright [2017-2024] Orpheus
#ifdef EV_ERROR
#undef EV_ERROR
#endif
#include <ev++.h>
class schedule {
private:
void load_config(config * conf);
private:
void load_config(config * conf);
unsigned int reap_peers_interval;
worker * work;
mysql * db;
site_comm * sc;
uint64_t last_opened_connections;
uint64_t last_request_count;
unsigned int counter;
int next_reap_peers;
std::shared_ptr<spdlog::logger> logger;
public:
schedule(config * conf, worker * worker_obj, mysql * db_obj, site_comm * sc_obj);
void reload_config(config * conf);
void handle(ev::timer &watcher, int events_flags);
unsigned int reap_peers_interval;
worker * work;
mysql * db;
site_comm * sc;
uint64_t last_opened_connections;
uint64_t last_request_count;
unsigned int counter;
int next_reap_peers;
std::shared_ptr<spdlog::logger> logger;
unsigned int schedule_interval;
public:
schedule(config * conf, worker * worker_obj, mysql * db_obj, site_comm * sc_obj);
void reload_config(config * conf);
void handle(ev::timer &watcher, int events_flags);
unsigned int schedule_interval;
};
#endif
#endif // SRC_SCHEDULE_H_
+98 -92
View File
@@ -1,3 +1,7 @@
// Copyright [2017-2024] Orpheus
#include <spdlog/spdlog.h>
#include <iostream>
#include <istream>
#include <ostream>
@@ -5,9 +9,9 @@
#include <sstream>
#include <queue>
#include <mutex>
#include <boost/asio.hpp>
#include <thread>
#include <spdlog/spdlog.h>
#include <boost/asio.hpp>
#include "config.h"
#include "site_comm.h"
@@ -15,124 +19,126 @@
using boost::asio::ip::tcp;
site_comm::site_comm(config * conf) : t_active(false) {
logger = spdlog::get("logger");
load_config(conf);
logger = spdlog::get("logger");
load_config(conf);
}
void site_comm::load_config(config * conf) {
site_host = conf->get_str("site_host");
site_service = conf->get_str("site_service");
site_path = conf->get_str("site_path");
site_password = conf->get_str("site_password");
readonly = conf->get_bool("readonly");
site_host = conf->get_str("site_host");
site_service = conf->get_str("site_service");
site_path = conf->get_str("site_path");
site_password = conf->get_str("site_password");
readonly = conf->get_bool("readonly");
}
void site_comm::reload_config(config * conf) {
load_config(conf);
load_config(conf);
}
bool site_comm::all_clear() {
return (token_queue.size() == 0);
return (token_queue.size() == 0);
}
void site_comm::expire_token(int torrent, int user) {
std::stringstream token_pair;
token_pair << user << ':' << torrent;
if (expire_token_buffer != "") {
expire_token_buffer += ",";
}
expire_token_buffer += token_pair.str();
if (expire_token_buffer.length() > 350) {
logger->info("Flushing overloaded token buffer");
if (!readonly) {
std::lock_guard<std::mutex> lock(expire_queue_lock);
token_queue.push(expire_token_buffer);
}
expire_token_buffer.clear();
}
std::stringstream token_pair;
token_pair << user << ':' << torrent;
if (!expire_token_buffer.empty()) {
expire_token_buffer += ",";
}
expire_token_buffer += token_pair.str();
if (expire_token_buffer.length() > 350) {
logger->info("Flushing overloaded token buffer");
if (!readonly) {
std::lock_guard<std::mutex> lock(expire_queue_lock);
token_queue.push(expire_token_buffer);
}
expire_token_buffer.clear();
}
}
void site_comm::flush_tokens()
{
if (readonly) {
expire_token_buffer.clear();
return;
}
std::lock_guard<std::mutex> lock(expire_queue_lock);
size_t qsize = token_queue.size();
if (verbose_flush || qsize > 0) {
logger->info("Token expire queue size: " + std::to_string(qsize));
}
if (expire_token_buffer == "") {
return;
}
token_queue.push(expire_token_buffer);
expire_token_buffer.clear();
if (!t_active) {
std::thread thread(&site_comm::do_flush_tokens, this);
thread.detach();
}
if (readonly) {
expire_token_buffer.clear();
return;
}
std::lock_guard<std::mutex> lock(expire_queue_lock);
size_t qsize = token_queue.size();
if (verbose_flush || qsize > 0) {
logger->info("Token expire queue size: " + std::to_string(qsize));
}
if (expire_token_buffer.empty()) {
return;
}
token_queue.push(expire_token_buffer);
expire_token_buffer.clear();
if (!t_active) {
std::thread thread(&site_comm::do_flush_tokens, this);
thread.detach();
}
}
void site_comm::do_flush_tokens()
{
t_active = true;
try {
while (token_queue.size() > 0) {
boost::asio::io_service io_service;
t_active = true;
try {
while (token_queue.size() > 0) {
boost::asio::io_service io_service;
tcp::resolver resolver(io_service);
tcp::resolver::query query(site_host, site_service);
tcp::resolver::iterator endpoint_iterator = resolver.resolve(query);
tcp::resolver::iterator end;
tcp::resolver resolver(io_service);
tcp::resolver::query query(site_host, site_service);
tcp::resolver::iterator endpoint_iterator = resolver.resolve(query);
tcp::resolver::iterator end;
tcp::socket socket(io_service);
boost::system::error_code error = boost::asio::error::host_not_found;
while (error && endpoint_iterator != end) {
socket.close();
socket.connect(*endpoint_iterator++, error);
}
if (error) {
throw boost::system::system_error(error);
}
tcp::socket socket(io_service);
boost::system::error_code error = boost::asio::error::host_not_found;
while (error && endpoint_iterator != end) {
socket.close();
socket.connect(*endpoint_iterator++, error);
}
if (error) {
throw boost::system::system_error(error);
}
boost::asio::streambuf request;
std::ostream request_stream(&request);
request_stream << "GET " << site_path << "/tools.php?key=" << site_password
<< "&type=expiretoken&action=ocelot&tokens=" << token_queue.front() << " HTTP/1.0\r\n"
<< "Host: " << site_host << "\r\n"
<< "Accept: */*\r\n"
<< "Connection: close\r\n\r\n";
boost::asio::streambuf request;
std::ostream request_stream(&request);
request_stream << "GET " << site_path << "/tools.php?key=" << site_password
<< "&type=expiretoken&action=ocelot&tokens=" << token_queue.front() << " HTTP/1.0\r\n"
<< "Host: " << site_host << "\r\n"
<< "Accept: */*\r\n"
<< "Connection: close\r\n\r\n";
boost::asio::write(socket, request);
boost::asio::write(socket, request);
boost::asio::streambuf response;
boost::asio::read_until(socket, response, "\r\n");
boost::asio::streambuf response;
boost::asio::read_until(socket, response, "\r\n");
std::istream response_stream(&response);
std::string http_version;
response_stream >> http_version;
unsigned int status_code;
response_stream >> status_code;
std::string status_message;
std::getline(response_stream, status_message);
std::istream response_stream(&response);
std::string http_version;
response_stream >> http_version;
unsigned int status_code;
response_stream >> status_code;
std::string status_message;
std::getline(response_stream, status_message);
if (!response_stream || http_version.substr(0, 5) != "HTTP/") {
logger->info("Invalid response");
continue;
}
if (!response_stream || http_version.substr(0, 5) != "HTTP/") {
logger->info("Invalid response");
continue;
}
if (status_code == 200) {
std::lock_guard<std::mutex> lock(expire_queue_lock);
token_queue.pop();
} else {
logger->error("Response returned with status code " + std::to_string(status_code) + " when trying to expire a token!");
}
}
} catch (std::exception &er) {
logger->error("Exception: " + std::string(er.what()));
}
t_active = false;
if (status_code == 200) {
std::lock_guard<std::mutex> lock(expire_queue_lock);
token_queue.pop();
} else {
logger->error("Response returned with status code " + std::to_string(status_code) + " when trying to expire a token!");
// TODO: add exponential backoff
std::this_thread::sleep_for(std::chrono::seconds(5));
}
}
} catch (std::exception &er) {
logger->error("Exception: " + std::string(er.what()));
}
t_active = false;
}
site_comm::~site_comm()
+36 -28
View File
@@ -1,36 +1,44 @@
#ifndef OCELOT_SITE_COMM_H
#define OCELOT_SITE_COMM_H
#include <string>
#include <boost/asio.hpp>
#include <queue>
#include <mutex>
#ifndef SRC_SITE_COMM_H_
#define SRC_SITE_COMM_H_
// Copyright [2017-2024] Orpheus
#include <spdlog/spdlog.h>
#include <string>
#include <queue>
#include <memory>
#include <mutex>
#include <boost/asio.hpp>
#include "config.h"
using boost::asio::ip::tcp;
class site_comm {
private:
std::string site_host;
std::string site_service;
std::string site_path;
std::string site_password;
std::mutex expire_queue_lock;
std::string expire_token_buffer;
std::queue<std::string> token_queue;
std::shared_ptr<spdlog::logger> logger;
bool readonly;
bool t_active;
void load_config(config * conf);
void do_flush_tokens();
private:
std::string site_host;
std::string site_service;
std::string site_path;
std::string site_password;
std::mutex expire_queue_lock;
std::string expire_token_buffer;
std::queue<std::string> token_queue;
std::shared_ptr<spdlog::logger> logger;
bool readonly;
bool t_active;
void load_config(config * conf);
void do_flush_tokens();
public:
bool verbose_flush;
site_comm(config * conf);
void reload_config(config * conf);
bool all_clear();
void expire_token(int torrent, int user);
void flush_tokens();
~site_comm();
public:
bool verbose_flush;
site_comm(config * conf);
void reload_config(config * conf);
bool all_clear();
void expire_token(int torrent, int user);
void flush_tokens();
~site_comm();
};
#endif
#endif // SRC_SITE_COMM_H_
+4 -2
View File
@@ -1,6 +1,8 @@
// Copyright [2017-2024] Orpheus
#include "user.h"
user::user(userid_t uid, bool leech, bool protect) : id(uid), deleted(false), leechstatus(leech), protect_ip(protect) {
stats.leeching = 0;
stats.seeding = 0;
stats.leeching = 0;
stats.seeding = 0;
}
+32 -27
View File
@@ -1,33 +1,38 @@
#ifndef USER_H
#define USER_H
#ifndef SRC_USER_H_
#define SRC_USER_H_
// Copyright [2017-2024] Orpheus
#include <atomic>
#include "ocelot.h"
class user {
private:
userid_t id;
bool deleted;
bool leechstatus;
bool protect_ip;
struct {
std::atomic<uint32_t> leeching;
std::atomic<uint32_t> seeding;
} stats;
public:
user(userid_t uid, bool leech, bool protect);
userid_t get_id() { return id; }
bool is_deleted() { return deleted; }
void set_deleted(bool status) { deleted = status; }
bool is_protected() { return protect_ip; }
void set_protected(bool status) { protect_ip = status; }
bool can_leech() { return leechstatus; }
void set_leechstatus(bool status) { leechstatus = status; }
void decr_leeching() { --stats.leeching; }
void decr_seeding() { --stats.seeding; }
void incr_leeching() { ++stats.leeching; }
void incr_seeding() { ++stats.seeding; }
uint32_t get_leeching() { return stats.leeching; }
uint32_t get_seeding() { return stats.seeding; }
private:
userid_t id;
bool deleted;
bool leechstatus;
bool protect_ip;
struct {
std::atomic<uint32_t> leeching;
std::atomic<uint32_t> seeding;
} stats;
public:
user(userid_t uid, bool leech, bool protect);
userid_t get_id() { return id; }
bool is_deleted() { return deleted; }
void set_deleted(bool status) { deleted = status; }
bool is_protected() { return protect_ip; }
void set_protected(bool status) { protect_ip = status; }
bool can_leech() { return leechstatus; }
void set_leechstatus(bool status) { leechstatus = status; }
void decr_leeching() { --stats.leeching; }
void decr_seeding() { --stats.seeding; }
void incr_leeching() { ++stats.leeching; }
void incr_seeding() { ++stats.seeding; }
uint32_t get_leeching() { return stats.leeching; }
uint32_t get_seeding() { return stats.seeding; }
};
#endif
#endif // SRC_USER_H_
+1230 -1046
View File
File diff suppressed because it is too large. Load diff
+54 -45
View File
@@ -1,63 +1,72 @@
#ifndef WORKER_H
#define WORKER_H
#ifndef SRC_WORKER_H_
#define SRC_WORKER_H_
// Copyright [2017-2026] Orpheus
#include <spdlog/spdlog.h>
#include <string>
#include <vector>
#include <list>
#include <unordered_map>
#include <iostream>
#include <memory>
#include <mutex>
#include <ctime>
#include <spdlog/spdlog.h>
#include "site_comm.h"
#include "ocelot.h"
#include <random>
enum tracker_status { OPEN, PAUSED, CLOSING }; // tracker status
#include "site_comm.h"
enum tracker_status { OPEN, PAUSED, CLOSING }; // tracker status
class worker {
private:
config * conf;
mysql * db;
site_comm * s_comm;
torrent_list &torrents_list;
user_list &users_list;
std::vector<std::string> &whitelist;
std::unordered_map<std::string, del_message> del_reasons;
tracker_status status;
bool reaper_active;
time_t cur_time;
std::shared_ptr<spdlog::logger> logger;
private:
config * conf;
mysql * db;
site_comm * s_comm;
torrent_list &torrents_list;
user_list &users_list;
std::vector<std::string> &whitelist;
std::unordered_map<std::string, del_message> del_reasons;
tracker_status status;
bool reaper_active;
time_t cur_time;
std::shared_ptr<spdlog::logger> logger;
std::mt19937 randgen;
std::uniform_int_distribution<int> jitter;
static std::mutex client_len_mutex;
unsigned int announce_interval;
unsigned int del_reason_lifetime;
unsigned int peers_timeout;
unsigned int numwant_limit;
bool keepalive_enabled;
std::string site_password;
std::string report_password;
unsigned int announce_interval;
unsigned int del_reason_lifetime;
unsigned int peers_timeout;
unsigned int numwant_limit;
bool keepalive_enabled;
std::string site_password;
std::string report_password;
std::mutex del_reasons_lock;
void load_config(config * conf);
void do_start_reaper();
void reap_peers();
void reap_del_reasons();
std::string get_del_reason(int code);
peer_list::iterator add_peer(peer_list &peer_list, const std::string &peer_id);
inline bool peer_is_visible(user_ptr &u, peer *p);
std::mutex del_reasons_lock;
void load_config(config * conf);
void do_start_reaper();
void reap_peers();
void reap_del_reasons();
std::string get_del_reason(int code);
peer_list::iterator add_peer(peer_list &peer_list, const std::string &peer_id);
inline bool peer_is_visible(user_ptr &u, peer *p);
public:
worker(config * conf_obj, torrent_list &torrents, user_list &users, std::vector<std::string> &_whitelist, mysql * db_obj, site_comm * sc);
void reload_config(config * conf);
std::string work(const std::string &input, std::string &ip, client_opts_t &client_opts);
std::string announce(const std::string &input, torrent &tor, user_ptr &u, params_type &params, params_type &headers, std::string &ip, client_opts_t &client_opts);
std::string scrape(const std::list<std::string> &infohashes, params_type &headers, client_opts_t &client_opts);
std::string update(params_type &params, client_opts_t &client_opts);
public:
worker(config * conf_obj, torrent_list &torrents, user_list &users, std::vector<std::string> &_whitelist, mysql * db_obj, site_comm * sc);
void reload_config(config * conf);
std::string work(const std::string &input, std::string &ip, client_opts_t &client_opts);
std::string announce(const std::string &input, const std::string &peer_id, torrent &tor, user_ptr &u, params_type &params, params_type &headers, std::string &ip, client_opts_t &client_opts);
std::string scrape(const std::list<std::string> &infohashes, params_type &headers, client_opts_t &client_opts);
std::string update(params_type &params, client_opts_t &client_opts);
void reload_lists();
bool shutdown();
void reload_lists();
bool shutdown();
tracker_status get_status() { return status; }
tracker_status get_status() { return status; }
void start_reaper();
void start_reaper();
};
#endif
#endif // SRC_WORKER_H_