initial commit

.gitignore

@@ -0,0 +1,4 @@
+venv/
+.DS_Store
+.idea/
+margay/__pycache__/

README.rst

@@ -0,0 +1,39 @@
+Margay
+======
+
+Margay is a BitTorrent tracker written in Python (though intended to be compiled with Cython) for the Gazelle project.
+The first release is aimed to be 1-to-1 translation of Ocelot into Python to be then stress tested to see a comparison
+of efficiency between the two projects (with Margay then having the edge in developer productivity).
+
+## Dependencies
+Python 3.5+
+mysqlclient
+
+## Installation
+```
+python setup.py install
+```
+
+### Standalone
+While its generally recommended that you run Margay in conjuction with a Gazelle instance, it's possible to run this
+in standalone fashion.
+
+### Gazelle
+
+## Usage
+```
+usage: runner.py [-h] [-v] [-d] [-c [CONFIG]] [-V]
+
+Python BitTorrent tracker
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -v, --verbose         Be more verbose in the output
+  -d, --daemonize       Run tracker as daemon
+  -c [CONFIG], --config [CONFIG]
+  -V, --version         show program's version number and exit
+```
+
+## See Also:
+* [pybtracker](https://github.com/elektito/pybtracker)
+* [Ocelot](https://github.com/ApolloRIP/Ocelot)

margay.conf

@@ -0,0 +1,43 @@
+# Ocelot config file
+# Lines starting with a # are ignored
+# A # anywhere else is treated like any other character
+
+[internal]
+listen_port         = 34000
+max_connections     = 128
+max_middlemen       = 20000
+max_read_buffer     = 4096
+connection_timeout  = 10
+# Keepalive is mostly useful if the tracker runs behind reverse proxies
+keepalive_timeout   = 0
+
+[tracker]
+announce_interval   = 1800
+max_request_size    = 4096
+numwant_limit       = 50
+request_log_size    = 500
+
+[mysql]
+mysql_host          = localhost
+mysql_username      = gazelle
+mysql_password      = password
+mysql_db            = gazelle
+
+[gazelle]
+# The passwords must be 32 characters and match the Gazelle config
+report_password     = 00000000000000000000000000000000
+site_password       = 00000000000000000000000000000000
+
+[timers]
+peers_timeout       = 7200
+del_reason_lifetime = 86400
+reap_peers_interval = 1800
+schedule_interval   = 3
+
+[logging]
+log                 = true
+log_level           = info
+log_path            = /tmp/ocelot
+
+[debug]
+readonly            = false

margay.sql

@@ -0,0 +1,108 @@
+CREATE TABLE IF NOT EXISTS `torrents` (
+  `ID` int(10) NOT NULL AUTO_INCREMENT,
+  `UserID` int(10) DEFAULT NULL,
+  `info_hash` blob NOT NULL,
+  `Leechers` int(6) NOT NULL DEFAULT '0',
+  `Seeders` int(6) NOT NULL DEFAULT '0',
+  `last_action` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `FreeTorrent` enum('0','1','2') NOT NULL DEFAULT '0',
+  `FreeLeechType` enum('0','1','2','3','4','5','6','7') NOT NULL DEFAULT '0',
+  `Time` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `Snatched` int(10) unsigned NOT NULL DEFAULT '0',
+  `balance` bigint(20) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`ID`),
+  UNIQUE KEY `InfoHash` (`info_hash`(40)),
+  KEY `UserID` (`UserID`),
+  KEY `Seeders` (`Seeders`),
+  KEY `Leechers` (`Leechers`),
+  KEY `Snatched` (`Snatched`),
+  KEY `last_action` (`last_action`),
+  KEY `Time` (`Time`),
+  KEY `FreeTorrent` (`FreeTorrent`)
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `users_freeleeches` (
+  `UserID` int(10) NOT NULL,
+  `TorrentID` int(10) NOT NULL,
+  `Time` datetime NOT NULL,
+  `Expired` tinyint(1) NOT NULL DEFAULT '0',
+  `Downloaded` bigint(20) NOT NULL DEFAULT '0',
+  `Uses` int(10) NOT NULL DEFAULT '1',
+  PRIMARY KEY (`UserID`,`TorrentID`),
+  KEY `Time` (`Time`),
+  KEY `Expired_Time` (`Expired`,`Time`)
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `users_main` (
+  `ID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `LastLogin` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `LastAccess` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `IP` varchar(15) NOT NULL DEFAULT '0.0.0.0',
+  `Uploaded` bigint(20) unsigned NOT NULL DEFAULT '0',
+  `Downloaded` bigint(20) unsigned NOT NULL DEFAULT '0',
+  `BonusPoints` float(20, 5) NOT NULL DEFAULT '0',
+  `Enabled` enum('0','1','2') NOT NULL DEFAULT '0',
+  `Visible` enum('1','0') NOT NULL DEFAULT '1',
+  `can_leech` tinyint(4) NOT NULL DEFAULT '1',
+  `torrent_pass` char(32) NOT NULL,
+  `FLTokens` int(10) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`ID`),
+  KEY `LastAccess` (`LastAccess`),
+  KEY `IP` (`IP`),
+  KEY `Uploaded` (`Uploaded`),
+  KEY `Downloaded` (`Downloaded`),
+  KEY `Enabled` (`Enabled`),
+  KEY `torrent_pass` (`torrent_pass`)
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `xbt_client_whitelist` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `peer_id` varchar(20) DEFAULT NULL,
+  `vstring` varchar(200) DEFAULT '',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `peer_id` (`peer_id`)
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `xbt_files_history` (
+  `uid` int(11) NOT NULL,
+  `fid` int(11) NOT NULL,
+  `seedtime` int(11) NOT NULL DEFAULT '0',
+  `downloaded` bigint(20) NOT NULL DEFAULT '0',
+  `uploaded` bigint(20) NOT NULL DEFAULT '0'
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `xbt_files_users` (
+  `uid` int(11) NOT NULL,
+  `active` tinyint(1) NOT NULL DEFAULT '1',
+  `announced` int(11) NOT NULL DEFAULT '0',
+  `completed` tinyint(1) NOT NULL DEFAULT '0',
+  `downloaded` bigint(20) NOT NULL DEFAULT '0',
+  `remaining` bigint(20) NOT NULL DEFAULT '0',
+  `uploaded` bigint(20) NOT NULL DEFAULT '0',
+  `upspeed` int(10) unsigned NOT NULL DEFAULT '0',
+  `downspeed` int(10) unsigned NOT NULL DEFAULT '0',
+  `corrupt` bigint(20) NOT NULL DEFAULT '0',
+  `timespent` int(10) unsigned NOT NULL DEFAULT '0',
+  `useragent` varchar(51) NOT NULL DEFAULT '',
+  `connectable` tinyint(4) NOT NULL DEFAULT '1',
+  `peer_id` binary(20) NOT NULL DEFAULT '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0',
+  `fid` int(11) NOT NULL,
+  `mtime` int(11) NOT NULL DEFAULT '0',
+  `ip` varchar(15) NOT NULL DEFAULT '',
+  PRIMARY KEY (`peer_id`,`fid`,`uid`),
+  KEY `remaining_idx` (`remaining`),
+  KEY `fid_idx` (`fid`),
+  KEY `mtime_idx` (`mtime`),
+  KEY `uid_active` (`uid`,`active`)
+) ENGINE=InnoDB CHARSET utf8;
+
+CREATE TABLE IF NOT EXISTS `xbt_snatched` (
+  `uid` int(11) NOT NULL DEFAULT '0',
+  `tstamp` int(11) NOT NULL,
+  `fid` int(11) NOT NULL,
+  `IP` varchar(15) NOT NULL,
+  `seedtime` int(11) NOT NULL DEFAULT '0',
+  KEY `fid` (`fid`),
+  KEY `tstamp` (`tstamp`),
+  KEY `uid_tstamp` (`uid`,`tstamp`)
+) ENGINE=InnoDB CHARSET utf8;

margay/__init__.py

@@ -0,0 +1,2 @@
+__author__ = 'itismadness'
+__version__ = '0.1.0'

margay/config.py

@@ -0,0 +1,86 @@
+"""
+Configuration Class
+"""
+
+from configparser import ConfigParser
+import logging
+
+
+class Config(object):
+    def __init__(self, config_file=None, daemonize=False):
+        self.config = {
+            'internal': {
+                'listen_port': 35000,
+                'max_connections': 1024,
+                'max_middlemen': 20000,
+                'max_read_buffer': 4096,
+                'connection_timeout': 10,
+                'keepalive_timeout': 0,
+                'daemonize': daemonize
+            },
+            'tracker': {
+                'announce_interval': 1800,
+                'max_request_size': 4096,
+                'numwant_limit': 50,
+                'request_log_size': 500
+            },
+            'timers': {
+                'del_reason_lifetime': 86400,
+                'peers_timeout': 7200,
+                'reap_peers_interval': 1800,
+                'schedule_interval': 3
+            },
+            # note: host=localhost will cause mysqlclient to use a socket regardless of port,
+            # use 127.0.0.1 for the host if you're trying to connect to something with a port
+            'mysql': {
+                'host': '127.0.0.1',
+                'db': 'gazelle',
+                'user': 'gazelle',
+                'passwd': 'password',
+                'port': 36000
+            },
+            'gazelle': {
+                'site_host': '127.0.0.1',
+                'site_path': '',
+                'site_password': '00000000000000000000000000000000',
+                'report_password': '00000000000000000000000000000000'
+            },
+            'logging': {
+                'log': True,
+                'log_level': logging.getLevelName(logging.INFO),
+                'log_console': True,
+                'log_file': False,
+                'log_path': '/tmp/margay'
+            },
+            'debug': {
+                'readonly': False
+            }
+        }
+        print(self.config)
+
+        if config_file is not None:
+            config = ConfigParser()
+            config.read(config_file)
+            for key in config:
+                for value in config[key]:
+                    if type(self.config[key][value]) == int:
+                        self.config[key][value] = int(self.config[key][value])
+                    elif type(self.config[key][value]) == bool:
+                        self.config[key][value] = config[key][value] in ('True', 'true', 'On',
+                                                                         'on')
+                    else:
+                        self.config[key][value] = config[key][value]
+        if self.config['logging']['log_level'] == 'debug':
+            self.print()
+
+    def reload(self):
+        pass
+
+    def print(self):
+        for key in self.config:
+            print(self.config[key])
+            for kkey in self.config[key]:
+                print(kkey, self.config[key][kkey])
+
+    def __getitem__(self, item):
+        return self.config[item]

margay/database.py

@@ -0,0 +1,360 @@
+import binascii
+from copy import copy
+import logging
+from time import time
+from typing import Dict
+import threading
+# noinspection PyPackageRequirements
+import MySQLdb
+
+from .structs import Torrent, User, LeechType
+import margay.stats as stats
+
+
+class Database(object):
+    def __init__(self, settings, readonly=False):
+        self.logger = logging.getLogger()
+        self.settings = settings
+        self.db = self.get_connection()
+
+        self.readonly = readonly
+
+        self.user_buffer = []
+        self.torrent_buffer = []
+        self.heavy_peer_buffer = []
+        self.light_peer_buffer = []
+        self.snatch_buffer = []
+        self.token_buffer = []
+
+        self.user_queue = []
+        self.torrent_queue = []
+        self.peer_queue = []
+        self.snatch_queue = []
+        self.token_queue = []
+
+        self.u_active = self.t_active = self.p_active = self.s_active = self.tok_active = False
+        self.user_lock = threading.RLock()
+        self.torrent_lock = threading.RLock()
+        self.peer_lock = threading.RLock()
+        self.snatch_lock = threading.RLock()
+        self.token_lock = threading.RLock()
+
+        self.torrent_list_lock = threading.RLock()
+        self.user_list_lock = threading.RLock()
+        self.whitelist_lock = threading.RLock()
+
+        if not self.readonly:
+            self.logger.info('Clearing xbt_files_users and resetting peer counts...')
+            self.flush()
+            self._clear_peer_data()
+            self.logger.info('done')
+
+    def get_connection(self):
+        return MySQLdb.connect(host=self.settings['host'], user=self.settings['user'],
+                               passwd=self.settings['passwd'], db=self.settings['db'],
+                               port=self.settings['port'])
+
+    def connected(self):
+        return self.db is not None
+
+    def load_torrents(self, torrents=None):
+        if torrents is None:
+            torrents = dict()
+        cur_keys = set(torrents.keys())
+
+        cursor = self.db.cursor()
+        # info_hash is a binary blob and using HEX converts it to a hex string and LCASE lowers it
+        # so that we don't have to worry about weird cases
+        cursor.execute('SELECT ID, info_hash, FreeTorrent, Snatched FROM torrents '
+                       'ORDER BY ID')
+        with self.torrent_list_lock:
+            for row in cursor.fetchall():
+                info_hash = row[1].decode('utf-8', 'replace')
+                if info_hash == '':
+                    continue
+                if info_hash not in torrents:
+                    torrents[info_hash] = Torrent(row[0], row[3])
+                else:
+                    torrents[info_hash].tokened_users.clear()
+                    cur_keys.remove(info_hash)
+                torrents[info_hash].free_torrent = LeechType.to_enum(row[2])
+            cursor.close()
+
+            for key in cur_keys:
+                stats.leechers -= torrents[key].leechers
+                stats.seeders -= torrents[key].seeders
+                for leecher in torrents[key].leechers:
+                    leecher.user.leeching -= 1
+                for seeder in torrents[key].seeders:
+                    seeder.user.seeding -= 1
+                del torrents[key]
+
+        self.logger.info(f'Loaded {len(torrents)} torrents')
+        self.load_tokens(torrents)
+        return torrents
+
+    def load_users(self, users=None):
+        if users is None:
+            users = dict()
+        cur_keys = set(users.keys())
+
+        cursor = self.db.cursor()
+        cursor.execute("SELECT ID, can_leech, torrent_pass, (Visible='0' OR IP='127.0.0.1') AS "
+                       "Protected FROM users_main WHERE Enabled='1'")
+        with self.user_list_lock:
+            for row in cursor.fetchall():
+                if row[2] not in users:
+                    users[row[2]] = User(row[0], row[1], row[3])
+                else:
+                    users[row[2]].leech = row[1]
+                    users[row[2]].protect = row[3]
+                    cur_keys.remove(row[2])
+            cursor.close()
+
+            for key in cur_keys:
+                del users[key]
+
+        self.logger.info(f'Loaded {len(users)} users')
+        return users
+
+    def load_tokens(self, torrents: Dict[str, Torrent]):
+        """
+
+        :param torrents:
+        :type torrents: Dict[str, Torrent]
+        :return:
+        """
+        cursor = self.db.cursor()
+        cursor.execute("SELECT uf.UserID, t.info_hash FROM users_freeleeches AS uf "
+                       "JOIN torrents AS t ON t.ID = uf.TorrentID "
+                       "WHERE uf.Expired = '0'")
+        for row in cursor.fetchall():
+            info_hash = str(row[1])
+            torrent = torrents[info_hash]
+            torrent.tokened_users.append(row[0])
+        logging.info(f'Loaded {cursor.rownumber} tokens')
+        cursor.close()
+
+    def load_whitelist(self):
+        cursor = self.db.cursor()
+        whitelist = list()
+        cursor.execute("SELECT peer_id FROM xbt_client_whitelist")
+        with self.whitelist_lock:
+            for result in cursor.fetchall():
+                whitelist.append(result[0])
+            cursor.close()
+
+        if len(whitelist) == 0:
+            self.logger.info('Assuming no whitelist desired, disabled')
+        else:
+            self.logger.info(f'Loaded {len(whitelist)} clients into the whitelist')
+        return whitelist
+
+    def record_token(self, user_id, torrent_id, downloaded):
+        self.token_buffer.append((user_id, torrent_id, downloaded))
+
+    def record_user(self, user_id, uploaded, downloaded):
+        self.user_buffer.append((user_id, uploaded, downloaded))
+
+    def record_torrent(self, torrent_id, seeders, leechers, snatched, balance):
+        self.torrent_buffer.append((torrent_id, seeders, leechers, snatched, balance))
+
+    def record_snatch(self, user_id, torrent_id, ipv4, ipv6):
+        self.snatch_buffer.append((user_id, torrent_id, ipv4, ipv6))
+
+    def record_peer_light(self, user_id, torrent_id, timespent, announced, peer_id):
+        self.light_peer_buffer.append((user_id, torrent_id, timespent, announced, peer_id,
+                                       int(time())))
+
+    def record_peer_heavy(self, user_id, torrent_id, active, uploaded, downloaded, upspeed,
+                          downspeed, remaining, corrupt, timespent, announced, ip, peer_id,
+                          user_agent):
+        self.heavy_peer_buffer.append((user_id, torrent_id, active, uploaded, downloaded, upspeed,
+                                       downspeed, remaining, corrupt, timespent, announced,
+                                       ip, peer_id, user_agent, int(time())))
+
+    def flush(self):
+        self._flush_users()
+        self._flush_torrents()
+        self._flush_snatches()
+        self._flush_peers()
+        self._flush_tokens()
+
+    def _flush_users(self):
+        if self.readonly:
+            self.user_buffer.clear()
+            return
+
+        with self.user_lock:
+            if len(self.user_queue) > 0:
+                self.logger.info(f'User flush queue size: {len(self.user_queue)}, '
+                                 f'next query length: {len(str(self.token_queue[0]))}')
+            if len(self.user_buffer) == 0:
+                return
+            self.user_queue.extend(copy(self.user_buffer))
+            self.user_buffer.clear()
+        if not self.u_active:
+            threading.Thread(target=self._do_flush_users).start()
+
+    def _do_flush_users(self):
+        self.u_active = True
+        conn = self.get_connection()
+        while len(self.user_queue) > 0:
+            cursor = conn.cursor()
+            cursor.executemany('INSERT INTO users_main (ID, Uploaded, Downloaded) '
+                               'VALUES(%s, %s, %s) '
+                               'ON DUPLICATE KEY UPDATE Uploaded = Uploaded + Values(Uploaded), '
+                               'Downloaded = Downloaded + Values(Downloaded)', self.user_queue[0])
+            cursor.close()
+            with self.user_lock:
+                self.user_queue.pop(0)
+        self.u_active = False
+
+    def _flush_torrents(self):
+        if self.readonly:
+            self.torrent_buffer.clear()
+            return
+
+        with self.torrent_lock:
+            if len(self.torrent_queue) > 0:
+                self.logger.info(f'Torrent flush queue size: {len(self.torrent_queue)}, '
+                                 f'next query length: {len(str(self.torrent_queue[0]))}')
+            if len(self.torrent_buffer) == 0:
+                return
+            self.torrent_queue.extend(copy(self.torrent_buffer))
+            self.torrent_buffer.clear()
+        if not self.t_active:
+            threading.Thread(target=self._do_flush_torrents).start()
+
+    def _do_flush_torrents(self):
+        self.t_active = True
+        conn = self.get_connection()
+        while len(self.torrent_queue) > 0:
+            cursor = conn.cursor()
+            cursor.executemany('INSERT INTO torrents (ID, Seeders, Leechers, Snatched, Balance) '
+                               'VALUES (%s, %s, %s, %s, %s) '
+                               'ON DUPLICATE KEY UPDATE Seeders=VALUES(Seeders), '
+                               'Leechers=VALUES(Leechers), '
+                               'Snatched = Snatched + VALUES(Snatched), '
+                               'Balance=VALUES(Balance), '
+                               'last_action=IF(VALUES(Seeders) > 0, NOW(), last_action)',
+                               self.torrent_queue[0])
+            cursor.execute("DELETE FROM torrents WHERE info_hash = ''")
+            cursor.close()
+            with self.torrent_lock:
+                self.torrent_queue.pop(0)
+        self.t_active = False
+
+    def _flush_snatches(self):
+        if self.readonly:
+            self.snatch_buffer.clear()
+            return
+
+        with self.snatch_lock:
+            if len(self.snatch_queue) > 0:
+                self.logger.info(f'Snatch flush queue size: {len(self.snatch_queue)}, '
+                                 f'next query length: {len(str(self.snatch_queue[0]))}')
+            if len(self.snatch_buffer) == 0:
+                return
+            self.snatch_queue.extend(copy(self.snatch_buffer))
+            self.snatch_buffer.clear()
+        if not self.s_active:
+            threading.Thread(target=self._do_flush_snatches).start()
+
+    def _do_flush_snatches(self):
+        self.s_active = True
+        conn = self.get_connection()
+        while len(self.snatch_queue) > 0:
+            cursor = conn.cursor()
+            cursor.executemany('INSERT INTO xbt_snatched (uid, fid, tstamp, IP) '
+                               'VALUES (%s, %s, %s, %s)', self.snatch_queue[0])
+
+            cursor.close()
+            with self.snatch_lock:
+                self.snatch_queue.pop(0)
+        self.s_active = False
+
+    def _flush_peers(self):
+        if self.readonly:
+            self.heavy_peer_buffer.clear()
+            self.light_peer_buffer.clear()
+            return
+
+        with self.peer_lock:
+            if len(self.peer_queue) > 0:
+                self.logger.info(f'Heavy peer queue size: {len(self.peer_queue)}, '
+                                 f'next query length: {len(str(self.peer_queue[0]))}')
+            if len(self.heavy_peer_buffer) > 0:
+                if len(self.peer_queue) > 1000:
+                    self.peer_queue.pop(0)
+                self.peer_queue.extend(copy(self.heavy_peer_buffer))
+                self.heavy_peer_buffer.clear()
+            if len(self.light_peer_buffer) > 0:
+                if len(self.peer_queue) > 1000:
+                    self.peer_queue.pop(0)
+                self.peer_queue.extend(copy(self.light_peer_buffer))
+                self.light_peer_buffer.clear()
+        if not self.p_active:
+            threading.Thread(target=self._do_flush_peers).start()
+
+    def _do_flush_peers(self):
+        self.p_active = True
+        conn = self.get_connection()
+
+        while len(self.peer_queue) > 0:
+            cursor = conn.cursor()
+            if len(self.peer_queue[0]) == 4:
+                cursor.executemany('INSERT INTO xbt_files_users (uid, fid, timespent, '
+                                   'announced, peer_id, mtime) '
+                                   'VALUES (%s, %s, %s, %s, %s, %s) '
+                                   'ON DUPLICATE KEY UPDATE upspeed=0, downspeed=0, '
+                                   'timespent=VALUES(timespent), announced=VALUES(announced), '
+                                   'mtime=VALUES(mtime)', self.peer_queue[0])
+            else:
+                cursor.executemany('INSERT INTO xbt_files_users (uid, fid, active, uploaded, '
+                                   'downloaded, upspeed, downspeed, remaining, corrupt, '
+                                   'timespent, announced, ip, peer_id, useragent, mtime) '
+                                   'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,'
+                                   ' %s, %s) ON DUPLICATE KEY UPDATE active=VALUES(active), '
+                                   'uploaded=VALUES(uploaded), downloaded=VALUES(downloaded), '
+                                   'upspeed=VALUES(upspeed), downspeed=VALUES(downspeed), '
+                                   'remaining=VALUES(remaining), corrupt=VALUES(corrupt), '
+                                   'timespent=VALUES(timespent), announced=VALUES(announced), '
+                                   'mtime=VALUES(mtime)', self.peer_queue[0])
+            cursor.close()
+            with self.peer_lock:
+                self.peer_queue.pop(0)
+        self.p_active = False
+
+    def _flush_tokens(self):
+        if self.readonly:
+            self.token_buffer.clear()
+            return
+
+        with self.token_lock:
+            if len(self.token_queue) > 0:
+                self.logger.info(f'Token flush queue size: {len(self.token_queue)}, '
+                                 f'next query length: {len(str(self.token_queue[0]))}')
+
+            if len(self.token_buffer) == 0:
+                return
+            self.token_queue.extend(copy(self.token_buffer))
+            self.token_buffer.clear()
+            if not self.tok_active:
+                threading.Thread(target=self._do_flush_tokens).start()
+
+    def _do_flush_tokens(self):
+        self.tok_active = True
+        conn = self.get_connection()
+        while len(self.token_queue) > 0:
+            cursor = conn.cursor()
+            cursor.executemany('INSERT INTO users_freeleeches (UserID, TorrentID, Downloaded) '
+                               'VALUES(%s, %s, %s)', self.token_queue[0])
+            cursor.close()
+            with self.token_lock:
+                self.token_queue.pop(0)
+        self.tok_active = False
+
+    def _clear_peer_data(self):
+        self.db.query('TRUNCATE xbt_files_users')
+        self.db.query('UPDATE torrents SET Seeders = 0, Leechers = 0')

margay/main.py

@@ -0,0 +1,75 @@
+from argparse import ArgumentParser
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import signal
+import sys
+import threading
+
+from . import __version__
+from .config import Config
+from .database import Database
+from .site_comm import SiteComm
+from .schedule import Schedule
+from .worker import Worker
+
+
+def run():
+    parser = ArgumentParser(description='Python BitTorrent tracker')
+    parser.add_argument('-d', '--daemonize', action='store_true', help='Run tracker as daemon')
+    parser.add_argument('-c', '--config', nargs='?')
+    parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__)
+    args = parser.parse_args()
+    config = Config(args.config, args.daemonize)
+
+    logger = logging.getLogger()
+    while logger.handlers:
+        logger.handlers.pop()
+    log_format = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
+    if config['logging']['log']:
+        if config['logging']['log_file']:
+            file_logger = TimedRotatingFileHandler(config['logging']['log_path'], when='d',
+                                                   backupCount=5)
+            file_logger.setFormatter(log_format)
+            file_logger.setLevel(config['logging']['log_level'])
+            #logger.addHandler(file_logger)
+        if config['logging']['log_console']:
+            console_logger = logging.StreamHandler(sys.stdout)
+            console_logger.setFormatter(log_format)
+            console_logger.setLevel(config['logging']['log_level'])
+            logger.addHandler(console_logger)
+    else:
+        logger.addHandler(logging.NullHandler())
+    logger.setLevel(config['logging']['log_level'])
+
+    database = Database(config['mysql'], config['debug']['readonly'])
+    schedule = Schedule(config['timers']['schedule_interval'],
+                        config['timers']['reap_peers_interval'],
+                        database)
+    site_comm = SiteComm(config)
+    worker = Worker(database, site_comm, config)
+
+    def sig_handler(sig, _):
+        print("help")
+        logger = logging.getLogger()
+        if sig == signal.SIGINT or sig == signal.SIGTERM:
+            logger.info('Caught SIGINT/SIGTERM')
+            if worker.shutdown():
+                raise SystemExit
+        elif sig == signal.SIGHUP:
+            logger.info('Reloading config')
+            config.reload()
+            # reload various classes
+        elif sig == signal.SIGUSR1:
+            logger.info('Reloading from database')
+            threading.Thread(target=worker.reload_lists)
+
+    signal.signal(signal.SIGINT, sig_handler)
+    signal.signal(signal.SIGTERM, sig_handler)
+    signal.signal(signal.SIGHUP, sig_handler)
+    signal.signal(signal.SIGUSR1, sig_handler)
+    signal.signal(signal.SIGUSR2, sig_handler)
+
+    try:
+        worker.create_server(config['internal']['listen_port'])
+    finally:
+        schedule.stop()

margay/schedule.py

@@ -0,0 +1,51 @@
+import logging
+from threading import Thread, Timer
+
+import margay.stats as stats
+
+
+class Schedule(Thread):
+    def __init__(self, interval, reap_interval, database):
+        super().__init__()
+        self.logger = logging.getLogger()
+        self._timer = None
+        self.is_running = False
+        self.counter = 0
+        self.last_opened_connections = 0
+        self.last_request_count = 0
+        self.interval = interval
+        self.reap_interval = reap_interval
+        self.database = database
+        self._reap = self.reap_interval
+        self.start()
+
+    def start(self):
+        if not self.is_running:
+            self._timer = Timer(self.interval, self._run)
+            self._timer.start()
+            self.is_running = True
+
+    def stop(self):
+        self._timer.cancel()
+        self.is_running = False
+
+    def _run(self):
+        self.is_running = False
+        self.start()
+
+        self._reap -= self.interval
+
+        if self.counter % 20 == 0:
+            self.logger.info(f'{stats.open_connections} open, '
+                             f'{stats.opened_connections} connections ({stats.connection_rate}/s) '
+                             f'{stats.requests} requests ({stats.request_rate}/s)')
+
+        self.last_opened_connections = stats.opened_connections
+        self.last_request_count = stats.requests
+
+        self.database.flush()
+
+        if self._reap <= 0:
+            self._reap = self.reap_interval
+
+        self.counter += 1

margay/site_comm.py

@@ -0,0 +1,80 @@
+import logging
+import threading
+
+import requests
+
+
+class SiteComm(object):
+    def __init__(self, config):
+        self.config = config
+
+        self.logger = logging.getLogger()
+
+        self.site_host = ''
+        self.site_path = ''
+        self.site_password = ''
+        self.readonly = False
+
+        self.expire_queue_lock = threading.RLock()
+        self.token_queue = list()
+        self.expire_token_buffer = ''
+        self.t_active = False
+        self.verbose_flush = False
+
+        self.load_config(self.config)
+
+    def load_config(self, config):
+        self.site_host = config['gazelle']['site_host']
+        self.site_path = config['gazelle']['site_path']
+        self.site_password = config['gazelle']['site_password']
+        self.readonly = config['debug']['readonly']
+
+    def reload_config(self, config):
+        self.load_config(config)
+
+    def all_clear(self) -> bool:
+        return len(self.token_queue) == 0
+
+    def expire_token(self, torrent: int, user: int):
+        token_pair = f'{user}:{torrent}'
+        if self.expire_token_buffer != '':
+            self.expire_token_buffer += ','
+        self.expire_token_buffer += token_pair
+        if len(self.expire_token_buffer) > 350:
+            self.logger.info('Flushing overloaded token buffer')
+            if not self.readonly:
+                with self.expire_queue_lock:
+                    self.token_queue.append(self.expire_token_buffer)
+            self.expire_token_buffer = ''
+
+    def flush_tokens(self) -> None:
+        if self.readonly:
+            self.expire_token_buffer = ''
+            return
+        with self.expire_queue_lock:
+            if self.verbose_flush or len(self.token_queue) > 0:
+                self.logger.info(f'Token expire queue size: {len(self.token_queue)}')
+            if self.expire_token_buffer == '':
+                return
+            self.token_queue.extend(self.expire_token_buffer)
+            self.expire_token_buffer = ''
+            if not self.t_active:
+                threading.Thread(target=self._do_flush_tokens)
+
+    def _do_flush_tokens(self):
+        self.t_active = True
+        while len(self.token_queue) > 0:
+            response = requests.get(f'https://{self.site_host}/tools.php', params={
+                'key': self.site_password,
+                'type': 'expiretoken',
+                'action': 'ocelot',
+                'tokens': self.token_queue[0]
+            })
+
+            if response.status_code == 200:
+                with self.expire_queue_lock:
+                    self.token_queue.pop(0)
+            else:
+                self.logger.error(f'Response returned with status code {response.status_code} '
+                                  f'when trying to expire a token!')
+        self.t_active = False

margay/stats.py

@@ -0,0 +1,15 @@
+import time
+
+open_connections = 0
+opened_connections = 0
+connection_rate = 0
+requests = 0
+request_rate = 0
+leechers = 0
+seeders = 0
+announcements = 0
+succ_announcements = 0
+scrapes = 0
+bytes_read = 0
+bytes_written = 0
+start_time = int(time.time())

margay/structs.py

@@ -0,0 +1,138 @@
+from collections import OrderedDict
+from enum import IntEnum
+
+
+class User(object):
+    def __init__(self, uid: int, leech: bool, protect: bool):
+        self.id = uid
+        self.leech = leech
+        self.protect = protect
+        self.leeching = 0
+        self.seeding = 0
+        self.deleted = False
+
+
+class Peer(object):
+    def __init__(self):
+        self.uploaded = 0
+        self.downloaded = 0
+        self.corrupt = 0
+        self.left = 0
+        self.last_announced = None
+        self.first_announced = None
+        self.announces = 0
+        self.port = None
+        self.visible = False
+        self.invalid_ip = False
+        self.user = None  # type: User
+        self.ip = None
+        self.ip_port = ''
+        self.port = None
+
+
+class Torrent(object):
+    def __init__(self, tid, completed):
+        self.id = tid
+        self.completed = completed
+        self.balance = 0
+        self.free_torrent = None
+        self.last_flushed = 0
+        self.seeders = OrderedDict()  # type: OrderedDict[str, Peer]
+        self.leechers = OrderedDict()  # type: OrderedDict[str, Peer]
+        self.last_selected_seeder = ''
+        self.tokened_users = []
+
+
+class ErrorCodes(IntEnum):
+    DUPE = 0
+    TRUMP = 1
+    BAD_FILE_NAMES = 2
+    BAD_FOLDER_NAMES = 3
+    BAD_TAGS = 4
+    BAD_FORMAT = 5
+    DISCS_MISSING = 6
+    DISCOGRAPHY = 7
+    EDITED_LOG = 8
+    INACCURATE_BITRATE = 9
+    LOW_BITRATE = 10
+    MUTT_RIP = 11
+    BAD_SOURCE = 12
+    ENCODE_ERRORS = 13
+    BANNED = 14
+    TRACKS_MISSING = 15
+    TRANSCODE = 16
+    CASSETTE = 17
+    UNSPLIT_ALBUM = 18
+    USER_COMPILATION = 19
+    WRONG_FORMAT = 20
+    WRONG_MEDIA = 21
+    AUDIENCE = 22
+
+    @classmethod
+    def get_del_reason(cls, value):
+        if value == cls.DUPE:
+            return 'Dupe'
+        elif value == cls.TRUMP:
+            return 'Trump'
+        elif value == cls.BAD_FILE_NAMES:
+            return 'Bad File Names'
+        elif value == cls.BAD_FOLDER_NAMES:
+            return 'Bad Folder Names'
+        elif value == cls.BAD_TAGS:
+            return 'Bad Tags'
+        elif value == cls.BAD_FORMAT:
+            return 'Disallowed Format'
+        elif value == cls.DISCS_MISSING:
+            return 'Discs Missing'
+        elif value == cls.DISCOGRAPHY:
+            return 'Discography'
+        elif value == cls.EDITED_LOG:
+            return 'Edited Log'
+        elif value == cls.INACCURATE_BITRATE:
+            return 'Inaccurate Bitrate'
+        elif value == cls.LOW_BITRATE:
+            return 'Low Bitrate'
+        elif value == cls.MUTT_RIP:
+            return 'Mutt Rip'
+        elif value == cls.BAD_SOURCE:
+            return 'Disallowed Source'
+        elif value == cls.ENCODE_ERRORS:
+            return 'Encode Errors'
+        elif value == cls.BANNED:
+            return 'Specifically Banned'
+        elif value == cls.TRACKS_MISSING:
+            return 'Tracks Missing'
+        elif value == cls.TRANSCODE:
+            return 'Transcode'
+        elif value == cls.CASSETTE:
+            return 'Unapproved Cassette'
+        elif value == cls.UNSPLIT_ALBUM:
+            return 'Unsplit Album'
+        elif value == cls.USER_COMPILATION:
+            return 'User Compilation'
+        elif value == cls.WRONG_FORMAT:
+            return 'Wrong Format'
+        elif value == cls.WRONG_MEDIA:
+            return 'Wrong Media'
+        elif value == cls.AUDIENCE:
+            return 'Audience Recording'
+        else:
+            return ''
+
+
+class LeechType(IntEnum):
+    NORMAL = 0
+    FREE = 1
+    NEUTRAL = 2
+
+    @classmethod
+    def to_enum(cls, value):
+        for enum in cls:
+            if value == str(enum.value):
+                return enum
+        raise ValueError('Invalid leech type')
+
+
+torrent_list = dict()
+user_list = dict()
+peer_list = dict()

margay/util.py

@@ -0,0 +1,57 @@
+import binascii
+
+
+def hex_decode(inp: str) -> str:
+    out = ''
+    i = 0
+    while i < len(inp):
+        if inp[i] == '%' and (i + 2) < len(inp):
+            # ord('0') == chr(48)
+            x = 48
+
+            i += 1
+            char = ord(inp[i])
+            if 'a' <= inp[i] <= 'f':
+                x = (char - 87) << 4
+            elif 'A' <= inp[i] <= 'F':
+                x = (char - 55) << 4
+            elif '0' <= inp[i] <= '9':
+                x = (char - 48) << 4
+
+            i += 1
+            char = ord(inp[i])
+            if 'a' <= inp[i] <= 'f':
+                x += (char - 87)
+            elif 'A' <= inp[i] <= 'F':
+                x += (char - 55)
+            elif '0' <= inp[i] <= '9':
+                x += (char - 48)
+            x = chr(x)
+        else:
+            x = inp[i]
+        out += x
+        i += 1
+    return out
+
+
+if __name__ == '__main__':
+    #a = 'A959693FCAC904B7247537B3327740BFCEAF7851'
+    #b = b'0xA959693FCAC904B7247537B3327740BFCEAF7851'
+    #print(a)
+    #print(b)
+    test = hex_decode('%a9Yi%3f%ca%c9%04%b7%24u7%b32w%40%bf%ce%afxQ')
+    print(''.join(hex(ord(c))[2:] for c in test))
+    #print(binascii.hexlify(bytearray(, 'utf8')))
+    #print(binascii.hexlify(hex_decode('%A9Yi?%CA%C9%04%B7$u7%B32w@%BF%CE%AFxQ')))
+    #print(hashlib.sha1(hex_decode('%a9Yi%3f%ca%c9%04%b7%24u7%b32w%40%bf%ce%afxQ').encode('utf-8')).digest())
+
+"""
+>>> import urllib
+>>> '%a9Yi%3f%ca%c9%04%b7%24u7%b32w%40%bf%ce%afxQ'
+'%a9Yi%3f%ca%c9%04%b7%24u7%b32w%40%bf%ce%afxQ'
+>>> urllib.unquote_plus(_)
+'\xa9Yi?\xca\xc9\x04\xb7$u7\xb32w@\xbf\xce\xafxQ'
+>>> _.encode('hex')
+'a959693fcac904b7247537b3327740bfceaf7851'
+>>> 'a959693fcac904b7247537b3327740bfceaf7851'
+"""

margay/worker.py

@@ -0,0 +1,750 @@
+import ipaddress
+from enum import Enum, auto
+import logging
+import re
+from time import time
+import threading
+from typing import Dict, List
+
+# noinspection PyPackageRequirements
+import bencode
+from aiohttp import web
+
+from .structs import ErrorCodes, LeechType, Peer, Torrent, User
+import margay.stats as stats
+
+REGEX = re.compile(r'info_hash=([%a-zA-Z0-9]+)')
+
+
+class Status(Enum):
+    OPEN = auto()
+    PAUSED = auto()
+    CLOSING = auto()
+
+
+class Worker(object):
+    def __init__(self, database, site_comm, config):
+        self.logger = logging.getLogger()
+        self.database = database
+        self.site_comm = site_comm
+        self.config = config
+        self.torrents = dict()  # type: Dict[str, Torrent]
+        self.users = dict()  # type: Dict[str, User]
+        self.whitelist = list()  # type: List[str]
+
+        self.del_reasons = dict()
+        self.del_reasons_lock = threading.RLock()
+
+        self.announce_interval = 0
+        self.del_reason_lifetime = 0
+        self.peers_timeout = 0
+        self.numwant_limit = 0
+        self.site_password = ''
+        self.report_password = ''
+
+        self.status = Status.OPEN
+
+        self.reaper_active = False
+
+        self.load_config(self.config)
+        self.reload_lists()
+
+    def load_config(self, config):
+        self.announce_interval = config['tracker']['announce_interval']
+        self.del_reason_lifetime = config['timers']['del_reason_lifetime']
+        self.peers_timeout = config['timers']['peers_timeout']
+        self.numwant_limit = config['tracker']['numwant_limit']
+        self.site_password = config['gazelle']['site_password']
+        self.report_password = config['gazelle']['report_password']
+
+    def reload_config(self, config):
+        self.load_config(config)
+
+    def shutdown(self):
+        if self.status == Status.OPEN:
+            self.status = Status.CLOSING
+            self.logger.info('closing tracker... press Ctrl+C again to terminate')
+            return False
+        elif self.status == Status.CLOSING:
+            self.logger.info('shutting down uncleanly')
+            return True
+        else:
+            return False
+
+    def reload_lists(self):
+        self.status = Status.PAUSED
+        self.torrents = self.database.load_torrents(self.torrents)
+        self.users = self.database.load_users(self.users)
+        self.whitelist = self.database.load_whitelist()
+        self.status = Status.OPEN
+
+    def create_server(self, port):
+        app = web.Application()
+        app.router.add_get('/', self.handler_null)
+        app.router.add_get('/{passkey}/{action}', self.handler_work)
+        self.logger.info(f'======== Running on http://127.0.0.1:{port} ========')
+        web.run_app(app, host='127.0.0.1', print=False, port=port, handle_signals=False)
+
+    async def handler_null(self):
+        return self.handle_null()
+
+    # noinspection PyMethodMayBeStatic
+    def handle_null(self):
+        return web.Response(text='Nothing to see here.')
+
+    def error(self, message):
+        response = {'failure reason': message, 'min interval': 5400, 'interval': 5400}
+        return self.response(response)
+
+    def warning(self, message):
+        return self.response({'warning message': message})
+
+    async def handler_work(self, request):
+        action = request.match_info.get('action').lower()
+        actions = ['announce', 'scrape', 'update', 'report']
+        if action not in actions:
+            return web.Response(text='Invalid action.')
+        if len(request.query) == 0:
+            return self.handle_null()
+
+        if self.status != Status.OPEN:
+            return self.error('The tracker is temporarily unavailable.')
+
+        passkey = request.match_info.get('passkey')
+        if action == 'update' or action == 'report':
+            if passkey != self.site_password:
+                return self.error('Authentication failure.')
+
+        if action == 'update':
+            return self.handle_update(request)
+        elif action == 'report':
+            return self.handle_report(request)
+
+        with self.database.user_list_lock:
+            if passkey not in self.users:
+                return self.error('Passkey not found')
+            user = self.users[passkey]
+
+        if action == 'announce':
+            return self.handle_announce(request, user)
+        elif action == 'scrape':
+            return self.handle_scrape(request)
+
+    def handle_announce(self, request, user):
+        params = request.query
+        with self.database.torrent_list_lock:
+            tor = self.torrents[params['info_hash']]  # type: Torrent
+        cur_time = int(time())
+        if params['compact'] != '1':
+            return self.error('Your client does not support compact announces')
+
+        left = max(0, int(params['left']))
+        uploaded = max(0, int(params['uploaded']))
+        downloaded = max(0, int(params['downloaded']))
+        corrupt = max(0, int(params['corrupt']))
+
+        snatched = 0
+        active = 1
+        inserted = False
+        update_torrent = False
+        completed_torrent = False
+        stopped_torrent = False
+        expire_token = False
+        peer_changed = False
+        invalid_ip = False
+        inc_l = inc_s = dec_l = dec_s = False
+
+        if 'peer_id' not in params:
+            return self.error('No peer ID')
+        elif len(params['peer_id']) != 20:
+            return self.error('Invalid peer ID')
+
+        with self.database.whitelist_lock:
+            if len(self.whitelist) > 0:
+                found = False
+                for client in self.whitelist:
+                    if params['peer_id'].startswith(client):
+                        found = True
+
+                if not found:
+                    return self.error('Your client is not on the whitelist')
+
+        peer_key = params['peer_id'][12 + (tor.id & 7)] + str(user.id) + params['peer_id']
+
+        if params['event'] == 'completed':
+            completed_torrent = left == 0
+        elif params['event'] == 'stopped':
+            stopped_torrent = True
+            peer_changed = True
+            update_torrent = True
+            active = 0
+
+        peer = None  # type: Peer
+        if left > 0:
+            if peer_key not in tor.leechers:
+                peer = Peer()
+                tor.leechers[peer_key] = peer
+                inserted = True
+                inc_l = True
+            else:
+                peer = tor.leechers[peer_key]
+        elif completed_torrent:
+            if peer_key not in tor.leechers:
+                if peer_key not in tor.seeders:
+                    peer = Peer()
+                    tor.seeders[peer_key] = peer
+                    inserted = True
+                    inc_s = True
+                else:
+                    peer = tor.seeders[peer_key]
+                    completed_torrent = False
+            elif peer_key in tor.seeders:
+                peer = tor.leechers[peer_key]
+                dec_s = True
+        else:
+            if peer_key not in tor.seeders:
+                if peer_key not in tor.leechers:
+                    peer = Peer()
+                    tor.seeders[peer_key] = peer
+                    inserted = True
+                else:
+                    peer = tor.leechers[peer_key]
+                    tor.seeders[peer_key] = peer
+                    del tor.leechers[peer_key]
+                    peer_changed = True
+                    dec_l = True
+
+        upspeed = 0
+        downspeed = 0
+
+        if inserted or params['event'] == 'started':
+            update_torrent = True
+            if inserted:
+                peer.user = user
+            peer.first_announced = cur_time
+            peer.last_announced = 0
+            peer.uploaded = uploaded
+            peer.downloaded = downloaded
+            peer.corrupt = corrupt
+            peer.announces = 1
+            peer_changed = True
+        elif uploaded < peer.uploaded or downloaded < peer.downloaded:
+            peer.announces += 1
+            peer.uploaded = uploaded
+            peer.downloaded = downloaded
+            peer_changed = True
+        else:
+            uploaded_change = 0
+            downloaded_change = 0
+            corrupt_change = 0
+            peer.announces += 1
+
+            if uploaded != peer.uploaded:
+                uploaded_change = uploaded - peer.uploaded
+                peer.uploaded = uploaded
+            if downloaded != peer.downloaded:
+                downloaded_change = downloaded - peer.downloaded
+                peer.downloaded = downloaded
+            if corrupt != peer.corrupt:
+                corrupt_change = corrupt - peer.corrupt
+                peer.corrupt = corrupt
+                tor.balance -= corrupt_change
+                update_torrent = True
+            peer_changed = peer_changed or uploaded_change or downloaded_change or corrupt_change
+
+            if uploaded_change or downloaded_change:
+                tor.balance += uploaded_change
+                tor.balance -= downloaded_change
+                update_torrent = True
+
+                if cur_time > peer.last_announced:
+                    upspeed = uploaded_change / (cur_time - peer.last_announced)
+                    downspeed = downloaded_change / (cur_time - peer.last_announced)
+
+                tokened = user.id in tor.tokened_users
+                if tor.free_torrent == LeechType.NEUTRAL:
+                    downloaded_change = 0
+                    uploaded_change = 0
+                elif tor.free_torrent == LeechType.FREE or tokened:
+                    if tokened:
+                        expire_token = True
+                        self.database.record_token(user.id, tor.id, downloaded_change)
+                    downloaded_change = 0
+
+                if uploaded_change or downloaded_change:
+                    self.database.record_user(user.id, uploaded_change, downloaded_change)
+
+        peer.left = left
+
+        if 'ip' in params:
+            ip = params['ip']
+        elif 'ipv4' in params:
+            ip = params['ipv4']
+        else:
+            ip = request.headers['x-forwarded-for'].split(',')[0]
+
+        port = int(params['port'])
+
+        if inserted or port != peer.port or ip != peer.ip:
+            peer.ip = ip
+            peer.port = port
+            parsed = ipaddress.ip_address(ip)
+            if parsed.is_private or parsed.is_unspecified or parsed.is_reserved or parsed.is_loopback:
+                invalid_ip = True
+            if not invalid_ip:
+                peer.ip_port = parsed.packed + port.to_bytes(length=2, byteorder='big')
+            if len(peer.ip_port) != 6:
+                peer.ip_port = ''
+                invalid_ip = True
+            peer.invalid_ip = invalid_ip
+        else:
+            invalid_ip = peer.invalid_ip
+
+        peer.last_announced = cur_time
+
+        # Peer is visible in the lists if they have their leech priviledges and they're not
+        # using an invalid IP address
+        peer.visible = (peer.left == 0 or user.leech) and not peer.invalid_ip
+
+        if peer_changed:
+            record_ip = '' if user.protect else ip
+            self.database.record_peer_heavy(user.id, tor.id, active, uploaded, downloaded,
+                                            upspeed, downspeed, left, corrupt,
+                                            (cur_time - peer.first_announced), peer.announces,
+                                            record_ip, params['peer_id'],
+                                            request.headers['user-agent'])
+        else:
+            self.database.record_peer_light(user.id, tor.id, (cur_time - peer.first_announced),
+                                            peer.announces, params['peer_id'])
+
+        numwant = self.numwant_limit
+        if 'numwant' in params:
+            numwant = min(params['numwant'], numwant)
+
+        if stopped_torrent:
+            numwant = 0
+            if left > 0:
+                dec_l = True
+            else:
+                dec_s = True
+        elif completed_torrent:
+            snatched = 1
+            update_torrent = True
+            tor.completed += 1
+
+            record_ip = '' if user.protect else ip
+            self.database.record_snatch(user.id, tor.id, cur_time, record_ip)
+
+            if not inserted:
+                tor.seeders[peer_key] = peer
+                del tor.leechers[peer_key]
+                dec_l = inc_s = True
+
+            if expire_token:
+                self.site_comm.expire_token(tor.id, user.id)
+                tor.tokened_users.remove(user.id)
+        elif not user.leech and left > 0:
+            numwant = 0
+
+        peers = b''
+        if numwant > 0:
+            found_peers = 0
+            if left > 0:
+                if len(tor.seeders) > 0:
+                    seeders_list = list(tor.seeders.keys())
+                    i = 0
+
+                    # Find out where to begin in the seeder list
+                    if tor.last_selected_seeder != '':
+                        try:
+                            i = seeders_list.index(tor.last_selected_seeder)
+                            i += 1
+                            if i == len(seeders_list):
+                                i = 0
+                        except ValueError:
+                            pass
+
+                    # Find out where to end in the seeder list
+                    end = len(seeders_list)
+                    if i != 0:
+                        end = i - 1
+                        if end == 0:
+                            end += 1
+                            i += 1
+
+                    while i != end and found_peers < numwant:
+                        if i == len(seeders_list):
+                            i = 0
+                        seeder = tor.seeders[seeders_list[i]]
+                        # Don't show users to themselves or leech disabled users
+                        if seeder.user.deleted or seeder.user.id == user.id or not seeder.visible:
+                            i += 1
+                            continue
+                        found_peers += 1
+                        peers += seeder.ip_port
+
+                if found_peers < numwant and len(tor.leechers) > 1:
+                    for key in tor.leechers:
+                        leecher = tor.leechers[key]
+                        if leecher.user.deleted or leecher.ip_port == peer.ip_port or leecher.user.id == user.id or not leecher.visible:
+                            continue
+                        found_peers += 1
+                        peers += leecher.ip_port
+                        if found_peers >= numwant:
+                            break
+            elif len(tor.leechers) > 0:
+                for key in tor.leechers:
+                    leecher = tor.leechers[key]
+                    if leecher.user.id == user.id or not leecher.visible:
+                        continue
+                    found_peers += 1
+                    peers += leecher.ip_port
+                    if found_peers >= numwant:
+                        break
+
+
+        stats.succ_announcements += 1
+        if dec_l or dec_s or inc_l or inc_s:
+            if inc_l:
+                peer.user.leeching += 1
+                stats.leechers += 1
+            if inc_s:
+                peer.user.seeding += 1
+                stats.seeders += 1
+            if dec_l:
+                peer.user.leeching -= 1
+                stats.leechers -= 1
+            if dec_s:
+                peer.user.seeding -= 1
+                stats.seeders -= 1
+
+        if peer.user != user:
+            if not stopped_torrent:
+                if left > 0:
+                    user.leeching += 1
+                    peer.user.leeching -= 1
+                else:
+                    user.seeding += 1
+                    peer.user.seeding -= 1
+            peer.user = user
+
+        if stopped_torrent:
+            if left > 0:
+                del tor.leechers[peer_key]
+            else:
+                del tor.seeders[peer_key]
+
+        if update_torrent or tor.last_flushed + 3600 < cur_time:
+            tor.last_flushed = cur_time
+
+            self.database.record_torrent(tor.id, len(tor.seeders), len(tor.leechers), snatched,
+                                         tor.balance)
+
+        if not user.leech and left > 0:
+            return self.error('Access denied, leeching forbidden')
+
+        response = {
+            'complete': len(tor.seeders),
+            'downloaded': tor.completed,
+            'incomplete': len(tor.leechers),
+            'interval': self.announce_interval + min(600, len(tor.seeders)),  # ensure a more even distribution of announces/second
+            'min interval': self.announce_interval,
+            'peers': peers
+        }
+
+        if invalid_ip:
+            response['warning message'] = 'Illegal character found in IP address. IPv6 is not ' \
+                                          'supported'
+
+        return web.Response(text=bencode.encode(response))
+
+    def handle_scrape(self, request):
+        response = {'files': {}}
+        for infohash in request.query['info_hash']:
+            if infohash not in self.torrents:
+                continue
+            t = self.torrents[infohash]
+            response['files'][infohash] = {
+                'complete': len(t.seeders),
+                'incomplete': len(t.leechers),
+                'downloaded': t.completed
+            }
+        return web.Response(text=bencode.encode(response))
+
+    def handle_update(self, request):
+        params = request.query
+        if params['action'] == 'change_passkey':
+            oldpasskey = params['oldpasskey']
+            newpasskey = params['newpasskey']
+            with self.database.user_list_lock:
+                if oldpasskey not in self.users:
+                    self.logger.warning(f'No user with passkey {oldpasskey} exists when '
+                                        f'attempting to change passkey to {newpasskey}')
+                else:
+                    self.users[newpasskey] = self.users[oldpasskey]
+                    del self.users[oldpasskey]
+                    self.logger.info(f'Changed passkey from {oldpasskey} to {newpasskey} for '
+                                     f'user {self.users[newpasskey].id}')
+        elif params['action'] == 'add_torrent':
+            info_hash = params['info_hash']
+            with self.database.torrent_list_lock:
+                if info_hash not in self.torrents:
+                    torrent = Torrent(params['id'], 0)
+                else:
+                    torrent = self.torrents[info_hash]
+                if params['freetorrent'] == '0':
+                    torrent.free_torrent = LeechType.NORMAL
+                elif params['freetorrent'] == '1':
+                    torrent.free_torrent = LeechType.FREE
+                else:
+                    torrent.free_torrent = LeechType.NEUTRAL
+                self.torrents[info_hash] = torrent
+                self.logger.info(f"Added torrent {torrent.id}. FL: {torrent.free_torrent} "
+                                 f"{params['freetorrent']}")
+        elif params['action'] == 'update_torrent':
+            info_hash = params['info_hash']
+            if params['freetorrent'] == '0':
+                fl = LeechType.NORMAL
+            elif params['freetorrent'] == '1':
+                fl = LeechType.FREE
+            else:
+                fl = LeechType.NEUTRAL
+            with self.database.torrent_list_lock:
+                if info_hash in self.torrents:
+                    self.torrents[info_hash].free_torrent = fl
+                    self.logger.info(f'Updated torrent {self.torrents[info_hash].id} to FL {fl}')
+                else:
+                    self.logger.warning(f'Failed to find torrent {info_hash} to FL {fl}')
+        elif params['action'] == 'update_torrents':
+            # Each decoded infohash is exactly 20 characters long
+            # TODO: this probably doesn't work and needs more work
+            info_hashes = params['info_hashes']
+            if params['freetorrent'] == '0':
+                fl = LeechType.NORMAL
+            elif params['freetorrent'] == '1':
+                fl = LeechType.FREE
+            else:
+                fl = LeechType.NEUTRAL
+            with self.database.torrent_list_lock:
+                pos = 0
+                while pos < len(info_hashes):
+                    info_hash = info_hashes[pos:pos+20]
+                    if info_hash in self.torrents:
+                        self.torrents[info_hash].free_torrent = fl
+                        self.logger.info(f'Updated torrent {self.torrents[info_hash].id} '
+                                         f'to FL {fl}')
+                    else:
+                        self.logger.warning(f'Failed to find torrent {info_hash} to FL {fl}')
+        elif params['action'] == 'add_token':
+            info_hash = params['info_hash']
+            userid = int(params['userid'])
+            with self.database.torrent_list_lock:
+                if info_hash in self.torrents:
+                    self.torrents[info_hash].tokened_users.remove(userid)
+                else:
+                    self.logger.warning(f'Failed to find torrent to add a token for user {userid}')
+        elif params['action'] == 'remove_token':
+            info_hash = params['info_hash']
+            userid = int(params['userid'])
+            with self.database.torrent_list_lock:
+                if info_hash in self.torrents:
+                    self.torrents[info_hash].tokened_users.remove(userid)
+                else:
+                    self.logger.warning(f'Failed to find torrent {info_hash} to remove token '
+                                        f'for user {userid}')
+        elif params['action'] == 'delete_torrent':
+            info_hash = params['info_hash']
+            reason = int(params['reason']) if 'reason' in params else -1
+            with self.database.torrent_list_lock:
+                if info_hash in self.torrents:
+                    torrent = self.torrents[info_hash]
+                    self.logger.info(f'Deleting torrent {torrent.id} for the '
+                                     f'reason {ErrorCodes.get_del_reason(reason)}')
+                    stats.leechers -= len(torrent.leechers)
+                    stats.seeders -= len(torrent.seeders)
+                    for peer_key in torrent.leechers:
+                        torrent.leechers[peer_key].user.leeching -= 1
+                    for peer_key in torrent.seeders:
+                        torrent.seeders[peer_key].user.seeding -= 1
+                    with self.del_reasons_lock:
+                        self.del_reasons[info_hash] = {'reason': reason, time: int(time())}
+                        del self.torrents[info_hash]
+                else:
+                    self.logger.warning(f'Failed to find torrent {info_hash} to delete')
+        elif params['action'] == 'add_user':
+            passkey = params['passkey']
+            userid = int(params['id'])
+            with self.database.user_list_lock:
+                if passkey not in self.users:
+                    self.users[passkey] = User(userid, True, params['visible'] == '0')
+                    self.logger.info(f'Added user {passkey} with id {userid}')
+                else:
+                    self.logger.warning(f'Tried to add already known user {passkey} '
+                                        f'with id {self.users[passkey].id}')
+                    self.users[passkey].deleted = True
+        elif params['action'] == 'remove_user':
+            passkey = params['passkey']
+            with self.database.user_list_lock:
+                if passkey in self.users:
+                    self.logger.info(f'Removed user {passkey} with id {self.users[passkey].id}')
+                    self.users[passkey].deleted = True
+                    del self.users[passkey]
+        elif params['action'] == 'remove_users':
+            # Each passkey is 32 characters long
+            passkeys = params['passkeys']
+            with self.database.user_list_lock:
+                i = 0
+                while i < len(passkeys):
+                    passkey = passkeys[i:i+32]
+                    if passkey in self.users:
+                        self.logger.info(f'Removed user {passkey}')
+                        self.users[passkey].deleted = True
+                        del self.users[passkey]
+                    i += 32
+        elif params['action'] == 'update_user':
+            passkey = params['passkey']
+            can_leech = False if params['can_leech'] == '0' else True
+            protect_ip = True if params['visible'] == '0' else False
+            with self.database.user_list_lock:
+                if passkey not in self.users:
+                    self.logger.warning(f'No user with passkey {passkey} found when attempting to '
+                                        f'change leeching status!')
+                else:
+                    self.users[passkey].protect = protect_ip
+                    self.users[passkey].leech = can_leech
+                    self.logger.info(f'Updated user {passkey}')
+        elif params['action'] == 'add_whitelist':
+            peer_id = params['peer_id']
+            with self.database.whitelist_lock:
+                self.whitelist.append(peer_id)
+                self.logger.info(f'Whitelisted {peer_id}')
+        elif params['action'] == 'remove_whitelist':
+            peer_id = params['peer_id']
+            with self.database.whitelist_lock:
+                try:
+                    self.whitelist.remove(peer_id)
+                except ValueError:
+                    pass
+                self.logger.info(f'De-whitelisted {peer_id}')
+        elif params['action'] == 'edit_whitelist':
+            new_peer_id = params['new_peer_id']
+            old_peer_id = params['old_peer_id']
+            with self.database.whitelist_lock:
+                try:
+                    self.whitelist.remove(old_peer_id)
+                except ValueError:
+                    pass
+                self.whitelist.append(new_peer_id)
+                self.logger.info(f'Edited whitelist item from {old_peer_id} to {new_peer_id}')
+        elif params['action'] == 'update_announce_interval':
+            self.announce_interval = int(params['announce_interval'])
+            self.config['tracker']['announce_interval'] = self.announce_interval
+            self.logger.info(f'Edited announce interval to {self.announce_interval}')
+        elif params['action'] == 'info_torrent':
+            info_hash = params['info_hash']
+            self.logger.info(f"Info for torrent '{info_hash}'")
+            with self.database.torrent_list_lock:
+                if info_hash in self.torrents:
+                    self.logger.info(f'Torrent {self.torrents[info_hash].id}, '
+                                     f'freetorrent = {self.torrents[info_hash].free_torrent}')
+                else:
+                    self.logger.warning(f'Failed to find torrent {info_hash}')
+                    
+        return web.Response(text='success')
+
+    def handle_report(self, request):
+        params = request.query
+        action = params['get']
+        output = ''
+        if action == '':
+            output += "Invalid action\n"
+        elif action == 'stats':
+            uptime = int(time()) - stats.start_time
+            up_d = uptime // 86400
+            uptime -= up_d * 86400
+            up_h = uptime // 3600
+            uptime -= up_h * 3600
+            up_m = uptime // 60
+            up_s = uptime - up_m * 60
+            output += f"Uptime {up_d} days, {up_h:02}:{up_m:02}:{up_s:02}\n" \
+                      f"{stats.opened_connections} connections opened\n" \
+                      f"{stats.open_connections} open connections\n" \
+                      f"{stats.connection_rate} connections/s\n" \
+                      f"{stats.requests} requests handled\n" \
+                      f"{stats.request_rate} requests/s\n" \
+                      f"{stats.succ_announcements} successful announcements\n" \
+                      f"{(stats.announcements - stats.succ_announcements)} failed announcements\n" \
+                      f"{stats.scrapes} scrapes\n" \
+                      f"{stats.leechers} leechers tracked\n" \
+                      f"{stats.seeders} seeders tracked\n" \
+                      f"{stats.bytes_read} bytes read\n" \
+                      f"{stats.bytes_written} bytes written\n"
+        elif action == 'user':
+            key = params['key']
+            if len(key) == 0:
+                output += "Invalid action\n"
+            else:
+                with self.database.user_list_lock:
+                    if key in self.users:
+                        output += f"{self.users[key].leeching} leeching\n" \
+                                  f"{self.users[key].seeding} seeding\n"
+        else:
+            output += "Invalid action\n"
+        return web.Response(text=output)
+
+    # noinspection PyMethodMayBeStatic
+    def response(self, response):
+        return web.Response(text=bencode.bencode(response))
+
+    def start_reaper(self):
+        if not self.reaper_active:
+            threading.Thread(target=self._do_start_reaper)
+
+    def _do_start_reaper(self):
+        self.reaper_active = True
+        self.reap_peers()
+        self.reap_del_reasons()
+        self.reaper_active = False
+
+    def reap_peers(self):
+        self.logger.info('Starting peer reaper')
+        cur_time = int(time())
+        reaped_l = reaped_s = 0
+        cleared_torrents = 0
+        for info_hash in self.torrents:
+            reaped_this = False
+            torrent = self.torrents[info_hash]
+            for peer_key in torrent.leechers:
+                if torrent.leechers[peer_key].last_announced + self.peers_timeout < cur_time:
+                    with self.database.torrent_list_lock:
+                        torrent.leechers[peer_key].user.leeching -= 1
+                        del torrent.leechers[peer_key]
+                        reaped_this = True
+                        reaped_l += 1
+            for peer_key in torrent.seeders:
+                if torrent.seeders[peer_key].last_announced + self.peers_timeout < cur_time:
+                    with self.database.torrent_list_lock:
+                        torrent.seeders[peer_key].user.seeding -= 1
+                        del torrent.seeders[peer_key]
+                        reaped_this = True
+                        reaped_s += 1
+            if reaped_this and len(torrent.seeders) == 0 and len(torrent.leechers) == 0:
+                self.database.record_torrent(torrent.id, 0, 0, 0, torrent.balance)
+                cleared_torrents += 1
+        if reaped_l > 0 or reaped_s > 0:
+            stats.leechers -= reaped_l
+            stats.seeders -= reaped_s
+        self.logger.info(f'Reaped {reaped_l} leechers and {reaped_s} seeders. '
+                         f'Reset {cleared_torrents} torrents')
+
+    def reap_del_reasons(self):
+        self.logger.info('Starting del reason reaper')
+        max_time = int(time()) - self.del_reason_lifetime
+        reaped = 0
+        for key in self.del_reasons:
+            if self.del_reasons[key]['time'] <= max_time:
+                with self.del_reasons_lock:
+                    del self.del_reasons[key]
+                    reaped += 1
+
+        self.logger.info(f'Reaped {reaped} del reasons')

runner.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from margay.main import run
+
+if __name__ == '__main__':
+    run()

setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup
+from margay import __author__, __version__
+
+setup(
+    name='Margay',
+    author=__author__,
+    version=__version__,
+    url='https://github.com/ApolloRIP/Margay',
+    description='A Python BitTorrent Server',
+    long_description=open('README.rst').read(),
+    classifiers=[
+        'Development Status :: 2 - Pre-Alpha',
+        'Environment :: Console',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Cython',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License'
+    ],
+    install_requires=[
+        'aiohttp',
+        'bencode.py',
+        'mysqlclient',
+        'requests'
+    ]
+)