2019-04-21 17:27:21 +00:00
|
|
|
|
using System;
|
|
|
|
|
using System.Collections.Concurrent;
|
2019-12-24 15:26:25 +00:00
|
|
|
|
using System.Diagnostics;
|
2020-03-21 10:16:03 +00:00
|
|
|
|
using System.Linq;
|
2020-09-09 17:17:54 +00:00
|
|
|
|
using System.Runtime;
|
2019-04-21 17:27:21 +00:00
|
|
|
|
using System.Threading.Tasks;
|
2019-04-21 17:38:20 +00:00
|
|
|
|
using CompatBot.Commands;
|
2020-03-21 10:16:03 +00:00
|
|
|
|
using CompatBot.Database.Providers;
|
2020-09-01 10:05:27 +00:00
|
|
|
|
using CompatBot.EventHandlers;
|
2019-04-21 17:27:21 +00:00
|
|
|
|
using DSharpPlus;
|
2020-07-03 08:57:27 +00:00
|
|
|
|
using Microsoft.ApplicationInsights;
|
2020-09-01 10:05:27 +00:00
|
|
|
|
using NLog;
|
2019-04-21 17:27:21 +00:00
|
|
|
|
|
|
|
|
|
namespace CompatBot
|
|
|
|
|
{
|
|
|
|
|
internal static class Watchdog
|
|
|
|
|
{
|
|
|
|
|
private static readonly TimeSpan CheckInterval = TimeSpan.FromSeconds(10);
|
2020-11-14 09:33:29 +00:00
|
|
|
|
public static readonly ConcurrentQueue<DateTime> DisconnectTimestamps = new();
|
2019-12-24 15:26:25 +00:00
|
|
|
|
public static readonly Stopwatch TimeSinceLastIncomingMessage = Stopwatch.StartNew();
|
2020-09-02 15:16:46 +00:00
|
|
|
|
private static bool IsOk => DisconnectTimestamps.IsEmpty && TimeSinceLastIncomingMessage.Elapsed < Config.IncomingMessageCheckIntervalInMin;
|
2020-11-11 19:11:36 +00:00
|
|
|
|
private static DiscordClient? discordClient = null;
|
2019-04-21 17:27:21 +00:00
|
|
|
|
|
|
|
|
|
public static async Task Watch(DiscordClient client)
|
|
|
|
|
{
|
2020-09-01 10:05:27 +00:00
|
|
|
|
discordClient = client;
|
2019-04-21 17:27:21 +00:00
|
|
|
|
do
|
|
|
|
|
{
|
|
|
|
|
await Task.Delay(CheckInterval, Config.Cts.Token).ConfigureAwait(false);
|
2020-03-21 10:16:03 +00:00
|
|
|
|
foreach (var sudoer in ModProvider.Mods.Values.Where(m => m.Sudoer))
|
|
|
|
|
{
|
|
|
|
|
var user = await client.GetUserAsync(sudoer.DiscordId).ConfigureAwait(false);
|
|
|
|
|
if (user?.Presence?.Activity?.CustomStatus?.Name is string cmd && cmd.StartsWith("restart"))
|
|
|
|
|
{
|
|
|
|
|
var instance = cmd.Split(' ', StringSplitOptions.RemoveEmptyEntries).LastOrDefault();
|
|
|
|
|
if (ulong.TryParse(instance, out var botId) && botId == client.CurrentUser.Id)
|
|
|
|
|
{
|
|
|
|
|
Config.Log.Warn($"Found request to restart on {user.Username}#{user.Discriminator}'s custom status");
|
|
|
|
|
Sudo.Bot.Restart(Program.InvalidChannelId, $"Restarted by request from {user.Username}#{user.Discriminator}'s custom status");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-03 10:15:19 +00:00
|
|
|
|
if (IsOk)
|
2019-04-21 17:27:21 +00:00
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
2020-07-03 08:57:27 +00:00
|
|
|
|
Config.TelemetryClient?.TrackEvent("socket-deadlock-potential");
|
2019-04-28 18:08:18 +00:00
|
|
|
|
Config.Log.Warn("Potential socket deadlock detected, reconnecting...");
|
2019-04-22 22:49:46 +00:00
|
|
|
|
await client.ReconnectAsync(true).ConfigureAwait(false);
|
|
|
|
|
await Task.Delay(CheckInterval, Config.Cts.Token).ConfigureAwait(false);
|
2020-01-03 10:15:19 +00:00
|
|
|
|
if (IsOk)
|
2019-04-24 11:04:52 +00:00
|
|
|
|
{
|
2019-04-28 18:08:18 +00:00
|
|
|
|
Config.Log.Info("Looks like we're back in business");
|
2019-04-22 22:49:46 +00:00
|
|
|
|
continue;
|
2019-04-24 11:04:52 +00:00
|
|
|
|
}
|
2020-01-03 10:15:19 +00:00
|
|
|
|
|
2020-07-03 08:57:27 +00:00
|
|
|
|
Config.TelemetryClient?.TrackEvent("socket-deadlock-for-sure");
|
2019-04-28 18:08:18 +00:00
|
|
|
|
Config.Log.Error("Hard reconnect failed, restarting...");
|
2020-03-21 11:45:04 +00:00
|
|
|
|
Sudo.Bot.Restart(Program.InvalidChannelId, $@"Restarted to reset potential socket deadlock (last incoming message event: {TimeSinceLastIncomingMessage.Elapsed:h\:mm\:ss} ago)");
|
2019-04-21 17:27:21 +00:00
|
|
|
|
}
|
|
|
|
|
catch (Exception e)
|
|
|
|
|
{
|
|
|
|
|
Config.Log.Error(e);
|
|
|
|
|
}
|
|
|
|
|
} while (!Config.Cts.IsCancellationRequested);
|
|
|
|
|
}
|
2020-07-03 08:57:27 +00:00
|
|
|
|
|
2020-09-01 10:05:27 +00:00
|
|
|
|
public static void OnLogHandler(string level, string message)
|
|
|
|
|
{
|
|
|
|
|
if (level == nameof(LogLevel.Info))
|
|
|
|
|
{
|
2020-11-11 19:11:36 +00:00
|
|
|
|
if (message.Contains("Session resumed"))
|
2020-09-01 10:05:27 +00:00
|
|
|
|
DisconnectTimestamps.Clear();
|
|
|
|
|
}
|
|
|
|
|
else if (level == nameof(LogLevel.Warn))
|
|
|
|
|
{
|
2020-11-11 19:11:36 +00:00
|
|
|
|
if (message.Contains("Dispatch:PRESENCES_REPLACE")
|
|
|
|
|
&& discordClient != null)
|
2020-09-01 10:05:27 +00:00
|
|
|
|
BotStatusMonitor.RefreshAsync(discordClient).ConfigureAwait(false).GetAwaiter().GetResult();
|
2020-11-11 19:11:36 +00:00
|
|
|
|
else if (message.Contains("Pre-emptive ratelimit triggered"))
|
2020-09-01 10:05:27 +00:00
|
|
|
|
Config.TelemetryClient?.TrackEvent("preemptive-rate-limit");
|
|
|
|
|
}
|
|
|
|
|
else if (level == nameof(LogLevel.Fatal))
|
|
|
|
|
{
|
2020-11-11 19:11:36 +00:00
|
|
|
|
if ((message.Contains("Socket connection terminated"))
|
|
|
|
|
|| (message.Contains("heartbeats were skipped. Issuing reconnect.")))
|
2020-09-01 10:05:27 +00:00
|
|
|
|
DisconnectTimestamps.Enqueue(DateTime.UtcNow);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-03 08:57:27 +00:00
|
|
|
|
public static async Task SendMetrics(DiscordClient client)
|
|
|
|
|
{
|
|
|
|
|
do
|
|
|
|
|
{
|
2020-09-02 15:16:46 +00:00
|
|
|
|
await Task.Delay(Config.MetricsIntervalInSec).ConfigureAwait(false);
|
2020-11-11 19:11:36 +00:00
|
|
|
|
var gcMemInfo = GC.GetGCMemoryInfo();
|
2020-09-09 17:17:54 +00:00
|
|
|
|
using var process = Process.GetCurrentProcess();
|
2020-11-14 10:25:20 +00:00
|
|
|
|
if (Config.TelemetryClient is not TelemetryClient tc)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
tc.TrackMetric("gw-latency", client.Ping);
|
|
|
|
|
tc.TrackMetric("time-since-last-incoming-message", TimeSinceLastIncomingMessage.ElapsedMilliseconds);
|
|
|
|
|
tc.TrackMetric("memory-gc-total", gcMemInfo.HeapSizeBytes);
|
|
|
|
|
tc.TrackMetric("memory-gc-load", gcMemInfo.MemoryLoadBytes);
|
|
|
|
|
tc.TrackMetric("memory-gc-commited", gcMemInfo.TotalCommittedBytes);
|
|
|
|
|
tc.TrackMetric("memory-process-private", process.PrivateMemorySize64);
|
|
|
|
|
tc.TrackMetric("memory-process-ws", process.WorkingSet64);
|
|
|
|
|
tc.TrackMetric("github-limit-remaining", GithubClient.Client.RateLimitRemaining);
|
|
|
|
|
tc.Flush();
|
2020-07-03 08:57:27 +00:00
|
|
|
|
} while (!Config.Cts.IsCancellationRequested);
|
|
|
|
|
}
|
2020-09-09 17:17:54 +00:00
|
|
|
|
|
|
|
|
|
public static async Task CheckGCStats()
|
|
|
|
|
{
|
|
|
|
|
do
|
|
|
|
|
{
|
2020-11-11 19:11:36 +00:00
|
|
|
|
var gcMemInfo = GC.GetGCMemoryInfo();
|
2020-09-09 17:17:54 +00:00
|
|
|
|
using var process = Process.GetCurrentProcess();
|
2020-09-10 17:16:10 +00:00
|
|
|
|
Config.Log.Info($"Process memory stats:\n" +
|
2020-11-11 19:11:36 +00:00
|
|
|
|
$"GC Heap: {gcMemInfo.HeapSizeBytes}\n" +
|
2020-09-10 17:16:10 +00:00
|
|
|
|
$"Private: {process.PrivateMemorySize64}\n" +
|
|
|
|
|
$"Working set: {process.WorkingSet64}\n" +
|
|
|
|
|
$"Virtual: {process.VirtualMemorySize64}\n" +
|
|
|
|
|
$"Paged: {process.PagedMemorySize64}\n" +
|
|
|
|
|
$"Paged sytem: {process.PagedSystemMemorySize64}\n" +
|
|
|
|
|
$"Non-pated system: {process.NonpagedSystemMemorySize64}");
|
|
|
|
|
await Task.Delay(TimeSpan.FromHours(1)).ConfigureAwait(false);
|
2020-09-09 17:17:54 +00:00
|
|
|
|
} while (!Config.Cts.IsCancellationRequested);
|
|
|
|
|
}
|
2019-04-21 17:27:21 +00:00
|
|
|
|
}
|
|
|
|
|
}
|