mirror of
https://github.com/RPCS3/discord-bot.git
synced 2026-01-31 01:25:22 +01:00
Log Analysis Implementation
7Z Stream reader is broken Zip Stream reader doesn't exist Supported formats: - .gz - .log Add manual library load report. Last problems still not fixed. Disable .zip and .7z logs. Turn on feedback for invalid/non-tested serials. Fixup Log Analysis
This commit is contained in:
Nicba1010
3
.gitignore
vendored
3
.gitignore
vendored
@@ -105,4 +105,5 @@ venv.bak/
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
.idea
|
||||
.idea
|
||||
discord.log
|
||||
152
bot.py
152
bot.py
@@ -7,19 +7,41 @@ import discord
|
||||
import requests
|
||||
from discord import Message
|
||||
from discord.ext.commands import Bot
|
||||
from requests import Response
|
||||
|
||||
from api import newline_separator, directions, regions, statuses, release_types
|
||||
from api.request import ApiRequest
|
||||
from bot_config import latest_limit, newest_header, invalid_command_text, oldest_header, boot_up_message
|
||||
from bot_utils import get_code
|
||||
from math_parse import NumericStringParser
|
||||
from utils import limit_int
|
||||
from math_utils import limit_int
|
||||
from phases import LogAnalyzer
|
||||
from stream_handlers import stream_text_log, stream_gzip_decompress
|
||||
|
||||
channel_id = "291679908067803136"
|
||||
bot_spam_id = "319224795785068545"
|
||||
rpcs3Bot = Bot(command_prefix="!")
|
||||
pattern = '[A-z]{4}\\d{5}'
|
||||
id_pattern = '[A-z]{4}\\d{5}'
|
||||
nsp = NumericStringParser()
|
||||
|
||||
file_handlers = (
|
||||
# {
|
||||
# 'ext': '.zip'
|
||||
# },
|
||||
{
|
||||
'ext': '.log',
|
||||
'handler': stream_text_log
|
||||
},
|
||||
{
|
||||
'ext': '.gz',
|
||||
'handler': stream_gzip_decompress
|
||||
},
|
||||
# {
|
||||
# 'ext': '.7z',
|
||||
# 'handler': stream_7z_decompress
|
||||
# }
|
||||
)
|
||||
|
||||
|
||||
@rpcs3Bot.event
|
||||
async def on_message(message: Message):
|
||||
@@ -27,24 +49,115 @@ async def on_message(message: Message):
|
||||
OnMessage event listener
|
||||
:param message: message
|
||||
"""
|
||||
# Self reply detect
|
||||
if message.author.name == "RPCS3 Bot":
|
||||
return
|
||||
# Command detect
|
||||
try:
|
||||
if message.content[0] == "!":
|
||||
return await rpcs3Bot.process_commands(message)
|
||||
except IndexError as ie:
|
||||
print(message.content)
|
||||
return
|
||||
codelist = []
|
||||
for matcher in re.finditer(pattern, message.content):
|
||||
print("Empty message! Could still have attachments.")
|
||||
|
||||
# Code reply
|
||||
code_list = []
|
||||
for matcher in re.finditer(id_pattern, message.content):
|
||||
code = str(matcher.group(0)).upper()
|
||||
if code not in codelist:
|
||||
codelist.append(code)
|
||||
if code not in code_list:
|
||||
code_list.append(code)
|
||||
print(code)
|
||||
for code in codelist:
|
||||
info = await get_code(code)
|
||||
if info is not None:
|
||||
await rpcs3Bot.send_message(message.channel, info)
|
||||
if len(code_list) > 0:
|
||||
for code in code_list:
|
||||
info = get_code(code)
|
||||
if info is not None:
|
||||
await rpcs3Bot.send_message(message.channel, '```{}```'.format(info))
|
||||
else:
|
||||
await rpcs3Bot.send_message(message.channel, '```Serial not found in compatibility database, possibly '
|
||||
'untested!```')
|
||||
return
|
||||
|
||||
# Log Analysis!
|
||||
if len(message.attachments) > 0:
|
||||
log = LogAnalyzer()
|
||||
print("Attachments present, looking for log file...")
|
||||
for attachment in filter(lambda a: any(e['ext'] in a['url'] for e in file_handlers), message.attachments):
|
||||
for handler in file_handlers:
|
||||
if attachment['url'].endswith(handler['ext']):
|
||||
print("Found log attachment, name: {name}".format(name=attachment['filename']))
|
||||
with requests.get(attachment['url'], stream=True) as response:
|
||||
print("Opened request stream!")
|
||||
# noinspection PyTypeChecker
|
||||
for row in stream_line_by_line_safe(response, handler['handler']):
|
||||
error_code = log.feed(row)
|
||||
if error_code == LogAnalyzer.ERROR_SUCCESS:
|
||||
continue
|
||||
elif error_code == LogAnalyzer.ERROR_PIRACY:
|
||||
await piracy_alert(message, log.get_trigger())
|
||||
break
|
||||
elif error_code == LogAnalyzer.ERROR_OVERFLOW:
|
||||
print("Possible Buffer Overflow Attack Detected!")
|
||||
break
|
||||
elif error_code == LogAnalyzer.ERROR_STOP:
|
||||
await rpcs3Bot.send_message(
|
||||
message.channel,
|
||||
log.get_report()
|
||||
)
|
||||
break
|
||||
elif error_code == LogAnalyzer.ERROR_FAIL:
|
||||
break
|
||||
print("Stopping stream!")
|
||||
del log
|
||||
|
||||
|
||||
async def piracy_alert(message: Message, trigger: str):
|
||||
await rpcs3Bot.send_message(
|
||||
message.channel,
|
||||
"Pirated release detected {author}!\n"
|
||||
"Please note that the RPCS3 community and it's developers do not support piracy!\n"
|
||||
"Most of the issues caused by pirated dumps is because they have been tampered with in such a way "
|
||||
"and therefore act unpredictably on RPCS3.\n"
|
||||
"If you need help obtaining legal dumps please read <https://rpcs3.net/quickstart>\n"
|
||||
"The trigger phrase was `{trigger}`, if you believe this was detected wrongly please contact a mod "
|
||||
"or {bot_admin}".format(
|
||||
author=message.author.mention,
|
||||
trigger=mask(trigger),
|
||||
bot_admin=discord.Object(id=267367850706993152)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def mask(string: str):
|
||||
return ''.join("*" if i % 2 == 0 else char for i, char in enumerate(string, 1))
|
||||
|
||||
|
||||
def stream_line_by_line_safe(stream: Response, func: staticmethod):
|
||||
buffer = ''
|
||||
chunk_buffer = b''
|
||||
for chunk in func(stream):
|
||||
try:
|
||||
chunk_buffer += chunk
|
||||
message = chunk_buffer.decode('UTF-8')
|
||||
chunk_buffer = b''
|
||||
if '\n' in message:
|
||||
parts = message.split('\n')
|
||||
yield buffer + parts[0]
|
||||
buffer = ''
|
||||
for part in parts[1:-1]:
|
||||
yield part
|
||||
buffer += parts[-1]
|
||||
elif len(buffer) > 1024 * 1024 or len(chunk_buffer) > 1024 * 1024:
|
||||
print('Possible overflow intended, piss off!')
|
||||
break
|
||||
else:
|
||||
buffer += message
|
||||
except UnicodeDecodeError as ude:
|
||||
if ude.end == len(chunk_buffer):
|
||||
pass
|
||||
else:
|
||||
print("{}\n{} {} {} {}".format(chunk_buffer, ude.reason, ude.start, ude.end, len(chunk_buffer)))
|
||||
break
|
||||
del chunk
|
||||
del buffer
|
||||
|
||||
|
||||
@rpcs3Bot.command()
|
||||
@@ -53,6 +166,7 @@ async def math(*args):
|
||||
return await rpcs3Bot.say(nsp.eval(''.join(map(str, args))))
|
||||
|
||||
|
||||
# noinspection PyShadowingBuiltins
|
||||
@rpcs3Bot.command()
|
||||
async def credits(*args):
|
||||
"""Author Credit"""
|
||||
@@ -178,20 +292,6 @@ async def latest(ctx, *args):
|
||||
)
|
||||
|
||||
|
||||
async def get_code(code: str) -> object:
|
||||
"""
|
||||
Gets the game data for a certain game code or returns None
|
||||
:param code: code to get data for
|
||||
:return: data or None
|
||||
"""
|
||||
result = ApiRequest().set_search(code).set_amount(10).request()
|
||||
if len(result.results) >= 1:
|
||||
for result in result.results:
|
||||
if result.game_id == code:
|
||||
return "```" + result.to_string() + "```"
|
||||
return None
|
||||
|
||||
|
||||
async def greet():
|
||||
"""
|
||||
Greets on boot!
|
||||
|
||||
@@ -28,3 +28,7 @@ boot_up_message = "Hello and welcome to CompatBot. \n" \
|
||||
"often as possible.\n" \
|
||||
"*Roberto Anic Banic AKA Nicba1010\n" \
|
||||
"https://github.com/RPCS3/discord-bot"
|
||||
|
||||
piracy_strings = {
|
||||
|
||||
}
|
||||
|
||||
15
bot_utils.py
Normal file
15
bot_utils.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from api.request import ApiRequest
|
||||
|
||||
|
||||
def get_code(code: str) -> str:
|
||||
"""
|
||||
Gets the game data for a certain game code or returns None
|
||||
:param code: code to get data for
|
||||
:return: data or None
|
||||
"""
|
||||
result = ApiRequest().set_search(code).set_amount(10).request()
|
||||
if len(result.results) >= 1:
|
||||
for result in result.results:
|
||||
if result.game_id == code:
|
||||
return result.to_string()
|
||||
return None
|
||||
166
phases.py
Normal file
166
phases.py
Normal file
@@ -0,0 +1,166 @@
|
||||
import re
|
||||
|
||||
from bot_config import piracy_strings
|
||||
from bot_utils import get_code
|
||||
|
||||
SERIAL_PATTERN = re.compile('Serial: (?P<id>[A-z]{4}\d{5})')
|
||||
LIBRARIES_PATTERN = re.compile('Load libraries:(?P<libraries>.*)', re.DOTALL | re.MULTILINE)
|
||||
|
||||
|
||||
class LogAnalyzer(object):
|
||||
ERROR_SUCCESS = 0
|
||||
ERROR_PIRACY = 1
|
||||
ERROR_STOP = 2
|
||||
ERROR_OVERFLOW = -1
|
||||
ERROR_FAIL = -2
|
||||
|
||||
def piracy_check(self):
|
||||
for trigger in piracy_strings:
|
||||
if trigger in self.buffer:
|
||||
self.trigger = trigger
|
||||
return self.ERROR_PIRACY
|
||||
return self.ERROR_SUCCESS
|
||||
|
||||
def done(self):
|
||||
return self.ERROR_STOP
|
||||
|
||||
def get_id(self):
|
||||
try:
|
||||
info = get_code(re.search(SERIAL_PATTERN, self.buffer).group('id'))
|
||||
if info is not None:
|
||||
self.report = info + '\n' + self.report
|
||||
return self.ERROR_SUCCESS
|
||||
except AttributeError:
|
||||
print("Could not detect serial! Aborting!")
|
||||
return self.ERROR_FAIL
|
||||
|
||||
def get_libraries(self):
|
||||
try:
|
||||
self.libraries = [lib.strip().replace('.sprx', '')
|
||||
for lib
|
||||
in re.search(LIBRARIES_PATTERN, self.buffer).group('libraries').strip()[1:].split('-')]
|
||||
if len(self.libraries) > 0:
|
||||
self.report += 'Selected Libraries: ' + ', '.join(self.libraries) + '\n\n'
|
||||
except KeyError as ke:
|
||||
print(ke)
|
||||
pass
|
||||
return self.ERROR_SUCCESS
|
||||
|
||||
"""
|
||||
End Trigger
|
||||
Regex
|
||||
Message To Print
|
||||
Special Return
|
||||
"""
|
||||
phase = (
|
||||
{
|
||||
'end_trigger': 'Compatibility notice:',
|
||||
'regex': re.compile('(?P<all>.*)', flags=re.DOTALL | re.MULTILINE),
|
||||
'string_format': '{all}\n\n'
|
||||
},
|
||||
{
|
||||
'end_trigger': 'Core:',
|
||||
'regex': None,
|
||||
'string_format': None,
|
||||
'function': [get_id, piracy_check]
|
||||
},
|
||||
{
|
||||
'end_trigger': 'VFS:',
|
||||
'regex': re.compile('Decoder: (?P<ppu_decoder>.*?)\n.*?'
|
||||
'Threads: (?P<ppu_threads>.*?)\n.*?'
|
||||
'scheduler: (?P<thread_scheduler>.*?)\n.*?'
|
||||
'Decoder: (?P<spu_decoder>.*?)\n.*?'
|
||||
'priority: (?P<spu_lower_thread_priority>.*?)\n.*?'
|
||||
'SPU Threads: (?P<spu_threads>.*?)\n.*?'
|
||||
'penalty: (?P<spu_delay_penalty>.*?)\n.*?'
|
||||
'detection: (?P<spu_loop_detection>.*?)\n.*?'
|
||||
'Loader: (?P<lib_loader>.*?)\n.*?'
|
||||
'functions: (?P<hook_static_functions>.*?)\n.*',
|
||||
flags=re.DOTALL | re.MULTILINE),
|
||||
'string_format':
|
||||
'PPU Decoder: {ppu_decoder:>21s} | PPU Threads: {ppu_threads}\n'
|
||||
'SPU Decoder: {spu_decoder:>21s} | SPU Threads: {spu_threads}\n'
|
||||
'SPU Lower Thread Priority: {spu_lower_thread_priority:>7s} | SPU Delay Penalty: {spu_delay_penalty}\n'
|
||||
'SPU Loop Detection: {spu_loop_detection:>14s} | Hook Static Functions: {hook_static_functions}\n'
|
||||
'Thread Scheduler: {thread_scheduler:>16s} | Lib Loader: {lib_loader}\n\n',
|
||||
'function': get_libraries
|
||||
},
|
||||
{
|
||||
'end_trigger': 'Video:',
|
||||
'regex': None,
|
||||
'string_format': None,
|
||||
'function': None
|
||||
},
|
||||
{
|
||||
'end_trigger': 'Audio:',
|
||||
'regex': re.compile('Renderer: (?P<renderer>.*?)\n.*?'
|
||||
'Resolution: (?P<resolution>.*?)\n.*?'
|
||||
'limit: (?P<frame_limit>.*?)\n.*?'
|
||||
'Color Buffers: (?P<write_color_buffers>.*?)\n.*?'
|
||||
'VSync: (?P<vsync>.*?)\n.*?'
|
||||
'Rendering Mode: (?P<strict_rendering_mode>.*?)\n.*?',
|
||||
flags=re.DOTALL | re.MULTILINE),
|
||||
'string_format':
|
||||
'Renderer: {renderer:>21s} | Resolution: {resolution}\n'
|
||||
'Frame Limit: {frame_limit:>18s} | Write Color Buffers: {write_color_buffers}\n'
|
||||
'VSync: {vsync:>24s} | Strict Rendering Mode: {strict_rendering_mode}\n'
|
||||
},
|
||||
{
|
||||
'end_trigger': 'Log:',
|
||||
'regex': None,
|
||||
'string_format': None,
|
||||
'function': done
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self.buffer = ''
|
||||
self.phase_index = 0
|
||||
self.report = ''
|
||||
self.trigger = ''
|
||||
self.libraries = []
|
||||
|
||||
def feed(self, data):
|
||||
if len(self.buffer) > 16 * 1024 * 1024:
|
||||
return self.ERROR_OVERFLOW
|
||||
if self.phase[self.phase_index]['end_trigger'] in data \
|
||||
or self.phase[self.phase_index]['end_trigger'] is data.strip():
|
||||
error_code = self.process_data()
|
||||
if error_code == self.ERROR_SUCCESS:
|
||||
self.buffer = ''
|
||||
self.phase_index += 1
|
||||
else:
|
||||
return error_code
|
||||
else:
|
||||
self.buffer += '\n' + data
|
||||
return self.ERROR_SUCCESS
|
||||
|
||||
def process_data(self):
|
||||
current_phase = self.phase[self.phase_index]
|
||||
if current_phase['regex'] is not None and current_phase['string_format'] is not None:
|
||||
try:
|
||||
self.report += current_phase['string_format'].format(
|
||||
**re.search(current_phase['regex'], self.buffer).groupdict()
|
||||
)
|
||||
except AttributeError as ae:
|
||||
print("Regex failed!")
|
||||
return self.ERROR_FAIL
|
||||
try:
|
||||
if current_phase['function'] is not None:
|
||||
if isinstance(current_phase['function'], list):
|
||||
for func in current_phase['function']:
|
||||
error_code = func(self)
|
||||
if error_code != self.ERROR_SUCCESS:
|
||||
return error_code
|
||||
return self.ERROR_SUCCESS
|
||||
else:
|
||||
return current_phase['function'](self)
|
||||
except KeyError:
|
||||
pass
|
||||
return self.ERROR_SUCCESS
|
||||
|
||||
def get_trigger(self):
|
||||
return self.trigger
|
||||
|
||||
def get_report(self):
|
||||
return '```\n{}```'.format(self.report)
|
||||
16
stream_handlers.py
Normal file
16
stream_handlers.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import zlib
|
||||
|
||||
|
||||
def stream_text_log(stream):
|
||||
for chunk in stream.iter_content(chunk_size=1024):
|
||||
yield chunk
|
||||
|
||||
|
||||
def stream_gzip_decompress(stream):
|
||||
dec = zlib.decompressobj(32 + zlib.MAX_WBITS) # offset 32 to skip the header
|
||||
for chunk in stream:
|
||||
rv = dec.decompress(chunk)
|
||||
if rv:
|
||||
yield rv
|
||||
del rv
|
||||
del dec
|
||||
Reference in New Issue
Block a user