niklasb/libc-database
1
0
Fork 0
mirror of https://github.com/niklasb/libc-database.git synced 2026-10-12 02:19:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

webservice v1

Browse Source
This commit is contained in:
Niklas Baumstark
2020-08-19 23:29:26 +02:00
parent 7b00212c79
commit 3dce67101f
11 changed files with 496 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
searchengine/.gitignore
+1
View File
@@ -0,0 +1 @@
__pycache__
searchengine/Dockerfile
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.8-slim-buster AS uwsgi
ENV PYTHONUNBUFFERED 1
WORKDIR /app
RUN apt-get -y update && apt-get install -y gcc
COPY requirements.txt /app/requirements.txt
RUN pip install -r requirements.txt
COPY . /app
EXPOSE 8000
CMD uwsgi --ini uwsgi.ini
searchengine/README.md
+46
View File
@@ -0,0 +1,46 @@
```
$ curl -X POST -H 'Content-Type: application/json' --data '{"symbols": {"strncpy": "db0", "strcat": "0x000000000d800"}}' 'https://libc.rip/api/find'
[
{
"buildid": "d3cf764b2f97ac3efe366ddd07ad902fb6928fd7",
"download_url": "https://libc.rip/download/libc6_2.27-3ubuntu1.2_amd64.so",
"id": "libc6_2.27-3ubuntu1.2_amd64",
"md5": "35ef4ffc9c6ad7ffd1fd8c16f14dc766",
"sha1": "a22321cd65f28f70cf321614fdfd22f36ecd0afe",
"sha256": "f0ad9639b2530741046e06c96270b25da2339b6c15a7ae46de8fb021b3c4f529",
"symbols": {
"__libc_start_main_ret": "0x21b97",
"dup2": "0x110ab0",
"printf": "0x64f00",
"puts": "0x80a30",
"read": "0x110180",
"str_bin_sh": "0x1b40fa",
"strcat": "0x9d800",
"strncpy": "0x9ddb0",
"system": "0x4f4e0",
"write": "0x110250"
}
}
]
$ curl -X POST -H 'Content-Type: application/json' --data '{"symbols": ["strcat"]}' 'https://libc.rip/api/libc/libc6_2.27-3ubuntu1.2_amd64'
{
"buildid": "d3cf764b2f97ac3efe366ddd07ad902fb6928fd7",
"download_url": "https://libc.rip/download/libc6_2.27-3ubuntu1.2_amd64.so",
"id": "libc6_2.27-3ubuntu1.2_amd64",
"md5": "35ef4ffc9c6ad7ffd1fd8c16f14dc766",
"sha1": "a22321cd65f28f70cf321614fdfd22f36ecd0afe",
"sha256": "f0ad9639b2530741046e06c96270b25da2339b6c15a7ae46de8fb021b3c4f529",
"symbols": {
"__libc_start_main_ret": "0x21b97",
"dup2": "0x110ab0",
"printf": "0x64f00",
"puts": "0x80a30",
"read": "0x110180",
"str_bin_sh": "0x1b40fa",
"strcat": "0x9d800",
"system": "0x4f4e0",
"write": "0x110250"
}
}
```
searchengine/api.yml
+92
View File
@@ -0,0 +1,92 @@
openapi: 3.0.3
info:
version: 1.0.0
title: libc-database search engine
description: ''
paths:
/find:
post:
operationId: app.find
tags:
- libcsearch
description: |-
Look up libc by various attributes
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Libc'
required: true
responses:
'200':
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/Libc'
description: ''
/libc/{id}:
post:
operationId: app.dump
tags:
- libcsearch
parameters:
- in: path
name: id
schema:
type: string
required: true
description: |-
Dump libc symbols
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/DumpRequest'
required: true
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/Libc'
description: ''
components:
schemas:
Libc:
type: object
properties:
md5:
type: string
sha1:
type: string
sha256:
type: string
buildid:
type: string
id:
type: string
symbols:
type: object
additionalProperties:
type: string
pattern: '^(0x)?[a-fA-F0-9]+$'
download_url:
type: string
format: url
readOnly: true
DumpRequest:
type: object
properties:
symbols:
type: array
items:
type: string
servers:
- url: https://libc.rip/api
searchengine/app.py
+99
View File
@@ -0,0 +1,99 @@
import logging
from functools import lru_cache
import connexion
from elasticsearch import Elasticsearch
import config
es = Elasticsearch(hosts=[config.ES_HOST])
log = logging.getLogger('wsgi')
log.info(f'Using elasticsearch server {config.ES_HOST}, index {config.ES_INDEX_NAME}')
@lru_cache(maxsize=2000)
def get_symbols(id):
syms = {}
with open(f'{config.DB_DIR}/{id}.symbols') as f:
for line in f:
if not line.strip():
continue
name, addr = line.split()
addr = int(addr, 16)
syms[name] = addr
return syms
def find(body, extra_symbols=[]):
filters = []
for h in ('id', 'md5', 'sha1', 'sha256', 'buildid'):
if h in body:
filters.append({'match': {h: body[h]}})
symbol_filters = body.get('symbols')
if symbol_filters:
terms = []
for sym, addr in symbol_filters.items():
addr = int(addr, 16)
term = f'{sym}@{addr & 0xfff:03x}'
filters.append({'term': {'symbols': term}})
if not filters:
return connexion.problem(
status=400,
title='Bad request',
detail='must provide at least one filter',
)
query = {"bool": {"filter": filters}}
res = es.search(index=config.ES_INDEX_NAME, body={"query": query})
libcs = []
for hit in res['hits']['hits']:
doc = hit['_source']
id = doc['id']
syms = get_symbols(id)
result_symbols = {}
names = list(config.DEFAULT_SYMBOLS) + extra_symbols
if symbol_filters:
names += symbol_filters.keys()
for name in names:
if name in syms:
result_symbols[name] = f'{syms[name]:#x}'
libcs.append({
'id': id,
'buildid': doc.get('buildid'),
'sha1': doc.get('sha1'),
'md5': doc.get('md5'),
'sha256': doc.get('sha256'),
'symbols': result_symbols,
'download_url': config.DOWNLOAD_URL.format(id),
})
return libcs
def dump(id, body):
res = find({'id': id}, extra_symbols=body['symbols'])
if not res:
return connexion.problem(
status=404,
title='Not found',
detail=f'Unknown ID: {id}'
)
return res[0]
app = connexion.App(__name__, specification_dir='.')
app.add_api('api.yml')
if __name__ == '__main__':
app.run(port=8080, host='127.0.0.1', debug=True)
searchengine/config.py
+21
View File
@@ -0,0 +1,21 @@
import os
ES_INDEX_NAME = 'libcsearch'
# ES_HOST = 'localhost'
ES_HOST = 'es01'
# DB_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../db'
DB_DIR = '/db'
DEFAULT_SYMBOLS = [
'__libc_start_main_ret',
'system',
'dup2',
'str_bin_sh',
'read',
'write',
'puts',
'printf',
]
DOWNLOAD_URL = 'https://libc.rip/download/{}.so'
searchengine/docker-compose.yml
+49
View File
@@ -0,0 +1,49 @@
version: '3.4'
services:
uwsgi:
build:
context: .
target: uwsgi
volumes:
- ../db:/db
ports:
- 127.0.0.1:8000:8000
networks:
- libcsearch
restart: always
logging:
driver: syslog
options:
tag: uwsgi
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- data01:/usr/share/elasticsearch/data
ports:
- 127.0.0.1:9200:9200
networks:
- libcsearch
restart: always
logging:
driver: syslog
options:
tag: elasticsearch
volumes:
data01:
driver: local
networks:
libcsearch:
searchengine/index.py
+111
View File
@@ -0,0 +1,111 @@
import argparse
import elasticsearch
import glob
import hashlib
import logging
import os
import re
import sys
from datetime import datetime
from elasticsearch import Elasticsearch
from subprocess import check_output, DEVNULL
import config
log = logging.getLogger('indexer')
BUILD_ID_REGEX = re.compile('Build ID: ([a-fA-F0-9]+)')
def get_build_id(fname):
try:
res = check_output(['readelf', '-n', fname], stderr=DEVNULL).decode()
except Exception:
return None
m = BUILD_ID_REGEX.search(res)
if m:
return m.group(1)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
p = argparse.ArgumentParser()
p.add_argument('--index', default=config.ES_INDEX_NAME)
p.add_argument('--fresh', action='store_true')
p.add_argument('dir')
args = p.parse_args()
es = Elasticsearch()
if args.fresh:
es.indices.delete(args.index, ignore=[404])
es.indices.create(args.index, ignore=[400])
es.indices.put_mapping(index=args.index, doc_type='libc', body={
'libc': {
"properties": {
"symbols": {
"type": "text",
"analyzer": "whitespace"
},
"sha1": {
"type": "keyword",
},
"md5": {
"type": "keyword",
},
"sha256": {
"type": "keyword",
},
"buildid": {
"type": "keyword",
},
"id": {
"type": "keyword",
},
}
}
}, include_type_name=True)
dir = os.path.abspath(args.dir)
for libc_fname in glob.glob(f'{dir}/*.so'):
id, _ = os.path.splitext(os.path.basename(libc_fname))
log.info(f'{id}: indexing')
res = es.get(index=args.index, id=id, ignore=[404])
if res['found']:
log.info(f'{id}: already exists')
continue
doc = {
'id': id,
'timestamp': datetime.now(),
}
syms = []
with open(f'{dir}/{id}.symbols') as f:
for line in f:
if not line.strip():
continue
name, addr = line.split()
addr = int(addr, 16)
syms.append(f'{name}@{addr & 0xfff:03x}')
doc['symbols'] = ' '.join(syms)
with open(libc_fname, 'rb') as f:
libc = f.read()
doc['sha1'] = hashlib.sha1(libc).hexdigest()
doc['sha256'] = hashlib.sha256(libc).hexdigest()
doc['md5'] = hashlib.md5(libc).hexdigest()
buildid = get_build_id(libc_fname)
if buildid:
doc['buildid'] = buildid
es.create(index=args.index, id=id, body=doc)
es.indices.refresh(index=args.index)
# res = es.search(index=args.index, body={"query": {"match": {"sha1": "102be3798e5d42044fb6b8f072ef609ef33ee5bf"}}})
# res = es.search(index=args.index, body={"query": {"match": {"buildid": "28a5cf977adc27c69ca78bedd595096dd1977a7d"}}})
# res = es.search(index=args.index, body={"query": {"term": {"symbols": "faccessat@190"}}})
# print("Got %d Hits:" % res['hits']['total']['value'])
# for hit in res['hits']['hits']:
# s = hit['_source']
# print(f"Found {s['id']}")
searchengine/nginx.conf
+48
View File
@@ -0,0 +1,48 @@
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
server {
listen 443;
server_name libc.rocks;
ssl on;
ssl_certificate /etc/letsencrypt/live/libc.rip/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/libc.rip/privkey.pem;
location /api/ {
proxy_pass http://127.0.0.1:8000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection keep-alive;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
location /download/ {
alias /home/niklas/libc-database/db/;
}
}
}
searchengine/requirements.txt
+3
View File
@@ -0,0 +1,3 @@
elasticsearch
connexion
uwsgi
searchengine/uwsgi.ini
+11
View File
@@ -0,0 +1,11 @@
[uwsgi]
module=app:app
master=true
workers=4
vacuum=true
http-socket=0.0.0.0:8000
die-on-term=True
buffer-size=262144
logger=syslog:uwsgi,local7
harakiri=30
enable-threads=true