initial commit

Readme.md

@@ -0,0 +1,55 @@
+This is a python script + nginx config to store and serve approved images with
+limited resizing capabilities. 
+
+Image links are signed and expire regularly. Images can be proxied or stored
+permanently.
+
+# Prerequisites
+ 
+ * nginx
+ * imgproxy (https://github.com/imgproxy/imgproxy)
+ * python >= 3.9
+
+# Install
+
+ * edit `/tmp/imgproxy` paths in `nginx/imgproxy.conf` and move file to nginx's `sites-enabled` dir
+ * Create directory structure
+ 
+       mkdir images ; for x in {a..z} {A..Z} {0..9} _ - ; do mkdir images/"$x" ; for y in {a..z} {A..Z} {0..9} _ - ; do mkdir images/"$x/$y" ; done ; done
+
+ * in `main.py` change `SIG_SECRET` (same as gazelle's `IMAGE_CACHE_SECRET`)
+ 
+# Run
+
+ * `uvicorn main:app` (or any other ASGI-compatible server)
+
+See `contrib/imageproxy.service` for a systemd service.
+
+# URL format description
+
+URI format: `/i/{img_size}/{signature}/{ext_url}{extension}`
+
+`img_size` can be one of: `full` or `{h}x{w}` (x denotes the literal character "x")
+`h` and `w` can be an integer or empty string. The integer must be one of the
+preconfigured and allowed sizes (in pixels) in the imageproxy server. If `w`
+and `h` are given, the image is resized to the maximum size that keeps the correct
+aspect ratio while not exceeding any of the two limits. Suggested values are 150/250/500.
+
+`signature` is the result of `urlsafe-base64(hmac-sha256-trunc(key={IMG_PROXY_SECRET}{current_year}{iso_current_week}, message={ext_url}{extension}))`.
+HMAC result is truncated to 12 bytes, producing a 16 characters encoded signature.
+
+`ext_url` is the result of `urlsafe-base64(image_url)`. Trailing `=` may be stripped.
+
+`extension` is either the empty string or the string `/proxy` which instructs
+the imageproxy server to not permanently store the source image. Resizing is
+not supported in proxy-only mode.
+
+## Signature test vector
+
+```
+IMG_PROXY_SECRET="1234"
+image_url="https://example_url/img.jpg"
+date=1640730000 // (2021-12-29)
+```
+
+-> `/i/full/ae0_4RXxKhh6_AwX/aHR0cHM6Ly9leGFtcGxlX3VybC9pbWcuanBn`

contrib/imageproxy.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=imageproxy-py image fetcher
+After=nginx.service
+
+[Service]
+User=imageproxy
+Group=imageproxy
+WorkingDirectory=/data/imageproxy-py
+ExecStart=/data/imageproxy-py/venv/bin/uvicorn \
+    --no-access-log --port 8091 main:app
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

downloader.py

@@ -0,0 +1,143 @@
+from typing import Mapping
+import httpx
+import time
+import logging
+from aiofiles import os, open as aio_open
+
+
+logger = logging.getLogger('uvicorn.application')
+
+
+class BadResponse(Exception):
+    def __init__(self, msg=None):
+        self.msg = msg
+        super().__init__()
+
+
+class DownloadInProgress(Exception):
+    pass
+
+
+class ImgproxyDownloader:
+    DEFAULT_OPTS = "max_bytes:2097152"  # 2MiB
+
+    def __init__(self, base_url: str):
+        self.base_url = base_url
+        self.client = httpx.AsyncClient(base_url=base_url,
+                                        follow_redirects=False)
+
+    @staticmethod
+    def _error_lock_name(file_path: str) -> str:
+        return file_path + '.err'
+
+    @staticmethod
+    def _headers_name(file_path: str) -> str:
+        return file_path + '.headers'
+
+    @classmethod
+    def _get_full_path(cls, img_url: str) -> str:
+        return cls.DEFAULT_OPTS + '/' + img_url
+
+    @classmethod
+    async def read_headers(cls, file_path: str) -> Mapping[str, str]:
+        headers = {}
+        headers_path = cls._headers_name(file_path)
+        async with aio_open(headers_path, mode='r') as fh:
+            for line in await fh.readlines():
+                k, v = line.rstrip('\n').split(' ', maxsplit=1)
+                headers[k] = v
+        return headers
+
+    @classmethod
+    async def _write_headers(cls, file_path: str, img_url: str,
+                             headers: Mapping[str, str]) -> Mapping[str, str]:
+        headers_path = cls._headers_name(file_path)
+        save_headers = {'content-type': headers['content-type'],
+                        'x-original-url': img_url}
+        async with aio_open(headers_path, mode='w') as fh:
+            for pair in save_headers.items():
+                await fh.write(' '.join(pair) + '\n')
+
+        return save_headers
+
+    async def save(self, img_url: str, file_path: str) -> Mapping[str, str]:
+        error_lock = ErrorLock(self._error_lock_name(file_path))
+        has_error_lock = await error_lock.is_locked()
+        if has_error_lock:
+            logger.debug("found valid lock file, skipping download")
+            raise BadResponse(f"backoff {error_lock.timeleft} sec")
+
+        tmp_path = file_path + '.tmp'
+        try:
+            fp = await aio_open(tmp_path, mode='xb')
+        except FileExistsError:
+            raise DownloadInProgress()
+
+        try:
+            async with self.client.stream(
+                    'GET', self._get_full_path(img_url)) as resp:
+                if not resp or resp.status_code != 200:
+                    raise BadResponse(f"bad http status: {resp.status_code}")
+
+                async for chunk in resp.aiter_bytes():
+                    await fp.write(chunk)
+
+                headers = await self._write_headers(
+                    file_path, img_url, resp.headers)
+        except Exception as e:
+            await fp.close()
+            await error_lock.update()
+            await os.remove(tmp_path)
+            logger.info(f"failed to download: {e}")
+            raise BadResponse(f"download error")
+        else:
+            await fp.close()
+            await error_lock.remove()
+            await os.rename(tmp_path, file_path)
+            return headers
+
+
+class ErrorLock:
+    INITIAL_LOCK_DURATION = 60  # 1 min
+    MAX_LOCK_DURATION = 60 * 60 * 24  # 1 day
+
+    def __init__(self, path: str):
+        self.path = path
+        self.lock_time = None
+        self.lock_duration = self.INITIAL_LOCK_DURATION
+        self._now = self._get_now()
+        self.exists = False
+
+    @property
+    def timeleft(self) -> int:
+        return self.lock_time - self._now
+
+    @staticmethod
+    def _get_now() -> int:
+        return int(time.time())
+
+    async def is_locked(self) -> bool:
+        try:
+            async with aio_open(self.path, 'r') as fh:
+                lock_time, lock_duration = (await fh.read()).split(' ')
+            self.lock_time = int(lock_time)
+            self.lock_duration = min(self.MAX_LOCK_DURATION, int(lock_duration) * 2)
+            self.exists = True
+            return self.lock_time > self._now
+        except FileNotFoundError:
+            self.exists = False
+        except ValueError:
+            # invalid lock file somehow
+            logger.warning("found invalid lock file, removing")
+            await self.remove()
+        return False
+
+    async def update(self) -> None:
+        lock_until = (self.lock_time or self._now) + self.lock_duration
+        async with aio_open(self.path, 'w') as fh:
+            await fh.write('{} {}'.format(lock_until, self.lock_duration))
+
+    async def remove(self) -> None:
+        if self.exists:
+            await os.remove(self.path)
+            self.exists = False

main.py

@@ -0,0 +1,189 @@
+import asyncio
+import datetime
+import hmac
+import logging
+from os.path import join as path_join
+from base64 import urlsafe_b64encode, urlsafe_b64decode
+from hashlib import sha224, sha256
+
+from aiofiles.os import path
+from blacksheep import Application, Request, Response, bad_request
+from blacksheep.server import responses
+
+from downloader import ImgproxyDownloader, DownloadInProgress, BadResponse
+
+
+logger = logging.getLogger('uvicorn.application')
+
+
+IMGPROXY_URL = 'http://127.0.0.1:8090/'
+SIG_SECRET = b'1234'
+SIG_LENGTH_BYTES = 12
+ALLOWED_SIZES = {b'150', b'250', b'500'}
+DOWNLOAD_TIMEOUT_SEC = 30
+
+
+app = Application()
+
+
+class BadOptions(Exception):
+    pass
+
+
+class SignatureValidator:
+    SIG_LENGTH_BASE64 = int(SIG_LENGTH_BYTES * 1.34)
+    WEEK_DELTA = datetime.timedelta(weeks=1)
+
+    def __init__(self, key: bytes):
+        self.key = key
+
+    def _get_now(self) -> datetime.datetime:
+        return datetime.datetime.now()
+
+    def _calc_sig(self, iso_tuple: tuple[int, int, int], msg: bytes) -> bytes:
+        key = self.key + "{:04}{:02}".format(iso_tuple.year, iso_tuple.week).encode('ascii')
+        return urlsafe_b64encode(hmac.HMAC(key, msg, digestmod=sha256)
+                                 .digest()[:SIG_LENGTH_BYTES])
+
+    def verify_signature(self, msg: bytes, sig: bytes) -> bool:
+        if len(sig) != self.SIG_LENGTH_BASE64:
+            return False
+
+        now = self._get_now()
+        iso_week = now.isocalendar()
+        # check for current week
+        real_sig = self._calc_sig(iso_week, msg)
+        if sig == real_sig:
+            return True
+
+        # check for last week
+        last_week = now - self.WEEK_DELTA
+        last_week_iso = last_week.isocalendar()
+        real_sig = self._calc_sig(last_week_iso, msg)
+        return sig == real_sig
+
+
+def parse_imgproxy_params(params: str) -> list[bytes]:
+    if params == 'full':
+        return []
+    params = params.encode('ascii')
+    sizes = params.split(b'x')
+    if not sizes:
+        raise BadOptions()
+    opts = []
+    try:
+        height, width = sizes[0], sizes[1]
+    except:
+        raise BadOptions()
+
+    if width and width in ALLOWED_SIZES:
+        opts.append(b'w:' + width)
+
+    if height and height in ALLOWED_SIZES:
+        opts.append(b'h:' + height)
+
+    if not opts:
+        raise BadOptions()
+
+    return opts
+
+
+def create_filename(img_url: bytes) -> bytes:
+    img_url_hash = sha224(img_url).digest()
+    hash_b64 = urlsafe_b64encode(img_url_hash)
+    return hash_b64[0:1] + b'/' + hash_b64[1:2] + b'/' + hash_b64
+
+
+requester = ImgproxyDownloader(IMGPROXY_URL)
+#requester = ImgproxyDownloader('')
+#requester._get_full_path = lambda x: urlsafe_b64decode(x).decode('ascii')
+
+
+async def download_file(img_url: str, file_path: str, requester=requester):
+    try:
+        return await requester.save(img_url, file_path)
+    except DownloadInProgress:
+        for i in range(DOWNLOAD_TIMEOUT_SEC):
+            await asyncio.sleep(1)
+            if await path.isfile(file_path):
+                return await requester.read_headers(file_path)
+        raise BadResponse('stalled download')
+
+
+validator = SignatureValidator(SIG_SECRET)
+
+
+@app.router.get("/i/*")
+async def handle_signed_request(request: Request) -> Response:
+    # check path
+    url = request.route_values['tail'].split('/')
+    if len(url) not in [3, 4]:
+        logger.debug("wrong number of path elements: %d", len(url))
+        return bad_request()
+
+    img_url = url[2].encode('ascii')
+    sig_msg = img_url
+    if len(url) == 4:
+        sig_msg += b'/' + url[3].encode('ascii')
+
+    # validate signature
+    if not validator.verify_signature(sig_msg, url[1].encode('ascii')):
+        logger.debug("failed to verify signature %s for %s", url[1], sig_msg)
+        return responses.status_code(410)
+
+    if logger.isEnabledFor(logging.INFO):
+        url_b64 = url[2] + ("=" * (len(url[2]) % 4))
+        log_url = urlsafe_b64decode(url_b64).decode('ascii', errors='ignore')
+        if len(log_url) > 100 and not logger.isEnabledFor(logging.DEBUG):
+            log_url = log_url[:95] + '<...>'
+    else:
+        log_url = url[2]
+
+    # parse options
+    is_proxy = len(url) == 4 and url[3] == 'proxy'
+
+    try:
+        imgproxy_opts = [] if is_proxy else parse_imgproxy_params(url[0])
+    except BadOptions:
+        logger.debug("invalid resize options: %s", url[0])
+        return bad_request()
+
+    is_resize = bool(imgproxy_opts)
+
+    img_path = create_filename(img_url)
+    imgproxy_opts.append(b'fn:' + img_url)
+
+    # download and perma-store image through imgproxy
+    base_path = request.get_first_header(b'root_dir').decode('ascii')
+    file_path = path_join(base_path, img_path.decode('ascii'))
+    if not is_proxy and not await path.isfile(file_path):
+        try:
+            headers = await download_file(url[2], file_path)
+        except BadResponse as e:
+            err_msg = "failed to fetch from remote server"
+            if e.msg:
+                err_msg += f" ({e.msg})"
+
+            if e.msg.startswith('backoff'):
+                logger.debug("failed to fetch %s: %s", log_url, e.msg)
+            else:
+                logger.info("failed to fetch %s: %s", log_url, e.msg)
+            return responses.status_code(520, err_msg)
+        logger.info("downloaded image %s", log_url)
+    else:
+        headers = await requester.read_headers(file_path)
+
+    # set headers and leave everything else to nginx
+    if is_proxy:
+        resp = responses.status_code(305)
+    elif is_resize:
+        resp = responses.see_other(b'/' + img_path)
+    else:
+        resp = responses.redirect(b'/' + img_path)
+    resp.set_header(b'imgproxy-opts', b'/'.join(imgproxy_opts) + b'/')
+    resp.set_header(b'imgproxy-path', img_path)
+    resp.set_header(b'imgproxy-url', img_url)
+    for k, v in headers.items():
+        resp.set_header(k.encode('ascii'), v.encode('ascii'))
+
+    return resp

nginx/imgproxy.conf

@@ -0,0 +1,103 @@
+proxy_cache_path /tmp/imgproxy/cache/ levels=2:2 keys_zone=imgproxy-cache:128m inactive=30d;
+
+upstream imgproxy {
+    server 127.0.0.1:8080;
+}
+
+upstream imgproxy-py {
+    server 127.0.0.1:8091;
+}
+
+server {
+    listen 127.0.0.1:34533;
+    root /tmp/imgproxy/images;
+    #log_subrequest on;
+    #rewrite_log on;
+
+    if ($request_method !~ ^(GET|HEAD)$) { return 405; }
+
+    set $is_resize "";
+    set $img_url "";
+    set $img_path "";
+    set $loc_prefix "imgproxy";
+    set $imgproxy_opts "max_bytes:2097152"; # 2MiB, used if no resize
+
+    add_header root_dir "";
+
+    proxy_hide_header imgproxy-opts;
+    proxy_hide_header imgproxy-url;
+    proxy_hide_header imgproxy-path;
+    
+    location / {
+        deny all;
+    }
+
+    # TODO: this is outdated
+    # workflow:
+    # 1. python code will be called
+    # 1.1 verify the signature
+    # 1.2 permanently save the image file through imgproxy if not yet
+    #     available (unless /proxy)
+    # 1.3 emit headers to serve the correct image file and correct content type
+    # 2. nginx will, based on status code, serve the local file or forward
+    #    the request to imgproxy
+
+    location /i/ {
+        proxy_pass http://imgproxy-py;
+        proxy_intercept_errors on;
+        proxy_set_header root_dir $document_root;
+        error_page 302 = @serve-local;
+        error_page 303 = @serve-imgproxy;
+        error_page 305 = @serve-proxy;
+    }
+
+    location @serve-imgproxy {
+        # for resized and non-cached images
+        set $img_path $upstream_http_imgproxy_path;
+        set $imgproxy_opts $imgproxy_opts/$upstream_http_imgproxy_opts;
+        set $img_url $upstream_http_imgproxy_url;
+        add_header Content-Type "";
+        add_header x-original-url $upstream_http_x_original_url;
+        rewrite .* /$loc_prefix/${imgproxy_opts}plain/local:///${img_path};
+    }
+
+    location @serve-proxy {
+        # serve from perma-cache if available, otherwise nocache
+        set $loc_prefix "imgproxy-nocache";
+        default_type "";
+        add_header Content-Type $upstream_http_content_type;
+        add_header x-original-url $upstream_http_x_original_url;
+        try_files $upstream_http_location @serve-imgproxy;
+    }
+
+    location @serve-local {
+        # serve locally stored image, downloaded by python
+        default_type "";
+        add_header Content-Type $upstream_http_content_type;
+        add_header x-original-url $upstream_http_x_original_url;
+        try_files $upstream_http_location =404;
+    }
+
+    location /imgproxy/ {
+        internal;
+        proxy_pass http://imgproxy;
+        proxy_temp_path /tmp/imgproxy/tmp 1;
+        proxy_cache imgproxy-cache;
+        proxy_cache_key $imgproxy_opts/$img_url;
+        proxy_cache_lock on;
+        proxy_cache_lock_age 10s;
+        proxy_cache_lock_timeout 10s;
+        proxy_cache_valid 200 1y;
+        proxy_cache_valid any 0s;
+        proxy_cache_use_stale updating timeout error;
+        proxy_ignore_client_abort on;
+        proxy_redirect off;
+    }
+
+    location /imgproxy-nocache/ {
+        internal;
+        proxy_pass http://imgproxy;
+        proxy_buffering off;
+        proxy_redirect off;
+    }
+}

nginx/nginx.conf

@@ -0,0 +1,16 @@
+pid /tmp/nginx.pid;
+error_log nginx.log debug;
+daemon off;
+
+events {}
+http {
+    access_log /dev/stdout;
+    error_log /dev/stderr debug;
+    
+    client_body_temp_path /tmp/nginx 1 2;
+    proxy_temp_path /tmp/nginx-proxy;
+    fastcgi_temp_path /tmp/nginx-fastcgi;
+    uwsgi_temp_path /tmp/nginx-uwsgi;
+
+    include imgproxy.conf;
+}

requirements.txt

@@ -0,0 +1,4 @@
+blacksheep~=1.2.14
+uvicorn~=0.22.0
+aiofiles~=23.1.0
+httpx~=0.24.0