From 799b435f2c4f35db309aa45db6352fe652b70c40 Mon Sep 17 00:00:00 2001 From: sheepish Date: Wed, 3 May 2023 11:44:43 +0000 Subject: [PATCH] initial commit --- .gitignore | 1240 ++++++++++++++++++++++++++++++++++++ Readme.md | 55 ++ contrib/imageproxy.service | 17 + downloader.py | 143 +++++ main.py | 189 ++++++ nginx/imgproxy.conf | 103 +++ nginx/nginx.conf | 16 + requirements.txt | 4 + 8 files changed, 1767 insertions(+) create mode 100644 .gitignore create mode 100644 Readme.md create mode 100644 contrib/imageproxy.service create mode 100644 downloader.py create mode 100644 main.py create mode 100644 nginx/imgproxy.conf create mode 100644 nginx/nginx.conf create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c725bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,1240 @@ +### MonoDevelop template +#User Specific +*.userprefs +*.usertasks + +#Mono Project Files +*.pidb +*.resources +test-results/ + +### NotepadPP template +# Notepad++ backups # +*.bak + +### VisualStudioCode template +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VirtualEnv template +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json + +### Diff template +*.patch +*.diff + +### Eclipse template +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +### EiffelStudio template +# The compilation directory +EIFGENs + +### Dreamweaver template +# DW Dreamweaver added files +_notes +_compareTemp +configs/ +dwsync.xml +dw_php_codehinting.config +*.mno + +### PuTTY template +# Private key +*.ppk + +### WebMethods template +**/IntegrationServer/datastore/ +**/IntegrationServer/db/ +**/IntegrationServer/DocumentStore/ +**/IntegrationServer/lib/ +**/IntegrationServer/logs/ +**/IntegrationServer/replicate/ +**/IntegrationServer/sdk/ +**/IntegrationServer/support/ +**/IntegrationServer/update/ +**/IntegrationServer/userFtpRoot/ +**/IntegrationServer/web/ +**/IntegrationServer/WmRepository4/ +**/IntegrationServer/XAStore/ +**/IntegrationServer/packages/Wm*/ + +### MATLAB template +# Windows default autosave extension +*.asv + +# OSX / *nix default autosave extension +*.m~ + +# Compiled MEX binaries (all platforms) +*.mex* + +# Packaged app and toolbox files +*.mlappinstall +*.mltbx + +# Generated helpsearch folders +helpsearch*/ + +# Simulink code generation folders +slprj/ +sccprj/ + +# Matlab code generation folders +codegen/ + +# Simulink autosave extension +*.autosave + +# Simulink cache files +*.slxc + +# Octave session info +octave-workspace + +### Vagrant template +# General +.vagrant/ + +# Log files (if you are creating logs in debug mode, uncomment this) +# *.log + +### MicrosoftOffice template +*.tmp + +# Word temporary +~$*.doc* + +# Word Auto Backup File +Backup of *.doc* + +# Excel temporary +~$*.xls* + +# Excel Backup File +*.xlk + +# PowerPoint temporary +~$*.ppt* + +# Visio autosave temporary files +*.~vsd* + +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +.idea + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### DartEditor template +.project +.buildlog + +### Ninja template +.ninja_deps +.ninja_log + +### Archives template +# It's better to unpack these files and commit the raw source because +# git has its own built in compression methods. +*.7z +*.jar +*.rar +*.zip +*.gz +*.gzip +*.tgz +*.bzip +*.bzip2 +*.bz2 +*.xz +*.lzma +*.cab +*.xar + +# Packing-only formats +*.iso +*.tar + +# Package management formats +*.dmg +*.xpi +*.gem +*.egg +*.deb +*.rpm +*.msi +*.msm +*.msp +*.txz + +### Syncthing template +# Syncthing caches +.stversions + +### SynopsysVCS template +# Waveform formats +*.vcd +*.vpd +*.evcd +*.fsdb + +# Default name of the simulation executable. A different name can be +# specified with this switch (the associated daidir database name is +# also taken from here): -o / +simv + +# Generated for Verilog and VHDL top configs +simv.daidir/ +simv.db.dir/ + +# Infrastructure necessary to co-simulate SystemC models with +# Verilog/VHDL models. An alternate directory may be specified with this +# switch: -Mdir= +csrc/ + +# Log file - the following switch allows to specify the file that will be +# used to write all messages from simulation: -l +*.log + +# Coverage results (generated with urg) and database location. The +# following switch can also be used: urg -dir .vdb +simv.vdb/ +urgReport/ + +# DVE and UCLI related files. +DVEfiles/ +ucli.key + +# When the design is elaborated for DirectC, the following file is created +# with declarations for C/C++ functions. +vc_hdrs.h + +### Tags template +# Ignore tags created by etags, ctags, gtags (GNU global) and cscope +TAGS +.TAGS +!TAGS/ +tags +.tags +!tags/ +gtags.files +GTAGS +GRTAGS +GPATH +GSYMS +cscope.files +cscope.out +cscope.in.out +cscope.po.out + + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### TortoiseGit template +# Project-level settings +/.tgitconfig + +### Momentics template +# Built files +x86/ +arm/ +arm-p/ +translations/*.qm + +# IDE settings +.settings/ + +### Images template +# JPEG +*.jpg +*.jpeg +*.jpe +*.jif +*.jfif +*.jfi + +# JPEG 2000 +*.jp2 +*.j2k +*.jpf +*.jpx +*.jpm +*.mj2 + +# JPEG XR +*.jxr +*.hdp +*.wdp + +# Graphics Interchange Format +*.gif + +# RAW +*.raw + +# Web P +*.webp + +# Portable Network Graphics +*.png + +# Animated Portable Network Graphics +*.apng + +# Multiple-image Network Graphics +*.mng + +# Tagged Image File Format +*.tiff +*.tif + +# Scalable Vector Graphics +*.svg +*.svgz + +# Portable Document Format +*.pdf + +# X BitMap +*.xbm + +# BMP +*.bmp +*.dib + +# ICO +*.ico + +# 3D Images +*.3dm +*.max + +### Octave template +# Windows default autosave extension +*.asv + +# OSX / *nix default autosave extension +*.m~ + +# Compiled MEX binaries (all platforms) +*.mex* + +# Packaged app and toolbox files +*.mlappinstall +*.mltbx + +# Generated helpsearch folders +helpsearch*/ + +# Simulink code generation folders +slprj/ +sccprj/ + +# Matlab code generation folders +codegen/ + +# Simulink autosave extension +*.autosave + +# Simulink cache files +*.slxc + +# Octave session info +octave-workspace + +### Stata template +# .gitignore file for git projects containing Stata files +# Commercial statistical software: http://www.stata.com + +# Stata dataset and output files +*.dta +*.gph +*.log +*.smcl +*.stpr +*.stsem + +# Graphic export files from Stata +# Stata command graph export: http://www.stata.com/manuals14/g-2graphexport.pdf +# +# You may add graphic export files to your .gitignore. However you should be +# aware that this will exclude all image files from this main directory +# and subdirectories. +# *.ps +# *.eps +# *.wmf +# *.emf +# *.pdf +# *.png +# *.tif + +### AL template +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ +*.app +.snapshots/* + +### KDevelop4 template +*.kdev4 +.kdev4/ + +### SlickEdit template +# SlickEdit workspace and project files are ignored by default because +# typically they are considered to be developer-specific and not part of a +# project. +*.vpw +*.vpj + +# SlickEdit workspace history and tag files always contain user-specific +# data so they should not be stored in a repository. +*.vpwhistu +*.vpwhist +*.vtg + +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Ensime template +# Ensime specific +.ensime +.ensime_cache/ +.ensime_lucene/ + +### Emacs template +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### XilinxISE template +# intermediate build files +*.bgn +*.bit +*.bld +*.cmd_log +*.drc +*.ll +*.lso +*.msd +*.msk +*.ncd +*.ngc +*.ngd +*.ngr +*.pad +*.par +*.pcf +*.prj +*.ptwx +*.rbb +*.rbd +*.stx +*.syr +*.twr +*.twx +*.unroutes +*.ut +*.xpi +*.xst +*_bitgen.xwbt +*_envsettings.html +*_map.map +*_map.mrp +*_map.ngm +*_map.xrpt +*_ngdbuild.xrpt +*_pad.csv +*_pad.txt +*_par.xrpt +*_summary.html +*_summary.xml +*_usage.xml +*_xst.xrpt + +# iMPACT generated files +_impactbatch.log +impact.xsl +impact_impact.xwbt +ise_impact.cmd +webtalk_impact.xml + +# Core Generator generated files +xaw2verilog.log + +# project-wide generated files +*.gise +par_usage_statistics.html +usage_statistics_webtalk.html +webtalk.log +webtalk_pn.xml + +# generated folders +iseconfig/ +xlnx_auto_0_xdb/ +xst/ +_ngo/ +_xmsgs/ + +### Ansible template +*.retry + +### Calabash template +# Calabash / Cucumber +rerun/ +reports/ +screenshots/ +screenshot*.png +test-servers/ + +# bundler +.bundle +vendor + +### Patch template +*.orig +*.rej + +### Cloud9 template +# Cloud9 IDE - http://c9.io +.c9revisions +.c9 + +### CVS template +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore + +### Xcode template +## User settings +xcuserdata/ + +## Xcode 8 and earlier +*.xcscmblueprint +*.xccheckout + +### Kate template +# Swap Files # +.*.kate-swp +.swp.* + +### Mercurial template +.hg/ +.hgignore +.hgsigs +.hgsub +.hgsubstate +.hgtags + +### Metals template + # Generated Metals (Scala Language Server) files + # Reference: https://scalameta.org/metals/ +.metals/ +.bloop/ +project/metals.sbt + +### Backup template +*.bak +*.gho +*.ori +*.orig +*.tmp + +### LyX template +# Ignore LyX backup and autosave files +# http://www.lyx.org/ +*.lyx~ +*.lyx# + +### SVN template +.svn/ + +### Windows template +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### FlexBuilder template +bin/ +bin-debug/ +bin-release/ + +### SublimeText template +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +### Lazarus template +# Lazarus compiler-generated binaries (safe to delete) +*.exe +*.dll +*.so +*.dylib +*.lrs +*.res +*.compiled +*.dbg +*.ppu +*.o +*.or +*.a + +# Lazarus autogenerated files (duplicated info) +*.rst +*.rsj +*.lrt + +# Lazarus local files (user-specific info) +*.lps + +# Lazarus backups and unit output folders. +# These can be changed by user in Lazarus/project options. +backup/ +*.bak +lib/ + +# Application bundle for Mac OS +*.app/ + +### CodeKit template +# General CodeKit files to ignore +config.codekit +config.codekit3 +/min + +### BricxCC template +# Bricx Command Center IDE +# http://bricxcc.sourceforge.net +*.bak +*.sym + +### Bazaar template +.bzr/ +.bzrignore + +### LibreOffice template +# LibreOffice locks +.~lock.*# + +### Dropbox template +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + +### PSoCCreator template +# Project Settings +*.cywrk.* +*.cyprj.* + +# Generated Assets and Resources +Debug/ +Release/ +Export/ +*/codegentemp +*/Generated_Source +*_datasheet.pdf +*_timing.html +*.cycdx +*.cyfit +*.rpt +*.svd +*.log +*.zip + +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### JEnv template +# JEnv local Java version configuration file +.java-version + +# Used by previous versions of JEnv +.jenv-version + +### SBT template +# Simple Build Tool +# http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control + +dist/* +target/ +lib_managed/ +src_managed/ +project/boot/ +project/plugins/project/ +.history +.cache +.lib/ + +### Otto template +.otto/ + +### Espresso template +*.esproj + +### Redcar template +.redcar + +### Redis template +# Ignore redis binary dump (dump.rdb) files + +*.rdb + +### Anjuta template +# Local configuration folder and symbol database +/.anjuta/ +/.anjuta_sym_db.db + +### NetBeans template +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +### TextMate template +*.tmproj +*.tmproject +tmtags + +### ModelSim template +# ignore ModelSim generated files and directories (temp files and so on) +[_@]* + +# ignore compilation output of ModelSim +*.mti +*.dat +*.dbs +*.psm +*.bak +*.cmp +*.jpg +*.html +*.bsf + +# ignore simulation output of ModelSim +wlf* +*.wlf +*.vstf +*.ucdb +cov*/ +transcript* +sc_dpiheader.h +vsim.dbg + +### Vim template +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### JDeveloper template +# default application storage directory used by the IDE Performance Cache feature +.data/ + +# used for ADF styles caching +temp/ + +# default output directories +classes/ +deploy/ +javadoc/ + +# lock file, a part of Oracle Credential Store Framework +cwallet.sso.lck +### Virtuoso template +# Gitignore for Cadence Virtuoso +################################################################ + +# Log files +*.log +panic*.log.* + +# OpenAccess database lock files +*.cdslck* + +# Run directories for layout vs. schematic and design rule check +lvsRunDir/* +drcRunDir/* + +# Abstract generation tool +abstract.log* +abstract.record* + + +### GPG template +secring.* + + diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..a73d823 --- /dev/null +++ b/Readme.md @@ -0,0 +1,55 @@ +This is a python script + nginx config to store and serve approved images with +limited resizing capabilities. + +Image links are signed and expire regularly. Images can be proxied or stored +permanently. + +# Prerequisites + + * nginx + * imgproxy (https://github.com/imgproxy/imgproxy) + * python >= 3.9 + +# Install + + * edit `/tmp/imgproxy` paths in `nginx/imgproxy.conf` and move file to nginx's `sites-enabled` dir + * Create directory structure + + mkdir images ; for x in {a..z} {A..Z} {0..9} _ - ; do mkdir images/"$x" ; for y in {a..z} {A..Z} {0..9} _ - ; do mkdir images/"$x/$y" ; done ; done + + * in `main.py` change `SIG_SECRET` (same as gazelle's `IMAGE_CACHE_SECRET`) + +# Run + + * `uvicorn main:app` (or any other ASGI-compatible server) + +See `contrib/imageproxy.service` for a systemd service. + +# URL format description + +URI format: `/i/{img_size}/{signature}/{ext_url}{extension}` + +`img_size` can be one of: `full` or `{h}x{w}` (x denotes the literal character "x") +`h` and `w` can be an integer or empty string. The integer must be one of the +preconfigured and allowed sizes (in pixels) in the imageproxy server. If `w` +and `h` are given, the image is resized to the maximum size that keeps the correct +aspect ratio while not exceeding any of the two limits. Suggested values are 150/250/500. + +`signature` is the result of `urlsafe-base64(hmac-sha256-trunc(key={IMG_PROXY_SECRET}{current_year}{iso_current_week}, message={ext_url}{extension}))`. +HMAC result is truncated to 12 bytes, producing a 16 characters encoded signature. + +`ext_url` is the result of `urlsafe-base64(image_url)`. Trailing `=` may be stripped. + +`extension` is either the empty string or the string `/proxy` which instructs +the imageproxy server to not permanently store the source image. Resizing is +not supported in proxy-only mode. + +## Signature test vector + +``` +IMG_PROXY_SECRET="1234" +image_url="https://example_url/img.jpg" +date=1640730000 // (2021-12-29) +``` + +-> `/i/full/ae0_4RXxKhh6_AwX/aHR0cHM6Ly9leGFtcGxlX3VybC9pbWcuanBn` diff --git a/contrib/imageproxy.service b/contrib/imageproxy.service new file mode 100644 index 0000000..0e33290 --- /dev/null +++ b/contrib/imageproxy.service @@ -0,0 +1,17 @@ +[Unit] +Description=imageproxy-py image fetcher +After=nginx.service + +[Service] +User=imageproxy +Group=imageproxy +WorkingDirectory=/data/imageproxy-py +ExecStart=/data/imageproxy-py/venv/bin/uvicorn \ + --no-access-log --port 8091 main:app +ExecReload=/bin/kill -s HUP $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID +PrivateTmp=true +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..584c687 --- /dev/null +++ b/downloader.py @@ -0,0 +1,143 @@ +from typing import Mapping +import httpx +import time +import logging +from aiofiles import os, open as aio_open + + +logger = logging.getLogger('uvicorn.application') + + +class BadResponse(Exception): + def __init__(self, msg=None): + self.msg = msg + super().__init__() + + +class DownloadInProgress(Exception): + pass + + +class ImgproxyDownloader: + DEFAULT_OPTS = "max_bytes:2097152" # 2MiB + + def __init__(self, base_url: str): + self.base_url = base_url + self.client = httpx.AsyncClient(base_url=base_url, + follow_redirects=False) + + @staticmethod + def _error_lock_name(file_path: str) -> str: + return file_path + '.err' + + @staticmethod + def _headers_name(file_path: str) -> str: + return file_path + '.headers' + + @classmethod + def _get_full_path(cls, img_url: str) -> str: + return cls.DEFAULT_OPTS + '/' + img_url + + @classmethod + async def read_headers(cls, file_path: str) -> Mapping[str, str]: + headers = {} + headers_path = cls._headers_name(file_path) + async with aio_open(headers_path, mode='r') as fh: + for line in await fh.readlines(): + k, v = line.rstrip('\n').split(' ', maxsplit=1) + headers[k] = v + return headers + + @classmethod + async def _write_headers(cls, file_path: str, img_url: str, + headers: Mapping[str, str]) -> Mapping[str, str]: + headers_path = cls._headers_name(file_path) + save_headers = {'content-type': headers['content-type'], + 'x-original-url': img_url} + async with aio_open(headers_path, mode='w') as fh: + for pair in save_headers.items(): + await fh.write(' '.join(pair) + '\n') + + return save_headers + + async def save(self, img_url: str, file_path: str) -> Mapping[str, str]: + error_lock = ErrorLock(self._error_lock_name(file_path)) + has_error_lock = await error_lock.is_locked() + if has_error_lock: + logger.debug("found valid lock file, skipping download") + raise BadResponse(f"backoff {error_lock.timeleft} sec") + + tmp_path = file_path + '.tmp' + try: + fp = await aio_open(tmp_path, mode='xb') + except FileExistsError: + raise DownloadInProgress() + + try: + async with self.client.stream( + 'GET', self._get_full_path(img_url)) as resp: + if not resp or resp.status_code != 200: + raise BadResponse(f"bad http status: {resp.status_code}") + + async for chunk in resp.aiter_bytes(): + await fp.write(chunk) + + headers = await self._write_headers( + file_path, img_url, resp.headers) + except Exception as e: + await fp.close() + await error_lock.update() + await os.remove(tmp_path) + logger.info(f"failed to download: {e}") + raise BadResponse(f"download error") + else: + await fp.close() + await error_lock.remove() + await os.rename(tmp_path, file_path) + return headers + + +class ErrorLock: + INITIAL_LOCK_DURATION = 60 # 1 min + MAX_LOCK_DURATION = 60 * 60 * 24 # 1 day + + def __init__(self, path: str): + self.path = path + self.lock_time = None + self.lock_duration = self.INITIAL_LOCK_DURATION + self._now = self._get_now() + self.exists = False + + @property + def timeleft(self) -> int: + return self.lock_time - self._now + + @staticmethod + def _get_now() -> int: + return int(time.time()) + + async def is_locked(self) -> bool: + try: + async with aio_open(self.path, 'r') as fh: + lock_time, lock_duration = (await fh.read()).split(' ') + self.lock_time = int(lock_time) + self.lock_duration = min(self.MAX_LOCK_DURATION, int(lock_duration) * 2) + self.exists = True + return self.lock_time > self._now + except FileNotFoundError: + self.exists = False + except ValueError: + # invalid lock file somehow + logger.warning("found invalid lock file, removing") + await self.remove() + return False + + async def update(self) -> None: + lock_until = (self.lock_time or self._now) + self.lock_duration + async with aio_open(self.path, 'w') as fh: + await fh.write('{} {}'.format(lock_until, self.lock_duration)) + + async def remove(self) -> None: + if self.exists: + await os.remove(self.path) + self.exists = False diff --git a/main.py b/main.py new file mode 100644 index 0000000..e32b29f --- /dev/null +++ b/main.py @@ -0,0 +1,189 @@ +import asyncio +import datetime +import hmac +import logging +from os.path import join as path_join +from base64 import urlsafe_b64encode, urlsafe_b64decode +from hashlib import sha224, sha256 + +from aiofiles.os import path +from blacksheep import Application, Request, Response, bad_request +from blacksheep.server import responses + +from downloader import ImgproxyDownloader, DownloadInProgress, BadResponse + + +logger = logging.getLogger('uvicorn.application') + + +IMGPROXY_URL = 'http://127.0.0.1:8090/' +SIG_SECRET = b'1234' +SIG_LENGTH_BYTES = 12 +ALLOWED_SIZES = {b'150', b'250', b'500'} +DOWNLOAD_TIMEOUT_SEC = 30 + + +app = Application() + + +class BadOptions(Exception): + pass + + +class SignatureValidator: + SIG_LENGTH_BASE64 = int(SIG_LENGTH_BYTES * 1.34) + WEEK_DELTA = datetime.timedelta(weeks=1) + + def __init__(self, key: bytes): + self.key = key + + def _get_now(self) -> datetime.datetime: + return datetime.datetime.now() + + def _calc_sig(self, iso_tuple: tuple[int, int, int], msg: bytes) -> bytes: + key = self.key + "{:04}{:02}".format(iso_tuple.year, iso_tuple.week).encode('ascii') + return urlsafe_b64encode(hmac.HMAC(key, msg, digestmod=sha256) + .digest()[:SIG_LENGTH_BYTES]) + + def verify_signature(self, msg: bytes, sig: bytes) -> bool: + if len(sig) != self.SIG_LENGTH_BASE64: + return False + + now = self._get_now() + iso_week = now.isocalendar() + # check for current week + real_sig = self._calc_sig(iso_week, msg) + if sig == real_sig: + return True + + # check for last week + last_week = now - self.WEEK_DELTA + last_week_iso = last_week.isocalendar() + real_sig = self._calc_sig(last_week_iso, msg) + return sig == real_sig + + +def parse_imgproxy_params(params: str) -> list[bytes]: + if params == 'full': + return [] + params = params.encode('ascii') + sizes = params.split(b'x') + if not sizes: + raise BadOptions() + opts = [] + try: + height, width = sizes[0], sizes[1] + except: + raise BadOptions() + + if width and width in ALLOWED_SIZES: + opts.append(b'w:' + width) + + if height and height in ALLOWED_SIZES: + opts.append(b'h:' + height) + + if not opts: + raise BadOptions() + + return opts + + +def create_filename(img_url: bytes) -> bytes: + img_url_hash = sha224(img_url).digest() + hash_b64 = urlsafe_b64encode(img_url_hash) + return hash_b64[0:1] + b'/' + hash_b64[1:2] + b'/' + hash_b64 + + +requester = ImgproxyDownloader(IMGPROXY_URL) +#requester = ImgproxyDownloader('') +#requester._get_full_path = lambda x: urlsafe_b64decode(x).decode('ascii') + + +async def download_file(img_url: str, file_path: str, requester=requester): + try: + return await requester.save(img_url, file_path) + except DownloadInProgress: + for i in range(DOWNLOAD_TIMEOUT_SEC): + await asyncio.sleep(1) + if await path.isfile(file_path): + return await requester.read_headers(file_path) + raise BadResponse('stalled download') + + +validator = SignatureValidator(SIG_SECRET) + + +@app.router.get("/i/*") +async def handle_signed_request(request: Request) -> Response: + # check path + url = request.route_values['tail'].split('/') + if len(url) not in [3, 4]: + logger.debug("wrong number of path elements: %d", len(url)) + return bad_request() + + img_url = url[2].encode('ascii') + sig_msg = img_url + if len(url) == 4: + sig_msg += b'/' + url[3].encode('ascii') + + # validate signature + if not validator.verify_signature(sig_msg, url[1].encode('ascii')): + logger.debug("failed to verify signature %s for %s", url[1], sig_msg) + return responses.status_code(410) + + if logger.isEnabledFor(logging.INFO): + url_b64 = url[2] + ("=" * (len(url[2]) % 4)) + log_url = urlsafe_b64decode(url_b64).decode('ascii', errors='ignore') + if len(log_url) > 100 and not logger.isEnabledFor(logging.DEBUG): + log_url = log_url[:95] + '<...>' + else: + log_url = url[2] + + # parse options + is_proxy = len(url) == 4 and url[3] == 'proxy' + + try: + imgproxy_opts = [] if is_proxy else parse_imgproxy_params(url[0]) + except BadOptions: + logger.debug("invalid resize options: %s", url[0]) + return bad_request() + + is_resize = bool(imgproxy_opts) + + img_path = create_filename(img_url) + imgproxy_opts.append(b'fn:' + img_url) + + # download and perma-store image through imgproxy + base_path = request.get_first_header(b'root_dir').decode('ascii') + file_path = path_join(base_path, img_path.decode('ascii')) + if not is_proxy and not await path.isfile(file_path): + try: + headers = await download_file(url[2], file_path) + except BadResponse as e: + err_msg = "failed to fetch from remote server" + if e.msg: + err_msg += f" ({e.msg})" + + if e.msg.startswith('backoff'): + logger.debug("failed to fetch %s: %s", log_url, e.msg) + else: + logger.info("failed to fetch %s: %s", log_url, e.msg) + return responses.status_code(520, err_msg) + logger.info("downloaded image %s", log_url) + else: + headers = await requester.read_headers(file_path) + + # set headers and leave everything else to nginx + if is_proxy: + resp = responses.status_code(305) + elif is_resize: + resp = responses.see_other(b'/' + img_path) + else: + resp = responses.redirect(b'/' + img_path) + resp.set_header(b'imgproxy-opts', b'/'.join(imgproxy_opts) + b'/') + resp.set_header(b'imgproxy-path', img_path) + resp.set_header(b'imgproxy-url', img_url) + for k, v in headers.items(): + resp.set_header(k.encode('ascii'), v.encode('ascii')) + + return resp diff --git a/nginx/imgproxy.conf b/nginx/imgproxy.conf new file mode 100644 index 0000000..f83e1cb --- /dev/null +++ b/nginx/imgproxy.conf @@ -0,0 +1,103 @@ +proxy_cache_path /tmp/imgproxy/cache/ levels=2:2 keys_zone=imgproxy-cache:128m inactive=30d; + +upstream imgproxy { + server 127.0.0.1:8080; +} + +upstream imgproxy-py { + server 127.0.0.1:8091; +} + +server { + listen 127.0.0.1:34533; + root /tmp/imgproxy/images; + #log_subrequest on; + #rewrite_log on; + + if ($request_method !~ ^(GET|HEAD)$) { return 405; } + + set $is_resize ""; + set $img_url ""; + set $img_path ""; + set $loc_prefix "imgproxy"; + set $imgproxy_opts "max_bytes:2097152"; # 2MiB, used if no resize + + add_header root_dir ""; + + proxy_hide_header imgproxy-opts; + proxy_hide_header imgproxy-url; + proxy_hide_header imgproxy-path; + + location / { + deny all; + } + + # TODO: this is outdated + # workflow: + # 1. python code will be called + # 1.1 verify the signature + # 1.2 permanently save the image file through imgproxy if not yet + # available (unless /proxy) + # 1.3 emit headers to serve the correct image file and correct content type + # 2. nginx will, based on status code, serve the local file or forward + # the request to imgproxy + + location /i/ { + proxy_pass http://imgproxy-py; + proxy_intercept_errors on; + proxy_set_header root_dir $document_root; + error_page 302 = @serve-local; + error_page 303 = @serve-imgproxy; + error_page 305 = @serve-proxy; + } + + location @serve-imgproxy { + # for resized and non-cached images + set $img_path $upstream_http_imgproxy_path; + set $imgproxy_opts $imgproxy_opts/$upstream_http_imgproxy_opts; + set $img_url $upstream_http_imgproxy_url; + add_header Content-Type ""; + add_header x-original-url $upstream_http_x_original_url; + rewrite .* /$loc_prefix/${imgproxy_opts}plain/local:///${img_path}; + } + + location @serve-proxy { + # serve from perma-cache if available, otherwise nocache + set $loc_prefix "imgproxy-nocache"; + default_type ""; + add_header Content-Type $upstream_http_content_type; + add_header x-original-url $upstream_http_x_original_url; + try_files $upstream_http_location @serve-imgproxy; + } + + location @serve-local { + # serve locally stored image, downloaded by python + default_type ""; + add_header Content-Type $upstream_http_content_type; + add_header x-original-url $upstream_http_x_original_url; + try_files $upstream_http_location =404; + } + + location /imgproxy/ { + internal; + proxy_pass http://imgproxy; + proxy_temp_path /tmp/imgproxy/tmp 1; + proxy_cache imgproxy-cache; + proxy_cache_key $imgproxy_opts/$img_url; + proxy_cache_lock on; + proxy_cache_lock_age 10s; + proxy_cache_lock_timeout 10s; + proxy_cache_valid 200 1y; + proxy_cache_valid any 0s; + proxy_cache_use_stale updating timeout error; + proxy_ignore_client_abort on; + proxy_redirect off; + } + + location /imgproxy-nocache/ { + internal; + proxy_pass http://imgproxy; + proxy_buffering off; + proxy_redirect off; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..7b4fa44 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,16 @@ +pid /tmp/nginx.pid; +error_log nginx.log debug; +daemon off; + +events {} +http { + access_log /dev/stdout; + error_log /dev/stderr debug; + + client_body_temp_path /tmp/nginx 1 2; + proxy_temp_path /tmp/nginx-proxy; + fastcgi_temp_path /tmp/nginx-fastcgi; + uwsgi_temp_path /tmp/nginx-uwsgi; + + include imgproxy.conf; +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..287b57e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +blacksheep~=1.2.14 +uvicorn~=0.22.0 +aiofiles~=23.1.0 +httpx~=0.24.0 \ No newline at end of file