From ef8d4455d119f1bc491ea790eaa7633349c52d26 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Sat, 18 Nov 2023 13:50:44 -0800 Subject: [PATCH 1/4] Working on docker build improvements --- docker/DOCKER.md | 46 ++++++++++++++++++++++++++++--------- docker/Dockerfile | 15 +++++------- docker/docker-compose.yml | 6 +++-- docker/docker-entrypoint.sh | 19 +++++++++++---- 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 29c5012..3e8b227 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -2,29 +2,53 @@ Running VectorAdmin in Docker is the easily way to get a locally hosted option running as quickly as possible. - ## Requirements - Install [Docker](https://www.docker.com/) on your computer or machine. +- A running Postgres DB (RDS, remote, docker, local host machine) -## How to install +**Note** Failure to use a valid DB connection string prior to building or starting vector-admin will result in a failure. +On boot the `vdbms` database will be created using the connection string. + +**Run containerized Postgres DB** +Run this command first to get a dockerized Postgres container running: +`docker-compose up -d --build postgres` + +## Run from Docker pre-built image +- `git clone git@github.com:Mintplex-Labs/vector-admin.git` +- `cd vector-admin` +- `cd docker/` +- `cp .env.example .env`. +- Edit `.env` file and update the variables. **please** update all of the following: +```shell +JWT_SECRET="some-random-string" +SYS_EMAIL="root@vectoradmin.com" +SYS_PASSWORD="password" +DATABASE_CONNECTION_STRING="postgresql://vectoradmin:password@host.docker.internal:5433/vdbms" # Valid PG Connection string. +INNGEST_SIGNING_KEY="some-random-string" +``` + + + + ## How to use the user interface diff --git a/docker/Dockerfile b/docker/Dockerfile index a6a0ede..1f22ac6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,19 +1,20 @@ # Setup base image FROM ubuntu:jammy-20230522 AS base -# Build arguments -ARG DATABASE_CONNECTION_STRING - # Install system dependencies RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - curl libgfortran5 python3 python3-pip tzdata netcat \ + curl gnupg libgfortran5 python3 python3-pip tzdata netcat \ libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release \ xdg-utils && \ - curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ apt-get install -yq --no-install-recommends nodejs && \ curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ && dpkg -i yarn_1.22.19_all.deb \ @@ -79,10 +80,6 @@ RUN cd /app/document-processor && \ . v-env/bin/activate && \ pip install --no-cache-dir -r requirements.txt -# Reown files and init Prisma Client -RUN cd ./backend && DATABASE_CONNECTION_STRING=$DATABASE_CONNECTION_STRING npx prisma generate --schema=./prisma/schema.prisma -RUN cd ./backend && DATABASE_CONNECTION_STRING=$DATABASE_CONNECTION_STRING npx prisma migrate deploy --schema=./prisma/schema.prisma - # Setup the environment ENV NODE_ENV=production ENV PATH=/app/document-processor/v-env/bin:$PATH diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 00651b0..a825676 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,8 +26,10 @@ services: build: context: ../. dockerfile: ./docker/Dockerfile - args: - DATABASE_CONNECTION_STRING: ${DATABASE_CONNECTION_STRING} + volumes: + - "./.env:/app/backend/.env" + - "../backend/storage:/app/backend/storage" + - "../document-processor/hotdir/:/app/document-processor/hotdir" ports: - "3001:3001" - "3355:3355" diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 8ca4b79..a47fa75 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,18 @@ #!/bin/bash + +# Migrate Postgres using and prisma client +cd /app/backend +DATABASE_CONNECTION_STRING=$DATABASE_CONNECTION_STRING npx prisma migrate deploy --schema=./prisma/schema.prisma +DATABASE_CONNECTION_STRING=$DATABASE_CONNECTION_STRING npx prisma generate --schema=./prisma/schema.prisma + +# Start backend server & workers node /app/backend/index.js & NODE_ENV=development node /app/workers/index.js & -{ cd /app/workers/ && NODE_ENV=development yarn run inngest-cli dev -u http://0.0.0.0:3355/background-workers; } & -{ FLASK_ENV=production FLASK_APP=wsgi.py cd document-processor && gunicorn --timeout 300 --workers 4 --bind 0.0.0.0:8888 wsgi:api; } & -wait -n -exit $? -yarn run inngest-cli dev -u http://127.0.0.1:3355/background-workers \ No newline at end of file +cd /app/workers/ +NODE_ENV=development yarn run inngest-cli dev -u http://0.0.0.0:3355/background-workers & + +cd /app/document-processor +FLASK_ENV=production FLASK_APP=wsgi.py gunicorn --timeout 300 --workers 4 --bind 0.0.0.0:8888 wsgi:api & +wait -n +exit $? \ No newline at end of file From 76cf68803395bc3482c75676b22548ed684d3c8d Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Sat, 18 Nov 2023 14:37:24 -0800 Subject: [PATCH 2/4] update docker build file --- docker/DOCKER.md | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 3e8b227..b167402 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -14,6 +14,21 @@ Run this command first to get a dockerized Postgres container running: `docker-compose up -d --build postgres` ## Run from Docker pre-built image +- `docker pull mintplexlabs/vectoradmin:master` to pull in latest image +- Run the command with env variables and image defined. +`docker run -d -p 3001:3001 \ +-e SERVER_PORT="3001" \ +-e JWT_SECRET="your-random-string-here" \ +-e SYS_EMAIL="root@vectoradmin.com" \ +-e SYS_PASSWORD="password" \ +-e INNGEST_EVENT_KEY="background_workers" \ +-e INNGEST_SIGNING_KEY="random-string-goes-here" \ +-e INNGEST_LANDING_PAGE="true" \ +-e DATABASE_CONNECTION_STRING="postgresql://vectoradmin:password@xxxxxxx:5432/vdbms" \ +mintplexlabs/vectoradmin:master` + + +## Build docker image from source - `git clone git@github.com:Mintplex-Labs/vector-admin.git` - `cd vector-admin` - `cd docker/` @@ -26,32 +41,10 @@ SYS_PASSWORD="password" DATABASE_CONNECTION_STRING="postgresql://vectoradmin:password@host.docker.internal:5433/vdbms" # Valid PG Connection string. INNGEST_SIGNING_KEY="some-random-string" ``` +- `docker-compose up -d --build vector-admin` - - - - -## How to use the user interface +## How to use the user interface and login for the first time. - To access the full application, visit `http://localhost:3001` in your browser. - You first login will require you to use the `SYS_EMAIL` and `SYS_PASSWORD` set in the `.env` file. After onboarding this login will be permanently disabled. From 4245722d147f6d6231e1d61cddc96efd0e93ee4b Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Sat, 18 Nov 2023 14:47:24 -0800 Subject: [PATCH 3/4] enable automatic builds --- .github/workflows/build-and-push-image.yaml | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/build-and-push-image.yaml diff --git a/.github/workflows/build-and-push-image.yaml b/.github/workflows/build-and-push-image.yaml new file mode 100644 index 0000000..f7cca55 --- /dev/null +++ b/.github/workflows/build-and-push-image.yaml @@ -0,0 +1,46 @@ +name: Publish Docker image and Github Registry + +on: + push: + branches: ['docker-builds'] + +jobs: + push_to_registries: + name: Push Docker image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: | + mintplexlabs/vectoradmin + ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 8680090f5280de0b43cc36111144ead252eb62f4 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Sat, 18 Nov 2023 14:56:45 -0800 Subject: [PATCH 4/4] target master branch --- .github/workflows/build-and-push-image.yaml | 2 +- docker/DOCKER.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push-image.yaml b/.github/workflows/build-and-push-image.yaml index f7cca55..37a7ceb 100644 --- a/.github/workflows/build-and-push-image.yaml +++ b/.github/workflows/build-and-push-image.yaml @@ -2,7 +2,7 @@ name: Publish Docker image and Github Registry on: push: - branches: ['docker-builds'] + branches: ['master'] jobs: push_to_registries: diff --git a/docker/DOCKER.md b/docker/DOCKER.md index b167402..69b1873 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -46,8 +46,7 @@ INNGEST_SIGNING_KEY="some-random-string" ## How to use the user interface and login for the first time. - To access the full application, visit `http://localhost:3001` in your browser. -- You first login will require you to use the `SYS_EMAIL` and `SYS_PASSWORD` set in the `.env` file. After onboarding this login will be permanently disabled. - +- You first login will require you to use the `SYS_EMAIL` and `SYS_PASSWORD` set via ENV during build or run. After onboarding this login will be permanently disabled. # Connecting to a Vector Database