Isolate the Redis to Postgres data migration tool.

Dockerfile

@@ -45,6 +45,4 @@ COPY ./backend .
 # Copy the frontend build
 COPY --from=builder /frontend/dist ./ui
 
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --start-interval=1s --retries=3 CMD [ "curl", "-f", "http://localhost:8000/health" ]
-
 ENTRYPOINT [ "uvicorn", "app.server:app", "--host", "0.0.0.0" ]

README.md

@@ -174,40 +174,7 @@ Navigate to [http://localhost:5173/](http://localhost:5173/) and enjoy!
 
 ## Migrating data from Redis to Postgres
 
-OpenGPTs previously used Redis for data persistence, but has since switched to Postgres. If you have data in Redis that you would like to migrate to Postgres, you can use the following steps:
-
-### With Docker
-
-Add Postgres' environment variables to the `.env` file (by following `.env.example`). Then, start the services with `docker compose` and run the following command to migrate data from Redis to Postgres:
-
-```shell
-docker exec -it opengpts-backend make redis_to_postgres
-```
-
-### Without Docker
-
-Make sure the following environment variables are set:
-
-```shell
-export POSTGRES_HOST=...
-export POSTGRES_PORT=...
-export POSTGRES_DB=...
-export POSTGRES_USER=...
-export POSTGRES_PASSWORD=...
-export REDIS_URL=...
-```
-
-Install [golang-migrate](https://github.com/golang-migrate/migrate) on your machine if you haven't already and run database schema migrations:
-
-```shell
-make migrate
-```
-
-Finally, run the following command to migrate data from Redis to Postgres:
-
-```shell
-make redis_to_postgres
-```
+Refer to this [guide](tools/redis_to_postgres/README.md) for migrating data from Redis to Postgres.
 
 ## Features
 

backend/Makefile

@@ -19,9 +19,6 @@ start:
 migrate:
 	migrate -database postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DB)?sslmode=disable -path ./migrations up
 
-redis_to_postgres:
-	poetry run python redis_to_postgres.py
-
 test:
 	# We need to update handling of env variables for tests
 	YDC_API_KEY=placeholder OPENAI_API_KEY=placeholder poetry run pytest $(TEST_FILE)

backend/poetry.lock

@@ -2994,24 +2994,6 @@ files = [
 [package.extras]
 full = ["numpy"]
 
-[[package]]
-name = "redis"
-version = "5.0.1"
-description = "Python client for Redis database and key-value store"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "redis-5.0.1-py3-none-any.whl", hash = "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f"},
-    {file = "redis-5.0.1.tar.gz", hash = "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f"},
-]
-
-[package.dependencies]
-async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""}
-
-[package.extras]
-hiredis = ["hiredis (>=1.0.0)"]
-ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"]
-
 [[package]]
 name = "regex"
 version = "2023.10.3"
@@ -4184,4 +4166,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9.0,<3.12"
-content-hash = "bc7c67284c2b95f2880a5bd9a1a6bff68ddb7484bfa04013874e9166899520e2"
+content-hash = "6aba3d05838348cb038b98803ed0d66e70caaa08a56d5a16fdbea3c5c63ec073"

backend/pyproject.toml

@@ -43,7 +43,6 @@ pgvector = "^0.2.5"
 psycopg2-binary = "^2.9.9"
 asyncpg = "^0.29.0"
 langchain-core = "^0.1.33"
-redis = "5.0.1"
 
 [tool.poetry.group.dev.dependencies]
 uvicorn = "^0.23.2"

docker-compose-prod.yml

@@ -1,13 +1,6 @@
 version: "3"
 
 services:
-  redis:
-    container_name: opengpts-redis
-    image: redis/redis-stack-server:latest
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-volume:/data
   postgres:
     image: pgvector/pgvector:pg16
     healthcheck:
@@ -44,4 +37,3 @@ services:
       - .env
     environment:
       POSTGRES_HOST: "postgres"
-      REDIS_URL: "redis://opengpts-redis:6379"

docker-compose.yml

@@ -1,13 +1,6 @@
 version: "3"
 
 services:
-  redis:
-    container_name: opengpts-redis
-    image: redis/redis-stack-server:latest
-    ports:
-      - "6379:6379"
-    volumes:
-      - ./redis-volume:/data
   postgres:
     image: pgvector/pgvector:pg16
     healthcheck:
@@ -47,7 +40,6 @@ services:
       - ./backend:/backend
     environment:
       POSTGRES_HOST: "postgres"
-      REDIS_URL: "redis://opengpts-redis:6379"
     command:
       - --reload
   frontend:

tools/redis_to_postgres/Dockerfile

@@ -0,0 +1,8 @@
+FROM langchain/open-gpts:latest
+
+RUN poetry add redis==5.0.1
+
+COPY migrate_data.py .
+
+# Run database schema migrations and then migrate data
+ENTRYPOINT sh -c "make migrate && python migrate_data.py"

tools/redis_to_postgres/README.md

@@ -0,0 +1,9 @@
+OpenGPTs previously used Redis for data persistence, but has since switched to Postgres. If you have data in Redis that you would like to migrate to Postgres, follow the instructions below.
+
+Navigate to the `tools/redis_to_postgres` directory and ensure that the environment variables in the docker-compose file are set correctly for your Redis and Postgres instances. Then, run the following command to perform the migration:
+
+```shell
+docker compose up --build --abort-on-container-exit
+```
+
+This will run database schema migrations for Postgres and then copy data from Redis to Postgres. Eventually all containers will be stopped.

tools/redis_to_postgres/docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3"
+
+services:
+  redis:
+    container_name: opengpts-redis
+    image: redis/redis-stack-server:latest
+    ports:
+      - "6380:6379"
+    volumes:
+      - ./../../redis-volume:/data
+  data-migrator:
+    build:
+      context: .
+    depends_on:
+      - redis
+    network_mode: "host"
+    environment:
+      REDIS_URL: "redis://localhost:6380"
+      POSTGRES_HOST: "localhost"
+      POSTGRES_PORT: 5433
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      OPENAI_API_KEY: ...