Move CI pipeline to Github actions (#70)

.github/workflows/build.yml

@@ -0,0 +1,173 @@
+name: Build
+
+on:
+  push:
+    branches: [ develop, master ]
+  pull_request:
+    branches: [ develop ]
+  release:
+    types: [ published ]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    services:
+      postgres:
+        image: postgres:13.2
+        env:
+          POSTGRES_DB: postgres        
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16.x
+
+    - name: Install Node
+      uses: actions/setup-node@v2
+      with:
+        node-version: '14'
+
+    - name: Checkout
+      run: git fetch --prune --unshallow --tags
+
+    - name: Cache node modules
+      uses: actions/cache@v2
+      env:
+        cache-name: cache-node_modules
+      with:
+        path: frontend/node_modules
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('frontend/yarn.lock') }}
+
+    - name: Cache UI build
+      uses: actions/cache@v2
+      id: cache-ui
+      env:
+        cache-name: cache-ui
+      with:
+        path: ui/v2.5/build
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('frontend/yarn.lock', 'frontend/public/**', 'frontend/src/**', 'graphql/**/*.graphql') }}
+
+    - name: Cache go build
+      uses: actions/cache@v2
+      env:
+        cache-name: cache-go-cache
+      with:
+        path: .go-cache
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+
+    - name: Pre-install
+      run: make pre-ui
+
+    - name: Validate UI
+      # skip UI validation for pull requests if UI is unchanged
+      if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
+      run: make ui-validate
+
+    - name: Generate
+      run: make generate
+
+    - name: Run tests
+      run: POSTGRES_DB=postgres:postgres@localhost/postgres?sslmode=disable make vet it
+
+    - name: Build UI
+      # skip UI build for pull requests if UI is unchanged (UI was cached)
+      # this means that the build version/time may be incorrect if the UI is
+      # not changed in a pull request
+      if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
+      run: make ui-only
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1 
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Crosscompile binaries
+      run: make cross-compile
+
+    - name: Generate checksums
+      run: |
+        git describe --tags --exclude latest_develop | tee CHECKSUMS_SHA1
+        sha1sum dist/*/stash-box-* | sed 's/dist\/.*\///g' | tee -a CHECKSUMS_SHA1
+        echo "STASH_BOX_VERSION=$(git describe --tags --exclude latest_develop)" >> $GITHUB_ENV
+        echo "RELEASE_DATE=$(date +'%Y-%m-%d %H:%M:%S %Z')" >> $GITHUB_ENV
+
+    - name: Upload Windows binary
+      # only upload binaries for pull requests
+      if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
+      uses: actions/upload-artifact@v2
+      with:
+        name: stash-box-win.exe
+        path: dist/stash-box_windows_amd64/stash-box-windows.exe
+    - name: Upload OSX binary
+      # only upload binaries for pull requests
+      if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
+      uses: actions/upload-artifact@v2
+      with:
+        name: stash-box-darwin
+        path: dist/stash-box_darwin_amd64/stash-box-darwin
+    - name: Upload Linux binary
+      # only upload binaries for pull requests
+      if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
+      uses: actions/upload-artifact@v2
+      with:
+        name: stash-box-linux
+        path: dist/stash-box_linux_amd64/stash-box-linux
+        
+    - name: Update latest_develop tag
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
+      run : git tag -f latest_develop; git push -f --tags
+    - name: Development Release
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}      
+      uses: meeDamian/github-release@2.0
+      with:
+        token: "${{ secrets.GITHUB_TOKEN }}"
+        prerelease: true
+        allow_override: true
+        tag: latest_develop
+        name: "${{ env.STASH_BOX_VERSION }}: Latest development build"
+        body: "**${{ env.RELEASE_DATE }}**\n This is always the latest committed version on the develop branch. Use as your own risk!"
+        files: |
+          dist/stash-box_windows_amd64/stash-box-windows.exe
+          dist/stash-box_linux_amd64/stash-box-linux
+          dist/stash-box_darwin_amd64/stash-box-darwin
+          CHECKSUMS_SHA1
+        gzip: false
+    - name: Master release
+      if: ${{ github.event_name == 'release' && github.ref != 'refs/tags/latest_develop' }}
+      uses: meeDamian/github-release@2.0
+      with:
+        token: "${{ secrets.GITHUB_TOKEN }}"
+        allow_override: true
+        files: |
+          dist/stash-box_windows_amd64/stash-box-windows.exe
+          dist/stash-box_linux_amd64/stash-box-linux
+          dist/stash-box_darwin_amd64/stash-box-darwin
+          CHECKSUMS_SHA1
+        gzip: false
+    
+    - name: Development Docker
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
+      run: |
+        docker build -t stashapp/stash-box:development -f ./docker/ci/x86_64/Dockerfile ./dist
+        docker push stashapp/stash-box:development
+
+    - name: Release Docker
+      if: ${{ github.event_name == 'release' && github.ref != 'refs/tags/latest_develop' }}
+      run: |
+        docker build -t stashapp/stash-box:latest -f ./docker/ci/x86_64/Dockerfile ./dist
+        docker push stashapp/stash-box:latest

.goreleaser.yml

@@ -0,0 +1,15 @@
+project_name: stash-box
+builds:
+- id: stash-box
+  binary: stash-box-{{.Os}}
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+  ldflags:
+    - -X github.com/stashapp/stash-box/pkg/api.version={{.Version}} -X github.com/stashapp/stash-box/pkg/api.buildstamp={{.Date}} -X github.com/stashapp/stash-box/pkg/api.githash={{.ShortCommit}}
+
+archives:
+- format: binary

.travis.yml

@@ -1,102 +0,0 @@
-if: tag != latest_develop # dont build for the latest_develop tagged version
-
-dist: xenial
-git:
-  depth: false
-language: go
-go:
-  - 1.13.x
-services:
-  - docker
-addons:
-  apt:
-    packages:
-      - postgresql-12
-      - postgresql-client-12
-env:
-  global:
-  - GO111MODULE=on
-  - PGVER=12
-  - PGPORT=5433
-  - PGUSER=postgres
-before_install:
-  - sudo cp /etc/postgresql/{9.6,12}/main/pg_hba.conf
-  - sudo pg_ctlcluster 12 main restart
-  - echo -e "machine github.com\n  login $CI_USER_TOKEN" > ~/.netrc
-  - nvm install 12
-  - travis_retry make pre-ui
-  - make generate
-  - CI=false make ui-only
-  #- go get -v github.com/mgechev/revive
-before_script:
-  - psql -c 'create database "stash-box-test";' -U postgres
-  - psql -c 'CREATE EXTENSION pg_trgm;' -U postgres stash-box-test
-script:
-  # left lint off to avoid getting extra dependency
-  #- make lint
-  - make vet it
-after_success:
-  - docker pull stashapp/box-compiler:1
-  - sh ./scripts/cross-compile.sh
-before_deploy:
-  # push the latest tag when on the develop branch
-  - if [ "$TRAVIS_BRANCH" = "develop" ]; then git tag -f latest_develop; git push -f --tags; fi
-  - export RELEASE_DATE=$(date +'%Y-%m-%d %H:%M:%S %Z')
-  - export STASH_VERSION=$(git describe --tags --exclude latest_develop)
-  # set TRAVIS_TAG explcitly to the version so that it doesn't pick up latest_develop
-  - if [ "$TRAVIS_BRANCH" = "master"]; then export TRAVIS_TAG=${STASH_VERSION}; fi
-deploy:
-  # latest develop release
-  - provider: releases
-    # use the v2 release provider for proper release note setting
-    edge: true
-    api_key:
-      secure: NTTT3AiH8zfwf69RBXm6r3abDnuL16be89BTUscdAuLVXvE+FHaPGDCjpyZHY9nIcYQRO5MvPl8UfK0Fauwh970yh6hBGZ9C9jwJ5FJPc9i2PfVxETL7HvMdqONaNyArFE1HirHnAO/ePxbUS1omb6EDaf/q5nQbZECL18rhtVdtuyceN6qMlZNsDahPyUwCtCaVvbDB0sOLR/Lvx+nReOyumNWquHHZrmP6Z7q6lLT1MeLzCakUiy7NG1zB4XZg1qUk9fM3hwfXGZM695wegQ7GyIRWu9fsT7Fb4h211OHiHF8Tw4cNSN5JPcamoGm58CHij7ivTchVF+HbMCc4I2BgeRuTGlD4EKmGlrEOEjN65Rfqmr49Sd8ymsNC/XMnj9edUJj0smPJyMc4uHjDVODr1CzuSLFmQsjF3wkIW529x5//NBAnxfe18sG1edQwphj08SMGav6rL4p57sKX0ea3UaOCHmBqKHa6CxBPlrYXxdQt8Q9Q3H0LzQvrInHi0koUpuA4ccj8T+q9pozX3SxYPlLtJTZWKLKKUBfDWsr4esAXzp7T+SpTtoehsYckroGANzkkNXrULr3JChr+Tl7bvSWWiuVu3cgjKYfpi/VMBaZNWb9Ferq6XyYmYWsxvwH3nKsMX7v67/3k+CWtev8z+iqQjwiy00TF770d+Bg=
-    file:
-    - dist/stash-box-osx
-    - dist/stash-box-win.exe
-    - dist/stash-box-linux
-    skip_cleanup: true
-    overwrite: true
-    name: "${STASH_VERSION}: Latest development build"
-    release_notes: "**${RELEASE_DATE}**\n This is always the latest committed version on the develop branch. Use as your own risk!"
-    prerelease: true
-    on:
-      repo: stashapp/stash-box
-      branch: develop
-  # docker image build for develop release
-  - provider: script
-    skip_cleanup: true
-    script: bash ./docker/ci/x86_64/docker_push.sh development-x86_64
-    on:
-      repo: stashapp/stash-box
-      branch: develop
-  # official master release - only build when tagged
-  - provider: releases
-    api_key:
-      secure: NTTT3AiH8zfwf69RBXm6r3abDnuL16be89BTUscdAuLVXvE+FHaPGDCjpyZHY9nIcYQRO5MvPl8UfK0Fauwh970yh6hBGZ9C9jwJ5FJPc9i2PfVxETL7HvMdqONaNyArFE1HirHnAO/ePxbUS1omb6EDaf/q5nQbZECL18rhtVdtuyceN6qMlZNsDahPyUwCtCaVvbDB0sOLR/Lvx+nReOyumNWquHHZrmP6Z7q6lLT1MeLzCakUiy7NG1zB4XZg1qUk9fM3hwfXGZM695wegQ7GyIRWu9fsT7Fb4h211OHiHF8Tw4cNSN5JPcamoGm58CHij7ivTchVF+HbMCc4I2BgeRuTGlD4EKmGlrEOEjN65Rfqmr49Sd8ymsNC/XMnj9edUJj0smPJyMc4uHjDVODr1CzuSLFmQsjF3wkIW529x5//NBAnxfe18sG1edQwphj08SMGav6rL4p57sKX0ea3UaOCHmBqKHa6CxBPlrYXxdQt8Q9Q3H0LzQvrInHi0koUpuA4ccj8T+q9pozX3SxYPlLtJTZWKLKKUBfDWsr4esAXzp7T+SpTtoehsYckroGANzkkNXrULr3JChr+Tl7bvSWWiuVu3cgjKYfpi/VMBaZNWb9Ferq6XyYmYWsxvwH3nKsMX7v67/3k+CWtev8z+iqQjwiy00TF770d+Bg=
-    file:
-    - dist/stash-box-osx
-    - dist/stash-box-win.exe
-    - dist/stash-box-linux
-    # make the release a draft so the maintainers can confirm before releasing
-    draft: true
-    skip_cleanup: true
-    overwrite: true
-    # don't write the body. To be done manually for now. In future we might 
-    # want to generate the changelog or get it from a file
-    name: ${STASH_VERSION}
-    on:
-      repo: stashapp/stash-box
-      tags: true
-      # make sure we don't release using the latest_develop tag
-      condition: $TRAVIS_TAG != latest_develop
-  # docker image build for master release
-  - provider: script
-    skip_cleanup: true
-    script: bash ./docker/ci/x86_64/docker_push.sh latest
-    on:
-      repo: stashapp/stash-box
-      tags: true
-      # make sure we don't release using the latest_develop tag
-      condition: $TRAVIS_TAG != latest_develop

Makefile

@@ -1,31 +1,8 @@
-IS_WIN =
-ifeq (${SHELL}, sh.exe)
-  IS_WIN = true
-endif
-ifeq (${SHELL}, cmd)
-  IS_WIN = true
-endif
-
-ifdef IS_WIN
-  SEPARATOR := &&
-  SET := set
-else 
-  SEPARATOR := ;
-  SET := export
-endif
-
-# set LDFLAGS environment variable to any extra ldflags required
-# set OUTPUT to generate a specific binary name
-
 LDFLAGS := $(LDFLAGS)
 ifdef OUTPUT
   OUTPUT := -o $(OUTPUT)
 endif
 
-.PHONY: release pre-build install clean 
-
-release: generate ui build-release
-
 pre-build:
 ifndef BUILD_DATE
 	$(eval BUILD_DATE := $(shell go run scripts/getDate.go))
@@ -40,16 +17,7 @@ ifndef STASH_BOX_VERSION
 endif
 
 build: pre-build
-	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash-box/pkg/api.version=$(STASH_BOX_VERSION)' -X 'github.com/stashapp/stash-box/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash-box/pkg/api.githash=$(GITHASH)')
-	$(SET) CGO_ENABLED=1 $(SEPARATOR) go build $(OUTPUT) -v -tags "osusergo netgo" -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS)"
-
-# strips debug symbols from the release build
-# consider -trimpath in go build if we move to go 1.13+
-build-release: EXTRA_LDFLAGS := -s -w
-build-release: build
-
-build-release-static: EXTRA_LDFLAGS := -extldflags=-static -s -w
-build-release-static: build
+	go build $(OUTPUT) -v -ldflags "-X 'github.com/stashapp/stash-box/pkg/api.version=$(STASH_BOX_VERSION)' -X 'github.com/stashapp/stash-box/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash-box/pkg/api.githash=$(GITHASH)'"
 
 install:
 	packr2 install
@@ -119,3 +87,16 @@ ui: ui-only
 .PHONY: packr
 packr:
 	packr2
+
+# runs tests and checks on the UI and builds it
+.PHONY: ui-validate
+ui-validate:
+	cd frontend && yarn run validate
+
+.PHONY: cross-compile
+cross-compile:
+	docker run --rm --privileged \
+				-v $(PWD):/go/src/github.com/stashapp/stash-box \
+				-v /var/run/docker.sock:/var/run/docker.sock \
+				-w /go/src/github.com/stashapp/stash-box \
+				goreng/golang-cross:latest --snapshot --rm-dist

README.md

@@ -28,7 +28,7 @@ Before building the binary the frontend project needs to be built.
 
 Stash-box requires access to a postgres database server. When stash-box is first run, or when it cannot find a configuration file (defaulting to `stash-box-config.yml` in the current working directory), then it generates a new configuration file with a default postgres connection string (`postgres@localhost/stash-box?sslmode=disable`). It prints a message indicating that the configuration file is generated, and allows you to adjust the default connection string as needed.
 
-The database must be created and available, and `CREATE EXTENSION pg_trgm;` needs to be run by a superuser in the database before rerunning stash-box. The schema will be created within the database if it is not already present. 
+The database must be created and available. If the postgres user is not a superuser, `CREATE EXTENSION pg_trgm;` needs to be run by a superuser before rerunning stash-box, otherwise you will get a migration error. The schema will be created within the database if it is not already present. 
 
 The `sslmode` parameter is documented in the [pq documentation](https://godoc.org/github.com/lib/pq). Use `sslmode=disable` to not use SSL for the database connection. The default value is `require`.
 

docker/ci/x86_64/Dockerfile

@@ -4,7 +4,7 @@
 FROM alpine as app
 LABEL MAINTAINER="https://discord.gg/Uz29ny"
 
-COPY /stash-box-linux /usr/bin/stash-box
+COPY stash-box_linux_amd64/stash-box-linux /usr/bin/stash-box
 
 EXPOSE 9998
 CMD ["stash-box", "--config_file", "/root/.stash-box/stash-box-config.yml"]

docker/compiler/Dockerfile

@@ -1,29 +0,0 @@
-FROM dockercore/golang-cross:1.13.14
-LABEL maintainer="github.com/stashapp/stash-box"
-
-ENV PACKR2_VERSION=2.0.2
-ENV PACKR2_SHA=f95ff4c96d7a28813220df030ad91700b8464fe292ab3e1dc9582305c2a338d2
-ENV PACKR2_DOWNLOAD_FILE=packr_${PACKR2_VERSION}_linux_amd64.tar.gz
-ENV PACKR2_DOWNLOAD_URL=https://github.com/gobuffalo/packr/releases/download/v${PACKR2_VERSION}/${PACKR2_DOWNLOAD_FILE}
-
-# Install tools
-RUN apt-get update && apt-get install -y apt-transport-https
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
-
-RUN apt-get update && \
-    apt-get install -y automake autogen \
-    libtool libxml2-dev uuid-dev libssl-dev bash \
-    patch make tar xz-utils bzip2 gzip sed cpio \
-	--no-install-recommends || exit 1; \
-	rm -rf /var/lib/apt/lists/*;
-
-RUN mkdir -p /root/.ssh; \
-    chmod 0700 /root/.ssh; \
-    ssh-keyscan github.com > /root/.ssh/known_hosts;
-
-RUN wget ${PACKR2_DOWNLOAD_URL}; \
-			echo "$PACKR2_SHA $PACKR2_DOWNLOAD_FILE" | sha256sum -c - || exit 1; \
-			tar -xzf $PACKR2_DOWNLOAD_FILE -C /usr/bin/ packr2; \
-			rm $PACKR2_DOWNLOAD_FILE;
-
-CMD ["packr2", "version"]

docker/compiler/Makefile

@@ -1,16 +0,0 @@
-user=stashapp
-repo=box-compiler
-version=1
-
-latest:
-	docker build -t ${user}/${repo}:latest .
-
-build:
-	docker build -t ${user}/${repo}:${version} -t ${user}/${repo}:latest .
-
-build-no-cache:
-	docker build --no-cache -t ${user}/${repo}:${version} -t ${user}/${repo}:latest .
-
-install: build
-	docker push ${user}/${repo}:${version}
-	docker push ${user}/${repo}:latest

frontend/package.json

@@ -8,11 +8,13 @@
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
+    "validate": "yarn lint && yarn format-check",
     "lint": "yarn lint:css && yarn lint:js",
     "lint:js": "eslint --cache src/**/*.{ts,tsx}",
     "lint:css": "stylelint \"src/**/*.scss\"",
     "generate": "apollo client:codegen --target typescript --outputFlat src/graphql/definitions",
-    "format": "prettier --write \"src/**/*.{js,jsx,ts,tsx,scss}\""
+    "format": "prettier --write \"src/**/*.{js,jsx,ts,tsx,scss}\"",
+    "format-check": "prettier --check \"src/**/!(generated-graphql).{js,jsx,ts,tsx,scss}\""
   },
   "devDependencies": {
     "@types/classnames": "^2.2.11",

pkg/database/migrations/postgres/2_create_search.up.sql

@@ -1,3 +1,10 @@
+DO $$
+BEGIN
+  IF current_setting('is_superuser') = 'on' THEN
+    CREATE EXTENSION IF NOT EXISTS pg_trgm;
+  END IF;
+END$$;
+
 CREATE TABLE scene_search AS 
 SELECT
 	S.id as scene_id,

scripts/cross-compile.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-BUILD_DATE=`go run -mod=vendor scripts/getDate.go`
-GITHASH=`git rev-parse --short HEAD`
-STASH_BOX_VERSION=`git describe --tags --exclude latest_develop`
-SETENV="BUILD_DATE=\"$BUILD_DATE\" GITHASH=$GITHASH STASH_BOX_VERSION=\"$STASH_BOX_VERSION\""
-SETUP="export GO111MODULE=on; export CGO_ENABLED=1; set -e; echo '=== Running packr ==='; make packr;"
-WINDOWS="echo '=== Building Windows binary ==='; $SETENV GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ LDFLAGS=\"-extldflags '-static' \" OUTPUT=\"dist/stash-box-win.exe\" make build-release;"
-DARWIN="echo '=== Building OSX binary ==='; $SETENV GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ OUTPUT=\"dist/stash-box-osx\" make build-release;"
-LINUX_AMD64="echo '=== Building Linux (amd64) binary ==='; $SETENV GOOS=linux GOARCH=amd64 OUTPUT=\"dist/stash-box-linux\" make build-release-static;"
-
-COMMAND="$SETUP $WINDOWS $DARWIN $LINUX_AMD64 echo '=== Build complete ==='"
-
-docker run --rm --mount type=bind,source="$(pwd)",target=/stash-box -w /stash-box stashapp/box-compiler:1 /bin/bash -c "$COMMAND"