Nixpkgs support (#228)

* Use env shebangs

* CMake cleanup, install() invocations, & CTest

`ctest` & the `test` target work now

* Nixpkgs support

.gitignore

@@ -1,6 +1,12 @@
+# for CMake
+/Testing
+
 # for clion
 cmake-build-debug/*
 .idea/*
 build/*
 decompiler_out/*
 logs/*
+
+# for Nix
+/result*

CMakeLists.txt

@@ -1,11 +1,8 @@
 # Top Level CMakeLists.txt
-if (UNIX)
 cmake_minimum_required(VERSION 3.10)
-else ()
-    cmake_minimum_required(VERSION 3.16)
-endif ()
 
 project(jak)
+include(CTest)
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Debug")
 endif()
@@ -36,17 +33,17 @@ else ()
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:10000000")
 endif(UNIX)
 
-IF (WIN32)
+if(WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
     set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
     set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-ENDIF ()
+endif()
 
-IF (ASAN_BUILD)
+if(ASAN_BUILD)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -O1")
     message(STATUS "Doing ASAN build")
-ENDIF ()
+endif()
 
 option(CODE_COVERAGE "Enable Code Coverage Compiler Flags" OFF)
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/third-party/cmake/modules/)
@@ -80,25 +77,30 @@ add_subdirectory(goalc)
 add_subdirectory(tools)
 
 # build the gtest libraries
-add_subdirectory(third-party/googletest)
+if(WIN32)
+  set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+endif()
+add_subdirectory(third-party/googletest EXCLUDE_FROM_ALL)
+include(GoogleTest)
 
 # build tests
-add_subdirectory(test)
+include(test/CMakeLists.txt)
 
 # build minilzo library
-add_subdirectory(third-party/minilzo)
+add_subdirectory(third-party/minilzo EXCLUDE_FROM_ALL)
 
 # build format library
-add_subdirectory(third-party/fmt)
+add_subdirectory(third-party/fmt EXCLUDE_FROM_ALL)
 
 # build zydis third party library for disassembling x86
-option(ZYDIS_BUILD_TOOLS "" OFF)
-option(ZYDIS_BUILD_EXAMPLES "" OFF)
-option(ZYDIS_BUILD_SHARED_LIB "" ON)
-add_subdirectory("third-party/zydis")
+# NOTE: Once under CMake 3.13's policy CMP0077, override with `set()` instead
+option(ZYDIS_BUILD_TOOLS "Zydis: Build tools" OFF)
+option(ZYDIS_BUILD_EXAMPLES "Zydis: Build examples" OFF)
+option(ZYDIS_BUILD_SHARED_LIB "Zydis: Build shared library" ON)
+add_subdirectory(third-party/zydis EXCLUDE_FROM_ALL)
 
 
 # windows memory management lib
-IF (WIN32)
+if(WIN32)
     add_subdirectory(third-party/mman)
-ENDIF ()
+endif()

README.md

@@ -37,46 +37,66 @@ We support both Linux and Windows on x86-64.
 
 ## Getting Started - Linux (Ubuntu)
 
-Install Packages and Init Repository
+Install packages and init repository:
 
-```bash
+```sh
 sudo apt install gcc make cmake build-essential g++ nasm clang-format
 git submodule update --init --recursive
 ```
 
-Compile
+Compile:
 
-```bash
-mkdir build && cd build && cmake .. && make -j
+```sh
+cmake -B build && cmake --build build -j 8
 ```
 
-Run Tests
+Run tests:
 
-```bash
+```sh
 ./test.sh
 ```
 
 ## Getting Started - Linux (Arch)
 
-Install Packages and Init Repository
+Install packages and init repository:
 
-```bash
+```sh
 sudo pacman -S gcc make cmake base-devel g++ nasm
 git submodule update --init --recursive
 ```
 
-Compile
+Compile:
 
-```bash
-mkdir build && cd build && cmake .. && make -j
+```sh
+cmake -B build && cmake --build build -j 8
 ```
 
-Run Tests
+Run tests:
 
-```bash
+```sh
 ./test.sh
 ```
 
+## Getting Started - Nixpkgs
+
+If your Nix supports flakes:
+
+```sh
+nix develop # development environment
+nix build # package
+nix develop '.#jak-asan-dev' # development environment with Clang
+nix build '.#jak-asan' # package with Clang ASan build
+```
+
+Otherwise, with traditional Nix:
+
+```sh
+nix-shell # development environment
+nix-build # package
+nix-shell -A packages.x86_64-linux.jak-asan-dev # development environment with Clang
+nix-build -A packages.x86_64-linux.jak-asan # package with Clang ASan build
+```
+
 
 ## Getting Started - Windows
 
@@ -86,13 +106,13 @@ On Windows, it's recommended to get Scoop to use as a package manager, making th
 
 Once Scoop is installed, run the following command:
 
-```ps
+```ps1
 scoop install llvm nasm
 ```
 
 Initialize the repository's third-party dependencies:
 
-```bash
+```sh
 git submodule update --init --recursive
 ```
 
@@ -204,7 +224,7 @@ Check out these files for more documentation. Some of it is still in progress
 
 ## ASan Build
 The project supports building with Address Sanitizer (https://github.com/google/sanitizers/wiki/AddressSanitizer) in Linux.
-```
+```sh
 export CXX=clang++
 cmake .. -DASAN_BUILD=TRUE
 ```

boot_game.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

boot_kernel.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

check.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 

common/CMakeLists.txt

@@ -23,8 +23,10 @@ add_library(common
 
 target_link_libraries(common fmt)
 
-IF(WIN32)
+if(WIN32)
     target_link_libraries(common wsock32 ws2_32)
-ELSE()
+else()
     target_link_libraries(common stdc++fs)
-ENDIF()
+endif()
+
+install(TARGETS common)

decomp.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

decompiler/CMakeLists.txt

@@ -65,3 +65,5 @@ target_link_libraries(decompiler
         common
         minilzo
         fmt)
+
+install(TARGETS decompiler)

default.nix

@@ -0,0 +1,4 @@
+(import (fetchTarball {
+  url = "https://github.com/edolstra/flake-compat/archive/99f1c2157fba4bfe6211a321fd0ee43199025dbf.tar.gz";
+  sha256 = "0x2jn3vrawwv9xp15674wjz9pixwjyj3j771izayl962zziivbx2";
+}) { src =  ./.; }).defaultNix

flake.lock

@@ -0,0 +1,25 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1612312445,
+        "narHash": "sha256-l8qdZFqwjAGzQrsO+jFqRJA7YnN3jMGe5LGXRMJfcPg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "6b8fca6b5752c5c6038dc9349d0086cf2c49c567",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,168 @@
+{
+  description = "Jak PC ports via OpenGOAL PC port of Naughty Dog's GOAL";
+
+  outputs = { self, nixpkgs }: let
+    getVersionPre = output:
+      "${nixpkgs.lib.substring 0 8 output.lastModifiedDate}.${output.shortRev or "dirty"}";
+    supportedSystems = [ "x86_64-linux" "x86_64-darwin" ];
+    forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system);
+
+    supportedPackages = [ "jak" "jak-dev" "jak-asan" "jak-asan-dev" ];
+    forAllPackages = f: nixpkgs.lib.genAttrs supportedPackages (pname: f pname);
+
+    release = false;
+    version = "0.5.0" + nixpkgs.lib.optionalString (!release) "-${getVersionPre self}";
+  in {
+
+    overlay = pkgs: pkgsSuper: {
+      jak = pkgs.callPackage (
+        { lib, stdenv, buildPackages, fetchFromGitHub, runCommand, writeText
+        , jak-googletest-src, jak-zydis-src
+        , llvm ? null
+        , python3Packages ? null
+        , enableDevInputs ? false
+        , enableAsan ? false
+        , enableFramePointer ? enableAsan
+        , enablePIE ? false
+        , enableSourceLevelDebug ? enableAsan
+        }:
+
+        let
+          sh = lib.escapeShellArg;
+          optionalFun = b: f: if b then f else x: x;
+          inherit (stdenv.cc) isClang;
+
+          llvm-symbolizer = if isClang then runCommand "llvm-symbolizer" {
+            allowSubstitutes = false;
+            preferLocalBuild = true;
+          } ''
+            mkdir -p "$out/bin"
+            cp ${sh (lib.getBin llvm)}/bin/llvm-symbolizer "$out/bin"/llvm-symbolizer
+          '' else null;
+
+          makeFlagArrayBody = prefix: flags:
+            lib.concatStrings
+              (lib.mapAttrsToList (n: v: " " + sh "${prefix}${n}=${v}" + " \\\n") flags);
+          makeFlagArray = name: prefix: flags:
+            "${name}Array+=( \\\n${makeFlagArrayBody prefix flags})";
+
+        in stdenv.mkDerivation {
+          pname = "jak";
+          inherit version;
+
+          # Workaround until `src = self;` works with Git submodules.
+          src = runCommand "source" {
+            allowSubstitutes = false;
+            preferLocalBuild = true;
+            src = self;
+          } ''
+            cp -R "$src" "$out"
+            chmod -R u+w "$out"
+            shopt -s nullglob dotglob
+            files=("$out/third-party/googletest"/*); if (( ''${#files[*]} == 0 )); then
+              echo "providing third-party/googletest submodule via Nix"
+              rm -df "$out/third-party/googletest"
+              cp -R ${sh jak-googletest-src} "$out/third-party/googletest"
+            fi
+            files=("$out/third-party/zydis"/*); if (( ''${#files[*]} == 0 )); then
+              echo "providing third-party/zydis submodule via Nix"
+              rm -df "$out/third-party/zydis"
+              cp -R ${sh jak-zydis-src} "$out/third-party/zydis"
+            fi
+          '';
+
+          nativeBuildInputs = [
+            buildPackages.cmake
+            buildPackages.nasm
+          ] ++ lib.optionals enableDevInputs [
+            buildPackages.clang-tools # clang-format
+            python3Packages.pyqt5
+            python3Packages.python
+          ];
+
+          preConfigure = let
+            cmakeFlags = lib.pipe {
+              # https://github.com/NixOS/nixpkgs/pull/108496
+              CMAKE_SKIP_BUILD_RPATH = "OFF";
+            } [
+              (optionalFun enableAsan (o: o // {
+                ASAN_BUILD = "TRUE";
+              }))
+              (optionalFun enableFramePointer (o: o // {
+                CMAKE_C_FLAGS = o.CMAKE_C_FLAGS or "" + " -fno-omit-frame-pointer";
+                CMAKE_CXX_FLAGS = o.CMAKE_CXX_FLAGS or "" + " -fno-omit-frame-pointer";
+              }))
+              (optionalFun enablePIE (o: o // {
+                POSITION_INDEPENDENT_CODE = "TRUE";
+              }))
+              (optionalFun enableSourceLevelDebug (o: o // {
+                CMAKE_C_FLAGS = o.CMAKE_C_FLAGS or "" + " -g";
+                CMAKE_CXX_FLAGS = o.CMAKE_CXX_FLAGS or "" + " -g";
+              }))
+            ];
+          in ''
+            ${makeFlagArray "cmakeFlags" "-D" cmakeFlags}
+          '';
+
+          doCheck = true;
+
+          dontStrip = enableAsan;
+
+          preFixup = ''
+          '' + lib.optionalString (enableAsan && llvm-symbolizer != null) ''
+            for f in "$out/bin"/*; do
+              wrapProgram "$f" --set LLVM_SYMBOLIZER ${sh llvm-symbolizer}/bin/llvm-symbolizer
+            done
+          '';
+
+          meta = with lib; {
+            description = "OpenGOAL port of Naughty Dog's GOAL";
+            homepage = "https://github.com/water111/jak-project";
+            license = lib.licenses.gpl3Plus;
+            maintainers = with maintainers; [ bb010g ];
+            platforms = platforms.all;
+          };
+        }
+      ) { };
+      jak-dev = pkgs.jak.override { enableDevInputs = true; };
+
+      jak-asan = pkgs.jak.override {
+        inherit (pkgs.llvmPackages) llvm;
+        enableAsan = true;
+        stdenv = pkgs.clangStdenv;
+      };
+      jak-asan-dev = pkgs.jak-asan.override { enableDevInputs = true; };
+
+      jak-googletest-src = pkgs.callPackage ({ fetchFromGitHub }: fetchFromGitHub {
+        owner = "google";
+        repo = "googletest";
+        rev = "adeef192947fbc0f68fa14a6c494c8df32177508";
+        sha256 = "1nsl1c5il6mzwggs5fqcp8gyddk9rs6257vlz0zgpik32miq3cgw";
+      }) { };
+
+      jak-zydis-src = pkgs.callPackage ({ fetchFromGitHub }: fetchFromGitHub {
+        owner = "zyantific";
+        repo = "zydis";
+        rev = "c88a4b0cc1271b516b9697af8d088c851745dd60";
+        sha256 = "1j28vxr3r6w8xawlagcicbd95dcs85shgsm11b98g3qj45bv3haq";
+        fetchSubmodules = true;
+      }) { };
+    };
+
+    packages = forAllSystems (system:
+      let systemPackages = (import nixpkgs {
+        inherit system;
+        overlays = [ self.overlay ];
+      }); in forAllPackages (pname: systemPackages.${pname})
+    );
+
+    defaultPackage = forAllSystems (system:
+      self.packages.${system}.jak
+    );
+
+    devShell = forAllSystems (system:
+      self.packages.${system}.jak-dev
+    );
+
+  };
+}

game/CMakeLists.txt

@@ -74,12 +74,13 @@ set(RUNTIME_SOURCE
 add_library(runtime ${RUNTIME_SOURCE})
 
 target_link_libraries(runtime common fmt)
-IF (WIN32)
+if(WIN32)
     target_link_libraries(runtime mman)
-ELSE()
+else()
     target_link_libraries(runtime pthread)
-ENDIF()
+endif()
 
 add_executable(gk main.cpp)
 target_link_libraries(gk runtime)
 
+install(TARGETS gk)

gc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

gk.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

goalc/CMakeLists.txt

@@ -40,11 +40,11 @@ add_library(compiler
 
 target_link_libraries(compiler common Zydis)
 
-IF (WIN32)
+if(WIN32)
     target_link_libraries(compiler mman)
-ENDIF ()
+endif()
 
 add_executable(goalc main.cpp)
 target_link_libraries(goalc common Zydis compiler)
 
-
+install(TARGETS goalc)

offline_test_git_branch.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

scripts/add-pragma-once-headers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import sys
 import os

shell.nix

@@ -0,0 +1,4 @@
+(import (fetchTarball {
+  url = "https://github.com/edolstra/flake-compat/archive/99f1c2157fba4bfe6211a321fd0ee43199025dbf.tar.gz";
+  sha256 = "0x2jn3vrawwv9xp15674wjz9pixwjyj3j771izayl962zziivbx2";
+}) { src =  ./.; }).shellNix

test.sh

@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-$DIR/build/test/goalc-test --gtest_color=yes "$@"
+$DIR/build/goalc-test --gtest_color=yes "$@"

test/CMakeLists.txt

@@ -1,40 +1,39 @@
 set(CMAKE_CXX_STANDARD 17)
 
-include("goalc/CMakeLists.txt")
+include(${CMAKE_CURRENT_LIST_DIR}/goalc/CMakeLists.txt)
 
 add_executable(goalc-test
-        test_main.cpp
-        test_test.cpp
-        test_reader.cpp
-        test_goos.cpp
-        test_listener_deci2.cpp
-        test_kernel.cpp
-        all_jak1_symbols.cpp
-        test_type_system.cpp
-        test_CodeTester.cpp
-        test_emitter.cpp
-        test_emitter_avx.cpp
-        test_common_util.cpp
-        test_pretty_print.cpp
-        test_zydis.cpp
-        goalc/test_goal_kernel.cpp
-        decompiler/FormRegressionTest.cpp
-        decompiler/test_AtomicOpBuilder.cpp
-        decompiler/test_FormBeforeExpressions.cpp
-        decompiler/test_FormExpressionBuild.cpp
-        decompiler/test_FormExpressionBuildLong.cpp
-        decompiler/test_InstructionParser.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_main.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_test.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_reader.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_goos.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_listener_deci2.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_kernel.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/all_jak1_symbols.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_type_system.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_CodeTester.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_emitter.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_emitter_avx.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_common_util.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_pretty_print.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/test_zydis.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/goalc/test_goal_kernel.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/FormRegressionTest.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_AtomicOpBuilder.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormBeforeExpressions.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormExpressionBuild.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_FormExpressionBuildLong.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_InstructionParser.cpp
         ${GOALC_TEST_FRAMEWORK_SOURCES}
         ${GOALC_TEST_CASES})
 
-enable_testing()
-
 target_link_libraries(goalc-test common runtime compiler gtest decomp Zydis)
 
-IF (WIN32)
-  set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+if(WIN32)
   target_link_libraries(goalc-test mman)
-ENDIF()
+endif()
+
+gtest_discover_tests(goalc-test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
 if(UNIX AND CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE)
   include(CodeCoverage)

test/goalc/CMakeLists.txt

@@ -1,12 +1,12 @@
 # TODO - probably a more cmakey way to do this
 
 set(GOALC_TEST_CASES
-		"goalc/all_goalc_template_tests.cpp"
-		goalc/test_debugger.cpp
-		goalc/test_game_no_debug.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/all_goalc_template_tests.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/test_debugger.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/test_game_no_debug.cpp
 )
 
 set(GOALC_TEST_FRAMEWORK_SOURCES
-		"goalc/framework/test_runner.cpp"
-		"goalc/framework/test_runner.h"
+    ${CMAKE_CURRENT_LIST_DIR}/framework/test_runner.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/framework/test_runner.h
 )

test_code_coverage.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Directory of this script
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

tools/CMakeLists.txt

@@ -5,3 +5,5 @@ target_link_libraries(dgo_unpacker common)
 add_executable(dgo_packer
         dgo_packer.cpp)
 target_link_libraries(dgo_packer common)
+
+install(TARGETS dgo_unpacker dgo_packer)