Enable qtFRED deploy builds

The CMake code did not specify that qwindows.cll also needed to be
distributed with the application which caused errors for users who had
not installed Qt on their system.

This should make qtFRED redistributable without needing a QT
installation.

.clang-tidy

@@ -0,0 +1,27 @@
+---
+Checks: "-*,\
+modernize-use-override,\
+performance-*,\
+-performance-type-promotion-in-math-fn,\
+misc-assert-side-effect,\
+misc-bool-pointer-implicit-conversion,\
+misc-definitions-in-headers,\
+misc-fold-init-type,\
+misc-forward-declaration-namespace,\
+misc-inaccurate-erase,\
+misc-inefficient-algorithm,\
+misc-macro-repeated-side-effects,\
+misc-misplaced-const,\
+misc-move-forwarding-reference,\
+misc-multiple-statement-macro,\
+misc-new-delete-overloads,\
+misc-noexcept-move-constructor,\
+"
+WarningsAsErrors: ''
+HeaderFilterRegex: '.*'
+AnalyzeTemporaryDtors: false
+CheckOptions:
+  - key:          'misc-assert-side-effect.AssertMacros'
+    value:        'assert,Assert,Assertion'
+...
+

.gitignore

@@ -256,3 +256,7 @@ CMakeSettings.json
 
 # Weird files created by Linux
 .fuse_hidden*
+
+# This gets ignored by one of the rules above for some reason...
+!/code/scripting/lua/LuaConvert.h
+!/code/debugconsole/

.travis.yml

@@ -3,7 +3,8 @@ sudo: required
 dist: trusty
 cache:
     directories:
-    - $HOME/cmake
+    - $HOME/cmake-3.6
+    - $HOME/.ccache
 addons:
     apt:
         packages: &global_deps
@@ -16,6 +17,7 @@ addons:
             - libjansson-dev
             - libsdl2-dev
             - valgrind
+            - qt57base
     coverity_scan:
         project:
             name: "scp-fs2open/fs2open.github.com"
@@ -69,19 +71,33 @@ matrix:
               apt:
                   sources:
                       - ubuntu-toolchain-r-test
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
                   packages:
-                      - g++-5
                       - *global_deps
+                      - g++-5
         - os: linux
-          env: CONFIGURATION="Debug" CC=clang-3.6 CXX=clang++-3.6
+          env: CONFIGURATION="Debug" CC=gcc-7 CXX=g++-7
           addons:
               apt:
                   sources:
-                      - llvm-toolchain-precise
+                      - ubuntu-toolchain-r-test
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
                   packages:
-                      - clang-3.6
-                      - libc++-dev
+                      - g++-7
                       - *global_deps
+        - os: linux
+          env: CONFIGURATION="Debug" CC=clang-4.0 CXX=clang++-4.0 CMAKE_OPTIONS="-DENABLE_CLANG_TIDY=ON -DCLANG_TIDY_EXE=/usr/bin/clang-tidy-4.0"
+          addons:
+              apt:
+                  sources:
+                      - ubuntu-toolchain-r-test
+                      - llvm-toolchain-trusty-4.0
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
+                  packages:
+                      - *global_deps
+                      - clang-4.0
+                      - clang-tidy-4.0
+                      - libc++-dev
         - os: osx
           env: CONFIGURATION="Debug"
 
@@ -91,19 +107,33 @@ matrix:
               apt:
                   sources:
                       - ubuntu-toolchain-r-test
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
                   packages:
-                      - g++-5
                       - *global_deps
+                      - g++-5
         - os: linux
-          env: CONFIGURATION="Release" CC=clang-3.6 CXX=clang++-3.6
+          env: CONFIGURATION="Release" CC=gcc-7 CXX=g++-7
           addons:
               apt:
                   sources:
-                      - llvm-toolchain-precise
+                      - ubuntu-toolchain-r-test
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
                   packages:
-                      - clang-3.6
-                      - libc++-dev
+                      - g++-7
                       - *global_deps
+        - os: linux
+          env: CONFIGURATION="Release" CC=clang-4.0 CXX=clang++-4.0 CMAKE_OPTIONS="-DENABLE_CLANG_TIDY=ON -DCLANG_TIDY_EXE=/usr/bin/clang-tidy-4.0"
+          addons:
+              apt:
+                  sources:
+                      - ubuntu-toolchain-r-test
+                      - llvm-toolchain-trusty-4.0
+                      - sourceline: 'ppa:beineri/opt-qt571-trusty'
+                  packages:
+                      - *global_deps
+                      - clang-4.0
+                      - clang-tidy-4.0
+                      - libc++-dev
         - os: osx
           env: CONFIGURATION="Release"
 before_install:
@@ -117,8 +147,9 @@ before_install:
     - "./ci/travis/before_install.sh"
 install:
     - "./ci/travis/install.sh"
-    - export PATH=$HOME/cmake/bin:$PATH
+    - export PATH=$HOME/cmake-3.6/bin:$PATH
 before_script:
+    - if [ "$TRAVIS_OS_NAME" = "linux" ]; then QT_ENV_SCRIPT=$(find /opt -name 'qt*-env.sh'); source $QT_ENV_SCRIPT; fi
     - "./ci/travis/before_script.sh"
 script:
     - if [ "$BUILD_DEPLOYMENT" = true ] ; then ./ci/travis/release.sh ; else ./ci/travis/script.sh ; fi

CMakeLists.txt

@@ -40,6 +40,14 @@ set(CMAKE_CONFIGURATION_TYPES Debug Release FastDebug CACHE STRING "Available co
 
 PROJECT(FS2_Open)
 
+find_program(CCACHE_PROGRAM ccache)
+# Currently only GCC support ccache properly without generating additional warnings
+if(CCACHE_PROGRAM AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+	message(STATUS "Using ccache for compilation: ${CCACHE_PROGRAM}")
+	set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
+	set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE_PROGRAM}")
+endif()
+
 IF(CMAKE_CROSSCOMPILING)
 	SET(IMPORT_EXE_DIR "IMPORT_EXE_DIR-NOTFOUND" CACHE FILEPATH "Required for a cross compiling build, point to binary directory of a cmake build for the real OS")
 	INCLUDE(${IMPORT_EXE_DIR}/ImportExecutables.cmake)
@@ -85,7 +93,7 @@ OPTION(FSO_BUILD_TESTS "Build unit tests" OFF)
 
 OPTION(FSO_DEVELOPMENT_MODE "Generate binaries in development mode, only use if you know what you're doing!" OFF)
 
-OPTION(FSO_BUILD_WXFRED2 "Build wxFRED2 binary" OFF)
+OPTION(FSO_BUILD_QTFRED "Build qtFRED2 binary" OFF)
 
 IF(WIN32 OR APPLE)
 	# On windows and mac the default should be to always build the included libraries
@@ -134,6 +142,8 @@ if(FSO_BUILD_APPIMAGE)
 	include(appimage)
 endif()
 
+include(clang-tidy)
+
 # This includes source code for some tools, either used in the build or also for something else
 ADD_SUBDIRECTORY(tools)
 
@@ -147,9 +157,9 @@ IF(FSO_BUILD_FRED2)
 	ADD_SUBDIRECTORY(fred2)
 ENDIF(FSO_BUILD_FRED2)
 
-IF(FSO_BUILD_WXFRED2)
-	ADD_SUBDIRECTORY(wxfred2)
-ENDIF(FSO_BUILD_WXFRED2)
+IF(FSO_BUILD_QTFRED)
+	ADD_SUBDIRECTORY(qtfred)
+ENDIF()
 
 if(FSO_BUILD_TESTS)
 	add_subdirectory(test)
@@ -160,7 +170,7 @@ if ("${CMAKE_VERSION}" VERSION_GREATER "3.5")
 	set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT Freespace2)
 endif()
 
-IF(MSVC_USE_RUNTIME_DLL)
+IF(MSVC_USE_RUNTIME_DLL OR FSO_BUILD_QTFRED)
 	SET(CMAKE_INSTALL_SYSTEM_RUNTIME_DESTINATION "${BINARY_DESTINATION}")
 
 	IF(FSO_BUILD_FRED2)
@@ -168,7 +178,7 @@ IF(MSVC_USE_RUNTIME_DLL)
 	ENDIF(FSO_BUILD_FRED2)
 
 	INCLUDE(InstallRequiredSystemLibraries)
-ENDIF(MSVC_USE_RUNTIME_DLL)
+ENDIF()
 
 include(package)
 
@@ -176,12 +186,11 @@ include(doxygen)
 
 # Print used options to log
 IF (WIN32)
-	message(STATUS "Building FRED2: ${FSO_BUILD_FRED2}")
 	message(STATUS "Using text to speech: ${FSO_USE_SPEECH}")
 	message(STATUS "Using voice recogition: ${FSO_USE_VOICEREC}")
 elseif(UNIX)
 	message(STATUS "Building AppImage: ${FSO_BUILD_APPIMAGE}")
 ENDIF()
 message(STATUS "Building FSO tools: ${FSO_BUILD_TOOLS}")
-message(STATUS "Building wxFRED: ${FSO_BUILD_WXFRED2}")
+message(STATUS "Building qtFRED: ${FSO_BUILD_QTFRED}")
 message(STATUS "Fatal warnings: ${FSO_FATAL_WARNINGS}")

appveyor.yml

@@ -8,9 +8,11 @@ environment:
     - BuildID: 0
       CMAKE_GENERATOR: Visual Studio 12 2013
       PlatformToolset: v120_xp
+      QT_DIR: C:\Qt\5.7\msvc2013
     - BuildID: 1
       CMAKE_GENERATOR: Visual Studio 14 2015 Win64
       PlatformToolset: v140_xp
+      QT_DIR: C:\Qt\5.7\msvc2015_64
 before_build:
   - git submodule update --init --recursive
 build_script:

ci/appveyor/build.ps1

@@ -4,6 +4,7 @@ class BuildConfig {
 	[string]$PackageType
 	[string]$Toolset
 	[string]$SimdType
+	[string]$QtDir
 	[Bool]$SourcePackage
 }
 
@@ -13,6 +14,7 @@ $NightlyConfigurations = @(
 		PackageType="Win64";
 		Toolset="v140_xp";
 		SimdType="SSE2";
+		QtDir="C:\Qt\5.7\msvc2015_64";
 		SourcePackage=$false;
 	},
 	[BuildConfig]@{ 
@@ -20,6 +22,7 @@ $NightlyConfigurations = @(
 		PackageType="Win32";
 		Toolset="v140_xp";
 		SimdType="SSE2";
+		QtDir="C:\Qt\5.7\msvc2015";
 		SourcePackage=$false;
 	}
 )
@@ -29,6 +32,7 @@ $ReleaseConfigurations = @(
 		PackageType="Win32";
 		Toolset="v140_xp";
 		SimdType="SSE2";
+		QtDir="C:\Qt\5.7\msvc2015";
 		SourcePackage=$true;
 	}
 	[BuildConfig]@{
@@ -36,6 +40,7 @@ $ReleaseConfigurations = @(
 		PackageType="Win32-AVX";
 		Toolset="v140_xp";
 		SimdType="AVX";
+		QtDir="C:\Qt\5.7\msvc2015";
 		SourcePackage=$false;
 	}
 	[BuildConfig]@{
@@ -43,6 +48,7 @@ $ReleaseConfigurations = @(
 		PackageType="Win64";
 		Toolset="v140_xp";
 		SimdType="SSE2";
+		QtDir="C:\Qt\5.7\msvc2015_64";
 		SourcePackage=$false;
 	}
 	[BuildConfig]@{
@@ -50,6 +56,7 @@ $ReleaseConfigurations = @(
 		PackageType="Win64-AVX";
 		Toolset="v140_xp";
 		SimdType="AVX";
+		QtDir="C:\Qt\5.7\msvc2015_64";
 		SourcePackage=$false;
 	}
 )
@@ -125,6 +132,7 @@ if ($DeployBuild) {
 	}
 	
 	cmake -DCMAKE_INSTALL_PREFIX="$env:APPVEYOR_BUILD_FOLDER/../install" -DFSO_USE_SPEECH="ON" `
+		-DFSO_BUILD_QTFRED=ON -DQT5_INSTALL_ROOT="$($buildConfig.QtDir)" `
 		-DFSO_USE_VOICEREC="ON" -DMSVC_SIMD_INSTRUCTIONS="$($buildConfig.SimdType)" `
 		-G "$($buildConfig.Generator)" -T "$($buildConfig.Toolset)" ..
 
@@ -141,6 +149,7 @@ if ($DeployBuild) {
     Push-AppveyorArtifact "$($PackageName)-builds-$($buildConfig.PackageType).zip"
 } else {
 	cmake -DFSO_USE_SPEECH="ON" -DFSO_FATAL_WARNINGS="ON" -DFSO_USE_VOICEREC="ON" -DFSO_BUILD_TESTS="ON" -DMSVC_SIMD_INSTRUCTIONS=SSE2 `
+	-DFSO_BUILD_QTFRED=ON -DQT5_INSTALL_ROOT="$env:QT_DIR" -DFSO_BUILD_FRED2="OFF" `
 	-G "$Env:CMAKE_GENERATOR" -T "$Env:PlatformToolset" ..
 
     cmake --build . --config "$Env:CONFIGURATION" -- /verbosity:minimal

ci/travis/before_script.sh

@@ -20,7 +20,7 @@ if [ "$TRAVIS_OS_NAME" = "linux" ]; then
 
     export CXXFLAGS
     export CFLAGS
-    CMAKE="cmake -G Ninja -DFSO_FATAL_WARNINGS=ON"
+    CMAKE="cmake -G Ninja -DFSO_FATAL_WARNINGS=ON $CMAKE_OPTIONS"
     if [ "$BUILD_DEPLOYMENT" = true ]; then
         for config in $BUILD_CONFIGS
         do
@@ -32,12 +32,12 @@ if [ "$TRAVIS_OS_NAME" = "linux" ]; then
             cd ..
         done
     else
-        eval $CMAKE -DCMAKE_BUILD_TYPE=$CONFIGURATION -DFSO_BUILD_TESTS=ON -DFSO_BUILD_INCLUDED_LIBS=ON ..
+        eval $CMAKE -DCMAKE_BUILD_TYPE=$CONFIGURATION -DFSO_BUILD_TESTS=ON -DFSO_BUILD_INCLUDED_LIBS=ON -DFSO_BUILD_QTFRED=ON ..
     fi
 elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
     if [ "$BUILD_DEPLOYMENT" = true ]; then
         cmake -G "Xcode" -DFSO_FATAL_WARNINGS=ON -DFSO_INSTALL_DEBUG_FILES=ON -DCMAKE_INSTALL_PREFIX="/tmp/release" ..
     else
-        cmake -G "Xcode" ..
+        cmake -G "Xcode" $CMAKE_OPTIONS ..
     fi
 fi

ci/travis/check_release.sh

@@ -49,7 +49,7 @@ if ([ "$RELEASE_BUILD" = true ] || [ "$NIGHTLY_BUILD" = true ] || [ "$TEST_BUILD
 fi
 
 if [ "$BUILD_DEPLOYMENT" = true ]; then
-    if ([ "$TRAVIS_OS_NAME" = "linux" ] && (! [[ "$CC" =~ ^gcc.*$ ]])); then
+    if ([ "$TRAVIS_OS_NAME" = "linux" ] && (! [[ "$CC" =~ ^gcc-5.*$ ]])); then
         echo "Skipping non-release compiler";
         exit 0;
     fi

ci/travis/install.sh

@@ -4,16 +4,14 @@ set -ex
 
 FILENAME=
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-    if [ ! -d "$HOME/cmake/bin" ]; then
+    if [ ! -d "$HOME/cmake-3.6/bin" ]; then
         # If the cache does not currently contain CMake then download and install it
-        FILENAME=cmake-3.4.3-Linux-x86_64
+    	mkdir -p $HOME/cmake-3.6/
 
-    	mkdir -p $HOME/cmake/
-
-    	wget -O /tmp/cmake.tar.gz --no-check-certificate https://www.cmake.org/files/v3.4/$FILENAME.tar.gz
-    	tar -xzf /tmp/cmake.tar.gz -C $HOME/cmake/ --strip-components=1
+    	wget -O /tmp/cmake.tar.gz --no-check-certificate https://cmake.org/files/v3.6/cmake-3.6.0-Linux-x86_64.tar.gz
+    	tar -xzf /tmp/cmake.tar.gz -C $HOME/cmake-3.6/ --strip-components=1
     fi
-	export PATH=$HOME/cmake/bin:$PATH
+	export PATH=$HOME/cmake-3.6/bin:$PATH
     
     cd $HOME
     

ci/travis/script.sh

@@ -5,7 +5,7 @@ set -e
 cd build
 
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-    ninja all
+    ninja -k 20 all
 
     valgrind --leak-check=full --error-exitcode=1 --gen-suppressions=all \
         --suppressions="$TRAVIS_BUILD_DIR/ci/travis/valgrind.supp" ./bin/unittests --gtest_shuffle

cmake/clang-tidy.cmake

@@ -0,0 +1,37 @@
+option(ENABLE_CLANG_TIDY "Enable source code checking using clang-tidy" OFF)
+
+find_program(
+    CLANG_TIDY_EXE
+    NAMES "clang-tidy"
+    DOC "Path to clang-tidy executable"
+)
+
+function(enable_clang_tidy target)
+    # Not enabled in this configuration
+endfunction()
+
+if(NOT ENABLE_CLANG_TIDY)
+    return()
+endif()
+
+if(NOT EXISTS "${CLANG_TIDY_EXE}")
+    message(SEND_ERROR "clang-tidy: Could not find clang-tidy executable (value of variable is \"${CLANG_TIDY_EXE}\".")
+    return()
+endif()
+
+if(CMAKE_VERSION VERSION_LESS "3.6.0")
+    message(SEND_ERROR "Clang-tidy support requires CMake 3.6")
+    return()
+endif()
+
+message(STATUS "clang-tidy: Using \"${CLANG_TIDY_EXE}\"")
+
+function(enable_clang_tidy target)
+    SET(TIDY_OPTIONS "${CLANG_TIDY_EXE}")
+    if(FSO_FATAL_WARNINGS)
+        set(TIDY_OPTIONS "${TIDY_OPTIONS};-warnings-as-errors=*")
+    endif()
+
+    set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "${TIDY_OPTIONS}")
+endfunction()
+

cmake/util.cmake

@@ -10,7 +10,7 @@ FUNCTION(ADD_IMPORTED_LIB NAME INCLUDES LIBS)
 
 	target_link_libraries(${NAME} INTERFACE "${LIBS}")
 
-	target_include_directories(${NAME} INTERFACE "${INCLUDES}")
+	target_include_directories(${NAME} SYSTEM INTERFACE "${INCLUDES}")
 ENDFUNCTION(ADD_IMPORTED_LIB)
 
 MACRO(PKG_CONFIG_LIB_RESOLVE NAME OUTVAR)
@@ -195,3 +195,31 @@ function(suppress_warnings _target)
 		target_compile_options(${_target} PRIVATE "-w")
     endif()
 endfunction(suppress_warnings)
+
+
+function(list_target_dependencies _target _out_var)
+	set(out_list)
+	set(work_libs ${_target})
+	list(LENGTH work_libs libs_length)
+
+	while(libs_length GREATER 0)
+		list(GET work_libs 0 current)
+		list(REMOVE_AT work_libs 0)
+		list(APPEND out_list ${current})
+
+		get_target_property(current_libs ${current} INTERFACE_LINK_LIBRARIES)
+
+		if(current_libs)
+			foreach(lib ${current_libs})
+				if (TARGET ${lib})
+					list(APPEND work_libs ${lib})
+				endif (TARGET ${lib})
+			endforeach(lib)
+		endif(current_libs)
+
+		list(LENGTH work_libs libs_length)
+	endwhile(libs_length GREATER 0)
+
+	list(REMOVE_DUPLICATES out_list)
+	set(${_out_var} ${out_list} PARENT_SCOPE)
+endfunction(list_target_dependencies)

code/CMakeLists.txt

@@ -37,6 +37,8 @@ TARGET_LINK_LIBRARIES(code PUBLIC compiler)
 
 target_link_libraries(code PUBLIC md5)
 
+enable_clang_tidy(code)
+
 IF (FSO_USE_SPEECH)
 	find_package(Speech REQUIRED)
 	target_link_libraries(code PUBLIC speech)

code/ai/ai_profiles.cpp

@@ -498,7 +498,8 @@ void parse_ai_profiles_tbl(const char *filename)
 				}
 
 				// find next valid option
-				skip_to_start_of_string_either("$", "#");
+				if (!skip_to_start_of_string_either("$", "#"))
+					break;
 				saved_Mp = Mp;
 			}
 		}

code/bmpman/bmpman.cpp

@@ -1914,14 +1914,15 @@ int bm_load_sub_slow(const char *real_filename, const int num_ext, const char **
 	char full_path[MAX_PATH];
 	size_t size = 0, offset = 0;
 	int rval = -1;
+	const void* file_data = nullptr;
 
-	rval = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type, sizeof(full_path) - 1, full_path, &size, &offset, 0);
+	rval = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type, sizeof(full_path) - 1, full_path, &size, &offset, 0, &file_data);
 
 	// could not be found, or is invalid for some reason
 	if ((rval < 0) || (rval >= num_ext))
 		return -1;
 
-	CFILE *test = cfopen_special(full_path, "rb", size, offset, dir_type);
+	CFILE *test = cfopen_special(full_path, "rb", size, offset, file_data, dir_type);
 
 	if (test != NULL) {
 		if (img_cfp != NULL)
@@ -3320,6 +3321,13 @@ bool bm_is_texture_array(const int handle) {
 }
 
 int bm_get_base_frame(const int handle, int* num_frames) {
+	if (handle == -1) {
+		if (num_frames != nullptr) {
+			*num_frames = 0;
+		}
+		return -1;
+	}
+
 	// If the bitmap is no animation then num_frames will still be at least 1
 	auto animation_begin = bm_get_info(handle, nullptr, nullptr, nullptr, num_frames);
 
@@ -3354,3 +3362,61 @@ int bmpman_count_bitmaps() {
 
 	return total_bitmaps;
 }
+
+bool bm_validate_filename(const SCP_string& file, bool single_frame, bool animation) {
+	Assertion(single_frame || animation, "At least one of single_frame or animation must be true!");
+
+	if (!VALID_FNAME(file.c_str())) {
+		return false;
+	}
+
+	int handle = -1;
+	if (single_frame && animation) {
+		// Caller has indicated that single frames and animations are valid
+		handle = bm_load_either(file.c_str());
+	} else if (single_frame) {
+		handle = bm_load(file);
+	} else {
+		handle = bm_load_animation(file.c_str());
+	}
+	if (handle != -1) {
+		bm_release(handle);
+		return true;
+	}
+	return false;
+}
+SDL_Surface* bm_to_sdl_surface(int handle) {
+	Assertion(bm_is_valid(handle), "%d is no valid bitmap handle!", handle);
+
+	int w;
+	int h;
+
+	bm_get_info(handle, &w, &h, nullptr, nullptr);
+	Uint32 rmask, gmask, bmask, amask;
+
+#if SDL_BYTEORDER == SDL_BIG_ENDIAN
+	rmask = 0x0000ff00;
+		gmask = 0x00ff0000;
+		bmask = 0xff000000;
+		amask = 0x000000ff;
+#else
+	rmask = 0x00ff0000;
+	gmask = 0x0000ff00;
+	bmask = 0x000000ff;
+	amask = 0xff000000;
+#endif
+
+	SDL_Surface* bitmapSurface = SDL_CreateRGBSurface(0, w, h, 32, rmask, gmask, bmask, amask);
+	if (SDL_LockSurface(bitmapSurface) < 0) {
+		return nullptr;
+	}
+	bitmap* bmp = bm_lock(handle, 32, BMP_TEX_XPARENT);
+
+	memcpy(bitmapSurface->pixels, reinterpret_cast<void*>(bmp->data), static_cast<size_t>(w * h * 4));
+
+	bm_unlock(handle);
+	SDL_UnlockSurface(bitmapSurface);
+
+	return bitmapSurface;
+
+}

code/bmpman/bmpman.h

@@ -741,4 +741,21 @@ int bm_get_array_index(const int handle);
  */
 int bmpman_count_bitmaps();
 
+/**
+ * @brief Checks if the given filename is a valid effect or texture file name
+ *
+ * @note At least one of @c single_frame or @c animation must be true since the result would make no sense otherwise. It
+ * is also valid to set both parmeters to @c true.
+ *
+ * @todo This currently just tries to load the file but could be improved by not actually loading the file in the future.
+ *
+ * @param file The file name to check
+ * @param single_frame If set to @c true then single frames are valid files
+ * @param animation If set to @c true then animations are valid files
+ * @return @c true if the file name is valid, @c false otherwise
+ */
+bool bm_validate_filename(const SCP_string& file, bool single_frame, bool animation);
+
+SDL_Surface* bm_to_sdl_surface(int handle);
+
 #endif

code/cfile/cfile.cpp

@@ -110,6 +110,7 @@ static const char *Cfile_cdrom_dir = NULL;
 static int cfget_cfile_block();
 static CFILE *cf_open_fill_cfblock(const char* source, int line, FILE * fp, int type);
 static CFILE *cf_open_packed_cfblock(const char* source, int line, FILE *fp, int type, size_t offset, size_t size);
+static CFILE *cf_open_memory_fill_cfblock(const char* source, int line, const void* data, size_t size, int dir_type);
 
 #if defined _WIN32
 static CFILE *cf_open_mapped_fill_cfblock(const char* source, int line, HANDLE hFile, int type);
@@ -764,16 +765,16 @@ CFILE *_cfopen(const char* source, int line, const char *file_path, const char *
 	char copy_file_path[MAX_PATH_LEN];  // FIX change in memory from cf_find_file_location
 	strcpy_s(copy_file_path, file_path);
 
-
-	if ( cf_find_file_location( copy_file_path, dir_type, sizeof(longname) - 1, longname, &size, &offset, localize ) )	{
+	const void* file_data = nullptr;
+	if ( cf_find_file_location( copy_file_path, dir_type, sizeof(longname) - 1, longname, &size, &offset, localize, &file_data ) )	{
 
 		// Fount it, now create a cfile out of it
 		nprintf(("CFileDebug", "Requested file %s found at: %s\n", file_path, longname));
 		
 		if ( type & CFILE_MEMORY_MAPPED ) {
 		
-			// Can't open memory mapped files out of pack files
-			if ( offset == 0 )	{
+			// Can't open memory mapped files out of pack or memory files
+			if ( offset == 0 && file_data != nullptr )	{
 #if defined _WIN32
 				HANDLE hFile;
 
@@ -791,18 +792,8 @@ CFILE *_cfopen(const char* source, int line, const char *file_path, const char *
 			} 
 
 		} else {
-
-			FILE *fp = fopen( longname, "rb" );
-
-			if ( fp )	{
-				if ( offset )	{
-					// Found it in a pack file
-					return cf_open_packed_cfblock(source, line, fp, dir_type, offset, size );
-				} else {
-					// Found it in a normal file
-					return cf_open_fill_cfblock(source, line, fp, dir_type);
-				} 
-			}
+			// since cfopen_special already has all the code to handle the opening we can just use that here
+			return _cfopen_special(source, line, longname, mode, size, offset, file_data, dir_type);
 		}
 
 	}
@@ -821,7 +812,7 @@ CFILE *_cfopen(const char* source, int line, const char *file_path, const char *
 // returns:		success	==> address of CFILE structure
 //				error	==> NULL
 //
-CFILE *_cfopen_special(const char* source, int line, const char *file_path, const char *mode, const size_t size, const size_t offset, int dir_type)
+CFILE *_cfopen_special(const char* source, int line, const char *file_path, const char *mode, const size_t size, const size_t offset, const void* data, int dir_type)
 {
 	if ( !cfile_inited) {
 		Int3();
@@ -837,19 +828,27 @@ CFILE *_cfopen_special(const char* source, int line, const char *file_path, cons
 		return NULL;
 	}
 
-	// "file_path" should already be a fully qualified path, so just try to open it
-	FILE *fp = fopen( file_path, "rb" );
-
-	if ( !fp )
-		return NULL;
-
-	if ( offset ) {
-		// it's in a pack file
-		return cf_open_packed_cfblock(source, line, fp, dir_type, offset, size);
-	} else {
-		// it's a normal file
-		return cf_open_fill_cfblock(source, line, fp, dir_type);
+	// In-Memory files are a bit different from normal files so we need to handle them separately
+	if (data != nullptr) {
+		return cf_open_memory_fill_cfblock(source, line, data, size, dir_type);
 	}
+	else {
+		// "file_path" should already be a fully qualified path, so just try to open it
+		FILE *fp = fopen(file_path, "rb");
+
+		if (fp) {
+			if (offset) {
+				// Found it in a pack file
+				return cf_open_packed_cfblock(source, line, fp, dir_type, offset, size);
+			}
+			else {
+				// Found it in a normal file
+				return cf_open_fill_cfblock(source, line, fp, dir_type);
+			}
+		}
+	}
+
+	return NULL;
 }
 
 
@@ -923,7 +922,7 @@ int cfclose( CFILE * cfile )
 	cb = &Cfile_block_list[cfile->id];	
 
 	result = 0;
-	if ( cb->data ) {
+	if ( cb->data && cb->mem_mapped ) {
 		// close memory mapped file
 #if defined _WIN32
 		result = UnmapViewOfFile((void*)cb->data);
@@ -937,7 +936,8 @@ int cfclose( CFILE * cfile )
 		// FIXME: result is wrong after munmap() but it is successful
 		//result = munmap(cb->data, cb->data_length);
 		//Assert(result);
-		munmap(cb->data, cb->data_length);
+		// This const_cast is safe since the pointer returned by mmap was also non-const
+		munmap(const_cast<void*>(cb->data), cb->data_length);
 		if ( cb->fp != NULL)
 			result = fclose(cb->fp);
 #endif
@@ -997,6 +997,7 @@ static CFILE *cf_open_fill_cfblock(const char* source, int line, FILE *fp, int t
 		cfp->id = cfile_block_index;
 		cfp->version = 0;
 		cfbp->data = NULL;
+		cfbp->mem_mapped = false;
 		cfbp->fp = fp;
 		cfbp->dir_type = type;
 		cfbp->max_read_len = 0;
@@ -1039,6 +1040,7 @@ static CFILE *cf_open_packed_cfblock(const char* source, int line, FILE *fp, int
 		cfp->version = 0;
 		cfbp->data = NULL;
 		cfbp->fp = fp;
+		cfbp->mem_mapped = false;
 		cfbp->dir_type = type;
 		cfbp->max_read_len = 0;
 
@@ -1084,6 +1086,7 @@ static CFILE *cf_open_mapped_fill_cfblock(const char* source, int line, FILE *fp
 		cfp->version = 0;
 		cfbp->max_read_len = 0;
 		cfbp->fp = NULL;
+		cfbp->mem_mapped = true;
 #if defined _WIN32
 		cfbp->hInFile = hFile;
 #endif
@@ -1118,6 +1121,37 @@ static CFILE *cf_open_mapped_fill_cfblock(const char* source, int line, FILE *fp
 	}
 }
 
+static CFILE *cf_open_memory_fill_cfblock(const char* source, int line, const void* data, size_t size, int dir_type)
+{
+	int cfile_block_index;
+
+	cfile_block_index = cfget_cfile_block();
+	if ( cfile_block_index == -1 ) {
+		return NULL;
+	}
+	else {
+		CFILE *cfp;
+		Cfile_block *cfbp;
+		cfbp = &Cfile_block_list[cfile_block_index];
+
+		cfp = &Cfile_list[cfile_block_index];
+		cfp->id = cfile_block_index;
+		cfp->version = 0;
+		cfbp->max_read_len = 0;
+		cfbp->fp = NULL;
+		cfbp->mem_mapped = false;
+		cfbp->dir_type = dir_type;
+
+		cfbp->source_file = source;
+		cfbp->line_num = line;
+
+		cf_init_lowlevel_read_code(cfp, 0, size, 0 );
+		cfbp->data = data;
+
+		return cfp;
+	}
+}
+
 int cf_get_dir_type(CFILE *cfile)
 {
 	return Cfile_block_list[cfile->id].dir_type;
@@ -1128,7 +1162,7 @@ int cf_get_dir_type(CFILE *cfile)
 //
 // 
 
-void *cf_returndata(CFILE *cfile)
+const void *cf_returndata(CFILE *cfile)
 {
 	Assert(cfile != NULL);
 	Cfile_block *cb;
@@ -1419,9 +1453,7 @@ int cfilelength( CFILE * cfile )
 	cb = &Cfile_block_list[cfile->id];	
 
 	// TODO: return length of memory mapped file
-	Assert( !cb->data );
-
-	Assert(cb->fp != NULL);
+	Assert( !cb->mem_mapped );
 
 	// cb->size gets set at cfopen
 	
@@ -1520,7 +1552,6 @@ int cfputc(int c, CFILE *cfile)
 	return result;	
 }
 
-
 // cfputs() writes a string to a file
 //
 // returns:   success ==> non-negative value

code/cfile/cfile.h

@@ -13,7 +13,7 @@
 #define __CFILE_H__
 
 
-#include <time.h>
+#include <ctime>
 #include "globalincs/pstypes.h"
 
 #include <stdexcept>
@@ -140,7 +140,7 @@ CFILE *_cfopen(const char* source_file, int line, const char *filename, const ch
 // like cfopen(), but it accepts a fully qualified path only (ie, the result of a cf_find_file_location() call)
 // NOTE: only supports reading files!!
 CFILE *_cfopen_special(const char* source_file, int line, const char *file_path, const char *mode,
-	const size_t size, const size_t offset, int dir_type = CF_TYPE_ANY);
+	const size_t size, const size_t offset, const void* data, int dir_type = CF_TYPE_ANY);
 #define cfopen_special(...) _cfopen_special(LOCATION, __VA_ARGS__) // Pass source location to the function
 
 // Flush the open file buffer
@@ -216,7 +216,7 @@ char *cfgets(char *buf, int n, CFILE *cfile);
 int cfeof(CFILE *cfile);
 
 // Return the data pointer associated with the CFILE structure (for memory mapped files)
-void *cf_returndata(CFILE *cfile);
+const void *cf_returndata(CFILE *cfile);
 
 // get the 2 byte checksum of the passed filename - return 0 if operation failed, 1 if succeeded
 int cf_chksum_short(const char *filename, ushort *chksum, int max_size = -1, int cf_type = CF_TYPE_ANY );
@@ -326,7 +326,7 @@ void cf_sort_filenames( SCP_vector<SCP_string> &list, int sort, SCP_vector<file_
 //         size        - File size
 //         offset      - Offset into pack file.  0 if not a packfile.
 // Returns: If not found returns 0.
-int cf_find_file_location( const char *filespec, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize = false);
+int cf_find_file_location( const char *filespec, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize = false, const void** data_out = nullptr);
 
 // Searches for a file.   Follows all rules and precedence and searches
 // CD's and pack files.  Searches all locations in order for first filename using ext filter list.
@@ -340,7 +340,7 @@ int cf_find_file_location( const char *filespec, int pathtype, int max_out, char
 //         offset      - Offset into pack file.  0 if not a packfile.
 // Returns: If not found returns -1, else returns offset into ext_list.
 // (NOTE: This function is exponentially slow, so don't use it unless truely needed!!)
-int cf_find_file_location_ext(const char *filename, const int ext_num, const char **ext_list, int pathtype, int max_out = 0, char *pack_filename = NULL, size_t *size = NULL, size_t *offset = NULL, bool localize = false);
+int cf_find_file_location_ext(const char *filename, const int ext_num, const char **ext_list, int pathtype, int max_out = 0, char *pack_filename = NULL, size_t *size = NULL, size_t *offset = NULL, bool localize = false, const void** data_out = nullptr);
 
 // Functions to change directories
 int cfile_chdir(const char *dir);
@@ -365,16 +365,13 @@ namespace cfile
 			{
 			}
 
-		cfile_error(const std::string &excuse) : m_excuse(excuse)
+			explicit cfile_error(const std::string &excuse) : m_excuse(excuse)
 			{
 			}
 
-			~cfile_error() throw()
-			{
-			}
+			~cfile_error() SCP_NOEXCEPT override {}
 
-			virtual const char *what() const throw()
-			{
+			const char *what() const SCP_NOEXCEPT override {
 				return m_excuse.c_str();
 			}
 
@@ -385,7 +382,7 @@ namespace cfile
 	class max_read_length : public cfile_error
 	{
 		public:
-		max_read_length(const std::string &excuse) : cfile_error(excuse)
+		explicit max_read_length(const std::string &excuse) : cfile_error(excuse)
 			{
 			}
 

code/cfile/cfilearchive.cpp

@@ -74,18 +74,13 @@ int cfeof(CFILE *cfile)
 	Assert(cfile->id >= 0 && cfile->id < MAX_CFILE_BLOCKS);
 	cb = &Cfile_block_list[cfile->id];	
 
-	int result;
-
-	result = 0;
-
-	// cfeof() not supported for memory-mapped files
-	Assert( !cb->data );
-
-	Assert(cb->fp != NULL);
+	int result = 0;
 
 	#if defined(CHECK_POSITION) && !defined(NDEBUG)
-	auto raw_position = ftell(cb->fp) - cb->lib_offset;
-	Assert(raw_position == cb->raw_position);
+    if (cb->fp) {
+		auto raw_position = ftell(cb->fp) - cb->lib_offset;
+		Assert(raw_position == cb->raw_position);
+	}
 	#endif
 		
 	if (cb->raw_position >= cb->size ) {
@@ -109,14 +104,11 @@ int cftell( CFILE * cfile )
 	Assert(cfile->id >= 0 && cfile->id < MAX_CFILE_BLOCKS);
 	cb = &Cfile_block_list[cfile->id];	
 
-	// Doesn't work for memory mapped files
-	Assert( !cb->data );
-
-	Assert(cb->fp != NULL);
-
 	#if defined(CHECK_POSITION) && !defined(NDEBUG)
-	auto raw_position = ftell(cb->fp) - cb->lib_offset;
-	Assert(raw_position == cb->raw_position);
+    if (cb->fp) {
+		auto raw_position = ftell(cb->fp) - cb->lib_offset;
+		Assert(raw_position == cb->raw_position);
+	}
 	#endif
 
 	// The rest of the code still uses ints, do an overflow check to detect cases where this fails
@@ -141,8 +133,7 @@ int cfseek( CFILE *cfile, int offset, int where )
 
 
 	// TODO: seek to offset in memory mapped file
-	Assert( !cb->data );
-	Assert( cb->fp != NULL );
+	Assert( !cb->mem_mapped );
 	
 	size_t goal_position;
 
@@ -166,14 +157,22 @@ int cfseek( CFILE *cfile, int offset, int where )
 	// Make sure we don't seek beyond the end of the file
 	CAP(goal_position, cb->lib_offset, cb->lib_offset + cb->size);
 
-	int result = fseek(cb->fp, (long)goal_position, SEEK_SET );
-	Assertion(goal_position >= cb->lib_offset, "Invalid offset values detected while seeking! Goal was " SIZE_T_ARG ", lib_offset is " SIZE_T_ARG ".", goal_position, cb->lib_offset);
+	int result = 0;
+
+	if (cb->fp) {
+		// If we have a file pointer we can also seek in that file
+		result = fseek(cb->fp, (long)goal_position, SEEK_SET );
+		Assertion(goal_position >= cb->lib_offset, "Invalid offset values detected while seeking! Goal was " SIZE_T_ARG ", lib_offset is " SIZE_T_ARG ".", goal_position, cb->lib_offset);
+	}
+	// If we only have a data pointer this will do all the work
 	cb->raw_position = goal_position - cb->lib_offset;
 	Assertion(cb->raw_position <= cb->size, "Invalid raw_position value detected!");
 
 	#if defined(CHECK_POSITION) && !defined(NDEBUG)
-	auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
-	Assert(tmp_offset==cb->raw_position);
+	if (cb->fp) {
+		auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
+		Assert(tmp_offset == cb->raw_position);
+	}
 	#endif
 
 	return result;	
@@ -197,13 +196,6 @@ int cfread(void *buf, int elsize, int nelem, CFILE *cfile)
 
 	Cfile_block *cb = &Cfile_block_list[cfile->id];	
 
-	// cfread() not supported for memory-mapped files
-	if(cb->data != NULL)
-	{
-		Warning(LOCATION, "Writing is not supported for mem-mapped files");
-		return 0;
-	}
-
 	if ( (cb->raw_position+size) > cb->size ) {
 		Assertion(cb->raw_position <= cb->size, "Invalid raw_position value detected!");
 		size = cb->size - cb->raw_position;
@@ -222,15 +214,24 @@ int cfread(void *buf, int elsize, int nelem, CFILE *cfile)
 		}
 	}
 
-	size_t bytes_read = fread( buf, 1, size, cb->fp );
+	size_t bytes_read;
+	if (cb->data != nullptr) {
+		// This is a file from memory
+		bytes_read = size;
+		memcpy(buf, reinterpret_cast<const char*>(cb->data) + cb->raw_position, size);
+	} else {
+		bytes_read = fread( buf, 1, size, cb->fp );
+	}
 	if ( bytes_read > 0 )	{
 		cb->raw_position += bytes_read;
 		Assertion(cb->raw_position <= cb->size, "Invalid raw_position value detected!");
 	}		
 
 	#if defined(CHECK_POSITION) && !defined(NDEBUG)
-	auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
-	Assert(tmp_offset==cb->raw_position);
+    if (cb->fp) {
+		auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
+		Assert(tmp_offset==cb->raw_position);
+	}
 	#endif
 
 	return (int)(bytes_read / elsize);
@@ -254,14 +255,31 @@ int cfread_lua_number(double *buf, CFILE *cfile)
 		return 0;
 	}
 
-	long orig_pos = ftell(cb->fp);
-	int items_read = fscanf(cb->fp, LUA_NUMBER_SCAN, buf);
-	cb->raw_position += ftell(cb->fp)-orig_pos;
+	size_t advance = 0;
+	int items_read;
+	if (cb->fp) {
+		long orig_pos = ftell(cb->fp);
+		items_read = fscanf(cb->fp, LUA_NUMBER_SCAN, buf);
+		advance = (size_t) (ftell(cb->fp)-orig_pos);
+	} else {
+		int read;
+		// %n returns the number of bytes currently read so we append that to the scan format at the end so it will return
+		// how many bytes we have consumed
+		items_read = sscanf(reinterpret_cast<const char*>(cb->data), LUA_NUMBER_SCAN "%n", buf, &read);
+		if (items_read == 2) {
+			// We need to correct the items read counter since we read one additional item
+			items_read = 1;
+		}
+		advance = (size_t) read;
+	}
+	cb->raw_position += advance;
 	Assertion(cb->raw_position <= cb->size, "Invalid raw_position value detected!");
 
 	#if defined(CHECK_POSITION) && !defined(NDEBUG)
-	auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
-	Assert(tmp_offset==cb->raw_position);
+    if (cb->fp) {
+		auto tmp_offset = ftell(cb->fp) - cb->lib_offset;
+		Assert(tmp_offset==cb->raw_position);
+	}
 	#endif
 
 	return items_read;

code/cfile/cfilearchive.h

@@ -26,7 +26,8 @@ typedef struct Cfile_block {
 	int		type;				// CFILE_BLOCK_UNUSED, CFILE_BLOCK_USED
 	int		dir_type;		// directory location
 	FILE		*fp;				// File pointer if opening an individual file
-	void		*data;			// Pointer for memory-mapped file access.  NULL if not mem-mapped.
+	const void	*data;			// Pointer for memory-mapped file access.  NULL if not mem-mapped.
+	bool 	mem_mapped;		// Flag for memory mapped files (if data is not null and this is false it means that it's an embedded file)
 #ifdef _WIN32
 	HANDLE	hInFile;			// Handle from CreateFile()
 	HANDLE	hMapFile;		// Handle from CreateFileMapping()

code/cfile/cfilesystem.cpp

@@ -37,12 +37,16 @@
 #include "cfile/cfilesystem.h"
 #include "cmdline/cmdline.h"
 #include "globalincs/pstypes.h"
+#include "def_files/def_files.h"
 #include "localization/localize.h"
 #include "osapi/osapi.h"
 #include "parse/parselo.h"
 
-#define CF_ROOTTYPE_PATH 0
-#define CF_ROOTTYPE_PACK 1
+enum CfileRootType {
+	CF_ROOTTYPE_PATH = 0,
+	CF_ROOTTYPE_PACK = 1,
+	CF_ROOTTYPE_MEMORY = 2,
+};
 
 // for a defined and specifically set location to get/send pilot/campaign files
 SCP_string Pilot_file_path;
@@ -53,8 +57,8 @@ SCP_string Pilot_file_path;
 //    specifying cd-rom tree
 //    searching for pack files on CD-rom tree
 typedef struct cf_root {
-	char				path[CF_MAX_PATHNAME_LENGTH];		// Contains something like c:\projects\freespace or c:\projects\freespace\freespace.vp
-	int				roottype;								// CF_ROOTTYPE_PATH  = Path, CF_ROOTTYPE_PACK =Pack file
+	char	path[CF_MAX_PATHNAME_LENGTH];	// Contains something like c:\projects\freespace or c:\projects\freespace\freespace.vp
+	int		roottype;						// CF_ROOTTYPE_PATH  = Path, CF_ROOTTYPE_PACK =Pack file, CF_ROOTTYPE_MEMORY=In memory
 } cf_root;
 
 // convenient type for sorting (see cf_build_pack_list())
@@ -79,13 +83,14 @@ static int Num_path_roots = 0;
 
 // Created by searching all roots in order.   This means Files is then sorted by precedence.
 typedef struct cf_file {
-	char	name_ext[CF_MAX_FILENAME_LENGTH];	// Filename and extension
-	int		root_index;							// Where in Roots this is located
-	int		pathtype_index;						// Where in Paths this is located
-	time_t	write_time;							// When it was last written
-	int		size;								// How big it is in bytes
-	int		pack_offset;						// For pack files, where it is at.   0 if not in a pack file.  This can be used to tell if in a pack file.
-	char*	real_name;							// For real files, the full path
+	char		name_ext[CF_MAX_FILENAME_LENGTH];	// Filename and extension
+	int			root_index;							// Where in Roots this is located
+	int			pathtype_index;						// Where in Paths this is located
+	time_t		write_time;							// When it was last written
+	int			size;								// How big it is in bytes
+	int			pack_offset;						// For pack files, where it is at.   0 if not in a pack file.  This can be used to tell if in a pack file.
+	char*		real_name;							// For real files, the full path
+	const void*	data;								// For in-memory files, the data pointer
 } cf_file;
 
 #define CF_NUM_FILES_PER_BLOCK   512
@@ -510,6 +515,11 @@ void cf_build_root_list(const char *cdrom_dir)
 
 	}
 
+	// The final root is the in-memory root
+	root = cf_create_root();
+	memset(root->path, 0, sizeof(root->path));
+	root->roottype = CF_ROOTTYPE_MEMORY;
+
 }
 
 // Given a lower case list of file extensions 
@@ -836,7 +846,48 @@ void cf_search_root_pack(int root_index)
 	mprintf(( "%i files\n", num_files ));
 }
 
+void cf_search_memory_root(int root_index) {
+	int num_files = 0;
+	mprintf(( "Searching memory root ... " ));
 
+	auto default_files = defaults_get_all();
+	for (auto& default_file : default_files) {
+		// Pure built in files have an empty path_type string
+		if (strlen(default_file.path_type) <= 0) {
+			// Ignore pure builtin files. They should only be accessible with defaults_get_file
+			continue;
+		}
+
+		int pathtype = -1;
+		for (int i = 0; i < CF_MAX_PATH_TYPES; ++i) {
+			if (Pathtypes[i].path == nullptr) {
+				continue;
+			}
+
+			if (!strcmp(Pathtypes[i].path, default_file.path_type)) {
+				pathtype = i;
+				break;
+			}
+		}
+		Assertion(pathtype != -1, "Default file '%s%s%s' does not use a valid path type!",
+				  default_file.path_type,
+				  DIR_SEPARATOR_STR,
+				  default_file.filename);
+
+		cf_file *file = cf_create_file();
+
+		strcpy_s( file->name_ext, default_file.filename );
+		file->root_index = root_index;
+		file->pathtype_index = pathtype;
+		file->write_time = time(nullptr); // Just assume that memory files were last written to just now
+		file->size = (int)default_file.size;
+		file->data = default_file.data;
+
+		num_files++;
+	}
+
+	mprintf(( "%i files\n", num_files ));
+}
 
 void cf_build_file_list()
 {
@@ -851,6 +902,8 @@ void cf_build_file_list()
 			cf_search_root_path(i);
 		} else if ( root->roottype == CF_ROOTTYPE_PACK )	{
 			cf_search_root_pack(i);
+		} else if (root->roottype == CF_ROOTTYPE_MEMORY) {
+			cf_search_memory_root(i);
 		}
 	}
 
@@ -932,7 +985,7 @@ void cf_free_secondary_filelist()
  *
  * @return If not found returns 0.
  */
-int cf_find_file_location( const char *filespec, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize )
+int cf_find_file_location( const char *filespec, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize, const void** data_out )
 {
 	int i;
     uint ui;
@@ -1067,8 +1120,16 @@ int cf_find_file_location( const char *filespec, int pathtype, int max_out, char
 					if (offset)
 						*offset = (size_t)f->pack_offset;
 
+					if (data_out)
+						*data_out = f->data;
+
 					if (pack_filename) {
-						if (f->pack_offset < 1) {
+						if (f->data != nullptr) {
+							// This is an in-memory file so we just copy the pathtype name + file name
+							strncpy(pack_filename, Pathtypes[f->pathtype_index].path, max_out);
+							strncat(pack_filename, DIR_SEPARATOR_STR, max_out);
+							strncat(pack_filename, f->name_ext, max_out);
+						} else if (f->pack_offset < 1) {
 							// This is a real file, return the actual file path
 							strncpy( pack_filename, f->real_name, max_out );
 						} else {
@@ -1092,8 +1153,16 @@ int cf_find_file_location( const char *filespec, int pathtype, int max_out, char
 			if (offset)
 				*offset = (size_t)f->pack_offset;
 
+			if (data_out)
+				*data_out = f->data;
+
 			if (pack_filename) {
-				if (f->pack_offset < 1) {
+				if (f->data != nullptr) {
+					// This is an in-memory file so we just copy the pathtype name + file name
+					strncpy(pack_filename, Pathtypes[f->pathtype_index].path, max_out);
+					strncat(pack_filename, DIR_SEPARATOR_STR, max_out);
+					strncat(pack_filename, f->name_ext, max_out);
+				} else if (f->pack_offset < 1) {
 					// This is a real file, return the actual file path
 					strncpy( pack_filename, f->real_name, max_out );
 				} else {
@@ -1132,7 +1201,7 @@ extern char *stristr(char *str, const char *substr);
  *
  * @return If not found returns -1, else returns offset into ext_list.
  */
-int cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize )
+int cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype, int max_out, char *pack_filename, size_t *size, size_t *offset, bool localize, const void** data_out)
 {
 	int cur_ext, i;
     uint ui;
@@ -1217,6 +1286,9 @@ int cf_find_file_location_ext( const char *filename, const int ext_num, const ch
 				if (size)
 					*size = findstruct.size;
 
+				if (data_out)
+					*data_out = nullptr;
+
 				_findclose(findhandle);
 
 				if (offset)
@@ -1235,6 +1307,9 @@ int cf_find_file_location_ext( const char *filename, const int ext_num, const ch
 					if (size)
 						*size = filelength( fileno(fp) );
 
+					if (data_out)
+						*data_out = nullptr;
+
 					fclose(fp);
 
 					if (offset)
@@ -1334,15 +1409,25 @@ int cf_find_file_location_ext( const char *filename, const int ext_num, const ch
 						if (offset)
 							*offset = f->pack_offset;
 
+						if (data_out)
+							*data_out = f->data;
+
 						if (pack_filename) {
-							if (f->pack_offset < 1) {
+							if (f->data != nullptr) {
+								// This is an in-memory file so we just copy the pathtype name + file name
+								strncpy(pack_filename, Pathtypes[f->pathtype_index].path, max_out);
+								strncat(pack_filename, DIR_SEPARATOR_STR, max_out);
+								strncat(pack_filename, f->name_ext, max_out);
+							}
+							else if (f->pack_offset < 1) {
 								// This is a real file, return the actual file path
-								strncpy( pack_filename, f->real_name, max_out );
-							} else {
+								strncpy(pack_filename, f->real_name, max_out);
+							}
+							else {
 								// File is in a pack file
 								cf_root *r = cf_get_root(f->root_index);
 
-								strncpy( pack_filename, r->path, max_out );
+								strncpy(pack_filename, r->path, max_out);
 							}
 						}
 
@@ -1362,15 +1447,25 @@ int cf_find_file_location_ext( const char *filename, const int ext_num, const ch
 				if (offset)
 					*offset = f->pack_offset;
 
+				if (data_out)
+					*data_out = f->data;
+
 				if (pack_filename) {
-					if (f->pack_offset < 1) {
+					if (f->data != nullptr) {
+						// This is an in-memory file so we just copy the pathtype name + file name
+						strncpy(pack_filename, Pathtypes[f->pathtype_index].path, max_out);
+						strncat(pack_filename, DIR_SEPARATOR_STR, max_out);
+						strncat(pack_filename, f->name_ext, max_out);
+					}
+					else if (f->pack_offset < 1) {
 						// This is a real file, return the actual file path
-						strncpy( pack_filename, f->real_name, max_out );
-					} else {
+						strncpy(pack_filename, f->real_name, max_out);
+					}
+					else {
 						// File is in a pack file
 						cf_root *r = cf_get_root(f->root_index);
 
-						strncpy( pack_filename, r->path, max_out );
+						strncpy(pack_filename, r->path, max_out);
 					}
 				}
 
@@ -1432,6 +1527,7 @@ int cf_matches_spec(const char *filespec, const char *filename)
 int (*Get_file_list_filter)(const char *filename) = NULL;
 const char *Get_file_list_child = NULL;
 int Skip_packfile_search = 0;
+bool Skip_memory_files = false;
 
 static bool verify_file_list_child()
 {
@@ -1635,6 +1731,11 @@ int cf_get_file_list( SCP_vector<SCP_string> &list, int pathtype, const char *fi
 				continue;
 			}
 
+			if (Skip_memory_files && f->data != nullptr) {
+				// If we want to skip memory files and this is a memory file then ignore it
+				continue;
+			}
+
 			if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext) ) {
 				//mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index ));
 
@@ -1847,6 +1948,11 @@ int cf_get_file_list( int max, char **list, int pathtype, const char *filter, in
 				continue;
 			}
 
+			if (Skip_memory_files && f->data != nullptr) {
+				// If we want to skip memory files and this is a memory file then ignore it
+				continue;
+			}
+
 			if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext) ) {
 
 				//mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index ));
@@ -2067,6 +2173,11 @@ int cf_get_file_list_preallocated( int max, char arr[][MAX_FILENAME_LEN], char *
 				continue;
 			}
 
+			if (Skip_memory_files && f->data != nullptr) {
+				// If we want to skip memory files and this is a memory file then ignore it
+				continue;
+			}
+
 			if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext) ) {
 
 				//mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index ));

code/cmdline/cmdline.cpp

@@ -160,7 +160,6 @@ Flag exe_params[] =
 	{ "-noenv",				"Disable environment maps",					true,	EASY_DEFAULT_MEM,	EASY_MEM_ALL_ON,	"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-env", },
 	{ "-nomotiondebris",	"Disable motion debris",					true,	EASY_ALL_ON,		EASY_DEFAULT,		"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-nomotiondebris",},
 	{ "-noscalevid",		"Disable scale-to-window for movies",		true,	0,					EASY_DEFAULT,		"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-noscalevid", },
-	{ "-missile_lighting",	"Apply lighting to missiles"	,			true,	EASY_ALL_ON,		EASY_DEFAULT,		"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-missile_lighting", },
 	{ "-nonormal",			"Disable normal maps",						true,	EASY_DEFAULT_MEM,	EASY_MEM_ALL_ON,	"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-normal" },
 	{ "-no_emissive_light",	"Disable emissive light from ships",		true,	0,					EASY_DEFAULT,		"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-no_emissive_light" },
 	{ "-noheight",			"Disable height/parallax maps",				true,	EASY_DEFAULT_MEM,	EASY_MEM_ALL_ON,	"Graphics",		"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-height" },
@@ -216,7 +215,6 @@ Flag exe_params[] =
 	{ "-use_gldrawelements","Don't use glDrawRangeElements",			true,	0,					EASY_DEFAULT,		"Troubleshoot",	"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-use_gldrawelements", },
 	{ "-old_collision",		"Use old collision detection system",		true,	EASY_DEFAULT,		EASY_ALL_ON,		"Troubleshoot",	"http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-old_collision", },
 	{ "-gl_finish",			"Fix input lag on some ATI+Linux systems",	true,	0,					EASY_DEFAULT,		"Troubleshoot", "http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-gl_finish", },
-	{ "-no_batching",		"Disable batched model rendering",			true,	0,					EASY_DEFAULT,		"Troubleshoot", "", },
 	{ "-no_geo_effects",	"Disable geometry shader for effects",		true,	0,					EASY_DEFAULT,		"Troubleshoot", "", },
 	{ "-set_cpu_affinity",	"Sets processor affinity to config value",	true,	0,					EASY_DEFAULT,		"Troubleshoot", "", },
 	{ "-nograb",			"Disables mouse grabbing",					true,	0,					EASY_DEFAULT,		"Troubleshoot", "http://www.hard-light.net/wiki/index.php/Command-Line_Reference#-nograb", },
@@ -315,7 +313,6 @@ cmdline_parm spec_static_arg("-spec_static", "Adjusts suns contribution to specu
 cmdline_parm spec_point_arg("-spec_point", "Adjusts laser weapons contribution to specular highlights", AT_FLOAT);
 cmdline_parm spec_tube_arg("-spec_tube", "Adjusts beam weapons contribution to specular highlights", AT_FLOAT);
 cmdline_parm ambient_factor_arg("-ambient_factor", "Adjusts ambient light applied to all parts of a ship", AT_INT);		// Cmdline_ambient_factor
-cmdline_parm missile_lighting_arg("-missile_lighting", NULL, AT_NONE);	// Cmdline_missile_lighting
 cmdline_parm env("-noenv", NULL, AT_NONE);								// Cmdline_env
 cmdline_parm glow_arg("-noglow", NULL, AT_NONE); 						// Cmdline_glow  -- use Bobs glow code
 cmdline_parm nomotiondebris_arg("-nomotiondebris", NULL, AT_NONE);		// Cmdline_nomotiondebris  -- Removes those ugly floating rocks -C
@@ -333,10 +330,10 @@ cmdline_parm fxaa_preset_arg("-fxaa_preset", "FXAA quality (0-9), requires -post
 cmdline_parm fb_explosions_arg("-fb_explosions", NULL, AT_NONE);
 cmdline_parm fb_thrusters_arg("-fb_thrusters", NULL, AT_NONE);
 cmdline_parm flightshaftsoff_arg("-nolightshafts", NULL, AT_NONE);
-cmdline_parm no_batching("-no_batching", NULL, AT_NONE);
 cmdline_parm shadow_quality_arg("-shadow_quality", NULL, AT_INT);
 cmdline_parm enable_shadows_arg("-enable_shadows", NULL, AT_NONE);
 cmdline_parm no_deferred_lighting_arg("-no_deferred", NULL, AT_NONE);	// Cmdline_no_deferred
+cmdline_parm anisotropy_level_arg("-anisotropic_filter", NULL, AT_INT);
 
 float Cmdline_clip_dist = Default_min_draw_distance;
 float Cmdline_fov = 0.75f;
@@ -344,7 +341,6 @@ float Cmdline_ogl_spec = 80.0f;
 int Cmdline_ambient_factor = 128;
 int Cmdline_env = 1;
 int Cmdline_mipmap = 0;
-int Cmdline_missile_lighting = 0;
 int Cmdline_glow = 1;
 int Cmdline_nomotiondebris = 0;
 int Cmdline_noscalevid = 0;
@@ -361,10 +357,10 @@ int Cmdline_fxaa_preset = 6;
 extern int Fxaa_preset_last_frame;
 bool Cmdline_fb_explosions = 0;
 bool Cmdline_fb_thrusters = false;
-bool Cmdline_no_batching = false;
 extern bool ls_force_off;
 int Cmdline_shadow_quality = 0;
 int Cmdline_no_deferred_lighting = 0;
+int Cmdline_aniso_level = 0;
 
 // Game Speed related
 cmdline_parm cache_bitmaps_arg("-cache_bitmaps", NULL, AT_NONE);	// Cmdline_cache_bitmaps
@@ -508,6 +504,7 @@ char *Cmdline_start_mission = NULL;
 int Cmdline_old_collision_sys = 0;
 int Cmdline_dis_collisions = 0;
 int Cmdline_dis_weapons = 0;
+bool Cmdline_output_sexp_info = false;
 int Cmdline_noparseerrors = 0;
 #ifdef Allow_NoWarn
 int Cmdline_nowarn = 0; // turn warnings off in FRED
@@ -548,6 +545,7 @@ cmdline_parm deprecated_jpgtga_arg("-jpgtga", "Deprecated", AT_NONE);
 cmdline_parm deprecated_htl_arg("-nohtl", "Deprecated", AT_NONE);
 cmdline_parm deprecated_brieflighting_arg("-brief_lighting", "Deprecated", AT_NONE);
 cmdline_parm deprecated_sndpreload_arg("-snd_preload", "Deprecated", AT_NONE);
+cmdline_parm deprecated_missile_lighting_arg("-missile_lighting", "Deprecated", AT_NONE);
 
 int Cmdline_deprecated_spec = 0;
 int Cmdline_deprecated_glow = 0;
@@ -557,6 +555,7 @@ int Cmdline_deprecated_tbp = 0;
 int Cmdline_deprecated_jpgtga = 0;
 int Cmdline_deprecated_nohtl = 0;
 bool Cmdline_deprecated_brief_lighting = 0;
+bool Cmdline_deprecated_missile_lighting = false;
 
 #ifndef NDEBUG
 // NOTE: this assumes that os_init() has already been called but isn't a fatal error if it hasn't
@@ -622,6 +621,11 @@ void cmdline_debug_print_cmdline()
 	{
 		mprintf(("Deprecated flag '-brief_lighting' found. Please remove from your cmdline.\n"));
 	}
+
+	if (Cmdline_deprecated_missile_lighting) 
+	{
+		mprintf(("Deprecated flag '-missile_lighting' found. Please remove from your cmdline.\n"));
+	}
 }
 #endif
 
@@ -1163,7 +1167,7 @@ bool cmdline_parm::has_param() {
 #ifdef SCP_UNIX
 // Return a vector with all filesystem names of "parent/dir" relative to parent.
 // dir must not contain a slash.
-static SCP_vector<SCP_string> unix_get_single_dir_names(SCP_string parent, SCP_string dir)
+static SCP_vector<SCP_string> unix_get_single_dir_names(const SCP_string& parent, const SCP_string& dir)
 {
 	SCP_vector<SCP_string> ret;
 
@@ -1186,7 +1190,7 @@ static SCP_vector<SCP_string> unix_get_single_dir_names(SCP_string parent, SCP_s
 
 // Return a vector with all filesystem names of "parent/dir" relative to parent.
 // Recurses to deal with slashes in dir.
-static SCP_vector<SCP_string> unix_get_dir_names(SCP_string parent, SCP_string dir)
+static SCP_vector<SCP_string> unix_get_dir_names(const SCP_string& parent, const SCP_string& dir)
 {
 	size_t slash = dir.find_first_of("/\\");
 
@@ -1387,6 +1391,15 @@ static json_t* json_get_v1() {
 
 			json_object_set_new(openal_obj, "capture_devices", capture_array);
 		}
+		{
+			auto efx_support_obj = json_object();
+
+			for (auto& pair : openal_info.efx_support) {
+				json_object_set_new(efx_support_obj, pair.first.c_str(), json_boolean(pair.second));
+			}
+
+			json_object_set_new(openal_obj, "efx_support", efx_support_obj);
+		}
 
 		json_object_set_new(root, "openal", openal_obj);
 	}
@@ -1890,7 +1903,7 @@ bool SetCmdlineParams()
 		Output_scripting_meta = true;
 
 	if (output_sexp_arg.found() ) {
-		output_sexps("sexps.html");
+		Cmdline_output_sexp_info = true;
 	}
 
 	if ( no_pbo_arg.found() )
@@ -1982,9 +1995,6 @@ bool SetCmdlineParams()
 	if ( rearm_timer_arg.found() )
 		Cmdline_rearm_timer = 1;
 
-	if ( missile_lighting_arg.found() )
-		Cmdline_missile_lighting = 1;
-
 	if ( save_render_targets_arg.found() )
 		Cmdline_save_render_targets = 1;
 	
@@ -2019,11 +2029,6 @@ bool SetCmdlineParams()
         Cmdline_fb_thrusters = true;
     }
 
-	if ( no_batching.found() ) 
-	{
-		Cmdline_no_batching = true;
-	}
-
 	if ( postprocess_arg.found() )
 	{
 		Cmdline_postprocess = 1;
@@ -2059,6 +2064,11 @@ bool SetCmdlineParams()
 		Cmdline_no_deferred_lighting = 1;
 	}
 
+	if (anisotropy_level_arg.found()) 
+	{
+		Cmdline_aniso_level = anisotropy_level_arg.get_int();
+	}
+
 	if (frame_profile_write_file.found())
 	{
 		Cmdline_profile_write_file = true;
@@ -2144,6 +2154,11 @@ bool SetCmdlineParams()
 		Cmdline_deprecated_brief_lighting = 1;
 	}
 
+	if (deprecated_missile_lighting_arg.found())
+	{
+		Cmdline_deprecated_missile_lighting = true;
+	}
+
 	return true; 
 }
 

code/cmdline/cmdline.h

@@ -62,7 +62,6 @@ extern float static_point_factor;
 extern float static_tube_factor;
 extern int Cmdline_ambient_factor;
 extern int Cmdline_env;
-extern int Cmdline_missile_lighting;
 extern int Cmdline_glow;
 extern int Cmdline_nomotiondebris;
 extern int Cmdline_noscalevid;	// disables fit-to-window for movies - taylor
@@ -77,10 +76,10 @@ extern bool Cmdline_fxaa;
 extern int Cmdline_fxaa_preset;
 extern bool Cmdline_fb_explosions;
 extern bool Cmdline_fb_thrusters;
-extern bool Cmdline_no_batching;
 extern int Cmdline_shadow_quality;
 extern int Cmdline_no_deferred_lighting;
 extern int Cmdline_no_emissive;
+extern int Cmdline_aniso_level;
 
 // Game Speed related
 extern int Cmdline_cache_bitmaps;
@@ -145,6 +144,7 @@ extern char *Cmdline_start_mission;
 extern int Cmdline_old_collision_sys;
 extern int Cmdline_dis_collisions;
 extern int Cmdline_dis_weapons;
+extern bool Cmdline_output_sexp_info;
 extern int Cmdline_noparseerrors;
 extern int Cmdline_extra_warn;
 extern int Cmdline_bmpman_usage;

code/cutscene/ffmpeg/AudioDecoder.h

@@ -25,11 +25,11 @@ class AudioDecoder: public FFMPEGStreamDecoder<AudioFrame> {
  public:
 	explicit AudioDecoder(DecoderStatus* status);
 
-	virtual ~AudioDecoder();
+	~AudioDecoder() override;
 
-	virtual void decodePacket(AVPacket* packet) SCP_OVERRIDE;
+	void decodePacket(AVPacket* packet) override;
 
-	virtual void finishDecoding() SCP_OVERRIDE;
+	void finishDecoding() override;
 };
 }
 }

code/cutscene/ffmpeg/FFMPEGDecoder.cpp

@@ -184,11 +184,13 @@ std::unique_ptr<DecoderStatus> initializeStatus(std::unique_ptr<InputStream>& st
 	for (uint32_t i = 0; i < ctx->nb_streams; ++i) {
 		auto test_stream = ctx->streams[i];
 
+#if LIBAVCODEC_VERSION_INT > AV_VERSION_INT(57, 24, 255)
 		auto& pars = test_stream->codecpar;
 
 		if (!(pars->codec_type == AVMEDIA_TYPE_SUBTITLE)) {
 			continue;
 		}
+#endif
 		AVDictionaryEntry* tag = nullptr;
 		if ((tag = av_dict_get(test_stream->metadata, "language", nullptr, 0)) == nullptr) {
 			continue;

code/cutscene/ffmpeg/FFMPEGDecoder.h

@@ -17,19 +17,19 @@ class FFMPEGDecoder: public Decoder {
  public:
 	FFMPEGDecoder();
 
-	virtual ~FFMPEGDecoder();
+	~FFMPEGDecoder() override;
 
-	bool initialize(const SCP_string& fileName) SCP_OVERRIDE;
+	bool initialize(const SCP_string& fileName) override;
 
-	MovieProperties getProperties() SCP_OVERRIDE;
+	MovieProperties getProperties() override;
 
-	void startDecoding() SCP_OVERRIDE;
+	void startDecoding() override;
 
-	bool hasAudio() SCP_OVERRIDE;
+	bool hasAudio() override;
 
-	bool hasSubtitles() SCP_OVERRIDE;
+	bool hasSubtitles() override;
 
-	void close() SCP_OVERRIDE;
+	void close() override;
 };
 }
 }

code/cutscene/ffmpeg/SubtitleDecoder.h

@@ -10,11 +10,11 @@ class SubtitleDecoder: public FFMPEGStreamDecoder<SubtitleFrame> {
  public:
     explicit SubtitleDecoder(DecoderStatus* status);
 
-    virtual ~SubtitleDecoder();
+	~SubtitleDecoder() override;
 
-    virtual void decodePacket(AVPacket* packet) SCP_OVERRIDE;
+	void decodePacket(AVPacket* packet) override;
 
-    virtual void finishDecoding() SCP_OVERRIDE;
+	void finishDecoding() override;
     void pushSubtitleFrame(AVPacket* subtitle, AVSubtitle* pSubtitle);
 };
 

code/cutscene/ffmpeg/VideoDecoder.cpp

@@ -22,14 +22,14 @@ class FFMPEGVideoFrame: public VideoFrame {
 	FFMPEGVideoFrame() : frame(nullptr) {
 	}
 
-	virtual ~FFMPEGVideoFrame() {
+	~FFMPEGVideoFrame() override {
 		if (frame != nullptr) {
 			av_freep(&frame->data[0]);
 			av_frame_free(&frame);
 		}
 	}
 
-	virtual DataPointers getDataPointers() {
+	DataPointers getDataPointers() override {
 		DataPointers ptrs;
 		ptrs.y = frame->data[0];
 		ptrs.u = frame->data[1];

code/cutscene/ffmpeg/VideoDecoder.h

@@ -15,11 +15,11 @@ class VideoDecoder: public FFMPEGStreamDecoder<VideoFrame> {
  public:
 	explicit VideoDecoder(DecoderStatus* status);
 
-	virtual ~VideoDecoder();
+	~VideoDecoder() override;
 
-	virtual void decodePacket(AVPacket* packet) SCP_OVERRIDE;
+	void decodePacket(AVPacket* packet) override;
 
-	virtual void finishDecoding() SCP_OVERRIDE;
+	void finishDecoding() override;
 };
 }
 }

code/decals/decals.cpp

@@ -0,0 +1,541 @@
+#include <nebula/neb.h>
+#include <utility>
+#include "decals/decals.h"
+
+#include "graphics/2d.h"
+#include "graphics/decal_draw_list.h"
+#include "graphics/util/uniform_structs.h"
+#include "parse/parselo.h"
+#include "tracing/tracing.h"
+#include "ship/ship.h"
+
+namespace {
+
+class DecalDefinition {
+	SCP_string _name;
+
+	SCP_string _diffuseFilename;
+	bool _loopDiffuse = true;
+
+	SCP_string _glowFilename;
+	bool _loopGlow = true;
+
+	SCP_string _normalMapFilename;
+	bool _loopNormal = true;
+
+	int _diffuseBitmap = -1;
+	int _glowBitmap = -1;
+	int _normalBitmap = -1;
+
+ public:
+	explicit DecalDefinition(SCP_string name) : _name(std::move(name)) {
+	}
+
+	~DecalDefinition() {
+		if (_diffuseBitmap >= 0) {
+			bm_release(_diffuseBitmap);
+		}
+		if (_glowBitmap >= 0) {
+			bm_release(_glowBitmap);
+		}
+		if (_normalBitmap >= 0) {
+			bm_release(_normalBitmap);
+		}
+	}
+
+	// Disallow copying
+	DecalDefinition(const DecalDefinition&) = delete;
+	DecalDefinition& operator=(const DecalDefinition&) = delete;
+
+	// Move constructor and operator
+	DecalDefinition(DecalDefinition&& other) SCP_NOEXCEPT {
+		*this = std::move(other); // Use operator implementation
+	}
+	DecalDefinition& operator=(DecalDefinition&& other) SCP_NOEXCEPT {
+		std::swap(_name, other._name);
+
+		std::swap(_diffuseFilename, other._diffuseFilename);
+		std::swap(_glowFilename, other._glowFilename);
+		std::swap(_normalMapFilename, other._normalMapFilename);
+
+		std::swap(_diffuseBitmap, other._diffuseBitmap);
+		std::swap(_glowBitmap, other._glowBitmap);
+		std::swap(_normalBitmap, other._normalBitmap);
+
+		std::swap(_loopDiffuse, other._loopDiffuse);
+		std::swap(_loopGlow, other._loopGlow);
+		std::swap(_loopNormal, other._loopNormal);
+
+		return *this;
+	}
+
+	void parse() {
+		if (optional_string("+Diffuse:")) {
+			stuff_string(_diffuseFilename, F_FILESPEC);
+
+			if (!bm_validate_filename(_diffuseFilename, true, true)) {
+				error_display(0, "Animation '%s' is not valid!", _diffuseFilename.c_str());
+				_diffuseFilename = "";
+			}
+
+			if (optional_string("+Loop:")) {
+				stuff_boolean(&_loopDiffuse);
+			}
+		}
+		if (optional_string("+Glow:")) {
+			stuff_string(_glowFilename, F_FILESPEC);
+
+			if (!bm_validate_filename(_glowFilename, true, true)) {
+				error_display(0, "Animation '%s' is not valid!", _glowFilename.c_str());
+				_glowFilename = "";
+			}
+
+			if (optional_string("+Loop:")) {
+				stuff_boolean(&_loopGlow);
+			}
+		}
+		if (optional_string("+Normal:")) {
+			stuff_string(_normalMapFilename, F_FILESPEC);
+
+			if (!bm_validate_filename(_normalMapFilename, true, true)) {
+				error_display(0, "Animation '%s' is not valid!", _normalMapFilename.c_str());
+				_normalMapFilename = "";
+			}
+
+			if (optional_string("+Loop:")) {
+				stuff_boolean(&_loopNormal);
+			}
+		}
+	}
+
+	void loadBitmaps() {
+		if (_diffuseBitmap == -1 && VALID_FNAME(_diffuseFilename)) {
+			_diffuseBitmap = bm_load_either(_diffuseFilename.c_str());
+			if (_diffuseBitmap == -1) {
+				Warning(LOCATION,
+						"Bitmap '%s' failed to load for decal definition %s!",
+						_diffuseFilename.c_str(),
+						_name.c_str());
+			}
+		}
+		if (_glowBitmap == -1 && VALID_FNAME(_glowFilename)) {
+			_glowBitmap = bm_load_either(_glowFilename.c_str());
+			if (_glowBitmap == -1) {
+				Warning(LOCATION,
+						"Bitmap '%s' failed to load for decal definition %s!",
+						_glowFilename.c_str(),
+						_name.c_str());
+			}
+		}
+		if (_normalBitmap == -1 && VALID_FNAME(_normalMapFilename)) {
+			_normalBitmap = bm_load_either(_normalMapFilename.c_str());
+			if (_normalBitmap == -1) {
+				Warning(LOCATION,
+						"Bitmap '%s' failed to load for decal definition %s!",
+						_normalMapFilename.c_str(),
+						_name.c_str());
+			}
+		}
+	}
+
+	void pageIn() {
+		if (_diffuseBitmap >= 0) {
+			bm_page_in_texture(_diffuseBitmap);
+		}
+		if (_glowBitmap >= 0) {
+			bm_page_in_texture(_glowBitmap);
+		}
+		if (_normalBitmap >= 0) {
+			bm_page_in_texture(_normalBitmap);
+		}
+	}
+
+	bool bitmapsLoaded() {
+		// Since both bitmap types are optional we need to check if either is loaded to determine if any bitmap is loaded
+		return _diffuseBitmap >= 0 || _glowBitmap >= 0 || _normalBitmap >= 0;
+	}
+
+	const SCP_string& getName() const {
+		return _name;
+	}
+	int getDiffuseBitmap() const {
+		return _diffuseBitmap;
+	}
+	int getGlowBitmap() const {
+		return _glowBitmap;
+	}
+	int getNormalBitmap() const {
+		return _normalBitmap;
+	}
+	bool isDiffuseLooping() const {
+		return _loopDiffuse;
+	}
+	bool isGlowLooping() const {
+		return _loopGlow;
+	}
+	bool isNormalLooping() const {
+		return _loopNormal;
+	}
+};
+
+SCP_vector<DecalDefinition> decalDefinitions;
+
+// Variable to indicate if the system is able to work correctly on the current system
+bool decal_system_active = true;
+
+void parse_decals_table(const char* filename) {
+	try {
+		read_file_text(filename, CF_TYPE_TABLES);
+		reset_parse();
+
+		required_string("#Decals");
+
+		while (optional_string("$Decal:")) {
+			SCP_string name;
+			stuff_string(name, F_NAME);
+
+			DecalDefinition def(name);
+			def.parse();
+
+			decalDefinitions.push_back(std::move(def));
+		}
+
+		required_string("#End");
+	} catch (const parse::ParseException& e) {
+		mprintf(("TABLES: Unable to parse '%s'!  Error message = %s.\n", filename, e.what()));
+		return;
+	}
+
+	if (!gr_is_capable(CAPABILITY_DEFERRED_LIGHTING)) {
+		// we need deferred lighting
+		decal_system_active = false;
+		mprintf(("Note: Decal system has been disabled due to lack of deferred lighting.\n"));
+	}
+	if (!gr_is_capable(CAPABILITY_NORMAL_MAP)) {
+		// We need normal mapping for the full feature range
+		decal_system_active = false;
+		mprintf(("Note: Decal system has been disabled due to lack of normal mapping.\n"));
+	}
+	if (!gr_is_capable(CAPABILITY_SEPARATE_BLEND_FUNCTIONS)) {
+		// WWe need separate blending functions for different color buffers
+		decal_system_active = false;
+		mprintf(("Note: Decal system has been disabled due to lack of separate color buffer blend functions.\n"));
+	}
+}
+
+struct Decal {
+	int definition_handle = -1;
+	object_h object;
+	int submodel = -1;
+
+	float creation_time = -1.0f; //!< The mission time at which this decal was created
+	float lifetime = -1.0f; //!< The time this decal is active. When negative it never expires
+
+	vec3d position = vmd_zero_vector;
+	vec3d scale;
+	matrix orientation = vmd_identity_matrix;
+
+	Decal() {
+		vm_vec_make(&scale, 1.f, 1.f, 1.f);
+	}
+
+	bool isValid() {
+		if (!object.IsValid()) {
+			return false;
+		}
+
+		if (lifetime > 0.0f) {
+			if (f2fl(Missiontime) >= creation_time + lifetime) {
+				// Decal has expired
+				return false;
+			}
+		}
+
+		auto objp = object.objp;
+		if (objp->type == OBJ_SHIP) {
+			auto shipp = &Ships[objp->instance];
+			auto model_instance = model_get_instance(shipp->model_instance_num);
+
+			Assertion(submodel >= 0 && submodel < model_get(object_get_model(objp))->n_models,
+					  "Invalid submodel number detected!");
+			auto smi = &model_instance->submodel[submodel];
+
+			if (smi->blown_off) {
+				return false;
+			}
+		}
+
+		return true;
+	}
+};
+
+SCP_vector<Decal> active_decals;
+
+bool required_string_if_new(const char* token, bool new_entry) {
+	if (!new_entry) {
+		return optional_string(token) == 1;
+	}
+
+	required_string(token);
+	return true;
+}
+
+float clamp(float x, float min, float max) {
+	return std::min(std::max(x, min), max);
+}
+
+/**
+ * @brief Produces a smoothstep value as specified by the GLSL specification
+ * @param edge0
+ * @param edge1
+ * @param x
+ * @return
+ */
+float smoothstep(float edge0, float edge1, float x) {
+	auto t = clamp((x - edge0) / (edge1 - edge0), 0.0f, 1.0f);
+	return t * t * (3.0f - 2.0f * t);
+}
+
+}
+
+namespace decals {
+
+void initialize() {
+	decalDefinitions.clear();
+	graphics::decal_draw_list::globalInit();
+
+	parse_modular_table(NOX("*-dcl.tbm"), parse_decals_table);
+}
+void shutdown() {
+	graphics::decal_draw_list::globalShutdown();
+
+	// Free allocated resources
+	decalDefinitions.clear();
+}
+
+int getDecalConfig(const SCP_string& name) {
+	int index = 0;
+	for (auto& def : decalDefinitions) {
+		if (!stricmp(def.getName().c_str(), name.c_str())) {
+			return index;
+		}
+		++index;
+	}
+
+	return -1;
+}
+
+void parseDecalReference(creation_info& dest_info, bool is_new_entry) {
+	SCP_string name;
+	stuff_string(name, F_NAME);
+
+	auto decalRef = getDecalConfig(name);
+
+	if (decalRef < 0) {
+		error_display(0, "Decal definition '%s' is unknown!", name.c_str());
+	}
+	dest_info.definition_handle = decalRef;
+
+	if (required_string_if_new("+Radius:", is_new_entry)) {
+		stuff_float(&dest_info.radius);
+
+		if (dest_info.radius <= 0.0f) {
+			error_display(0, "Invalid radius of %f! Must be a positive number.", dest_info.radius);
+			dest_info.radius = 1.0f;
+		}
+	}
+
+	if (required_string_if_new("+Lifetime:", is_new_entry)) {
+		if (optional_string("Eternal")) {
+			dest_info.lifetime = util::UniformFloatRange(-1.0f);
+		} else {
+			// Require at least a small lifetime so that the calculations don't have to deal with div-by-zero
+			dest_info.lifetime = util::parseUniformRange(0.0001f);
+		}
+	}
+}
+void loadBitmaps(const creation_info& info) {
+	if (!decal_system_active) {
+		return;
+	}
+	// Silently ignore invalid definition handle since weapons use the default values if the decal option is not present
+	if (info.definition_handle < 0) {
+		return;
+	}
+
+	Assertion(info.definition_handle >= 0 && info.definition_handle < (int) decalDefinitions.size(),
+			  "Invalid decal handle detected!");
+
+	auto& def = decalDefinitions[info.definition_handle];
+
+	def.loadBitmaps();
+}
+void pageInDecal(const creation_info& info) {
+	if (!decal_system_active) {
+		return;
+	}
+	// Silently ignore invalid definition handle since weapons use the default values if the decal option is not present
+	if (info.definition_handle < 0) {
+		return;
+	}
+
+	Assertion(info.definition_handle >= 0 && info.definition_handle < (int) decalDefinitions.size(),
+			  "Invalid decal handle detected!");
+
+	decalDefinitions[info.definition_handle].pageIn();
+}
+
+void initializeMission() {
+	active_decals.clear();
+}
+
+matrix4 getDecalTransform(Decal& decal) {
+	Assertion(decal.object.objp->type == OBJ_SHIP, "Only ships are currently supported for decals!");
+
+	auto objp = decal.object.objp;
+	auto ship = &Ships[objp->instance];
+
+	vec3d worldPos;
+	model_instance_find_world_point(&worldPos,
+									&decal.position,
+									ship->model_instance_num,
+									decal.submodel,
+									&objp->orient,
+									&objp->pos);
+
+	vec3d worldDir;
+	model_instance_find_world_dir(&worldDir,
+								  &decal.orientation.vec.fvec,
+								  ship->model_instance_num,
+								  decal.submodel,
+								  &objp->orient);
+
+	vec3d worldUp;
+	model_instance_find_world_dir(&worldUp,
+								  &decal.orientation.vec.fvec,
+								  ship->model_instance_num,
+								  decal.submodel,
+								  &objp->orient);
+
+	matrix worldOrient;
+	vm_vector_2_matrix(&worldOrient, &worldDir, &worldUp);
+
+	// Apply scaling
+	worldOrient.a2d[0][0] *= decal.scale.xyz.x;
+	worldOrient.a2d[0][1] *= decal.scale.xyz.x;
+	worldOrient.a2d[0][2] *= decal.scale.xyz.x;
+	worldOrient.a2d[1][0] *= decal.scale.xyz.y;
+	worldOrient.a2d[1][1] *= decal.scale.xyz.y;
+	worldOrient.a2d[1][2] *= decal.scale.xyz.y;
+	worldOrient.a2d[2][0] *= decal.scale.xyz.z;
+	worldOrient.a2d[2][1] *= decal.scale.xyz.z;
+	worldOrient.a2d[2][2] *= decal.scale.xyz.z;
+
+	matrix4 mat4;
+	vm_matrix4_set_transform(&mat4, &worldOrient, &worldPos);
+
+	return mat4;
+}
+
+void renderAll() {
+	if (!decal_system_active) {
+		return;
+	}
+
+	// Clear out any invalid decals
+	for (auto iter = active_decals.begin(); iter != active_decals.end(); ++iter) {
+		if (!iter->isValid()) {
+			// if we're sitting on the very last element, popping-back will invalidate the iterator!
+			if (iter + 1 == active_decals.end()) {
+				active_decals.pop_back();
+				break;
+			}
+
+			*iter = active_decals.back();
+			active_decals.pop_back();
+			continue;
+		}
+	}
+
+	if (active_decals.empty()) {
+		return;
+	}
+
+
+	graphics::decal_draw_list draw_list;
+	auto mission_time = f2fl(Missiontime);
+
+	for (auto& decal : active_decals) {
+		int diffuse_bm = -1;
+		int glow_bm = -1;
+		int normal_bm = -1;
+
+		Assertion(decal.definition_handle >= 0 && decal.definition_handle < (int) decalDefinitions.size(),
+				  "Invalid decal handle detected!");
+		auto& decalDef = decalDefinitions[decal.definition_handle];
+
+		auto decal_time = mission_time - decal.creation_time;
+		auto progress = decal_time / decal.lifetime;
+
+		float alpha = 1.0f;
+		if (progress > 0.8) {
+			// Fade the decal out for the last 20% of its lifetime
+			alpha = 1.0f - smoothstep(0.8f, 1.0f, progress);
+		}
+
+		if (decalDef.getDiffuseBitmap() >= 0) {
+			diffuse_bm = decalDef.getDiffuseBitmap()
+				+ bm_get_anim_frame(decalDef.getDiffuseBitmap(), decal_time, 0.0f, decalDef.isDiffuseLooping());
+		}
+
+		if (decalDef.getGlowBitmap() >= 0) {
+			glow_bm = decalDef.getGlowBitmap()
+				+ bm_get_anim_frame(decalDef.getGlowBitmap(), decal_time, 0.0f, decalDef.isGlowLooping());
+		}
+
+		if (decalDef.getNormalBitmap() >= 0) {
+			normal_bm = decalDef.getNormalBitmap()
+				+ bm_get_anim_frame(decalDef.getNormalBitmap(), decal_time, 0.0f, decalDef.isNormalLooping());
+		}
+
+		draw_list.add_decal(diffuse_bm, glow_bm, normal_bm, decal_time, getDecalTransform(decal), alpha);
+	}
+
+	draw_list.render();
+}
+void
+addDecal(creation_info& info, object* host, int submodel, const vec3d& local_pos, const matrix& local_orient) {
+	if (!decal_system_active) {
+		return;
+	}
+	// Silently ignore invalid definition handle since weapons use the default values if the decal option is not present
+	if (info.definition_handle < 0) {
+		return;
+	}
+
+	Assertion(info.definition_handle >= 0 && info.definition_handle < (int) decalDefinitions.size(),
+			  "Invalid decal handle detected!");
+	auto& def = decalDefinitions[info.definition_handle];
+
+	if (!def.bitmapsLoaded()) {
+		// If this decal was never used before then the bitmaps are not loaded so we need to do that here.
+		def.loadBitmaps();
+	}
+
+	Decal newDecal;
+	newDecal.definition_handle = info.definition_handle;
+	newDecal.object = object_h(host);
+	newDecal.submodel = submodel;
+	newDecal.creation_time = f2fl(Missiontime);
+	newDecal.lifetime = info.lifetime.next();
+
+	newDecal.position = local_pos;
+	newDecal.orientation = local_orient;
+	newDecal.scale.xyz.x = info.radius;
+	newDecal.scale.xyz.y = info.radius;
+	newDecal.scale.xyz.z = info.radius;
+
+	active_decals.push_back(newDecal);
+}
+
+}

code/decals/decals.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#include "globalincs/pstypes.h"
+
+#include "object/object.h"
+
+#include "utils/RandomRange.h"
+
+namespace decals {
+
+/**
+ * @brief A reference to a decal definition
+ */
+typedef int DecalReference;
+
+/**
+ * @brief A structure containing all information for creating a decal
+ */
+struct creation_info {
+	DecalReference definition_handle = -1;
+	float radius = -1.0f;
+	util::UniformFloatRange lifetime = ::util::UniformFloatRange(-1.0f);
+};
+
+/**
+ * @brief Initializes the global state of the decal system. Call once at game init.
+ */
+void initialize();
+
+
+/**
+ * @brief Parses the information for a decal reference
+ *
+ * @details This function expects that the initial token has already been consumed by the parse system.
+ *
+ * @param dest_info The creation information struct to parse the information into
+ * @param is_new_entry Set to @c false if the currently parsed object used +nocreate
+ */
+void parseDecalReference(creation_info& dest_info, bool is_new_entry = true);
+
+/**
+ * @brief Loads the bitmaps specified by the creation info into texture memory
+ *
+ * This should be called when the bitmaps of the object using this decal are loaded.
+ *
+ * @param info The creation information containing the decal definition handle
+ */
+void loadBitmaps(const creation_info& info);
+
+/**
+ * @brief Page in the bitmap data of the specified creation information. Needs to be called after loadBitmaps
+ * @param info The information to page in
+ */
+void pageInDecal(const creation_info& info);
+
+/**
+ * @brief Shut down the decal system. Must be called from game_shutdown
+ */
+void shutdown();
+
+/**
+ * @brief Initialize the decal system for the mission. Must be called before every mission is started
+ */
+void initializeMission();
+
+/**
+ * @brief Renders all currently active decals and removes invalid decals from the active list.
+ */
+void renderAll();
+
+/**
+ * @brief Creates a new decal on the specified object at the specified location
+ *
+ * @param info The decal to create
+ * @param host The object host on which to create this decal. Must be a ship.
+ * @param submodel The submodel of the object model on which to create the decal.
+ * @param local_pos The position in the frame of reference of the submodel at which the decal should be created.
+ * @param local_orient The orientation of the decal in the frame of reference of the submodel. The forward vector of the
+ * orientation should look along the direction
+ */
+void addDecal(creation_info& info,
+			  object* host,
+			  int submodel,
+			  const vec3d& local_pos,
+			  const matrix& local_orient);
+
+}

code/def_files/controlconfigdefaults.tbl

@@ -1,3 +0,0 @@
-                                    #ControlConfigOverride 
-                                                                        #End
-                                    

code/def_files/data/effects/batched-f.sdr

@@ -1,15 +1,18 @@
+
+#include "gamma.sdr"
+
 in vec4 fragTexCoord;
 in vec4 fragColor;
 out vec4 fragOut0;
 uniform sampler2DArray baseMap;
 uniform float intensity;
-#define SRGB_GAMMA 2.2
+
 void main()
 {
 	vec4 baseColor = texture(baseMap, fragTexCoord.xyz);
 
-	baseColor.rgb = pow(baseColor.rgb, vec3(SRGB_GAMMA));
-	vec4 blendColor = vec4(pow(fragColor.rgb, vec3(SRGB_GAMMA)), fragColor.a);
+	baseColor.rgb = srgb_to_linear(baseColor.rgb);
+	vec4 blendColor = vec4(srgb_to_linear(fragColor.rgb), fragColor.a);
 
 	fragOut0 = baseColor * blendColor * intensity;
 }

code/def_files/data/effects/decal-f.sdr

@@ -0,0 +1,180 @@
+
+// NOTE: The technique and some of this code is based on this tutorial:
+// http://martindevans.me/game-development/2015/02/27/Drawing-Stuff-On-Other-Stuff-With-Deferred-Screenspace-Decals/
+
+#include "lighting.sdr"
+#include "normals.sdr"
+#include "gamma.sdr"
+
+out vec4 fragOut0; // Diffuse buffer
+out vec4 fragOut1; // Normal buffer
+out vec4 fragOut2; // Emissive buffer
+
+uniform sampler2D gDepthBuffer;
+uniform sampler2D gNormalBuffer;
+
+uniform sampler2DArray diffuseMap;
+uniform sampler2DArray glowMap;
+uniform sampler2DArray normalMap;
+
+layout (std140) uniform decalGlobalData {
+	mat4 viewMatrix;
+	mat4 projMatrix;
+	mat4 invViewMatrix;
+	mat4 invProjMatrix;
+
+	vec3 ambientLight;
+
+	vec2 viewportSize;
+};
+layout (std140) uniform decalInfoData {
+	mat4 model_matrix;
+	mat4 inv_model_matrix;
+
+	vec3 decal_direction;
+	float normal_angle_cutoff;
+
+	int diffuse_index;
+	int glow_index;
+	int normal_index;
+	float angle_fade_start;
+
+	float alpha_scale;
+	int diffuse_blend_mode;
+	int glow_blend_mode;
+};
+
+vec3 computeViewPosition(vec2 textureCoord) {
+	vec4 clipSpaceLocation;
+	vec2 normalizedCoord = textureCoord / viewportSize;
+
+	clipSpaceLocation.xy = normalizedCoord * 2.0f - 1.0f;
+	clipSpaceLocation.z = texelFetch(gDepthBuffer, ivec2(textureCoord), 0).r * 2.0f - 1.0f;
+	clipSpaceLocation.w = 1.0f;
+
+	vec4 homogenousLocation = invProjMatrix * clipSpaceLocation;
+
+	return homogenousLocation.xyz / homogenousLocation.w;
+}
+
+vec3 getPixelNormal(vec3 frag_position, vec2 tex_coord, inout float alpha, out vec3 binormal, out vec3 tangent) {
+#ifdef USE_NORMAL_MAP
+	// If we can then we just use the existing normal buffer
+    vec3 normal = texelFetch(gNormalBuffer, ivec2(tex_coord), 0).xyz;
+
+	// If we use the normal map then we don't really need these values so we don't need to compute them here
+    binormal = vec3(0.0);
+    tangent = vec3(0.0);
+#else
+	// Use some fancy screen-space derivates to determine the normal of the current pixel by looking at the surrounding pixels
+	vec3 pos_dx = dFdx(frag_position);
+	vec3 pos_dy = dFdy(frag_position);
+
+    vec3 normal = normalize(cross(pos_dx, pos_dy));
+
+    binormal = normalize(pos_dx);
+    tangent = normalize(pos_dy);
+#endif
+
+	//Calculate angle between surface normal and decal direction
+	float angle = acos(dot(normal, decal_direction));
+
+	if (angle > normal_angle_cutoff) {
+		// The angle between surface normal and decal direction is too big
+		discard;
+	}
+
+	// Make a smooth alpha transition leading up to an edge
+	alpha = alpha * (1 - smoothstep(angle_fade_start, normal_angle_cutoff, angle));
+
+	return normal;
+}
+
+vec2 getDecalTexCoord(vec3 view_pos, inout float alpha) {
+	vec4 object_pos = inv_model_matrix * invViewMatrix * vec4(view_pos, 1.0);
+
+	bvec3 invalidComponents = greaterThan(abs(object_pos.xyz), vec3(0.5));
+	bvec4 nanComponents = isnan(object_pos); // nan can happen some times if we have an infinite depth value
+
+	if (any(invalidComponents) || any(nanComponents)) {
+		// Fragment is out of the box
+		discard;
+	}
+
+	// Fade out the texture when it gets close to the top or bottom of the decal box
+	alpha = alpha * (1.0 - smoothstep(0.4, 0.5, abs(object_pos.z)));
+
+	return object_pos.xy + 0.5;
+}
+
+void main() {
+	vec3 frag_position = computeViewPosition(gl_FragCoord.xy);
+
+	float alpha = alpha_scale;
+	vec2 tex_coord = getDecalTexCoord(frag_position, alpha);
+
+	vec3 binormal;
+	vec3 tangent;
+	vec3 normal = getPixelNormal(frag_position, gl_FragCoord.xy, alpha, binormal, tangent);
+
+	vec4 diffuse_out = vec4(0.0);
+	vec4 emissive_out = vec4(0.0, 0.0, 0.0, 0.0);
+	vec3 normal_out = vec3(0.0);
+
+	if (diffuse_index >= 0) {
+		// We have a valid diffuse map
+		vec4 color = texture(diffuseMap, vec3(tex_coord, float(diffuse_index)));
+
+		color.rgb = srgb_to_linear(color.rgb);
+
+		if (diffuse_blend_mode == 0) {
+			// Normal alpha blending
+			diffuse_out = vec4(color.rgb, color.a * alpha);
+		} else {
+			// Additive blending
+			diffuse_out = vec4(color.rgb * alpha, 1.0);
+		}
+
+		// The main model shader applies ambient lighting by drawing the ambient part of the texture into the emissive
+		// texture. We do the same here to make sure the decal material is applied correctly
+		if (glow_blend_mode == 0) {
+			// Normal alpha blending
+			emissive_out = vec4(color.rgb * ambientLight, color.a * alpha);
+		} else {
+			// Additive blending
+			emissive_out = vec4(alpha * color.rgb * ambientLight, 1.0);
+		}
+	}
+
+	if (glow_index >= 0) {
+		// We have a valid diffuse map
+		vec4 color = texture(glowMap, vec3(tex_coord, float(glow_index)));
+
+		color.rgb = srgb_to_linear(color.rgb) * GLOW_MAP_SRGB_MULTIPLIER;
+
+		color.rgb *= GLOW_MAP_INTENSITY;
+
+		if (glow_blend_mode == 0) {
+			// Normal alpha blending
+			emissive_out = vec4(color.rgb + emissive_out.rgb * emissive_out.a, color.a * alpha);
+		} else {
+			// Additive blending
+			emissive_out.rgb += color.rgb * alpha;
+		}
+	}
+
+	if (normal_index >= 0) {
+		vec3 decalNormal = unpackNormal(texture(normalMap, vec3(tex_coord, float(normal_index))).ag);
+
+		mat3 tangentToView;
+		tangentToView[0] = tangent;
+		tangentToView[1] = binormal;
+		tangentToView[2] = normal;
+
+		normal_out = tangentToView * decalNormal * alpha;
+	}
+
+	fragOut0 = diffuse_out;
+	fragOut1 = vec4(normal_out, 0.0);
+	fragOut2 = emissive_out;
+}

code/def_files/data/effects/decal-v.sdr

@@ -0,0 +1,33 @@
+
+in vec4 vertPosition;
+
+layout (std140) uniform decalGlobalData {
+	mat4 viewMatrix;
+	mat4 projMatrix;
+	mat4 invViewMatrix;
+	mat4 invProjMatrix;
+
+	vec3 ambientLight;
+
+	vec2 viewportSize;
+};
+layout (std140) uniform decalInfoData {
+	mat4 model_matrix;
+	mat4 inv_model_matrix;
+
+	vec3 decal_direction;
+	float normal_angle_cutoff;
+
+	int diffuse_index;
+	int glow_index;
+	int normal_index;
+	float angle_fade_start;
+
+	float alpha_scale;
+	int diffuse_blend_mode;
+	int glow_blend_mode;
+};
+
+void main() {
+	gl_Position = projMatrix * viewMatrix * model_matrix * vertPosition;
+}

code/def_files/data/effects/default-material-f.sdr

@@ -1,6 +1,10 @@
+
+#include "gamma.sdr"
+
 in vec4 fragTexCoord;
 in vec4 fragColor;
 out vec4 fragOut0;
+
 uniform sampler2DArray baseMap;
 uniform int baseMapIndex;
 uniform int alphaTexture;
@@ -8,12 +12,12 @@ uniform int noTexturing;
 uniform int srgb;
 uniform float intensity;
 uniform float alphaThreshold;
-#define SRGB_GAMMA 2.2
+
 void main()
 {
 	vec4 baseColor = texture(baseMap, vec3(fragTexCoord.xy, float(baseMapIndex)));
 	if(alphaThreshold > baseColor.a) discard;
-	baseColor.rgb = (srgb == 1) ? pow(baseColor.rgb, vec3(SRGB_GAMMA)) : baseColor.rgb;
-	vec4 blendColor = (srgb == 1) ? vec4(pow(fragColor.rgb, vec3(SRGB_GAMMA)), fragColor.a) : fragColor;
+	baseColor.rgb = (srgb == 1) ? srgb_to_linear(baseColor.rgb) : baseColor.rgb;
+	vec4 blendColor = (srgb == 1) ? vec4(srgb_to_linear(fragColor.rgb), fragColor.a) : fragColor;
 	fragOut0 = mix(mix(baseColor * blendColor, vec4(blendColor.rgb, baseColor.r * blendColor.a), float(alphaTexture)), blendColor, float(noTexturing)) * intensity;
 }

code/def_files/data/effects/deferred-clear-f.sdr

@@ -2,10 +2,12 @@ out vec4 fragOut0;
 out vec4 fragOut1;
 out vec4 fragOut2;
 out vec4 fragOut3;
+out vec4 fragOut4;
 void main()
 {
 	fragOut0 = vec4(0.0, 0.0, 0.0, 1.0); // color
 	fragOut1 = vec4(0.0, 0.0, -1000000.0, 1.0); // position
 	fragOut2 = vec4(0.0, 0.0, 0.0, 1.0); // normal
 	fragOut3 = vec4(0.0, 0.0, 0.0, 0.0); // specular
-}
+	fragOut4 = vec4(0.0, 0.0, 0.0, 1.0); // emissive
+}

code/def_files/data/effects/deferred-f.sdr

@@ -0,0 +1,138 @@
+
+#include "lighting.sdr"
+
+#include "shadows.sdr"
+
+in vec3 beamVec;
+in vec3 lightPosition;
+out vec4 fragOut0;
+
+uniform sampler2D ColorBuffer;
+uniform sampler2D NormalBuffer;
+uniform sampler2D PositionBuffer;
+uniform sampler2D SpecBuffer;
+uniform sampler2DArray shadow_map;
+
+layout (std140) uniform globalDeferredData {
+	mat4 shadow_mv_matrix;
+	mat4 shadow_proj_matrix[4];
+
+	mat4 inv_view_matrix;
+
+	float veryneardist;
+	float neardist;
+	float middist;
+	float fardist;
+
+	float specFactor;
+	float invScreenWidth;
+	float invScreenHeight;
+};
+
+layout (std140) uniform lightData {
+	vec3 diffuseLightColor;
+	float coneAngle;
+
+	vec3 specLightColor;
+	float coneInnerAngle;
+
+	vec3 coneDir;
+	bool dualCone;
+
+	vec3 scale;
+	float lightRadius;
+
+	vec3 lightDir;
+	int lightType;
+
+	bool enable_shadows;
+};
+
+void GetLightInfo(vec3 position, out vec3 lightDir, out float attenuation)
+{
+	if (lightType == LT_DIRECTIONAL) {
+		lightDir = normalize(lightPosition);
+		attenuation = 1.0;
+	} else {
+		// Positional light source
+		lightDir = lightPosition - position.xyz;
+		float dist = length(lightDir);
+		attenuation = 1.0 - clamp(dist / lightRadius, 0.0, 1.0);
+
+		if(dist > lightRadius && lightType != LT_TUBE) {
+			discard;
+		}
+
+		if (lightType == LT_TUBE) {  // Tube light
+			float beamLength = length(beamVec);
+			vec3 beamDir = beamVec / beamLength;
+			// Get nearest point on line
+			float neardist = clamp(dot(lightDir, beamDir), 0.0, beamLength);
+			// Move back from the endpoint of the beam along the beam by the distance we calculated
+			vec3 nearest = lightPosition - beamDir * neardist;
+			lightDir = nearest - position.xyz;
+			dist = length(lightDir);
+			if(dist > lightRadius) {
+				discard;
+			}
+		} else if (lightType == LT_CONE) {
+			float coneDot = dot(normalize(-lightDir), coneDir);
+			if(dualCone) {
+				if(abs(coneDot) < coneAngle) {
+					discard;
+				} else {
+					attenuation *= smoothstep(coneAngle, coneInnerAngle, abs(coneDot));
+				}
+			} else {
+				if (coneDot < coneAngle) {
+					discard;
+				} else {
+					attenuation *= smoothstep(coneAngle, coneInnerAngle, coneDot);
+				}
+			}
+		}
+
+		lightDir = normalize(lightDir);
+	}
+}
+
+void main()
+{
+	vec2 screenPos = gl_FragCoord.xy * vec2(invScreenWidth, invScreenHeight);
+	vec3 position = texture(PositionBuffer, screenPos).xyz;
+
+	if(abs(dot(position, position)) < 0.1)
+		discard;
+
+	vec3 color = texture(ColorBuffer, screenPos).rgb;
+	vec4 normalData = texture(NormalBuffer, screenPos);
+	vec4 specColor = texture(SpecBuffer, screenPos);
+	// The vector in the normal buffer could be longer than the unit vector since decal rendering only adds to the normal buffer
+	vec3 normal = normalize(normalData.xyz);
+	float gloss = normalData.a;
+	float fresnel = specColor.a;
+	vec3 eyeDir = normalize(-position);
+
+	vec3 lightDir;
+	float attenuation;
+
+	GetLightInfo(position, lightDir, attenuation);
+
+	if (enable_shadows) {
+		vec4 fragShadowPos = shadow_mv_matrix * inv_view_matrix * vec4(position, 1.0);
+		vec4 fragShadowUV[4];
+		fragShadowUV[0] = transformToShadowMap(shadow_proj_matrix[0], 0, fragShadowPos);
+		fragShadowUV[1] = transformToShadowMap(shadow_proj_matrix[1], 1, fragShadowPos);
+		fragShadowUV[2] = transformToShadowMap(shadow_proj_matrix[2], 2, fragShadowPos);
+		fragShadowUV[3] = transformToShadowMap(shadow_proj_matrix[3], 3, fragShadowPos);
+
+		attenuation *= getShadowValue(shadow_map, -position.z, fragShadowPos.z, fragShadowUV, fardist, middist,
+								neardist, veryneardist);
+	}
+
+	vec3 halfVec = normalize(lightDir + eyeDir);
+	float NdotL = clamp(dot(normal, lightDir), 0.0, 1.0);
+	vec4 fragmentColor = vec4(color * (diffuseLightColor * NdotL * attenuation), 1.0);
+	fragmentColor.rgb += computeLighting(specColor.rgb, lightDir, normal.xyz, halfVec, eyeDir, gloss, fresnel, NdotL).rgb * specLightColor * attenuation;
+	fragOut0 = max(fragmentColor, vec4(0.0));
+}

code/def_files/data/effects/deferred-v.sdr

@@ -0,0 +1,40 @@
+
+#include "lighting.sdr"
+
+in vec4 vertPosition;
+uniform mat4 modelViewMatrix;
+uniform mat4 projMatrix;
+
+layout (std140) uniform lightData {
+	vec3 diffuseLightColor;
+	float coneAngle;
+
+	vec3 specLightColor;
+	float coneInnerAngle;
+
+	vec3 coneDir;
+	bool dualCone;
+
+	vec3 scale;
+	float lightRadius;
+
+	vec3 lightDir;
+	int lightType;
+
+	bool enable_shadows;
+};
+
+out vec3 lightPosition;
+out vec3 beamVec;
+void main()
+{
+	if (lightType == LT_DIRECTIONAL) {
+		gl_Position = vec4(vertPosition.xyz, 1.0);
+		lightPosition = lightDir;
+	} else {
+		gl_Position = projMatrix * modelViewMatrix * vec4(vertPosition.xyz * scale, 1.0);
+		lightPosition = modelViewMatrix[3].xyz;
+		if(lightType == LT_TUBE)
+			beamVec = vec3(modelViewMatrix * vec4(0.0, 0.0, -scale.z, 0.0));
+	}
+}

code/def_files/data/effects/effect-particle-f.sdr

@@ -1,3 +1,6 @@
+
+#include "gamma.sdr"
+
 uniform sampler2DArray baseMap;
 uniform sampler2D depthMap;
 uniform float window_width;
@@ -12,12 +15,12 @@ in vec4 fragPosition;
 in vec4 fragTexCoord;
 in vec4 fragColor;
 out vec4 fragOut0;
-#define SRGB_GAMMA 2.2
+
 void main()
 {
 	vec4 fragmentColor = texture(baseMap, fragTexCoord.xyz);
-	fragmentColor.rgb = mix(fragmentColor.rgb, pow(fragmentColor.rgb, vec3(SRGB_GAMMA)), float(srgb));
-	fragmentColor *= mix(fragColor, vec4(pow(fragColor.rgb, vec3(SRGB_GAMMA)), fragColor.a), float(srgb));
+	fragmentColor.rgb = mix(fragmentColor.rgb, srgb_to_linear(fragmentColor.rgb), float(srgb));
+	fragmentColor *= mix(fragColor, vec4(srgb_to_linear(fragColor.rgb), fragColor.a), float(srgb));
 	vec2 offset = vec2(fragRadius * abs(0.5 - fragTexCoord.x) * 2.0, fragRadius * abs(0.5 - fragTexCoord.y) * 2.0);
 	float offset_len = length(offset);
 	if ( offset_len > fragRadius ) {

code/def_files/data/effects/fog-f.sdr

@@ -0,0 +1,34 @@
+
+#include "gamma.sdr"
+
+in vec4 fragTexCoord;
+out vec4 fragOut0;
+
+uniform sampler2D tex;
+uniform sampler2D depth_tex;
+
+uniform float fog_start;
+uniform float fog_scale;
+uniform vec3 fog_color;
+uniform float zNear;
+uniform float zFar;
+
+void main()
+{
+	vec4 color_in = texture(tex, fragTexCoord.xy);
+
+	// If the scene is in fog then we change the color we operate on based on the depth of the pixel
+	float depth_val = texture(depth_tex, fragTexCoord.xy).x;
+	// Transform depth value from [-1, 1] to [0, 1]
+	float depth_normalized = 2.0 * depth_val - 1.0;
+	// Now we compute the depth value in projection space using the clipping plane information
+	float view_depth = 2.0 * zNear * zFar / (zFar + zNear - depth_normalized * (zFar - zNear));
+
+	// This is the same formula that is also used by the main shader
+	float fog_dist = isinf(view_depth) ? 1.0 : clamp((view_depth - fog_start) * 0.75 * fog_scale, 0.0, 1.0);
+
+	vec3 finalFogColor = srgb_to_linear(fog_color);
+
+	fragOut0.rgb = mix(color_in.rgb, finalFogColor, fog_dist);
+	fragOut0.a = 1.0;
+}

code/def_files/data/effects/gamma.sdr

@@ -0,0 +1,29 @@
+
+const float SRGB_GAMMA = 2.2;
+const float SRGB_GAMMA_INVERSE = 1.0 / SRGB_GAMMA;
+
+float srgb_to_linear(float val) {
+	return pow(val, SRGB_GAMMA);
+}
+vec2 srgb_to_linear(vec2 val) {
+	return pow(val, vec2(SRGB_GAMMA));
+}
+vec3 srgb_to_linear(vec3 val) {
+	return pow(val, vec3(SRGB_GAMMA));
+}
+vec4 srgb_to_linear(vec4 val) {
+	return pow(val, vec4(SRGB_GAMMA));
+}
+
+float linear_to_srgb(float val) {
+	return pow(val, SRGB_GAMMA_INVERSE);
+}
+vec2 linear_to_srgb(vec2 val) {
+	return pow(val, vec2(SRGB_GAMMA_INVERSE));
+}
+vec3 linear_to_srgb(vec3 val) {
+	return pow(val, vec3(SRGB_GAMMA_INVERSE));
+}
+vec4 linear_to_srgb(vec4 val) {
+	return pow(val, vec4(SRGB_GAMMA_INVERSE));
+}

code/def_files/data/effects/lighting.sdr

@@ -0,0 +1,58 @@
+
+const int LT_DIRECTIONAL	= 0;		// A light like a sun
+const int LT_POINT			= 1;		// A point light, like an explosion
+const int LT_TUBE			= 2;		// A tube light, like a fluorescent light
+const int LT_CONE			= 3;		// A cone light, like a flood light
+
+const float SPEC_INTENSITY_POINT		= 5.3 ;// Point light
+const float SPEC_INTENSITY_DIRECTIONAL	= 3.0 ;// Directional light
+const float SPECULAR_FACTOR				= 1.75;
+const float SPECULAR_ALPHA				= 0.1;
+const float SPEC_FACTOR_NO_SPEC_MAP		= 0.6;
+const float ENV_ALPHA_FACTOR			= 0.3;
+const float GLOW_MAP_INTENSITY			= 1.5;
+const float AMBIENT_LIGHT_BOOST			= 1.0;
+const float GLOW_MAP_SRGB_MULTIPLIER	= 3.0;
+
+vec3 FresnelSchlick(vec3 specColor, vec3 light, vec3 halfVec)
+{
+	return specColor + (vec3(1.0) - specColor) * pow(1.0 - clamp(dot(light, halfVec), 0.0, 1.0), 5.0);
+}
+
+vec3 SpecularBlinnPhong(vec3 specColor, vec3 light, vec3 normal, vec3 halfVec, float specPower, float fresnel, float dotNL)
+{
+	return mix(specColor, FresnelSchlick(specColor, light, halfVec), fresnel) * ((specPower + 2.0) / 8.0 ) * pow(clamp(dot(normal, halfVec), 0.0, 1.0), specPower) * dotNL;
+}
+
+vec3 SpecularGGX(vec3 specColor, vec3 light, vec3 normal, vec3 halfVec, vec3 view, float gloss, float fresnelFactor, float dotNL)
+{
+	float roughness = clamp(1.0f - gloss, 0.0f, 1.0f);
+	float alpha = roughness * roughness;
+
+	float dotNH = clamp(dot(normal, halfVec), 0.0f, 1.0f);
+	float dotNV = clamp(dot(normal, view), 0.0f, 1.0f);
+
+	float alphaSqr = alpha * alpha;
+	float pi = 3.14159f;
+	float denom = dotNH * dotNH * (alphaSqr - 1.0f) + 1.0f;
+	float distribution = alphaSqr / (pi * denom * denom);
+
+	vec3 fresnel = mix(specColor, FresnelSchlick(specColor, light, halfVec), fresnelFactor);
+
+	float alphaPrime = roughness + 1.0f;
+	float k = alphaPrime * alphaPrime / 8.0f;
+	float g1vNL = 1.0f / (dotNL * (1.0f - k) + k);
+	float g1vNV = 1.0f / (dotNV * (1.0f - k) + k);
+	float visibility = g1vNL * g1vNV;
+
+	return distribution * fresnel * visibility * dotNL;
+}
+
+vec3 computeLighting(vec3 specColor, vec3 light, vec3 normal, vec3 halfVec, vec3 view, float gloss, float fresnelFactor, float dotNL) 
+{
+	#ifdef FLAG_LIGHT_MODEL_BLINN_PHONG
+	return SpecularBlinnPhong(specColor, light, normal, halfVec, 32, fresnelFactor, dotNL);
+	#else
+	return SpecularGGX(specColor, light, normal, halfVec, view, gloss, fresnelFactor, dotNL);
+	#endif
+}

code/def_files/data/effects/main-f.sdr

@@ -1,3 +1,8 @@
+
+#include "shadows.sdr"
+#include "lighting.sdr"
+#include "gamma.sdr"
+
 #define MAX_LIGHTS 8
 #define LT_DIRECTIONAL		0
 #define LT_POINT			1
@@ -91,6 +96,30 @@ layout (std140) uniform modelData {
 	int sMiscmapIndex;
 };
 
+in VertexOutput {
+#ifdef FLAG_ENV_MAP
+	vec3 envReflect;
+#endif
+#ifdef FLAG_NORMAL_MAP
+	mat3 tangentMatrix;
+#endif
+#ifdef FLAG_FOG
+	float fogDist;
+#endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	float notVisible;
+ #endif
+#endif
+	vec4 position;
+	vec3 normal;
+	vec4 texCoord;
+#ifdef FLAG_SHADOWS
+	vec4 shadowUV[4];
+	vec4 shadowPos;
+#endif
+} vertIn;
+
 #ifdef FLAG_DIFFUSE_MAP
 uniform sampler2DArray sBasemap;
 #endif
@@ -102,18 +131,13 @@ uniform sampler2DArray sSpecmap;
 #endif
 #ifdef FLAG_ENV_MAP
 uniform samplerCube sEnvmap;
-in vec3 fragEnvReflect;
 #endif
 #ifdef FLAG_NORMAL_MAP
 uniform sampler2DArray sNormalmap;
-in mat3 fragTangentMatrix;
 #endif
 #ifdef FLAG_AMBIENT_MAP
 uniform sampler2DArray sAmbientmap;
 #endif
-#ifdef FLAG_FOG
-in float fragFogDist;
-#endif
 #ifdef FLAG_ANIMATED
 uniform sampler2D sFramebuffer;
 #endif
@@ -124,150 +148,39 @@ vec2 teamMask = vec2(0.0, 0.0);
 uniform sampler2DArray sMiscmap;
 #endif
 #ifdef FLAG_SHADOWS
-in vec4 fragShadowUV[4];
-in vec4 fragShadowPos;
 uniform sampler2DArray shadow_map;
 #endif
-#define SPEC_INTENSITY_POINT      5.3 // Point light
-#define SPEC_INTENSITY_DIRECTIONAL   3.0 // Directional light
-#define SPECULAR_FACTOR         1.75
-#define SPEC_FACTOR_NO_SPEC_MAP   0.6
-#define ENV_ALPHA_FACTOR         0.3
-#define GLOW_MAP_INTENSITY         1.5
-#define AMBIENT_LIGHT_BOOST      1.0
-#define VARIANCE_SHADOW_SCALE		1000000.0
-#define VARIANCE_SHADOW_SCALE_INV	1.0/VARIANCE_SHADOW_SCALE
-#define SRGB_GAMMA 2.2
-in vec4 fragPosition;
-in vec3 fragNormal;
-in vec4 fragTexCoord;
+
 out vec4 fragOut0;
 out vec4 fragOut1;
 out vec4 fragOut2;
 out vec4 fragOut3;
+out vec4 fragOut4;
 
-#ifdef FLAG_SHADOWS
-vec2 sampleShadowMap(vec2 uv, vec2 offset_uv, int cascade, float shadowMapSizeInv)
-{
-	return texture(shadow_map, vec3(uv + offset_uv * shadowMapSizeInv, float(cascade))).xy;
-}
-
-float computeShadowFactor(vec2 moments, float bias)
-{
-	float shadow = 1.0;
-	if((moments.x - bias) > fragShadowPos.z)
-	{
-		// variance shadow mapping using Chebychev's Formula
-		float variance = moments.y * VARIANCE_SHADOW_SCALE - moments.x * moments.x;
-		float mD = moments.x - bias - fragShadowPos.z;
-		shadow = variance / (variance + mD * mD);
-		shadow = clamp(shadow, 0.0, 1.0);
-	}
-	return shadow;
-}
-
-float sampleNoPCF(int cascade)
-{
-	return computeShadowFactor(sampleShadowMap(fragShadowUV[cascade].xy, vec2(0.0, 0.0), cascade, 1.0/1024.0), 0.05);
-}
-
-float samplePoissonPCF(int cascade)
-{
-	if(cascade > 3 || cascade < 0) return 1.0;
-	vec2 poissonDisc[16];
-	poissonDisc[0] = vec2(-0.76275, -0.3432573);
-	poissonDisc[1] = vec2(-0.5226235, -0.8277544);
-	poissonDisc[2] = vec2(-0.3780261, 0.01528688);
-	poissonDisc[3] = vec2(-0.7742821, 0.4245702);
-	poissonDisc[4] = vec2(0.04196143, -0.02622231);
-	poissonDisc[5] = vec2(-0.2974772, -0.4722782);
-	poissonDisc[6] = vec2(-0.516093, 0.71495);
-	poissonDisc[7] = vec2(-0.3257416, 0.3910343);
-	poissonDisc[8] = vec2(0.2705966, 0.6670476);
-	poissonDisc[9] = vec2(0.4918377, 0.1853267);
-	poissonDisc[10] = vec2(0.4428544, -0.6251478);
-	poissonDisc[11] = vec2(-0.09204347, 0.9267113);
-	poissonDisc[12] = vec2(0.391505, -0.2558275);
-	poissonDisc[13] = vec2(0.05605913, -0.7570801);
-	poissonDisc[14] = vec2(0.81772, -0.02475523);
-	poissonDisc[15] = vec2(0.6890262, 0.5191521);
-	float maxUVOffset[4];
-	maxUVOffset[0] = 1.0/300.0;
-	maxUVOffset[1] = 1.0/250.0;
-	maxUVOffset[2] = 1.0/200.0;
-	maxUVOffset[3] = 1.0/200.0;
-	vec2 sum = sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[0], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[1], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[2], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[3], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[4], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[5], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[6], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[7], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[8], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[9], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[10], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[11], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[12], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[13], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[14], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	sum += sampleShadowMap(fragShadowUV[cascade].xy, poissonDisc[15], cascade, maxUVOffset[cascade])*(1.0/16.0);
-	return computeShadowFactor(sum, 0.1);
-}
-float getShadowValue()
-{
-	// Valathil's Shadows
-	float depth = -fragPosition.z;
-	int cascade = 4;
-	cascade -= int(step(depth, fardist));
-	cascade -= int(step(depth, middist));
-	cascade -= int(step(depth, neardist));
-	cascade -= int(step(depth, veryneardist));
-	float cascade_start_dist[5];
-	cascade_start_dist[0] = 0.0;
-	cascade_start_dist[1] = veryneardist;
-	cascade_start_dist[2] = neardist;
-	cascade_start_dist[3] = middist;
-	cascade_start_dist[4] = fardist;
-	if(cascade > 3 || cascade < 0) return 1.0;
-	float dist_threshold = (cascade_start_dist[cascade+1] - cascade_start_dist[cascade])*0.2;
-	if(cascade_start_dist[cascade+1] - dist_threshold > depth)
-		return samplePoissonPCF(cascade);
-	return mix(samplePoissonPCF(cascade), samplePoissonPCF(cascade+1), smoothstep(cascade_start_dist[cascade+1] - dist_threshold, cascade_start_dist[cascade+1], depth));
-}
-#endif
 vec3 FresnelLazarovEnv(vec3 specColor, vec3 view, vec3 normal, float gloss)
 {
 	// Fresnel for environment lighting 
 	// Equation referenced from Dimitar Lazarov's presentation titled Physically Based Rendering in Call of Duty: Black Ops
 	return specColor + (vec3(1.0) - specColor) * pow(1.0 - clamp(dot(view, normal), 0.0, 1.0), 5.0) / (4.0 - 3.0 * gloss);
 }
-vec3 FresnelSchlick(vec3 specColor, vec3 light, vec3 halfVec)
-{
-	return specColor + (vec3(1.0) - specColor) * pow(1.0 - clamp(dot(light, halfVec), 0.0, 1.0), 5.0);
-}
-vec3 SpecularBlinnPhong(vec3 specColor, vec3 light, vec3 normal, vec3 halfVec, float specPower, float fresnel, float dotNL)
-{
-	return mix(specColor, FresnelSchlick(specColor, light, halfVec), fresnel) * ((specPower + 2.0) / 8.0 ) * pow(clamp(dot(normal, halfVec), 0.0, 1.0), specPower) * dotNL;
-}
-#ifdef FLAG_LIGHT
+
 void GetLightInfo(int i, out vec3 lightDir, out float attenuation)
 {
 	lightDir = normalize(lights[i].position.xyz);
 	attenuation = 1.0;
 	if (lights[i].light_type != LT_DIRECTIONAL) {
 		// Positional light source
-		float dist = distance(lights[i].position.xyz, fragPosition.xyz);
-		lightDir = (lights[i].position.xyz - fragPosition.xyz);
+		float dist = distance(lights[i].position.xyz, vertIn.position.xyz);
+		lightDir = (lights[i].position.xyz - vertIn.position.xyz);
 
 		if (lights[i].light_type == LT_TUBE) {  // Tube light
 			float beamlength = length(lights[i].direction);
 			vec3 beamDir = normalize(lights[i].direction);
 			// Get nearest point on line
-			float neardist = dot(fragPosition.xyz - lights[i].position.xyz, beamDir);
+			float neardist = dot(vertIn.position.xyz - lights[i].position.xyz, beamDir);
 			// Move back from the endpoint of the beam along the beam by the distance we calculated
 			vec3 nearest = lights[i].position.xyz - beamDir * abs(neardist);
-			lightDir = nearest - fragPosition.xyz;
+			lightDir = nearest - vertIn.position.xyz;
 			dist = length(lightDir);
 		}
 
@@ -277,7 +190,7 @@ void GetLightInfo(int i, out vec3 lightDir, out float attenuation)
 }
 vec3 CalculateLighting(vec3 normal, vec3 diffuseMaterial, vec3 specularMaterial, float gloss, float fresnel, float shadow, float aoFactor)
 {
-	vec3 eyeDir = vec3(normalize(-fragPosition).xyz);
+	vec3 eyeDir = vec3(normalize(-vertIn.position).xyz);
 	vec3 lightAmbient = (emissionFactor + ambientFactor * ambientFactor) * aoFactor; // ambientFactor^2 due to legacy OpenGL compatibility behavior
 	vec3 lightDiffuse = vec3(0.0, 0.0, 0.0);
 	vec3 lightSpecular = vec3(0.0, 0.0, 0.0);
@@ -292,25 +205,33 @@ vec3 CalculateLighting(vec3 normal, vec3 diffuseMaterial, vec3 specularMaterial,
 		// gather light params
 		GetLightInfo(i, lightDir, attenuation);
 		vec3 halfVec = normalize(lightDir + eyeDir);
-		float NdotL = max(dot(normal, lightDir), 0.0);
+		float NdotL = clamp(dot(normal, lightDir), 0.0f, 1.0f);
 		// Ambient, Diffuse, and Specular
 		lightDiffuse += (lights[i].diffuse_color.rgb * diffuseFactor * NdotL * attenuation) * shadow;
-		lightSpecular += lights[i].spec_color * SpecularBlinnPhong(specularMaterial, lightDir, normal, halfVec, exp2(10.0 * gloss + 1.0), fresnel, NdotL) * attenuation * shadow;
+		lightSpecular += lights[i].spec_color * computeLighting(specularMaterial, lightDir, normal, halfVec, eyeDir, gloss, fresnel, NdotL) * attenuation * shadow;
 	}
 	return diffuseMaterial * (lightAmbient + lightDiffuse) + lightSpecular;
 }
-#endif
+
 void main()
 {
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	if(vertIn.notVisible >= 0.9) { discard; }
+ #endif
+#endif
 #ifdef FLAG_SHADOW_MAP
 	// need depth and depth squared for variance shadow maps
-	fragOut0 = vec4(fragPosition.z, fragPosition.z * fragPosition.z * VARIANCE_SHADOW_SCALE_INV, 0.0, 1.0);
-	return;
+	fragOut0 = vec4(vertIn.position.z, vertIn.position.z * vertIn.position.z * VARIANCE_SHADOW_SCALE_INV, 0.0, 1.0);
+	if (true) {
+		return;
+	}
 #endif
-	vec3 eyeDir = vec3(normalize(-fragPosition).xyz);
-	vec2 texCoord = fragTexCoord.xy;
+	vec3 eyeDir = vec3(normalize(-vertIn.position).xyz);
+	vec2 texCoord = vertIn.texCoord.xy;
 	vec4 baseColor = color;
 	vec4 specColor = vec4(0.0, 0.0, 0.0, 1.0);
+	vec4 emissiveColor = vec4(0.0, 0.0, 0.0, 1.0);
 	float fresnelFactor = 0.0;
 	float glossData = 0.6;
 #ifdef FLAG_LIGHT
@@ -322,14 +243,14 @@ void main()
 	// green is cavity occlusion factor which only affects diffuse and specular lighting.
 	aoFactors = texture(sAmbientmap, vec3(texCoord, float(sAmbientmapIndex))).xy;
 #endif
-	vec3 unitNormal = normalize(fragNormal);
+	vec3 unitNormal = normalize(vertIn.normal);
 	vec3 normal = unitNormal;
 #ifdef FLAG_NORMAL_MAP
    // Normal map - convert from DXT5nm
    vec2 normalSample;
    normal.rg = normalSample = (texture(sNormalmap, vec3(texCoord, float(sNormalmapIndex))).ag * 2.0) - 1.0;
    normal.b = clamp(sqrt(1.0 - dot(normal.rg, normal.rg)), 0.0001, 1.0);
-   normal = fragTangentMatrix * normal;
+   normal = vertIn.tangentMatrix * normal;
    float norm = length(normal);
    // prevent breaking of normal maps
    if (norm > 0.0)
@@ -339,7 +260,7 @@ void main()
 #endif
 
 #ifdef FLAG_ANIMATED
-   vec2 distort = vec2(cos(fragPosition.x*fragPosition.w*0.005+anim_timer*20.0)*sin(fragPosition.y*fragPosition.w*0.005),sin(fragPosition.x*fragPosition.w*0.005+anim_timer*20.0)*cos(fragPosition.y*fragPosition.w*0.005))*0.03;
+   vec2 distort = vec2(cos(vertIn.position.x*vertIn.position.w*0.005+anim_timer*20.0)*sin(vertIn.position.y*vertIn.position.w*0.005),sin(vertIn.position.x*vertIn.position.w*0.005+anim_timer*20.0)*cos(vertIn.position.y*vertIn.position.w*0.005))*0.03;
 #endif
 
 #ifdef FLAG_DIFFUSE_MAP
@@ -358,7 +279,7 @@ void main()
 #endif
 	specColor = vec4(baseColor.rgb * SPEC_FACTOR_NO_SPEC_MAP, glossData);
 #ifdef FLAG_HDR
-	baseColor.rgb = pow(baseColor.rgb, vec3(SRGB_GAMMA));
+	baseColor.rgb = srgb_to_linear(baseColor.rgb);
 #endif
 #ifdef FLAG_SPEC_MAP
 	specColor = texture(sSpecmap, vec3(texCoord, float(sSpecmapIndex)));
@@ -368,10 +289,10 @@ void main()
 		fresnelFactor = 1.0;
 	}
 	if(gammaSpec) {
+		specColor.rgb = max(specColor.rgb, vec3(0.03f)); // hardcoded minimum specular value. read John Hable's blog post titled 'Everything Is Shiny'. 
  #ifdef FLAG_HDR
-		specColor.rgb = pow(specColor.rgb, vec3(SRGB_GAMMA));
+		specColor.rgb = srgb_to_linear(specColor.rgb);
  #endif
-		specColor.rgb = max(specColor.rgb, vec3(0.04)); // hardcoded minimum specular value. read John Hable's blog post titled 'Everything Is Shiny'. 
 		fresnelFactor = 1.0;
 	}
 #endif
@@ -381,62 +302,88 @@ void main()
  #ifdef FLAG_TEAMCOLOR
 	vec4 teamMask = vec4(0.0, 0.0, 0.0, 0.0);
 	teamMask = texture(sMiscmap, vec3(texCoord, float(sMiscmapIndex)));
-	vec3 base = max(base_color - vec3(0.5), vec3(0.0));
-	vec3 stripe = max(stripe_color - vec3(0.5), vec3(0.0));
-	baseColor.rgb += (base * teamMask.x) + (stripe * teamMask.y);
-	baseColor.rgb = max(baseColor.rgb, vec3(0.0));
+	//For team colors applied to a diffuse or spec map, we assume that the base color of the diffuse 
+	//at this point is vec3(0.5). To get accurate results, we subtract 0.5 from the team colors
+	vec3 team_color = ((base_color - vec3(0.5)) * teamMask.x) + ((stripe_color - vec3(0.5)) * teamMask.y);
+	vec3 team_color_glow = (base_color * teamMask.b) + (stripe_color * teamMask.a);
+
+	#ifdef FLAG_HDR
+	team_color = srgb_to_linear(team_color);
+	team_color_glow = srgb_to_linear(team_color_glow) * GLOW_MAP_SRGB_MULTIPLIER;
+	#endif
+
+	baseColor.rgb += team_color;					
+	baseColor.rgb = max(baseColor.rgb, vec3(0.0));	// We need to make sure that nothing here ever goes negative
+	specColor.rgb += team_color;
+	specColor.rgb = max(specColor.rgb, vec3(0.03));
  #endif
 #endif
-	float shadow = 1.0;
-#ifdef FLAG_SHADOWS
-	shadow = getShadowValue();
-#endif
-#ifdef FLAG_LIGHT
-	baseColor.rgb = CalculateLighting(normal, baseColor.rgb, specColor.rgb, glossData, fresnelFactor, shadow, aoFactors.x);
-#else
- #ifdef FLAG_SPEC_MAP
+	// Lights aren't applied when we are rendering to the G-buffers since that gets handled later
+#ifdef FLAG_DEFERRED
+ #ifdef FLAG_LIGHT
+	// Ambient lighting still needs to be done since that counts as an "emissive" color
+	vec3 lightAmbient = (emissionFactor + ambientFactor * ambientFactor) * aoFactors.x; // ambientFactor^2 due to legacy OpenGL compatibility behavior
+	emissiveColor.rgb += baseColor.rgb * lightAmbient;
+ #else
+  #ifdef FLAG_SPEC_MAP
 	baseColor.rgb += pow(1.0 - clamp(dot(eyeDir, normal), 0.0, 1.0), 5.0 * clamp(glossData, 0.01, 1.0)) * specColor.rgb;
+  #endif
+	// If there is no lighting then we copy the color data so far into the
+	emissiveColor.rgb += baseColor.rgb;
+ #endif
+#else
+ #ifdef FLAG_LIGHT
+	float shadow = 1.0;
+  #ifdef FLAG_SHADOWS
+	shadow = getShadowValue(shadow_map, -vertIn.position.z, vertIn.shadowPos.z, vertIn.shadowUV, fardist, middist, neardist, veryneardist);
+  #endif
+	baseColor.rgb = CalculateLighting(normal, baseColor.rgb, specColor.rgb, glossData, fresnelFactor, shadow, aoFactors.x);
+ #else
+  #ifdef FLAG_SPEC_MAP
+	baseColor.rgb += pow(1.0 - clamp(dot(eyeDir, normal), 0.0, 1.0), 5.0 * clamp(glossData, 0.01, 1.0)) * specColor.rgb;
+  #endif
  #endif
 #endif
 #ifdef FLAG_ENV_MAP
-   vec3 envReflectNM = fragEnvReflect;
+	vec3 envReflectNM = vertIn.envReflect;
  #ifdef FLAG_NORMAL_MAP
-   envReflectNM += vec3(normalSample, 0.0);
+	envReflectNM += vec3(normalSample, 0.0);
 	envReflectNM = normalize(envReflectNM);
  #endif
 	float mip = (envGloss) ? (1.0 - glossData) * 7.0 : 0.0;
-   vec4 envColour = textureLod(sEnvmap, envReflectNM, mip);
+	vec4 envColour = textureLod(sEnvmap, envReflectNM, mip);
  #ifdef FLAG_HDR
-	envColour.rgb = pow(envColour.rgb, vec3(SRGB_GAMMA));
+	envColour.rgb = srgb_to_linear(envColour.rgb);
  #endif
 	envColour.rgb *= (envGloss) ? 1.0 : specColor.a;
-	baseColor.rgb += envColour.rgb * FresnelLazarovEnv(specColor.rgb, eyeDir, normal, glossData);
+	emissiveColor.rgb += envColour.rgb * FresnelLazarovEnv(specColor.rgb, eyeDir, normal, glossData);
 #endif
 #ifdef FLAG_GLOW_MAP
 	vec3 glowColor = texture(sGlowmap, vec3(texCoord, float(sGlowmapIndex))).rgb;
 	if(overrideGlow) glowColor = glowClr;
  #ifdef FLAG_HDR
-	glowColor = pow(glowColor, vec3(SRGB_GAMMA)) * 3.0f;
+	glowColor = srgb_to_linear(glowColor) * GLOW_MAP_SRGB_MULTIPLIER;
  #endif
  #ifdef FLAG_MISC_MAP
   #ifdef FLAG_TEAMCOLOR
 	float glowColorLuminance = dot(glowColor, vec3(0.299, 0.587, 0.114));
-	glowColor = team_glow_enabled ? mix((base * teamMask.b) + (stripe * teamMask.a), glowColor, clamp(glowColorLuminance - teamMask.b - teamMask.a, 0.0, 1.0)) : glowColor;
+	glowColor = team_glow_enabled ? mix(max(team_color_glow, vec3(0.0)), glowColor, clamp(glowColorLuminance - teamMask.b - teamMask.a, 0.0, 1.0)) : glowColor;
   #endif
  #endif
-   baseColor.rgb += glowColor * GLOW_MAP_INTENSITY;
+   emissiveColor.rgb += glowColor * GLOW_MAP_INTENSITY;
 #endif
 
 #ifdef FLAG_FOG
 	vec3 finalFogColor = fogColor.rgb;
  #ifdef FLAG_HDR
-	finalFogColor = pow(finalFogColor, vec3(SRGB_GAMMA));
+	finalFogColor = srgb_to_linear(finalFogColor);
  #endif
  #ifdef FLAG_DIFFUSE_MAP
 	if(blend_alpha == 1) finalFogColor *= baseColor.a;
  #endif
-	baseColor.rgb = mix(baseColor.rgb, finalFogColor, fragFogDist);
-	specColor.rgb *= fragFogDist;
+	baseColor.rgb = mix(baseColor.rgb, finalFogColor, vertIn.fogDist);
+	emissiveColor.rgb = mix(emissiveColor.rgb, finalFogColor, vertIn.fogDist);
+	specColor.rgb *= vertIn.fogDist;
 #endif
 
 #ifdef FLAG_DIFFUSE_MAP
@@ -447,38 +394,41 @@ void main()
 #ifdef FLAG_ANIMATED
    if (effect_num == 0) {
       float shinefactor = 1.0/(1.0 + pow(abs((fract(abs(texCoord.x))-anim_timer) * 1000.0), 2.0)) * 1000.0;
-      baseColor.rgb = baseColor.rgb + vec3(shinefactor);
+      emissiveColor.rgb += vec3(shinefactor);
       baseColor.a = baseColor.a * clamp(shinefactor * (fract(abs(texCoord.x))-anim_timer) * -10000.0,0.0,1.0);
    }
    if (effect_num == 1) {
-      float shinefactor = 1.0/(1.0 + pow(abs(fragPosition.y-anim_timer), 2.0));
-      baseColor.rgb = baseColor.rgb + vec3(shinefactor);
- #ifdef FLAG_LIGHT
-      baseColor.a = baseColor.a;
- #else
+      float shinefactor = 1.0/(1.0 + pow(abs(vertIn.position.y-anim_timer), 2.0));
+      emissiveColor.rgb += vec3(shinefactor);
+ #ifndef FLAG_LIGHT
       // ATI Wireframe fix *grumble*
-      baseColor.a = clamp((fragPosition.y-anim_timer) * 10000.0,0.0,1.0);
+      baseColor.a = clamp((vertIn.position.y-anim_timer) * 10000.0,0.0,1.0);
  #endif
    }
    if (effect_num == 2) {
       vec2 screenPos = gl_FragCoord.xy * vec2(vpwidth,vpheight);
       baseColor.a = baseColor.a;
-      float cloak_interp = (sin(fragPosition.x*fragPosition.w*0.005+anim_timer*20.0)*sin(fragPosition.y*fragPosition.w*0.005)*0.5)-0.5;
+      float cloak_interp = (sin(vertIn.position.x*vertIn.position.w*0.005+anim_timer*20.0)*sin(vertIn.position.y*vertIn.position.w*0.005)*0.5)-0.5;
  #ifdef FLAG_LIGHT
       baseColor.rgb = mix(texture(sFramebuffer, screenPos + distort*anim_timer + anim_timer*0.1*normal.xy).rgb,baseColor.rgb,clamp(cloak_interp+anim_timer*2.0,0.0,1.0));
  #else
-      baseColor.rgb = mix(texture(sFramebuffer, screenPos + distort*anim_timer + anim_timer*0.1*fragNormal.xy).rgb,baseColor.rgb,clamp(cloak_interp+anim_timer*2.0,0.0,1.0));
+      baseColor.rgb = mix(texture(sFramebuffer, screenPos + distort*anim_timer + anim_timer*0.1*vertIn.normal.xy).rgb,baseColor.rgb,clamp(cloak_interp+anim_timer*2.0,0.0,1.0));
  #endif
    }
 #endif
 #ifdef FLAG_NORMAL_ALPHA
 	float normViewOffset = dot(vec3(0.0, 0.0, 1.0), normal);
 	baseColor.a = smoothstep(min(normalAlphaMinMax.x, normalAlphaMinMax.y), max(normalAlphaMinMax.x, normalAlphaMinMax.y), clamp(normalAlphaMinMax.x > normalAlphaMinMax.y ? normViewOffset : 1.0 - normViewOffset, 0.0, 1.0));
+#endif
+#ifndef FLAG_DEFERRED
+	// emissive colors won't be added later when we are using forward rendering so we need to do that here
+	baseColor.rgb += emissiveColor.rgb;
 #endif
 	fragOut0 = baseColor;
 #ifdef FLAG_DEFERRED
-	fragOut1 = vec4(fragPosition.xyz, 1.0);
+	fragOut1 = vec4(vertIn.position.xyz, 1.0);
 	fragOut2 = vec4(normal, glossData);
 	fragOut3 = vec4(specColor.rgb, fresnelFactor);
+	fragOut4 = emissiveColor;
 #endif
 }

code/def_files/data/effects/main-g.sdr

@@ -96,33 +96,104 @@ layout (std140) uniform modelData {
 	int sMiscmapIndex;
 };
 
-#if !defined(GL_ARB_gpu_shader5)
-in float geoInstance[];
+in VertexOutput {
+#ifdef FLAG_ENV_MAP
+	vec3 envReflect;
 #endif
-in vec3 geoNormal[];
-in vec4 geoTexCoord[];
-out vec4 fragPosition;
-out vec3 fragNormal;
-out vec4 fragTexCoord;
+#ifdef FLAG_NORMAL_MAP
+	mat3 tangentMatrix;
+#endif
+#ifdef FLAG_FOG
+	float fogDist;
+#endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	float notVisible;
+ #endif
+#endif
+#ifdef FLAG_SHADOW_MAP
+ #if !defined(GL_ARB_gpu_shader5)
+	float instance;
+ #endif
+#else
+	vec4 position;
+#endif
+	vec3 normal;
+	vec4 texCoord;
+#ifdef FLAG_SHADOWS
+	vec4 shadowUV[4];
+	vec4 shadowPos;
+#endif
+} vertIn[];
+
+out VertexOutput {
+#ifdef FLAG_ENV_MAP
+	vec3 envReflect;
+#endif
+#ifdef FLAG_NORMAL_MAP
+	mat3 tangentMatrix;
+#endif
+#ifdef FLAG_FOG
+	float fogDist;
+#endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	float notVisible;
+ #endif
+#endif
+	vec4 position;
+	vec3 normal;
+	vec4 texCoord;
+#ifdef FLAG_SHADOWS
+	vec4 shadowUV[4];
+	vec4 shadowPos;
+#endif
+} vertOut;
+
 void main(void)
 {
 #ifdef GL_ARB_gpu_shader5
 	int instanceID = gl_InvocationID;
 #else
-	int instanceID = int(geoInstance[0]);
+	int instanceID = int(vertIn[0].instance);
 #endif
 	for(int vert = 0; vert < gl_in.length(); vert++)
 	{
 		gl_Position = shadow_proj_matrix[instanceID] * gl_in[vert].gl_Position;
 		if(gl_Position.z < -1.0)
 			gl_Position.z = -1.0;
-		fragPosition = gl_in[vert].gl_Position;
-		fragNormal = geoNormal[vert];
-		fragTexCoord = geoTexCoord[vert];
+		vertOut.position = gl_in[vert].gl_Position;
+		vertOut.normal = vertIn[vert].normal;
+		vertOut.texCoord = vertIn[vert].texCoord;
 
 		gl_Layer = instanceID;
-#if defined(FLAG_CLIP) || defined(FLAG_TRANSFORM)
+#ifdef FLAG_ENV_MAP
+		vertOut.envReflect = vertIn[vert].envReflect;
+#endif
+#ifdef FLAG_NORMAL_MAP
+		vertOut.tangentMatrix = vertIn[vert].tangentMatrix;
+#endif
+#ifdef FLAG_FOG
+		vertOut.fogDist = vertIn[vert].fogDist;
+#endif
+#ifdef FLAG_SHADOWS
+		vertOut.shadowUV[0] = vertIn[vert].shadowUV[0];
+		vertOut.shadowUV[1] = vertIn[vert].shadowUV[1];
+		vertOut.shadowUV[2] = vertIn[vert].shadowUV[2];
+		vertOut.shadowUV[3] = vertIn[vert].shadowUV[3];
+		vertOut.shadowPos = vertIn[vert].shadowPos;
+#endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+		vertOut.notVisible = vertIn[vert].notVisible;
+ #endif
+ #ifdef FLAG_CLIP
 		gl_ClipDistance[0] = gl_in[vert].gl_ClipDistance[0];
+ #endif
+#else
+ #if defined(FLAG_CLIP) || defined(FLAG_TRANSFORM)
+		gl_ClipDistance[0] = gl_in[vert].gl_ClipDistance[0];
+ #endif
 #endif
 		EmitVertex();
 	}

code/def_files/data/effects/main-v.sdr

@@ -1,5 +1,13 @@
 #extension GL_ARB_gpu_shader5: enable
 
+#include "shadows.sdr"
+
+in vec4 vertPosition;
+in vec4 vertTexCoord;
+in vec3 vertNormal;
+in vec4 vertTangent;
+in float vertModelID;
+
 #define MAX_LIGHTS 8
 
 struct model_light {
@@ -91,38 +99,40 @@ layout (std140) uniform modelData {
 	int sMiscmapIndex;
 };
 
-in vec4 vertPosition;
-in vec4 vertTexCoord;
-in vec3 vertNormal;
-in vec4 vertTangent;
-in float vertModelID;
-#ifdef FLAG_ENV_MAP
-out vec3 fragEnvReflect;
-#endif
-#ifdef FLAG_NORMAL_MAP
-out mat3 fragTangentMatrix;
-#endif
-#ifdef FLAG_FOG
-out float fragFogDist;
-#endif
 #ifdef FLAG_TRANSFORM
 uniform samplerBuffer transform_tex;
 #endif
+
+out VertexOutput {
+#ifdef FLAG_ENV_MAP
+	vec3 envReflect;
+#endif
+#ifdef FLAG_NORMAL_MAP
+	mat3 tangentMatrix;
+#endif
+#ifdef FLAG_FOG
+	float fogDist;
+#endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	float notVisible;
+ #endif
+#endif
 #ifdef FLAG_SHADOW_MAP
-#if !defined(GL_ARB_gpu_shader5)
-out float geoInstance;
-#endif
-out vec3 geoNormal;
-out vec4 geoTexCoord;
+ #if !defined(GL_ARB_gpu_shader5)
+	float instance;
+ #endif
 #else
-out vec4 fragPosition;
-out vec3 fragNormal;
-out vec4 fragTexCoord;
+	vec4 position;
 #endif
+	vec3 normal;
+	vec4 texCoord;
 #ifdef FLAG_SHADOWS
-out vec4 fragShadowUV[4];
-out vec4 fragShadowPos;
+	vec4 shadowUV[4];
+	vec4 shadowPos;
 #endif
+} vertOut;
+
 #ifdef FLAG_TRANSFORM
 #define TEXELS_PER_MATRIX 4
 void getModelTransform(inout mat4 transform, out bool invisible, int id, int matrix_offset)
@@ -135,18 +145,6 @@ void getModelTransform(inout mat4 transform, out bool invisible, int id, int mat
 	transform[3].w = 1.0;
 }
 #endif
-#ifdef FLAG_SHADOWS
-vec4 transformToShadowMap(int i, vec4 pos)
-{
-	vec4 shadow_proj;
-	shadow_proj = shadow_proj_matrix[i] * pos;
-	shadow_proj += 1.0;
-	shadow_proj *= 0.5;
-	shadow_proj.w = shadow_proj.z;
-	shadow_proj.z = float(i);
-	return shadow_proj;
-}
-#endif
 void main()
 {
 	vec4 position;
@@ -154,12 +152,20 @@ void main()
 	vec4 texCoord;
 	mat4 orient = mat4(1.0);
 	mat4 scale = mat4(1.0);
+#ifdef WORKAROUND_CLIPPING_PLANES
+ #ifdef FLAG_TRANSFORM
+	bool clipModel;
+	getModelTransform(orient, clipModel, int(vertModelID), buffer_matrix_offset);
+	vertOut.notVisible = clipModel ? 1.0 : 0.0;
+ #endif
+#else
  #ifdef FLAG_TRANSFORM
 	bool clipModel;
 	getModelTransform(orient, clipModel, int(vertModelID), buffer_matrix_offset);
  #else
 	bool clipModel = false;
  #endif
+#endif
 	texCoord = textureMatrix * vertTexCoord;
 	vec4 vertex = vertPosition;
  #ifdef FLAG_THRUSTER
@@ -172,15 +178,13 @@ void main()
 	position = modelViewMatrix * orient * vertex;
  #ifdef FLAG_SHADOW_MAP
 	gl_Position = position;
-
- #if !defined(GL_ARB_gpu_shader5)
-  #ifdef APPLE
-	geoInstance = float(gl_InstanceIDARB);
-  #else
-	geoInstance = float(gl_InstanceID);
+  #if !defined(GL_ARB_gpu_shader5)
+   #ifdef APPLE
+	vertOut.instance = float(gl_InstanceIDARB);
+   #else
+	vertOut.instance = float(gl_InstanceID);
+   #endif
   #endif
- #endif
-
  #else
 	gl_Position = projMatrix * position;
  #endif
@@ -188,26 +192,34 @@ void main()
 	gl_Position.xy += (mat3(projMatrix) * normal.xyz).xy * gl_Position.z * extrudeWidth;
  #endif
  #ifdef FLAG_SHADOWS
-	fragShadowPos = shadow_mv_matrix * modelMatrix * orient * vertPosition;
-	fragShadowUV[0] = transformToShadowMap(0, fragShadowPos);
-	fragShadowUV[1] = transformToShadowMap(1, fragShadowPos);
-	fragShadowUV[2] = transformToShadowMap(2, fragShadowPos);
-	fragShadowUV[3] = transformToShadowMap(3, fragShadowPos);
+	vec4 shadowPos = shadow_mv_matrix * modelMatrix * orient * vertPosition;
+	vertOut.shadowPos = shadow_mv_matrix * modelMatrix * orient * vertPosition;
+	vertOut.shadowUV[0] = transformToShadowMap(shadow_proj_matrix[0], 0, shadowPos);
+	vertOut.shadowUV[1] = transformToShadowMap(shadow_proj_matrix[1], 1, shadowPos);
+	vertOut.shadowUV[2] = transformToShadowMap(shadow_proj_matrix[2], 2, shadowPos);
+	vertOut.shadowUV[3] = transformToShadowMap(shadow_proj_matrix[3], 3, shadowPos);
  #endif
  #ifdef FLAG_NORMAL_MAP
  // Setup stuff for normal maps
 	vec3 t = normalize(mat3(modelViewMatrix) * mat3(orient) * vertTangent.xyz);
 	vec3 b = cross(normal, t) * vertTangent.w;
-	fragTangentMatrix = mat3(t, b, normal);
+	vertOut.tangentMatrix = mat3(t, b, normal);
  #endif
  #ifdef FLAG_ENV_MAP
  // Environment mapping reflection vector.
-	fragEnvReflect = reflect(normalize(position.xyz), normal);
-	fragEnvReflect = vec3(envMatrix * vec4(fragEnvReflect, 0.0));
+	vec3 envReflect = reflect(normalize(position.xyz), normal);
+	vertOut.envReflect = vec3(envMatrix * vec4(envReflect, 0.0));
  #endif
  #ifdef FLAG_FOG
-	fragFogDist = clamp((gl_Position.z - fogStart) * 0.75 * fogScale, 0.0, 1.0);
+	vertOut.fogDist = clamp((gl_Position.z - fogStart) * 0.75 * fogScale, 0.0, 1.0);
  #endif
+#ifdef WORKAROUND_CLIPPING_PLANES
+	if(use_clip_plane) {
+ 		gl_ClipDistance[0] = dot(clip_equation, modelMatrix * orient * vertex);
+ 	} else {
+		gl_ClipDistance[0] = 1.0;
+ 	}
+#else
  #ifdef FLAG_CLIP
 	if (clipModel) {
 		gl_ClipDistance[0] = -1.0;
@@ -219,12 +231,13 @@ void main()
  #elif defined(FLAG_TRANSFORM)
 	gl_ClipDistance[0] = clipModel ? -1.0 : 1.0;
  #endif
+#endif
  #ifndef FLAG_SHADOW_MAP
-	fragPosition = position;
-	fragNormal = normal;
-	fragTexCoord = texCoord;
+	vertOut.position = position;
+	vertOut.normal = normal;
+	vertOut.texCoord = texCoord;
  #else
-	geoNormal = normal;
-	geoTexCoord = texCoord;
+	vertOut.normal = normal;
+	vertOut.texCoord = texCoord;
  #endif
 }

code/def_files/data/effects/normals.sdr

@@ -0,0 +1,12 @@
+
+/**
+ * @brief Unpacks a DXT5nm encoded normal
+ * @param normal_sample The Alpha and Green channel of the DXT5 texture
+ * @return The unpacked normal
+ */
+vec3 unpackNormal(vec2 normal_sample) {
+	vec3 normal;
+	normal.xy = normal_sample * 2.0 - 1.0;
+	normal.z = sqrt(1 - normal_sample.x * normal_sample.x - normal_sample.y * normal_sample.y);
+	return normal;
+}

code/def_files/data/effects/passthrough-f.sdr

@@ -1,15 +1,18 @@
+
+#include "gamma.sdr"
+
 in vec4 fragTexCoord;
 in vec4 fragColor;
 out vec4 fragOut0;
 uniform sampler2D baseMap;
 uniform int noTexturing;
 uniform int srgb;
-#define SRGB_GAMMA 2.2
+
 void main()
 {
 	vec4 baseColor = texture(baseMap, fragTexCoord.xy);
 
-	baseColor.rgb = (srgb == 1) ? pow(baseColor.rgb, vec3(SRGB_GAMMA)) : baseColor.rgb;
-	vec4 blendColor = (srgb == 1) ? vec4(pow(fragColor.rgb, vec3(SRGB_GAMMA)), fragColor.a) : fragColor;
+	baseColor.rgb = (srgb == 1) ? srgb_to_linear(baseColor.rgb) : baseColor.rgb;
+	vec4 blendColor = (srgb == 1) ? vec4(srgb_to_linear(fragColor.rgb), fragColor.a) : fragColor;
 	fragOut0 = mix(baseColor * blendColor, blendColor, float(noTexturing));
 }

code/def_files/data/effects/shadows.sdr

@@ -0,0 +1,105 @@
+
+const float VARIANCE_SHADOW_SCALE = 1000000.0;
+const float VARIANCE_SHADOW_SCALE_INV = 1.0/VARIANCE_SHADOW_SCALE;
+
+vec2 sampleShadowMap(sampler2DArray shadow_map, vec2 uv, vec2 offset_uv, int cascade, float shadowMapSizeInv)
+{
+	return texture(shadow_map, vec3(uv + offset_uv * shadowMapSizeInv, float(cascade))).xy;
+}
+
+float computeShadowFactor(float shadowDepth, vec2 moments, float bias)
+{
+	float shadow = 1.0;
+	if((moments.x - bias) > shadowDepth)
+	{
+		// variance shadow mapping using Chebychev's Formula
+		float variance = moments.y * VARIANCE_SHADOW_SCALE - moments.x * moments.x;
+		float mD = moments.x - bias - shadowDepth;
+		shadow = variance / (variance + mD * mD);
+		shadow = clamp(shadow, 0.0, 1.0);
+	}
+	return shadow;
+}
+
+float sampleNoPCF(sampler2DArray shadow_map, float shadowDepth, int cascade, vec4 shadowUV[4])
+{
+	return computeShadowFactor(shadowDepth, sampleShadowMap(shadow_map, shadowUV[cascade].xy, vec2(0.0, 0.0), cascade, 1.0/1024.0), 0.05);
+}
+
+float samplePoissonPCF(sampler2DArray shadow_map, float shadowDepth, int cascade, vec4 shadowUV[4])
+{
+	if(cascade > 3 || cascade < 0) return 1.0;
+	vec2 poissonDisc[16];
+	poissonDisc[0] = vec2(-0.76275, -0.3432573);
+	poissonDisc[1] = vec2(-0.5226235, -0.8277544);
+	poissonDisc[2] = vec2(-0.3780261, 0.01528688);
+	poissonDisc[3] = vec2(-0.7742821, 0.4245702);
+	poissonDisc[4] = vec2(0.04196143, -0.02622231);
+	poissonDisc[5] = vec2(-0.2974772, -0.4722782);
+	poissonDisc[6] = vec2(-0.516093, 0.71495);
+	poissonDisc[7] = vec2(-0.3257416, 0.3910343);
+	poissonDisc[8] = vec2(0.2705966, 0.6670476);
+	poissonDisc[9] = vec2(0.4918377, 0.1853267);
+	poissonDisc[10] = vec2(0.4428544, -0.6251478);
+	poissonDisc[11] = vec2(-0.09204347, 0.9267113);
+	poissonDisc[12] = vec2(0.391505, -0.2558275);
+	poissonDisc[13] = vec2(0.05605913, -0.7570801);
+	poissonDisc[14] = vec2(0.81772, -0.02475523);
+	poissonDisc[15] = vec2(0.6890262, 0.5191521);
+	float maxUVOffset[4];
+	maxUVOffset[0] = 1.0/300.0;
+	maxUVOffset[1] = 1.0/250.0;
+	maxUVOffset[2] = 1.0/200.0;
+	maxUVOffset[3] = 1.0/200.0;
+	vec2 sum = sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[0], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[1], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[2], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[3], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[4], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[5], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[6], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[7], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[8], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[9], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[10], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[11], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[12], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[13], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[14], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	sum += sampleShadowMap(shadow_map, shadowUV[cascade].xy, poissonDisc[15], cascade, maxUVOffset[cascade])*(1.0/16.0);
+	return computeShadowFactor(shadowDepth, sum, 0.1);
+}
+
+float getShadowValue(sampler2DArray shadow_map, float depth, float shadowDepth, vec4 shadowUV[4], float fardist,
+					 float middist, float neardist, float veryneardist)
+{
+	// Valathil's Shadows
+	int cascade = 4;
+	cascade -= int(step(depth, fardist));
+	cascade -= int(step(depth, middist));
+	cascade -= int(step(depth, neardist));
+	cascade -= int(step(depth, veryneardist));
+	float cascade_start_dist[5];
+	cascade_start_dist[0] = 0.0;
+	cascade_start_dist[1] = veryneardist;
+	cascade_start_dist[2] = neardist;
+	cascade_start_dist[3] = middist;
+	cascade_start_dist[4] = fardist;
+	if(cascade > 3 || cascade < 0) return 1.0;
+	float dist_threshold = (cascade_start_dist[cascade+1] - cascade_start_dist[cascade])*0.2;
+	if(cascade_start_dist[cascade+1] - dist_threshold > depth)
+		return samplePoissonPCF(shadow_map, shadowDepth, cascade, shadowUV);
+	return mix(samplePoissonPCF(shadow_map, shadowDepth, cascade, shadowUV), samplePoissonPCF(shadow_map, shadowDepth, cascade+1, shadowUV),
+			smoothstep(cascade_start_dist[cascade+1] - dist_threshold, cascade_start_dist[cascade+1], depth));
+}
+
+vec4 transformToShadowMap(mat4 shadow_proj_matrix, int i, vec4 pos)
+{
+	vec4 shadow_proj;
+	shadow_proj = shadow_proj_matrix * pos;
+	shadow_proj += 1.0;
+	shadow_proj *= 0.5;
+	shadow_proj.w = shadow_proj.z;
+	shadow_proj.z = float(i);
+	return shadow_proj;
+}

code/def_files/data/effects/shield-impact-f.sdr

@@ -1,3 +1,6 @@
+
+#include "gamma.sdr"
+
 in vec4 fragImpactUV;
 in float fragNormOffset;
 uniform vec4 color;
@@ -5,15 +8,14 @@ uniform sampler2DArray shieldMap;
 uniform int shieldMapIndex;
 uniform int srgb;
 out vec4 fragOut0;
-#define SRGB_GAMMA 2.2
 #define EMISSIVE_GAIN 2.0
 void main()
 {
 	if(fragNormOffset < 0.0) discard;
 	if(fragImpactUV.x < 0.0 || fragImpactUV.x > 1.0 || fragImpactUV.y < 0.0 || fragImpactUV.y > 1.0) discard;
 	vec4 shieldColor = texture(shieldMap, vec3(fragImpactUV.xy, float(shieldMapIndex)));
-	shieldColor.rgb = (srgb == 1) ? pow(shieldColor.rgb, vec3(SRGB_GAMMA)) * EMISSIVE_GAIN : shieldColor.rgb;
+	shieldColor.rgb = (srgb == 1) ? srgb_to_linear(shieldColor.rgb) * EMISSIVE_GAIN : shieldColor.rgb;
 	vec4 blendColor = color;
-	blendColor.rgb = (srgb == 1) ? pow(blendColor.rgb, vec3(SRGB_GAMMA)) * EMISSIVE_GAIN : blendColor.rgb;
+	blendColor.rgb = (srgb == 1) ? srgb_to_linear(blendColor.rgb) * EMISSIVE_GAIN : blendColor.rgb;
 	fragOut0 = shieldColor * blendColor;
 }

code/def_files/data/effects/tonemapping-f.sdr

@@ -1,3 +1,6 @@
+
+#include "gamma.sdr"
+
 in vec4 fragTexCoord;
 out vec4 fragOut0;
 uniform sampler2D tex;
@@ -12,13 +15,13 @@ vec3 Uncharted2Tonemapping(vec3 hdr_color)
 	float F = 0.30;
 	return ((hdr_color*(A*hdr_color+C*B)+D*E)/(hdr_color*(A*hdr_color+B)+D*F))-E/F;
 }
-#define SRGB_GAMMA 2.2
+
 void main()
 {
 	vec4 color = texture(tex, fragTexCoord.xy);
 	// Tone mapping using John Hable's Uncharted 2 tonemapping algorithm.
 	float whitepoint = 11.2f; // hardcoded whitepoint value from Hable's algo
 	color.rgb = Uncharted2Tonemapping(color.rgb * exposure) / Uncharted2Tonemapping(vec3(whitepoint));
-	color.rgb = pow(color.rgb, vec3(1.0/SRGB_GAMMA)); // return from linear color space to SRGB color space
+	color.rgb = linear_to_srgb(color.rgb); // return from linear color space to SRGB color space
 	fragOut0 = vec4(color.rgb, 1.0);
-}
+}

code/def_files/data/tables/controlconfigdefaults.tbl

@@ -0,0 +1,2 @@
+#ControlConfigOverride
+#End

code/def_files/def_files-generic.cpp

@@ -10,12 +10,13 @@
 
 struct def_file
 {
+	const char* path_type;
 	const char* filename;
 	const void* contents;
 	const size_t size;
 };
 
-#include "def_files/generated_def_files-generic.h"
+#include "def_files/generated_def_files-generic.inc"
 
 default_file defaults_get_file(const char *filename)
 {
@@ -26,6 +27,8 @@ default_file defaults_get_file(const char *filename)
 	{
 		if (!stricmp(iter->filename, filename))
 		{
+			def.path_type = iter->path_type;
+			def.filename = iter->filename;
 			def.data = iter->contents;
 			def.size = iter->size;
 
@@ -37,3 +40,19 @@ default_file defaults_get_file(const char *filename)
 	Error(LOCATION, "Default table '%s' missing from executable - contact a coder.", filename);
 	return def;
 }
+
+SCP_vector<default_file> defaults_get_all() {
+	SCP_vector<default_file> files;
+
+	for (auto& file : Default_files) {
+		default_file def;
+		def.path_type = file.path_type;
+		def.filename = file.filename;
+		def.data = file.contents;
+		def.size = file.size;
+
+		files.push_back(def);
+	}
+
+	return files;
+}

code/def_files/def_files-win32.cpp

@@ -4,6 +4,7 @@
  */
 
 #include "globalincs/pstypes.h"
+
 #include "def_files/def_files.h"
 
 #include <iterator>
@@ -13,13 +14,14 @@
 
 struct def_file
 {
+	const char* path_type;
 	const char* filename;
 	const TCHAR* resource_name;
 };
 
 def_file Default_files[] =
 {
-#include "def_files/generated_def_files-win32.h"
+#include "def_files/generated_def_files-win32.inc"
 };
 
 default_file defaults_get_file(const char *filename)
@@ -45,6 +47,8 @@ default_file defaults_get_file(const char *filename)
 				continue;
 			}
 
+			def.path_type = iter->path_type;
+			def.filename = iter->filename;
 			def.data = LockResource(resHandle);
 			def.size = SizeofResource(nullptr, resource);
 
@@ -59,3 +63,34 @@ default_file defaults_get_file(const char *filename)
 	def.data = 0;
 	return def;
 }
+
+SCP_vector<default_file> defaults_get_all() {
+	SCP_vector<default_file> files;
+
+	for (auto& file : Default_files) {
+		default_file def;
+
+		HRSRC resource = FindResource(nullptr, file.resource_name, RT_RCDATA);
+
+		if (resource == nullptr)
+		{
+			continue;
+		}
+
+		HGLOBAL resHandle = LoadResource(nullptr, resource);
+
+		if (resHandle == nullptr)
+		{
+			continue;
+		}
+
+		def.path_type = file.path_type;
+		def.filename = file.filename;
+		def.data = LockResource(resHandle);
+		def.size = SizeofResource(nullptr, resource);
+
+		files.push_back(def);
+	}
+
+	return files;
+}

code/def_files/def_files.h

@@ -15,6 +15,8 @@
 
 struct default_file
 {
+	const char* path_type;
+	const char* filename;
 	const void* data;
 	size_t size;
 };
@@ -22,11 +24,6 @@ struct default_file
 //Used to retrieve pointer to file data from def_files.cpp
 default_file defaults_get_file(const char *filename);
 
-//WMC -
-//There are three parts to adding a file
-//:PART 1: Add variable declaration for new file
-//:PART 2: Add filename of default file to Default_files[] array, along with content variable
-//:PART 3: Define the content using the variable declared in part 1.
-//Do a search in def_files.cpp for the individual part labels for examples and locations.
+SCP_vector<default_file> defaults_get_all();
 
 #endif

code/def_files/deferred-f.sdr

@@ -1,106 +0,0 @@
-in vec3 beamVec;
-in vec3 lightPosition;
-out vec4 fragOut0;
-
-uniform sampler2D ColorBuffer;
-uniform sampler2D NormalBuffer;
-uniform sampler2D PositionBuffer;
-uniform sampler2D SpecBuffer;
-
-uniform float specFactor;
-uniform float invScreenWidth;
-uniform float invScreenHeight;
-
-layout (std140) uniform lightData {
-	vec3 diffuseLightColor;
-	float coneAngle;
-
-	vec3 specLightColor;
-	float coneInnerAngle;
-
-	vec3 coneDir;
-	bool dualCone;
-
-	vec3 scale;
-	float lightRadius;
-
-	int lightType;
-};
-
-#define SPEC_INTENSITY_POINT			5.3 // Point light
-#define SPEC_INTENSITY_DIRECTIONAL		3.0 // Directional light
-#define SPECULAR_FACTOR				1.75
-#define SPECULAR_ALPHA					0.1
-#define SPEC_FACTOR_NO_SPEC_MAP		0.6
-#define ENV_ALPHA_FACTOR				0.3
-#define GLOW_MAP_INTENSITY				1.5
-#define AMBIENT_LIGHT_BOOST			1.0
-vec3 FresnelSchlick(vec3 specColor, vec3 light, vec3 halfVec)
-{
-	return specColor + (vec3(1.0) - specColor) * pow(1.0 - clamp(dot(light, halfVec), 0.0, 1.0), 5.0);
-}
-vec3 SpecularBlinnPhong(vec3 specColor, vec3 light, vec3 normal, vec3 halfVec, float specPower, float fresnel, float dotNL)
-{
-	return mix(specColor, FresnelSchlick(specColor, light, halfVec), fresnel) * ((specPower + 2.0) / 8.0 ) * pow(clamp(dot(normal, halfVec), 0.0, 1.0), specPower) * dotNL;
-}
-vec4 SpecularLegacy(vec4 specColor, vec3 normal, vec3 halfVec, float specPower)
-{
-	return specColor * pow(clamp(dot(normal, halfVec), 0.0, 1.0), specPower);
-}
-void main()
-{
-	vec2 screenPos = gl_FragCoord.xy * vec2(invScreenWidth, invScreenHeight);
-	vec3 position = texture(PositionBuffer, screenPos).xyz;
-
-	if(abs(dot(position, position)) < 0.1)
-		discard;
-	vec3 lightDir = lightPosition - position.xyz;
-	float dist = length(lightDir);
-	if(dist > lightRadius && lightType != 1)
-		discard;
-	vec3 color = texture(ColorBuffer, screenPos).rgb;
-	vec4 normal = texture(NormalBuffer, screenPos);
-	vec4 specColor = texture(SpecBuffer, screenPos);
-	float gloss = normal.a;
-	float fresnel = specColor.a;
-	vec3 eyeDir = normalize(-position);
-
-	if(lightType == 1)
-	{
-		float beamLength = length(beamVec);
-		vec3 beamDir = beamVec / beamLength;
-		// Get nearest point on line
-		float neardist = clamp(dot(lightDir, beamDir), 0.0, beamLength);
-		// Move back from the endpoint of the beam along the beam by the distance we calculated
-		vec3 nearest = lightPosition - beamDir * neardist;
-		lightDir = nearest - position.xyz;
-		dist = length(lightDir);
-		if(dist > lightRadius)
-			discard;
-	}
-
-	float attenuation = 1.0 - clamp(dist / lightRadius, 0.0, 1.0);
-
-	if(lightType == 2)
-	{
-		float coneDot = dot(normalize(-lightDir), coneDir);
-		if(dualCone) {
-			if(abs(coneDot) < coneAngle)
-				discard;
-			else
-				attenuation *= smoothstep(coneAngle, coneInnerAngle, abs(coneDot));
-		} else {
-			if(coneDot < coneAngle)
-				discard;
-			else
-				attenuation *= smoothstep(coneAngle, coneInnerAngle, coneDot);
-		}
-	}
-
-	lightDir /= dist;
-	vec3 halfVec = normalize(lightDir + eyeDir);
-	float NdotL = clamp(dot(normal.xyz, lightDir), 0.0, 1.0);
-	vec4 fragmentColor = vec4(color * (diffuseLightColor * NdotL * attenuation), 1.0);
-	fragmentColor.rgb += SpecularBlinnPhong(specColor.rgb, lightDir, normal.xyz, halfVec, exp2(10.0 * gloss + 1.0), fresnel, NdotL).rgb * specLightColor * attenuation;
-	fragOut0 = max(fragmentColor, vec4(0.0));
-}

code/def_files/deferred-v.sdr

@@ -1,29 +0,0 @@
-in vec4 vertPosition;
-uniform mat4 modelViewMatrix;
-uniform mat4 projMatrix;
-
-layout (std140) uniform lightData {
-	vec3 diffuseLightColor;
-	float coneAngle;
-
-	vec3 specLightColor;
-	float coneInnerAngle;
-
-	vec3 coneDir;
-	bool dualCone;
-
-	vec3 scale;
-	float lightRadius;
-
-	int lightType;
-};
-
-out vec3 lightPosition;
-out vec3 beamVec;
-void main()
-{
-	gl_Position = projMatrix * modelViewMatrix * vec4(vertPosition.xyz * scale, 1.0);
-	lightPosition = modelViewMatrix[3].xyz;
-	if(lightType == 1)
-		beamVec = vec3(modelViewMatrix * vec4(0.0, 0.0, -scale.z, 0.0));
-}

code/default_files.cmake

@@ -8,23 +8,28 @@ IF(MSVC)
 	FOREACH(file ${file_root_def_files_files})
 		set(INPUT_NAME "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
 		SET(OUTPUT "${GENERATED_SOURCE_DIR}/code/${file}")
+		file(RELATIVE_PATH RELATIVE_FILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/def_files" "${INPUT_NAME}")
 
-		get_filename_component(FILENAME ${file} NAME)
 
-		string(MAKE_C_IDENTIFIER "${FILENAME}" FIELD_NAME)
+		# Retrieve the path type name from the relative path
+		GET_FILENAME_COMPONENT(PATH_TYPE "${RELATIVE_FILE_PATH}" DIRECTORY)
+		file(TO_NATIVE_PATH "${PATH_TYPE}" PATH_TYPE)
+		string(REPLACE "\\" "\\\\" PATH_TYPE "${PATH_TYPE}")
+		GET_FILENAME_COMPONENT(FILE_NAME "${RELATIVE_FILE_PATH}" NAME)
+
+		string(MAKE_C_IDENTIFIER "${RELATIVE_FILE_PATH}" FIELD_NAME)
+
 		set(FIELD_NAME "Default_${FIELD_NAME}")
 		string(TOUPPER "${FIELD_NAME}" FIELD_NAME)
 
-		file(TO_NATIVE_PATH "${INPUT_NAME}" NATIVE_PATH)
-
-		set(ARRAY_ELEMENTS "${ARRAY_ELEMENTS}\n\t{ \"${FILENAME}\" , TEXT(\"${FIELD_NAME}\") },")
+		set(ARRAY_ELEMENTS "${ARRAY_ELEMENTS}\n\t{ \"${PATH_TYPE}\", \"${FILE_NAME}\" , TEXT(\"${FIELD_NAME}\") },")
 
 		set(RES_CONTENT "${RES_CONTENT}\n${FIELD_NAME} RCDATA \"${INPUT_NAME}\"")
 	ENDFOREACH(file)
 
 	file(WRITE "${GENERATED_SOURCE_DIR}/code/default_files.rc" "${RES_CONTENT}")
 
-	configure_file("def_files/generated_def_files-win32.h.in" "${GENERATED_SOURCE_DIR}/code/def_files/generated_def_files-win32.h")
+	configure_file("def_files/generated_def_files-win32.inc.in" "${GENERATED_SOURCE_DIR}/code/def_files/generated_def_files-win32.inc")
 
 	target_sources(code INTERFACE "${GENERATED_SOURCE_DIR}/code/default_files.rc")
 else()
@@ -37,35 +42,41 @@ else()
 	FOREACH(file ${file_root_def_files_files})
 		set(INPUT_NAME "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
 		SET(OUTPUT "${GENERATED_SOURCE_DIR}/code/${file}")
+		file(RELATIVE_PATH RELATIVE_FILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/def_files" "${INPUT_NAME}")
 
 		# For some reason this is needed...
-		GET_FILENAME_COMPONENT(DIRECTORY_PATH ${OUTPUT} PATH)
+		GET_FILENAME_COMPONENT(DIRECTORY_PATH ${OUTPUT} DIRECTORY)
 		FILE(MAKE_DIRECTORY ${DIRECTORY_PATH})
 
-		get_filename_component(FILENAME ${file} NAME)
+		# Retrieve the path type name from the relative path
+		GET_FILENAME_COMPONENT(PATH_TYPE "${RELATIVE_FILE_PATH}" DIRECTORY)
+		file(TO_NATIVE_PATH "${PATH_TYPE}" PATH_TYPE)
+		GET_FILENAME_COMPONENT(FILE_NAME "${RELATIVE_FILE_PATH}" NAME)
 
-		string(MAKE_C_IDENTIFIER "${FILENAME}" FIELD_NAME)
+		string(MAKE_C_IDENTIFIER "${RELATIVE_FILE_PATH}" FIELD_NAME)
 
 		set(HEADER_FILE "${OUTPUT}.h")
 		set(SOURCE_FILE "${OUTPUT}.cpp")
 
+		set_source_files_properties("${HEADER_FILE}" "${SOURCE_FILE}" PROPERTIES GENERATED TRUE)
+
 		set(ALL_OUTPUTS "${HEADER_FILE}" "${SOURCE_FILE}")
 		set(FIELD_NAME "Default_${FIELD_NAME}")
 
 		set(INCLUDE_LIST "${INCLUDE_LIST}\n#include \"${file}.h\"")
-		set(ARRAY_ELEMENTS "${ARRAY_ELEMENTS}\n\t{ \"${FILENAME}\" , ${FIELD_NAME} , ${FIELD_NAME}_size },")
+		set(ARRAY_ELEMENTS "${ARRAY_ELEMENTS}\n\t{ \"${PATH_TYPE}\", \"${FILE_NAME}\" , ${FIELD_NAME} , ${FIELD_NAME}_size },")
 
 		ADD_CUSTOM_COMMAND(
 			OUTPUT ${ALL_OUTPUTS}
 			COMMAND embedfile "${INPUT_NAME}" "${OUTPUT}" "${FIELD_NAME}"
-			DEPENDS ${INPUT_NAME}
+			DEPENDS "${INPUT_NAME}"
 			COMMENT "Generating string file for ${INPUT_NAME}"
 			)
 
 		LIST(APPEND DEF_OUT_FILES ${ALL_OUTPUTS})
 	ENDFOREACH(file)
 
-	configure_file("def_files/generated_def_files-generic.h.in" "${GENERATED_SOURCE_DIR}/code/def_files/generated_def_files-generic.h")
+	configure_file("def_files/generated_def_files-generic.inc.in" "${GENERATED_SOURCE_DIR}/code/def_files/generated_def_files-generic.inc")
 	target_sources(code PRIVATE ${DEF_OUT_FILES})
 	SOURCE_GROUP("Generated Files\\Default Files" FILES ${DEF_OUT_FILES})
 

code/fs2netd/fs2netd_client.cpp

@@ -615,6 +615,10 @@ static void fs2netd_handle_ping()
 
 static void fs2netd_handle_messages()
 {
+	if (!Is_connected) {
+		return;
+	}
+
 	int buffer_size = 0, buffer_offset = 0;
 	int bytes_read = 0;
 	char tbuf[256];
@@ -809,6 +813,12 @@ void fs2netd_do_frame()
 	// do ping/pong and ident
 	fs2netd_handle_ping();
 
+	if (!Is_connected) {
+		// The ping handling above made us realize that we are disconnected. We will reconnect in some time so we can
+		// just return here
+		return;
+	}
+
 	// handle gameserver updates
 	fs2netd_gameserver_update();
 

code/gamesnd/eventmusic.cpp

@@ -29,15 +29,14 @@
 #pragma optimize("", off)
 #endif
 
-#define DEFAULT_MASTER_EVENT_MUSIC_VOLUME	0.5f
-
 #define HULL_VALUE_TO_PLAY_INTENSE_BATTLE_MUSIC 0.75f
 
 ////////////////////////////
 // Globals
 ////////////////////////////
 int Event_Music_battle_started = 0;
-float Master_event_music_volume = DEFAULT_MASTER_EVENT_MUSIC_VOLUME;			// range is 0->1
+float Default_music_volume = 0.5f;							// range is 0->1
+float Master_event_music_volume = Default_music_volume;
 
 typedef struct tagSNDPATTERN {
 	int default_next_pattern;	// Needed so the next_pattern member can be reset

code/gamesnd/eventmusic.h

@@ -41,6 +41,7 @@
 
 extern int Event_Music_battle_started;	// flag that will tell us if we've started a battle in the current mission
 extern int Event_music_enabled;
+extern float Default_music_volume;				// range is 0->1
 extern float Master_event_music_volume;			// range is 0->1
 
 

code/gamesnd/gamesnd.cpp

@@ -1322,7 +1322,10 @@ float gamesnd_get_max_duration(game_snd* gs) {
 	int max_length = 0;
 
 	for (auto& entry : gs->sound_entries) {
-		Assertion(entry.id != -1, "Game sound must be loaded to determine maximum duration!");
+		if (entry.id < 0) {
+			// Lazily load unloaded sound entries when required
+			entry.id = snd_load(&entry, gs->flags);
+		}
 
 		max_length = std::max(max_length, snd_get_duration(entry.id));
 	}

code/globalincs/globals.h

@@ -82,9 +82,6 @@
 // object.h
 #define MAX_OBJECTS			3500		
 
-// from lighting.cpp
-#define MAX_LIGHTS 256
-
 // from weapon.h (and beam.h)
 #define MAX_BEAM_SECTIONS				5
 

code/globalincs/pstypes.h

@@ -19,10 +19,10 @@
 #include "globalincs/toolchain.h"
 #include "utils/strings.h"
 
-#include <stdio.h>	// For NULL, etc
-#include <stdlib.h>
+#include <cstdio>    // For NULL, etc
+#include <cstdlib>
 #include <memory.h>
-#include <string.h>
+#include <cstring>
 #include <algorithm>
 #include <cstdint>
 
@@ -112,6 +112,11 @@ inline bool operator==(const vec3d &self, const vec3d &other)
 	);
 }
 
+inline bool operator!=(const vec3d &self, const vec3d &other)
+{
+	return !(self == other);
+}
+
 typedef struct vec2d {
 	float x, y;
 } vec2d;
@@ -363,15 +368,6 @@ typedef struct lod_checker {
 } lod_checker;
 
 
-// check to see that a passed sting is valid, ie:
-//  - has >0 length
-//  - is not "none"
-//  - is not "<none>"
-inline bool VALID_FNAME(const char* x) {
-	return strlen((x)) && stricmp((x), "none") && stricmp((x), "<none>");
-}
-
-
 // Callback Loading function.
 // If you pass a function to this, that function will get called
 // around 10x per second, so you can update the screen.
@@ -433,11 +429,42 @@ public:
 #include "globalincs/vmallocator.h"
 #include "globalincs/safe_strings.h"
 
+// check to see that a passed sting is valid, ie:
+//  - has >0 length
+//  - is not "none"
+//  - is not "<none>"
+inline bool VALID_FNAME(const char* x) {
+	return strlen((x)) && stricmp((x), "none") && stricmp((x), "<none>");
+}
+/**
+ * @brief Checks if the specified string may be a valid file name
+ *
+ * @warning This only does a quick check against an empty string and a few known invalid names. It does not check if the
+ * file actually exists.
+ *
+ * @param x The file name to check
+ * @return @c true if the name is valid, @c false otherwise
+ */
+inline bool VALID_FNAME(const SCP_string& x) {
+	if (x.empty()) {
+		return false;
+	}
+	if (!stricmp(x.c_str(), "none")) {
+		return false;
+	}
+	if (!stricmp(x.c_str(), "<none>")) {
+		return false;
+	}
+	return true;
+}
+
 // Function to generate a stacktrace
 SCP_string dump_stacktrace();
 
 // DEBUG compile time catch for dangerous uses of memset/memcpy/memmove
-#ifndef NDEBUG
+// would prefer std::is_trivially_copyable but it's not supported by gcc yet
+// ref: http://gcc.gnu.org/onlinedocs/libstdc++/manual/status.html
+#if !defined(NDEBUG) && !defined(USING_THIRD_PARTY_LIBS)
 	#if SCP_COMPILER_CXX_AUTO_TYPE && SCP_COMPILER_CXX_STATIC_ASSERT && defined(HAVE_STD_IS_TRIVIALLY_COPYABLE)
 	// feature support seems to be: gcc   clang   msvc
 	// auto                         4.4   2.9     2010