media-gfx/renderdoc: filter lto

Also:
* rebase the system-glslang patch as one header from the vendored copy
  of glslang was used, causing issues due to the differing versions.
* delete the vendored glslang directory in src_prepare to remove the
  possibility of the above happening again.

Closes: https://bugs.gentoo.org/859799
Signed-off-by: Matthew Smith <matthew@gentoo.org>

2 files changed, 206 insertions, 2 deletions

diff --git a/media-gfx/renderdoc/files/renderdoc-1.20-system-glslang.patch b/media-gfx/renderdoc/files/renderdoc-1.20-system-glslang.patch
new file mode 100644
index 000000000000..be808131c362
--- /dev/null
+++ b/media-gfx/renderdoc/files/renderdoc-1.20-system-glslang.patch
@@ -0,0 +1,197 @@
+From 70442b7a3952350429872e95d843374f0aaa6a2f Mon Sep 17 00:00:00 2001
+From: Matthew Smith <matthew@gentoo.org>
+Date: Sun, 30 Jan 2022 17:19:47 +0000
+Subject: [PATCH] build: Use system glslang
+
+Requires -DGLSLANG_TARGET_DIR option.
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -437,6 +437,17 @@ elseif(UNIX)
+     endif()
+ endif()
+ 
++# glslang package does not export a -config.cmake file.
++option(GLSLANG_TARGET_DIR "Absolute path to glslangTargets.cmake directory")
++include("${GLSLANG_TARGET_DIR}/OSDependentTargets.cmake")
++include("${GLSLANG_TARGET_DIR}/OGLCompilerTargets.cmake")
++include("${GLSLANG_TARGET_DIR}/HLSLTargets.cmake")
++include("${GLSLANG_TARGET_DIR}/glslangTargets.cmake")
++include("${GLSLANG_TARGET_DIR}/SPIRVTargets.cmake")
++if(NOT TARGET glslang OR NOT TARGET SPIRV)
++  message(FATAL_ERROR "glslang or SPIRV target not found")
++endif()
++
+ add_subdirectory(renderdoc)
+ 
+ # these variables are handled within the CMakeLists.txt in qrenderdoc,
+--- a/renderdoc/CMakeLists.txt
++++ b/renderdoc/CMakeLists.txt
+@@ -595,7 +595,7 @@ endif()
+ add_library(renderdoc SHARED ${renderdoc_objects})
+ target_compile_definitions(renderdoc ${RDOC_DEFINITIONS})
+ target_include_directories(renderdoc ${RDOC_INCLUDES})
+-target_link_libraries(renderdoc ${RDOC_LIBRARIES})
++target_link_libraries(renderdoc ${RDOC_LIBRARIES} glslang SPIRV)
+ 
+ add_dependencies(renderdoc renderdoc_libentry)
+ 
+--- a/renderdoc/data/glsl_shaders.cpp
++++ b/renderdoc/data/glsl_shaders.cpp
+@@ -26,7 +26,7 @@
+ #include "common/common.h"
+ #include "common/formatting.h"
+ #include "driver/shaders/spirv/glslang_compile.h"
+-#include "glslang/glslang/Public/ShaderLang.h"
++#include <glslang/Public/ShaderLang.h>
+ #include "os/os_specific.h"
+ 
+ #define GLSL_HEADERS(HEADER) \
+--- a/renderdoc/driver/gl/gl_shader_refl.cpp
++++ b/renderdoc/driver/gl/gl_shader_refl.cpp
+@@ -26,7 +26,7 @@
+ #include <algorithm>
+ #include <functional>
+ #include "driver/shaders/spirv/glslang_compile.h"
+-#include "glslang/glslang/Public/ShaderLang.h"
++#include <glslang/Public/ShaderLang.h>
+ #include "gl_driver.h"
+ 
+ template <>
+--- a/renderdoc/driver/shaders/spirv/CMakeLists.txt
++++ b/renderdoc/driver/shaders/spirv/CMakeLists.txt
+@@ -1,91 +1,3 @@
+-set(glslang_dir ${RDOC_SOURCE_DIR}/3rdparty/glslang)
+-set(glslang_sources
+-    ${glslang_dir}/OGLCompilersDLL/InitializeDll.cpp
+-    ${glslang_dir}/OGLCompilersDLL/InitializeDll.h
+-    ${glslang_dir}/SPIRV/GlslangToSpv.cpp
+-    ${glslang_dir}/SPIRV/GlslangToSpv.h
+-    ${glslang_dir}/SPIRV/GLSL.std.450.h
+-    ${glslang_dir}/SPIRV/GLSL.ext.AMD.h
+-    ${glslang_dir}/SPIRV/GLSL.ext.EXT.h
+-    ${glslang_dir}/SPIRV/GLSL.ext.KHR.h
+-    ${glslang_dir}/SPIRV/GLSL.ext.NV.h
+-    ${glslang_dir}/SPIRV/hex_float.h
+-    ${glslang_dir}/SPIRV/InReadableOrder.cpp
+-    ${glslang_dir}/SPIRV/Logger.cpp
+-    ${glslang_dir}/SPIRV/Logger.h
+-    ${glslang_dir}/SPIRV/SpvBuilder.cpp
+-    ${glslang_dir}/SPIRV/SpvBuilder.h
+-    ${glslang_dir}/SPIRV/SpvTools.cpp
+-    ${glslang_dir}/SPIRV/SpvTools.h
+-    ${glslang_dir}/SPIRV/SpvPostProcess.cpp
+-    ${glslang_dir}/SPIRV/spvIR.h
+-    ${glslang_dir}/glslang/GenericCodeGen/CodeGen.cpp
+-    ${glslang_dir}/glslang/GenericCodeGen/Link.cpp
+-    ${glslang_dir}/glslang/Include/arrays.h
+-    ${glslang_dir}/glslang/Include/BaseTypes.h
+-    ${glslang_dir}/glslang/Include/Common.h
+-    ${glslang_dir}/glslang/Include/ConstantUnion.h
+-    ${glslang_dir}/glslang/Include/InfoSink.h
+-    ${glslang_dir}/glslang/Include/InitializeGlobals.h
+-    ${glslang_dir}/glslang/Include/intermediate.h
+-    ${glslang_dir}/glslang/Include/PoolAlloc.h
+-    ${glslang_dir}/glslang/Include/ResourceLimits.h
+-    ${glslang_dir}/glslang/Include/revision.h
+-    ${glslang_dir}/glslang/Include/ShHandle.h
+-    ${glslang_dir}/glslang/Include/Types.h
+-    ${glslang_dir}/glslang/MachineIndependent/Constant.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/glslang_tab.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/glslang_tab.cpp.h
+-    ${glslang_dir}/glslang/MachineIndependent/gl_types.h
+-    ${glslang_dir}/glslang/MachineIndependent/iomapper.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/iomapper.h
+-    ${glslang_dir}/glslang/MachineIndependent/gl_types.h
+-    ${glslang_dir}/glslang/MachineIndependent/InfoSink.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/Initialize.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/Initialize.h
+-    ${glslang_dir}/glslang/MachineIndependent/Intermediate.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/intermOut.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/IntermTraverse.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/limits.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/linkValidate.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/LiveTraverser.h
+-    ${glslang_dir}/glslang/MachineIndependent/localintermediate.h
+-    ${glslang_dir}/glslang/MachineIndependent/parseConst.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/ParseContextBase.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/ParseHelper.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/ParseHelper.h
+-    ${glslang_dir}/glslang/MachineIndependent/PoolAlloc.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/propagateNoContraction.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/propagateNoContraction.h
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpAtom.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpContext.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpContext.h
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/Pp.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpScanner.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpTokens.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/preprocessor/PpTokens.h
+-    ${glslang_dir}/glslang/MachineIndependent/reflection.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/reflection.h
+-    ${glslang_dir}/glslang/MachineIndependent/RemoveTree.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/RemoveTree.h
+-    ${glslang_dir}/glslang/MachineIndependent/ScanContext.h
+-    ${glslang_dir}/glslang/MachineIndependent/Scan.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/Scan.h
+-    ${glslang_dir}/glslang/MachineIndependent/ShaderLang.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/SymbolTable.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/SymbolTable.h
+-    ${glslang_dir}/glslang/MachineIndependent/Versions.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/Versions.h
+-    ${glslang_dir}/glslang/MachineIndependent/attribute.cpp
+-    ${glslang_dir}/glslang/MachineIndependent/attribute.h
+-    ${glslang_dir}/glslang/OSDependent/osinclude.h
+-    ${glslang_dir}/glslang/Public/ShaderLang.h)
+-
+-if(UNIX)
+-    list(APPEND glslang_sources
+-        ${glslang_dir}/glslang/OSDependent/Unix/ossource.cpp)
+-endif()
+-
+ set(sources
+     glslang_compile.cpp
+     glslang_compile.h
+@@ -108,8 +20,7 @@ set(sources
+     spirv_processor.h
+     spirv_disassemble.cpp
+     spirv_stringise.cpp
+-    var_dispatch_helpers.h
+-    ${glslang_sources})
++    var_dispatch_helpers.h)
+ 
+ add_definitions(-DAMD_EXTENSIONS)
+ add_definitions(-DNV_EXTENSIONS)
+--- a/renderdoc/driver/shaders/spirv/glslang_compile.cpp
++++ b/renderdoc/driver/shaders/spirv/glslang_compile.cpp
+@@ -29,8 +29,8 @@
+ #undef min
+ #undef max
+ 
+-#include "glslang/glslang/Include/Types.h"
+-#include "glslang/glslang/Public/ShaderLang.h"
++#include <glslang/Include/Types.h>
++#include <glslang/Public/ShaderLang.h>
+ 
+ static bool glslang_inited = false;
+ rdcarray<glslang::TShader *> *allocatedShaders = NULL;
+@@ -129,6 +129,7 @@ static TBuiltInResource DefaultResources = {
+     /*.maxTaskWorkGroupSizeY_NV =*/1,
+     /*.maxTaskWorkGroupSizeZ_NV =*/1,
+     /*.maxMeshViewCountNV =*/4,
++    /*.maxDualSourceDrawBuffersEXT =*/1,
+ 
+     /*.limits*/
+     {
+--- a/renderdoc/driver/shaders/spirv/spirv_compile.cpp
++++ b/renderdoc/driver/shaders/spirv/spirv_compile.cpp
+@@ -31,8 +31,8 @@
+ #undef min
+ #undef max
+ 
+-#include "glslang/SPIRV/GlslangToSpv.h"
+-#include "glslang/glslang/Public/ShaderLang.h"
++#include <glslang/SPIRV/GlslangToSpv.h>
++#include <glslang/Public/ShaderLang.h>
+ 
+ rdcstr rdcspv::Compile(const rdcspv::CompilationSettings &settings, const rdcarray<rdcstr> &sources,
+                        rdcarray<uint32_t> &spirv)
diff --git a/media-gfx/renderdoc/renderdoc-1.20.ebuild b/media-gfx/renderdoc/renderdoc-1.20-r1.ebuild
index 1503f8651e20..547e21513366 100644
--- a/media-gfx/renderdoc/renderdoc-1.20.ebuild
+++ b/media-gfx/renderdoc/renderdoc-1.20-r1.ebuild
@@ -13,7 +13,7 @@ AUTOTOOLS_AUTO_DEPEND="no"
 DOCS_BUILDER="sphinx"
 DOCS_DIR="docs"
 PYTHON_COMPAT=( python3_{9,10} )
-inherit autotools cmake optfeature python-single-r1 docs qmake-utils verify-sig xdg
+inherit autotools cmake flag-o-matic optfeature python-single-r1 docs qmake-utils verify-sig xdg
 
 DESCRIPTION="A stand-alone graphics debugging tool"
 HOMEPAGE="https://renderdoc.org https://github.com/baldurk/renderdoc"
@@ -99,7 +99,7 @@ PATCHES=(
 	# Needed to prevent sandbox violations during build.
 	"${FILESDIR}"/${PN}-1.18-env-home.patch
 
-	"${FILESDIR}"/${PN}-1.18-system-glslang.patch
+	"${FILESDIR}"/${PN}-1.20-system-glslang.patch
 	"${FILESDIR}"/${PN}-1.18-system-compress.patch
 )
 
@@ -124,6 +124,10 @@ src_unpack() {
 src_prepare() {
 	cmake_src_prepare
 
+	# Ensure that we use the system glslang headers instead of the
+	# vendored copy.
+	rm -r renderdoc/3rdparty/glslang || die 'rm vendored glslang failed'
+
 	# Remove the calls to install the documentation files.  Instead,
 	# install them with einstalldocs.
 	sed -i '/share\/doc\/renderdoc/d' \
@@ -180,6 +184,9 @@ src_configure() {
 
 	use pyside2 && mycmakeargs+=( -DPYTHON_CONFIG_SUFFIX=-${EPYTHON} )
 
+	# Lots of type mismatch issues.
+	filter-lto
+
 	cmake_src_configure
 }