From 35275e0916bf41852a48f21b4f53b1fae855df8a Mon Sep 17 00:00:00 2001 From: Ivan 'provod' Avdeev Date: Sat, 22 May 2021 14:01:43 -0700 Subject: [PATCH] add aftermath sdk for gpu dump generation unfortunately, it's not very helpful tho --- ref_vk/shaders/rtx.comp | 10 +++- ref_vk/vk_core.c | 29 +++++++++- ref_vk/vk_nv_aftermath.c | 112 +++++++++++++++++++++++++++++++++++++++ ref_vk/vk_nv_aftermath.h | 5 ++ ref_vk/vk_rtx.c | 38 ++++++++++++- ref_vk/wscript | 37 ++++++++++++- 6 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 ref_vk/vk_nv_aftermath.c create mode 100644 ref_vk/vk_nv_aftermath.h diff --git a/ref_vk/shaders/rtx.comp b/ref_vk/shaders/rtx.comp index 78b51b44..7b42391f 100644 --- a/ref_vk/shaders/rtx.comp +++ b/ref_vk/shaders/rtx.comp @@ -171,7 +171,7 @@ float printTiledNumber(vec2 p, int n) { // if (pc.y > tlen * .6) return 0.; pc.y = mod(pc.y, 2.); - pc.x = mod(pc.x, 3.); + pc.x = mod(pc.x, 5.); PUTN(n); return col; } @@ -323,14 +323,20 @@ void main() { const int instance_index = rayQueryGetIntersectionInstanceIdEXT(rayQuery, true); const int geom_index = rayQueryGetIntersectionGeometryIndexEXT(rayQuery, true); const int kusok_index = instance_kusochki_offset + geom_index; + + // vec2 pix = vec2(1.,-1.) * vec2(gl_GlobalInvocationID.xy) + vec2(0., imageSize(image).y); + // C = mix(C*.5, vec3(0., 1., 0.), printTiledNumber(pix*2., kusok_index)); + // break; + const Kusok kusok = kusochki[kusok_index]; + + //const uint leaf = kusochki[kusok_index].leaf-1; //C = fract(pos / LIGHT_GRID_CELL_SIZE); break; //C = vec3(hash(float(geom_index)), hash(float(geom_index)+15.43), hash(float(geom_index)+34.)); break; // C = vec3(hash(float(instance_index)), hash(float(instance_index)+15.43), hash(float(instance_index)+34.)) + .1 * fract(pos/LIGHT_GRID_CELL_SIZE); - // vec2 pix = vec2(1.,-1.) * vec2(gl_GlobalInvocationID.xy) + vec2(0., imageSize(image).y); // C = mix(C*.5, vec3(0., 1., 0.), printTiledNumber(pix, instance_index)); // break; diff --git a/ref_vk/vk_core.c b/ref_vk/vk_core.c index 2cb7b6da..ba48ef7d 100644 --- a/ref_vk/vk_core.c +++ b/ref_vk/vk_core.c @@ -14,6 +14,7 @@ #include "vk_studio.h" #include "vk_rtx.h" #include "vk_descriptor.h" +#include "vk_nv_aftermath.h" #include "xash3d_types.h" #include "cvardef.h" @@ -132,6 +133,12 @@ static const char* device_extensions[] = { //VK_KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME, VK_KHR_RAY_QUERY_EXTENSION_NAME, VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME, + + // FIXME make this not depend on RTX +#ifdef USE_AFTERMATH + VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME, + VK_NV_DEVICE_DIAGNOSTICS_CONFIG_EXTENSION_NAME, +#endif }; VkBool32 debugCallback( @@ -396,9 +403,18 @@ static qboolean pickAndCreateDevice( void ) .rayQuery = VK_TRUE, .pNext = &sixteen_bit_feature, }; + void *head = &ray_query_feature; +#ifdef USE_AFTERMATH + VkDeviceDiagnosticsConfigCreateInfoNV diag_config_nv = { + .sType = VK_STRUCTURE_TYPE_DEVICE_DIAGNOSTICS_CONFIG_CREATE_INFO_NV, + .flags = VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_AUTOMATIC_CHECKPOINTS_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_RESOURCE_TRACKING_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_SHADER_DEBUG_INFO_BIT_NV, + .pNext = head, + }; + head = &diag_config_nv; +#endif VkDeviceCreateInfo create_info = { .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, - .pNext = vk_core.rtx ? &ray_query_feature : NULL, + .pNext = vk_core.rtx ? head : NULL, .flags = 0, .queueCreateInfoCount = 1, .pQueueCreateInfos = &queue_info, @@ -549,6 +565,12 @@ qboolean R_VkInit( void ) return false; } +#if USE_AFTERMATH + if (!VK_AftermathInit()) { + gEngine.Con_Printf( S_ERROR "Cannot initialize Nvidia Nsight Aftermath SDK\n" ); + } +#endif + if (!pickAndCreateDevice()) return false; @@ -645,6 +667,11 @@ void R_VkShutdown( void ) vkDestroyDevice(vk_core.device, NULL); +#if USE_AFTERMATH + VK_AftermathShutdown(); +#endif + + if (vk_core.debug_messenger) { vkDestroyDebugUtilsMessengerEXT(vk_core.instance, vk_core.debug_messenger, NULL); diff --git a/ref_vk/vk_nv_aftermath.c b/ref_vk/vk_nv_aftermath.c new file mode 100644 index 00000000..c8ccec06 --- /dev/null +++ b/ref_vk/vk_nv_aftermath.c @@ -0,0 +1,112 @@ +#include "vk_nv_aftermath.h" + +#ifdef USE_AFTERMATH +#include "vk_common.h" + +#include "xash3d_types.h" + +#include "GFSDK_Aftermath.h" +#include "GFSDK_Aftermath_GpuCrashDump.h" +#include "GFSDK_Aftermath_GpuCrashDumpDecoding.h" + +#include + +static const char *aftermathErrorName(GFSDK_Aftermath_Result result) { + switch (result) { +#define CASE(c) case c: return #c; + CASE(GFSDK_Aftermath_Result_NotAvailable) + CASE(GFSDK_Aftermath_Result_Fail) + CASE(GFSDK_Aftermath_Result_FAIL_VersionMismatch) + CASE(GFSDK_Aftermath_Result_FAIL_NotInitialized) + CASE(GFSDK_Aftermath_Result_FAIL_InvalidAdapter) + CASE(GFSDK_Aftermath_Result_FAIL_InvalidParameter) + CASE(GFSDK_Aftermath_Result_FAIL_Unknown) + CASE(GFSDK_Aftermath_Result_FAIL_ApiError) + CASE(GFSDK_Aftermath_Result_FAIL_NvApiIncompatible) + CASE(GFSDK_Aftermath_Result_FAIL_GettingContextDataWithNewCommandList) + CASE(GFSDK_Aftermath_Result_FAIL_AlreadyInitialized) + CASE(GFSDK_Aftermath_Result_FAIL_D3DDebugLayerNotCompatible) + CASE(GFSDK_Aftermath_Result_FAIL_DriverInitFailed) + CASE(GFSDK_Aftermath_Result_FAIL_DriverVersionNotSupported) + CASE(GFSDK_Aftermath_Result_FAIL_OutOfMemory) + CASE(GFSDK_Aftermath_Result_FAIL_GetDataOnBundle) + CASE(GFSDK_Aftermath_Result_FAIL_GetDataOnDeferredContext) + CASE(GFSDK_Aftermath_Result_FAIL_FeatureNotEnabled) + CASE(GFSDK_Aftermath_Result_FAIL_NoResourcesRegistered) + CASE(GFSDK_Aftermath_Result_FAIL_ThisResourceNeverRegistered) + CASE(GFSDK_Aftermath_Result_FAIL_NotSupportedInUWP) + CASE(GFSDK_Aftermath_Result_FAIL_D3dDllNotSupported) + CASE(GFSDK_Aftermath_Result_FAIL_D3dDllInterceptionNotSupported) + CASE(GFSDK_Aftermath_Result_FAIL_Disabled) +#undef CASE + } + + return "UNKNOWN"; +} + +#define AM_CHECK(F) \ +do { \ + GFSDK_Aftermath_Result result = F; \ + if (!GFSDK_Aftermath_SUCCEED(result)) { \ + gEngine.Con_Printf( S_ERROR "%s:%d " #F " failed (%#x): %s\n", \ + __FILE__, __LINE__, result, aftermathErrorName(result)); \ + } \ +} while (0) + +static qboolean writeFile(const char *filename, const void *data, size_t size) { + FILE *f = fopen(filename, "wb"); + qboolean result = false; + if (!f) + return result; + result = fwrite(data, 1, size, f) == size; + fclose(f); + return result; +} + +static void callbackGpuCrashDump(const void* pGpuCrashDump, const uint32_t gpuCrashDumpSize, void* pUserData) { + gEngine.Con_Printf(S_ERROR "AFTERMATH GPU CRASH DUMP: %p, size=%d\n", pGpuCrashDump, gpuCrashDumpSize); + writeFile("ref_vk.nv-gpudmp", pGpuCrashDump, gpuCrashDumpSize); +} + +static void callbackShaderDebugInfo(const void* pShaderDebugInfo, const uint32_t shaderDebugInfoSize, void* pUserData) { + GFSDK_Aftermath_ShaderDebugInfoIdentifier identifier = {0}; + gEngine.Con_Printf(S_ERROR "AFTERMATH Shader Debug Info: %p, size=%d\n", pShaderDebugInfo, shaderDebugInfoSize); + + AM_CHECK(GFSDK_Aftermath_GetShaderDebugInfoIdentifier( + GFSDK_Aftermath_Version_API, + pShaderDebugInfo, + shaderDebugInfoSize, + &identifier)); + + char filename[64]; + Q_snprintf(filename, sizeof(filename), "shader-%016llX-%016llX.nvdbg", identifier.id[0], identifier.id[1]); + writeFile(filename, pShaderDebugInfo, shaderDebugInfoSize); +} + +static void callbackGpuCrashDumpDescription(PFN_GFSDK_Aftermath_AddGpuCrashDumpDescription addValue, void* pUserData) { + gEngine.Con_Printf(S_ERROR "AFTERMATH asks for crash dump description\n"); + addValue(GFSDK_Aftermath_GpuCrashDumpDescriptionKey_ApplicationName, "xash3d-fwgs-ref-vk"); + addValue(GFSDK_Aftermath_GpuCrashDumpDescriptionKey_ApplicationVersion, "v0.0.1"); +} + +static qboolean initialized = false; +qboolean VK_AftermathInit() { + AM_CHECK(GFSDK_Aftermath_EnableGpuCrashDumps( + GFSDK_Aftermath_Version_API, + GFSDK_Aftermath_GpuCrashDumpWatchedApiFlags_Vulkan, + GFSDK_Aftermath_GpuCrashDumpFeatureFlags_DeferDebugInfoCallbacks, + callbackGpuCrashDump, + callbackShaderDebugInfo, + callbackGpuCrashDumpDescription, + NULL)); + + initialized = true; + return true; +} + +void VK_AftermathShutdown() { + if (initialized) { + GFSDK_Aftermath_DisableGpuCrashDumps(); + } +} +#endif //ifdef USE_AFTERMATH diff --git a/ref_vk/vk_nv_aftermath.h b/ref_vk/vk_nv_aftermath.h new file mode 100644 index 00000000..5c57615f --- /dev/null +++ b/ref_vk/vk_nv_aftermath.h @@ -0,0 +1,5 @@ +#pragma once +#include "xash3d_types.h" + +qboolean VK_AftermathInit(); +void VK_AftermathShutdown(); diff --git a/ref_vk/vk_rtx.c b/ref_vk/vk_rtx.c index 8560ab98..f56529d2 100644 --- a/ref_vk/vk_rtx.c +++ b/ref_vk/vk_rtx.c @@ -458,7 +458,6 @@ void VK_RayFrameBegin( void ) // HACK: blas caching requires persistent memory // proper fix would need some other memory allocation strategy // VK_RingBuffer_ClearFrame(&g_rtx.accels_buffer_alloc); - VK_RingBuffer_ClearFrame(&g_rtx.kusochki_alloc); } @@ -478,6 +477,40 @@ static void createPipeline( void ) ASSERT(g_rtx.pipeline); } +static void validateModelData( void ) { + const vk_kusok_data_t* kusochki = g_rtx.kusochki_buffer.mapped; + ASSERT(g_rtx.frame.num_models <= ARRAYSIZE(g_rtx.frame.models)); + for (int i = 0; i < g_rtx.frame.num_models; ++i) { + const vk_ray_model_t *model = g_rtx.frame.models + i; + int num_geoms = 1; // TODO can't validate non-dynamic models because this info is lost + ASSERT(model->accel != VK_NULL_HANDLE); + ASSERT(model->kusochki_offset < MAX_KUSOCHKI); + + if (model->dynamic) { + ASSERT(model->cache_accel); + ASSERT(model->cache_accel->as == model->accel); + ASSERT(model->cache_accel->geoms); + ASSERT(model->cache_accel->num_geoms > 0); + ASSERT(model->cache_accel->taken); + num_geoms = model->cache_accel->num_geoms; + } else { + ASSERT(!model->cache_accel); + } + + for (int j = 0; j < num_geoms; j++) { + const vk_kusok_data_t *kusok = kusochki + j; + const vk_texture_t *tex = findTexture(kusok->texture); + ASSERT(tex); + ASSERT(tex->vk.image_view != VK_NULL_HANDLE); + + // uint32_t index_offset; + // uint32_t vertex_offset; + // uint32_t triangles; + // uint32_t debug_is_emissive; + } + } +} + void VK_RayFrameEnd(const vk_ray_frame_render_args_t* args) { const VkCommandBuffer cmdbuf = args->cmdbuf; @@ -489,6 +522,9 @@ void VK_RayFrameEnd(const vk_ray_frame_render_args_t* args) // FIXME pass these matrices explicitly to let RTX module handle ubo itself ASSERT(args->ubo.size == sizeof(float) * 16 * 2); + if (vk_core.debug) + validateModelData(); + g_rtx.frame_number++; if (g_rtx.reload_pipeline) { diff --git a/ref_vk/wscript b/ref_vk/wscript index 907b685b..164f5d52 100644 --- a/ref_vk/wscript +++ b/ref_vk/wscript @@ -9,6 +9,8 @@ top = '.' def options(opt): grp = opt.add_option_group('ref_vk options') + grp.add_option('', '--aftermath', action='store', dest = 'NSIGHT_AFTERMATH_SDK', default = None, + help = 'Path to Nvidia Nsight Aftermath SDK (optional)') def configure(conf): # check for dedicated server build @@ -26,8 +28,31 @@ def configure(conf): conf.env.VULKAN_SDK = conf.environ['VULKAN_SDK'] conf.end_msg('found at ' + conf.env.VULKAN_SDK) + if conf.options.NSIGHT_AFTERMATH_SDK: + conf.start_msg('Nvidia Nsight Aftermath SDK') + conf.env.HAVE_AFTERMATH = 1 + path = conf.options.NSIGHT_AFTERMATH_SDK + conf.env.INCLUDES_AFTERMATH = [os.path.abspath(os.path.join(path, 'include'))] + libdir = 'lib' + lib = 'GFSDK_Aftermath_Lib' + if conf.env.COMPILER_CC == 'msvc': + if conf.env.DEST_CPU == 'x86_64': + libdir = 'lib/x64' + lib += '.x64' + else: + libdir = 'lib/' + conf.env.DEST_CPU + lib += conf.env.DEST_CPU + libdir = os.path.abspath(os.path.join(path, libdir)) + conf.env.LIBPATH_AFTERMATH = [libdir] + conf.env.LIB_AFTERMATH = [lib] + conf.end_msg('SDK: {0}, includes: {1}, libpath: {2}, lib: {3}'.format(path, conf.env.INCLUDES_AFTERMATH, conf.env.LIBPATH_AFTERMATH, conf.env.LIB_AFTERMATH)) + + conf.env.GLSLCFLAGS += ['-g'] + def build(bld): libs = [ 'public', 'M' ] + defines = [] + libpath = [] source = bld.path.ant_glob(['*.c']) glsl_source = bld.path.ant_glob(['shaders/*.vert', 'shaders/*.frag', 'shaders/*.comp']) @@ -44,13 +69,21 @@ def build(bld): if bld.env.DEST_OS == 'win32': includes.append(bld.env.VULKAN_SDK + '\\Include') + if bld.env.HAVE_AFTERMATH: + defines.append('USE_AFTERMATH') + libs.append('AFTERMATH') + # includes.append(bld.env.AFTERMATH_INCLUDE) + # libs.append(bld.env.AFTERMATH_LIB) + # libpath.append(bld.env.AFTERMATH_LIBDIR) + bld.shlib( source = source, target = 'ref_vk', features = 'c', includes = includes, - use = libs, #+ (['GL'] if bld.env.GL_STATIC else []), - defines = [], # ['XASH_GL_STATIC'] if bld.env.GL_STATIC else [], + use = libs, + defines = defines, + libpath = libpath, install_path = bld.env.LIBDIR, subsystem = bld.env.MSVC_SUBSYSTEM )