vk: add experimental nv checkpoints support

doesn't really work reliably

ref_vk/vk_core.c

@@ -510,7 +510,7 @@ static qboolean createDevice( void ) {
 		VkDeviceDiagnosticsConfigCreateInfoNV diag_config_nv = {
 			.sType = VK_STRUCTURE_TYPE_DEVICE_DIAGNOSTICS_CONFIG_CREATE_INFO_NV,
 			.pNext = head,
-			.flags = VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_AUTOMATIC_CHECKPOINTS_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_RESOURCE_TRACKING_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_SHADER_DEBUG_INFO_BIT_NV,
+			.flags = VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_AUTOMATIC_CHECKPOINTS_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_RESOURCE_TRACKING_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_SHADER_DEBUG_INFO_BIT_NV | VK_DEVICE_DIAGNOSTICS_CONFIG_ENABLE_SHADER_ERROR_REPORTING_BIT_NV
 		};
 		head = &diag_config_nv;
 #endif

ref_vk/vk_core.h

@@ -107,6 +107,20 @@ do { \
 	} \
 } while (0)
 
+#if USE_AFTERMATH
+void R_VK_NV_CheckpointF(VkCommandBuffer cmdbuf, const char *fmt, ...);
+void R_Vk_NV_Checkpoint_Dump(void);
+#define DEBUG_NV_CHECKPOINTF(cmdbuf, fmt, ...) \
+	do { \
+		if (vk_core.debug) { \
+			R_Vk_NV_CheckpointF(cmdbuf, fmt, ##__VA_ARGS__); \
+		} \
+	} while(0)
+#else
+#define DEBUG_CHECKPOINTF(...)
+#define R_Vk_NV_Checkpoint_Dump()
+#endif
+
 #define DEBUG_BEGIN(cmdbuf, msg) \
 	do { \
 		if (vk_core.debug) { \
@@ -115,6 +129,7 @@ do { \
 				.pLabelName = msg, \
 			}; \
 			vkCmdBeginDebugUtilsLabelEXT(cmdbuf, &label); \
+			DEBUG_NV_CHECKPOINTF(cmdbuf, "begin %s", msg); \
 		} \
 	} while(0)
 
@@ -128,6 +143,7 @@ do { \
 				.pLabelName = buf, \
 			}; \
 			vkCmdBeginDebugUtilsLabelEXT(cmdbuf, &label); \
+			DEBUG_NV_CHECKPOINTF(cmdbuf, "begin " fmt, ##__VA_ARGS__); \
 		} \
 	} while(0)
 
@@ -135,6 +151,7 @@ do { \
 	do { \
 		if (vk_core.debug) { \
 			vkCmdEndDebugUtilsLabelEXT(cmdbuf); \
+			DEBUG_NV_CHECKPOINTF(cmdbuf, "end "); /* TODO: find corresponding begin */ \
 		} \
 	} while(0)
 
@@ -145,6 +162,8 @@ do { \
 		if (result != VK_SUCCESS) { \
 			gEngine.Con_Printf( S_ERROR "%s:%d " #f " failed (%d): %s\n", \
 				__FILE__, __LINE__, result, R_VkResultName(result)); \
+			if (vk_core.debug) \
+				R_Vk_NV_Checkpoint_Dump(); \
 			gEngine.Host_Error( S_ERROR "%s:%d " #f " failed (%d): %s\n", \
 				__FILE__, __LINE__, result, R_VkResultName(result)); \
 		} \
@@ -254,6 +273,8 @@ do { \
 	X(vkCmdClearColorImage) \
 	X(vkCmdCopyImage) \
 	X(vkGetImageSubresourceLayout) \
+	X(vkCmdSetCheckpointNV) \
+	X(vkGetQueueCheckpointDataNV) \
 
 #define DEVICE_FUNCS_RTX(X) \
 	X(vkGetAccelerationStructureBuildSizesKHR) \

ref_vk/vk_nv_aftermath.c

@@ -2,6 +2,7 @@
 #include "vk_nv_aftermath.h"
 
 #include "vk_common.h"
+#include "vk_core.h"
 
 #include "xash3d_types.h"
 
@@ -66,6 +67,7 @@ static qboolean writeFile(const char *filename, const void *data, size_t size) {
 static void callbackGpuCrashDump(const void* pGpuCrashDump, const uint32_t gpuCrashDumpSize, void* pUserData) {
 	gEngine.Con_Printf(S_ERROR "AFTERMATH GPU CRASH DUMP: %p, size=%d\n", pGpuCrashDump, gpuCrashDumpSize);
 	writeFile("ref_vk.nv-gpudmp", pGpuCrashDump, gpuCrashDumpSize);
+	R_Vk_NV_Checkpoint_Dump();
 }
 
 static void callbackShaderDebugInfo(const void* pShaderDebugInfo, const uint32_t shaderDebugInfoSize, void* pUserData) {
@@ -109,4 +111,57 @@ void VK_AftermathShutdown() {
 		GFSDK_Aftermath_DisableGpuCrashDumps();
 	}
 }
+
+#define MAX_NV_CHECKPOINTS 2048
+
+typedef struct {
+	unsigned sequence;
+	char message[256];
+} vk_nv_checkpoint_entry_t;
+
+static struct {
+	unsigned sequence;
+	vk_nv_checkpoint_entry_t entries[MAX_NV_CHECKPOINTS];
+} g_nv_checkpoint = {0};
+
+void R_Vk_NV_CheckpointF(VkCommandBuffer cmdbuf, const char *fmt, ...) {
+	va_list argptr;
+
+	++g_nv_checkpoint.sequence;
+
+	vk_nv_checkpoint_entry_t *entry = g_nv_checkpoint.entries + (g_nv_checkpoint.sequence % MAX_NV_CHECKPOINTS);
+	entry->sequence = g_nv_checkpoint.sequence;
+
+	va_start( argptr, fmt );
+	vsnprintf( entry->message, sizeof entry->message, fmt, argptr );
+	va_end( argptr );
+
+	const uintptr_t marker = entry->sequence;
+	vkCmdSetCheckpointNV(cmdbuf, (const void*)marker);
+}
+
+void R_Vk_NV_Checkpoint_Dump(void) {
+	uint32_t checkpoints_count = 0;
+	vkGetQueueCheckpointDataNV(vk_core.queue, &checkpoints_count, NULL);
+
+	VkCheckpointDataNV checkpoints[32];
+	if (checkpoints_count > COUNTOF(checkpoints))
+		checkpoints_count = COUNTOF(checkpoints);
+
+	for (int i = 0; i < checkpoints_count; ++i) {
+		checkpoints[i].pNext = NULL;
+		checkpoints[i].sType = VK_STRUCTURE_TYPE_CHECKPOINT_DATA_NV;
+	}
+
+	vkGetQueueCheckpointDataNV(vk_core.queue, &checkpoints_count, checkpoints);
+
+	gEngine.Con_Reportf(S_ERROR "Checkpoints: %d\n", checkpoints_count);
+	for (int i = 0; i < checkpoints_count; ++i) {
+		const VkCheckpointDataNV *const checkpoint = checkpoints + i;
+		const unsigned sequence = (uintptr_t)checkpoint->pCheckpointMarker;
+		const vk_nv_checkpoint_entry_t *const entry = g_nv_checkpoint.entries + (sequence % MAX_NV_CHECKPOINTS);
+		gEngine.Con_Reportf(S_ERROR "\t%u: stage=%04x msg: %s\n", sequence, checkpoint->stage, entry->sequence == sequence ? entry->message : "[OBSOLETE]");
+	}
+}
+
 #endif //ifdef USE_AFTERMATH

ref_vk/vk_staging.c

@@ -121,6 +121,7 @@ static void commitBuffers(VkCommandBuffer cmdbuf) {
 			continue;
 
 		if (prev_buffer != VK_NULL_HANDLE) {
+			DEBUG_NV_CHECKPOINTF(cmdbuf, "staging dst_buffer=%p count=%d", prev_buffer, i-first_copy);
 			vkCmdCopyBuffer(cmdbuf, g_staging.buffer.buffer,
 				prev_buffer,
 				i - first_copy, g_staging.buffers.copy + first_copy);