vk: rt: investigate more nans

Found that some studio models end up producing zero-length normals. See #731

ref/vk/TODO.md

@@ -1,4 +1,6 @@
-## Next
+## 2024-01-19 E366
+- [x] investigate more shading nans
+	- found zero normals in studio models, see #731
 - [ ] bounce diffuse is still way darker than before
 
 # Previously

ref/vk/shaders/ray_primary_hit.glsl

@@ -77,6 +77,21 @@ void primaryRayHit(rayQueryEXT rq, inout RayPayloadPrimary payload) {
 			tnorm.z = sqrt(max(0., 1. - dot(tnorm.xy, tnorm.xy)));
 
 			geom.normal_shading = normalize(TBN * tnorm);
+
+#ifdef DEBUG_VALIDATE_EXTRA
+			if (IS_INVALIDV(geom.normal_shading)) {
+				debugPrintfEXT("ray_primary_hit.glsl:%d geom.tangent=(%f,%f,%f) T=(%f,%f,%f) nscale=%f tnorm=(%f,%f,%f) INVALID nshade=(%f,%f,%f)",
+					__LINE__,
+					PRIVEC3(geom.tangent),
+					PRIVEC3(T),
+					material.normal_scale,
+					PRIVEC3(tnorm),
+					PRIVEC3(geom.normal_shading)
+				);
+				// TODO ???
+				geom.normal_shading = geom.normal_geometry;
+			}
+#endif
 		}
 #endif
 	}

ref/vk/shaders/rt_geometry.glsl

@@ -121,6 +121,24 @@ Geometry readHitGeometry(vec2 bary, float ray_cone_width) {
 		GET_VERTEX(vi2).normal,
 		GET_VERTEX(vi3).normal,
 		bary));
+
+#ifdef DEBUG_VALIDATE_EXTRA
+	if (IS_INVALIDV(geom.normal_shading)) {
+		debugPrintfEXT("rt_geometry.glsl:%d model=%d geom=%d prim=%d v1n=(%f,%f,%f) v2n=(%f,%f,%f) v3n=(%f,%f,%f) nt=(%f,%f,%f;%f,%f,%f;%f,%f,%f) INVALID nshade=(%f,%f,%f)",
+			__LINE__,
+			model_index, geometry_index, primitive_index,
+			PRIVEC3(GET_VERTEX(vi1).normal),
+			PRIVEC3(GET_VERTEX(vi2).normal),
+			PRIVEC3(GET_VERTEX(vi3).normal),
+			PRIVEC3(normalTransform[0]),
+			PRIVEC3(normalTransform[1]),
+			PRIVEC3(normalTransform[2])
+		);
+		// TODO ???
+		geom.normal_shading = geom.normal_geometry;
+	}
+#endif
+
 	geom.tangent = normalize(normalTransform * baryMix(
 		GET_VERTEX(vi1).tangent,
 		GET_VERTEX(vi2).tangent,

ref/vk/vk_brush.c

@@ -1509,6 +1509,18 @@ static qboolean fillBrushSurfaces(fill_geometries_args_t args) {
 					VectorCopy(surf_normal, vertex.normal);
 				}
 
+				{
+					const float normal_len2 = DotProduct(vertex.normal, vertex.normal);
+					if (normal_len2 < .9f) {
+						ERR("model=%s surf=%d vert=%d surf_normal=(%f, %f, %f) vertex.normal=(%f,%f,%f) INVALID len2=%f",
+							args.mod->name, surface_index, k,
+							surf_normal[0], surf_normal[1], surf_normal[2],
+							vertex.normal[0], vertex.normal[1], vertex.normal[2],
+							normal_len2
+						);
+					}
+				}
+
 				VectorCopy(tangent, vertex.tangent);
 
 				Vector4Set(vertex.color, 255, 255, 255, 255);

ref/vk/vk_studio.c

@@ -1715,6 +1715,23 @@ static void buildSubmodelMeshGeometry( build_submodel_mesh_t args ) {
 			VectorCopy(args.prev_verts[vi], dst_vtx->prev_pos);
 
 			VectorCopy(g_studio.norms[vi], dst_vtx->normal);
+
+			{
+				const float normal_len2 = DotProduct(g_studio.norms[vi], g_studio.norms[vi]);
+				if (normal_len2 < .9f) {
+					ERR("model=%s bodypart=%d vert=%d+%d=%d vi=%d normal=(%f,%f,%f) INVALID len2=%f",
+						g_studio_current.entmodel->studio_header->name,
+						g_studio_current.bodypart_index,
+						j, vertex_offset, j + vertex_offset, vi,
+						g_studio.norms[0],
+						g_studio.norms[1],
+						g_studio.norms[2],
+						normal_len2
+					);
+				}
+			}
+
+
 			VectorCopy(g_studio.tangents[vi], dst_vtx->tangent);
 			dst_vtx->lm_tc[0] = dst_vtx->lm_tc[1] = 0.f;