Fix: additive soft particles, skybox shadow

Add another flag to kusochki for glow geometry. Move flags to a new kusok.flags field.
- [x] Fixes #231 (or at least makes it not stand out too much)

Fix the way we check for environment/skybox lights, explicitly look for closest hit instead of any.
- [x] Fixes #424 and #413

ref/vk/shaders/light.glsl

@@ -104,9 +104,14 @@ void computePointLights(vec3 P, vec3 N, uint cluster_index, vec3 throughput, vec
 		if (dot(combined,combined) < color_culling_threshold)
 			continue;
 
-		// FIXME split environment and other lights
-		if (shadowed(P, light_dir_norm, light_dist + shadow_offset_fudge, is_environment))
-			continue;
+		// TODO split environment and other lights
+		if (is_environment) {
+			if (shadowedSky(P, light_dir_norm))
+				continue;
+		} else {
+			if (shadowed(P, light_dir_norm, light_dist + shadow_offset_fudge))
+				continue;
+		}
 
 		diffuse += ldiffuse;
 		specular += lspecular;

ref/vk/shaders/light_common.glsl

@@ -29,7 +29,7 @@ uint traceShadowRay(vec3 pos, vec3 dir, float dist, uint flags) {
 bool shadowTestAlphaMask(vec3 pos, vec3 dir, float dist) {
 	rayQueryEXT rq;
 	const uint flags =  0
-		| gl_RayFlagsCullFrontFacingTrianglesEXT
+		//| gl_RayFlagsCullFrontFacingTrianglesEXT
 		//| gl_RayFlagsNoOpaqueEXT
 		| gl_RayFlagsTerminateOnFirstHitEXT
 		;
@@ -71,7 +71,7 @@ bool shadowTestAlphaMask(vec3 pos, vec3 dir, float dist) {
 }
 #endif
 
-bool shadowed(vec3 pos, vec3 dir, float dist, bool check_sky) {
+bool shadowed(vec3 pos, vec3 dir, float dist) {
 #ifdef RAY_TRACE
 	const uint flags =  0
 		//| gl_RayFlagsCullFrontFacingTrianglesEXT
@@ -80,11 +80,11 @@ bool shadowed(vec3 pos, vec3 dir, float dist, bool check_sky) {
 		| gl_RayFlagsSkipClosestHitShaderEXT
 		;
 	const uint hit_type = traceShadowRay(pos, dir, dist, flags);
-	return check_sky ? payload_shadow.hit_type != SHADOW_SKY : payload_shadow.hit_type == SHADOW_HIT;
+	return payload_shadow.hit_type == SHADOW_HIT;
 #elif defined(RAY_QUERY)
 	{
 		const uint flags =  0
-			| gl_RayFlagsCullFrontFacingTrianglesEXT
+			//| gl_RayFlagsCullFrontFacingTrianglesEXT
 			| gl_RayFlagsOpaqueEXT
 			| gl_RayFlagsTerminateOnFirstHitEXT
 			;
@@ -92,16 +92,8 @@ bool shadowed(vec3 pos, vec3 dir, float dist, bool check_sky) {
 		rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, pos, 0., dir, dist - shadow_offset_fudge);
 		while (rayQueryProceedEXT(rq)) {}
 
-		if (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionTriangleEXT) {
-			if (!check_sky)
-				return true;
-
-			const int instance_kusochki_offset = rayQueryGetIntersectionInstanceCustomIndexEXT(rq, true);
-			const int kusok_index = instance_kusochki_offset + rayQueryGetIntersectionGeometryIndexEXT(rq, true);
-			const uint tex_base_color = getKusok(kusok_index).tex_base_color;
-			if ((tex_base_color & KUSOK_MATERIAL_FLAG_SKYBOX) == 0)
-				return true;
-		}
+		if (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionTriangleEXT)
+			return true;
 	}
 
 	return shadowTestAlphaMask(pos, dir, dist);
@@ -111,6 +103,49 @@ bool shadowed(vec3 pos, vec3 dir, float dist, bool check_sky) {
 #endif
 }
 
+bool shadowedSky(vec3 pos, vec3 dir) {
+#ifdef RAY_TRACE
+	const uint flags =  0
+		//| gl_RayFlagsCullFrontFacingTrianglesEXT
+		//| gl_RayFlagsOpaqueEXT
+		| gl_RayFlagsTerminateOnFirstHitEXT
+		| gl_RayFlagsSkipClosestHitShaderEXT
+		;
+	const uint hit_type = traceShadowRay(pos, dir, dist, flags);
+	return payload_shadow.hit_type != SHADOW_SKY;
+#elif defined(RAY_QUERY)
+
+	rayQueryEXT rq;
+	const uint flags = 0
+		| gl_RayFlagsCullFrontFacingTrianglesEXT
+		| gl_RayFlagsOpaqueEXT
+		//| gl_RayFlagsTerminateOnFirstHitEXT
+		//| gl_RayFlagsSkipClosestHitShaderEXT
+		;
+	const float L = 10000.; // TODO Why 10k?
+	rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, pos, 0., dir, L);
+
+	// Find closest intersection, and then check whether that was a skybox
+	while (rayQueryProceedEXT(rq)) {}
+
+	if (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionTriangleEXT) {
+		const uint instance_kusochki_offset = rayQueryGetIntersectionInstanceCustomIndexEXT(rq, true);
+		const uint geometry_index = rayQueryGetIntersectionGeometryIndexEXT(rq, true);
+		const uint kusok_index = instance_kusochki_offset + geometry_index;
+		const Kusok kusok = getKusok(kusok_index);
+		if ((kusok.flags & KUSOK_MATERIAL_FLAG_SKYBOX) == 0)
+			return true;
+	}
+
+	// check for alpha-masked surfaces separately
+	const float hit_t = rayQueryGetIntersectionTEXT(rq, true);
+	return shadowTestAlphaMask(pos, dir, hit_t);
+
+#else
+#error RAY_TRACE or RAY_QUERY
+#endif
+}
+
 // This is an entry point for evaluation of all other BRDFs based on selected configuration (for direct light)
 void evalSplitBRDF(vec3 N, vec3 L, vec3 V, MaterialProperties material, out vec3 diffuse, out vec3 specular) {
 	// Prepare data needed for BRDF evaluation - unpack material properties and evaluate commonly used terms (e.g. Fresnel, NdotL, ...)

ref/vk/shaders/light_polygon.glsl

@@ -184,7 +184,7 @@ void sampleSinglePolygonLight(in vec3 P, in vec3 N, in vec3 view_dir, in SampleC
 
 	const float dist = - dot(vec4(P, 1.f), poly.plane) / dot(light_sample_dir.xyz, poly.plane.xyz);
 
-	if (shadowed(P, light_sample_dir.xyz, dist, false))
+	if (shadowed(P, light_sample_dir.xyz, dist))
 		return;
 
 	vec3 poly_diffuse = vec3(0.), poly_specular = vec3(0.);
@@ -251,7 +251,7 @@ void sampleEmissiveSurfaces(vec3 P, vec3 N, vec3 throughput, vec3 view_dir, Mate
 		const float dist = - plane_dist / dot(light_sample_dir.xyz, poly.plane.xyz);
 		const vec3 emissive = poly.emissive;
 
-		if (!shadowed(P, light_sample_dir.xyz, dist, false)) {
+		if (!shadowed(P, light_sample_dir.xyz, dist)) {
 			//const float estimate = total_contrib;
 			const float estimate = light_sample_dir.w;
 			vec3 poly_diffuse = vec3(0.), poly_specular = vec3(0.);
@@ -327,7 +327,7 @@ void sampleEmissiveSurfaces(vec3 P, vec3 N, vec3 throughput, vec3 view_dir, Mate
 	const vec3 emissive = poly.emissive;
 
 	//if (true) {//!shadowed(P, light_sample_dir.xyz, dist)) {
-	if (!shadowed(P, light_sample_dir.xyz, dist, false)) {
+	if (!shadowed(P, light_sample_dir.xyz, dist)) {
 		//const float estimate = total_contrib;
 		const float estimate = light_sample_dir.w;
 		vec3 poly_diffuse = vec3(0.), poly_specular = vec3(0.);

ref/vk/shaders/ray_interop.h

@@ -63,7 +63,8 @@ LIST_SPECIALIZATION_CONSTANTS(DECLARE_SPECIALIZATION_CONSTANT)
 #define SHADER_OFFSET_HIT_REGULAR_BASE 0
 #define SHADER_OFFSET_HIT_SHADOW_BASE 3
 
-#define KUSOK_MATERIAL_FLAG_SKYBOX 0x80000000
+#define KUSOK_MATERIAL_FLAG_SKYBOX (1<<0)
+#define KUSOK_MATERIAL_FLAG_FIXME_GLOW (1<<1)
 
 struct Kusok {
 	uint index_offset;
@@ -85,7 +86,9 @@ struct Kusok {
 
 	float roughness;
 	float metalness;
-	PAD(2)
+	uint flags;
+
+	PAD(1)
 
 	mat4 prev_transform;
 };

ref/vk/shaders/ray_primary_hit.glsl

@@ -28,13 +28,12 @@ void primaryRayHit(rayQueryEXT rq, inout RayPayloadPrimary payload) {
 	payload.prev_pos_t = vec4(geom.prev_pos, 0.);
 
 	const Kusok kusok = getKusok(geom.kusok_index);
-	const uint tex_base_color = kusok.tex_base_color;
 
-	if ((tex_base_color & KUSOK_MATERIAL_FLAG_SKYBOX) != 0) {
+	if ((kusok.flags & KUSOK_MATERIAL_FLAG_SKYBOX) != 0) {
 		payload.emissive.rgb = SRGBtoLINEAR(texture(skybox, rayDirection).rgb);
 		return;
 	} else {
-		payload.base_color_a = sampleTexture(tex_base_color, geom.uv, geom.uv_lods);
+		payload.base_color_a = sampleTexture(kusok.tex_base_color, geom.uv, geom.uv_lods);
 		payload.material_rmxx.r = (kusok.tex_roughness > 0) ? sampleTexture(kusok.tex_roughness, geom.uv, geom.uv_lods).r : kusok.roughness;
 		payload.material_rmxx.g = (kusok.tex_metalness > 0) ? sampleTexture(kusok.tex_metalness, geom.uv, geom.uv_lods).r : kusok.metalness;
 

ref/vk/shaders/trace_additive.glsl

@@ -20,7 +20,11 @@ vec3 traceAdditive(vec3 pos, vec3 dir, float L) {
 
 		const float hit_t = rayQueryGetIntersectionTEXT(rq, false);
 		const float overshoot = hit_t - L;
-		ret += color * smoothstep(additive_soft_overshoot, 0., overshoot);
+
+		if (overshoot < 0.)
+			ret += color;
+		else if ((kusok.flags & KUSOK_MATERIAL_FLAG_FIXME_GLOW) != 0)
+			ret += color * smoothstep(additive_soft_overshoot, 0., overshoot);
 	}
 	return ret;
 }

ref/vk/vk_ray_model.c

@@ -175,6 +175,8 @@ static void applyMaterialToKusok(vk_kusok_data_t* kusok, const vk_render_geometr
 	kusok->roughness = mat->roughness;
 	kusok->metalness = mat->metalness;
 
+	kusok->flags = 0;
+
 	// HACK until there is a proper mechanism for patching materials, see https://github.com/w23/xash3d-fwgs/issues/213
 	// FIXME also this erases previous roughness unconditionally
 	if (HACK_reflective) {
@@ -184,7 +186,10 @@ static void applyMaterialToKusok(vk_kusok_data_t* kusok, const vk_render_geometr
 	}
 
 	if (geom->material == kXVkMaterialSky)
-		kusok->tex_base_color |= KUSOK_MATERIAL_FLAG_SKYBOX;
+		kusok->flags |= KUSOK_MATERIAL_FLAG_SKYBOX;
+
+	if (geom->material == kXVkMaterialEmissiveGlow)
+		kusok->flags |= KUSOK_MATERIAL_FLAG_FIXME_GLOW;
 
 	{
 		vec4_t gcolor;
@@ -195,7 +200,7 @@ static void applyMaterialToKusok(vk_kusok_data_t* kusok, const vk_render_geometr
 		Vector4Copy(gcolor, kusok->color);
 	}
 
-	if (geom->material == kXVkMaterialEmissive) {
+	if (geom->material == kXVkMaterialEmissive || geom->material == kXVkMaterialEmissiveGlow) {
 		VectorCopy(geom->emissive, kusok->emissive);
 	} else {
 		RT_GetEmissiveForTexture( kusok->emissive, geom->texture );
@@ -308,13 +313,8 @@ vk_ray_model_t* VK_RayModelCreate( vk_ray_model_init_t args ) {
 		/* 	); */
 		/* } */
 
-		if (mg->material == kXVkMaterialSky) {
-			kusochki[i].tex_base_color |= KUSOK_MATERIAL_FLAG_SKYBOX;
-		} else {
-			kusochki[i].tex_base_color &= (~KUSOK_MATERIAL_FLAG_SKYBOX);
-		}
 
-		if (HACK_additive_emissive && mg->material != kXVkMaterialEmissive) {
+		if (HACK_additive_emissive && mg->material != kXVkMaterialEmissive && mg->material != kXVkMaterialEmissiveGlow) {
 			mg->material = kXVkMaterialEmissive;
 			VectorCopy(args.model->color, mg->emissive);
 		}
@@ -517,7 +517,7 @@ void VK_RayFrameAddModel( vk_ray_model_t *model, const vk_render_model_t *render
 			vk_render_geometry_t *geom = render_model->geometries + i;
 
 			// FIXME an impedance mismatch: render_type is per-model, while materials and emissive color are per-geom
-			if (HACK_additive_emissive && geom->material != kXVkMaterialEmissive) {
+			if (HACK_additive_emissive && geom->material != kXVkMaterialEmissive && geom->material != kXVkMaterialEmissiveGlow) {
 				geom->material = kXVkMaterialEmissive;
 				VectorCopy(render_model->color, geom->emissive);
 			}

ref/vk/vk_render.h

@@ -30,6 +30,7 @@ typedef enum {
 	kXVkMaterialWater,
 	kXVkMaterialSky,
 	kXVkMaterialEmissive,
+	kXVkMaterialEmissiveGlow,
 	kXVkMaterialConveyor,
 	kXVkMaterialChrome,
 } XVkMaterialType;

ref/vk/vk_sprite.c

@@ -718,7 +718,7 @@ static void R_DrawSpriteQuad( const char *debug_name, mspriteframe_t *frame, vec
 	{
 		const vk_render_geometry_t geometry = {
 			.texture = texture,
-			.material = kXVkMaterialEmissive,
+			.material = render_mode == kRenderGlow ? kXVkMaterialEmissiveGlow : kXVkMaterialEmissive,
 
 			.max_vertex = 4,
 			.vertex_offset = buffer.vertices.unit_offset,