rt: implement mix blending with undefined order

Known issues:
- apparently BLEND_MIX surfaces should also participate in lighting. Figure out how.
- sensitive to ray tracing order. Need proper OIT (or hacks)

ref/vk/shaders/bounce.comp

@@ -167,7 +167,10 @@ void computeBounce(ivec2 pix, vec3 direction, out vec3 diffuse, out vec3 specula
 		hit_material.roughness = payload.material_rmxx.r;
 		computeLighting(hit_pos, hit_shading_normal, throughput, -bounce_direction, hit_material, ldiffuse, lspecular);
 
-		const vec3 final_color = ldiffuse + lspecular + traceAdditive(pos, bounce_direction, payload.hit_t.w);
+		vec3 background = ldiffuse + lspecular;
+		vec3 emissive = vec3(0.);
+		traceAdditive(pos, bounce_direction, payload.hit_t.w, emissive, background);
+		const vec3 final_color = emissive + background;
 
 		if (brdf_type == DIFFUSE_TYPE)
 			diffuse += final_color;

ref/vk/shaders/ray_interop.h

@@ -72,6 +72,10 @@ LIST_SPECIALIZATION_CONSTANTS(DECLARE_SPECIALIZATION_CONSTANT)
 #define MATERIAL_MODE_SKYBOX 6
 
 struct Material {
+	// TODO this should be moved to ModelMetadata, as granularity of this is per-model
+	// EXCEPT for skybox. Sky surfaces are all over the place in worldbrush.
+	// Maybe tex_base_color should have a special value for skybox.
+	// Moving this to model would allow us to make kusochki mostly static (except for animated textures)
 	uint mode;
 
 	uint tex_base_color;

ref/vk/shaders/ray_primary.comp

@@ -102,7 +102,7 @@ void main() {
 		L = rayQueryGetIntersectionTEXT(rq, true);
 	}
 
-	payload.emissive.rgb += traceAdditive(ray.origin, ray.direction, L);
+	traceAdditive(ray.origin, ray.direction, L, payload.emissive.rgb, payload.base_color_a.rgb);
 
 	imageStore(out_position_t, pix, payload.hit_t);
 	imageStore(out_base_color_a, pix, payload.base_color_a);

ref/vk/shaders/trace_additive.glsl

@@ -1,9 +1,8 @@
 #ifndef TRACE_ADDITIVE_GLSL_INCLUDED
 #define TRACE_ADDITIVE_GLSL_INCLUDED
 
-vec3 traceAdditive(vec3 pos, vec3 dir, float L) {
+void traceAdditive(vec3 pos, vec3 dir, float L, inout vec3 emissive, inout vec3 background) {
 	const float additive_soft_overshoot = 16.;
-	vec3 ret = vec3(0., 0., 0.);
 	rayQueryEXT rq;
 	const uint flags = 0
 		| gl_RayFlagsCullFrontFacingTrianglesEXT
@@ -16,7 +15,8 @@ vec3 traceAdditive(vec3 pos, vec3 dir, float L) {
 		const Kusok kusok = getKusok(geom.kusok_index);
 
 		const vec4 texture_color = texture(textures[nonuniformEXT(kusok.material.tex_base_color)], geom.uv);
-		const vec3 color = texture_color.rgb * kusok.emissive * texture_color.a * kusok.model.color.a * SRGBtoLINEAR(geom.color.rgb * geom.color.a);
+		const float alpha = texture_color.a * kusok.model.color.a * geom.color.a;
+		const vec3 color = texture_color.rgb * SRGBtoLINEAR(geom.color.rgb) * alpha;
 
 		const float hit_t = rayQueryGetIntersectionTEXT(rq, false);
 		const float overshoot = hit_t - L;
@@ -36,13 +36,23 @@ vec3 traceAdditive(vec3 pos, vec3 dir, float L) {
 			ret += vec3(1., 1., 1.);
 		}
 #else
-		if (overshoot < 0.)
-			ret += color;
-		else if (kusok.material.mode == MATERIAL_MODE_BLEND_GLOW)
-			ret += color * smoothstep(additive_soft_overshoot, 0., overshoot);
+		if (kusok.material.mode == MATERIAL_MODE_BLEND_GLOW) {
+			// Glow is additive + small overshoot
+			emissive += color * kusok.emissive * smoothstep(additive_soft_overshoot, 0., overshoot);
+		} else if (overshoot < 0.) {
+			if (kusok.material.mode == MATERIAL_MODE_BLEND_ADD) {
+				emissive += color * kusok.emissive;
+			} else if (kusok.material.mode == MATERIAL_MODE_BLEND_MIX) {
+				// FIXME OIT incorrect
+				emissive = emissive * (1. - alpha) + color;
+				background *= (1. - alpha);
+			} else {
+				// Signal unhandled blending type
+				emissive += vec3(1., 0., 1.);
+			}
+		}
 #endif
 	}
-	return ret;
 }
 
 #endif //ifndef TRACE_ADDITIVE_GLSL_INCLUDED

ref/vk/vk_ray_model.c

@@ -377,9 +377,11 @@ void VK_RayFrameAddModel( vk_ray_model_t *model, const vk_render_model_t *render
 	uint32_t material_mode = MATERIAL_MODE_OPAQUE;
 	switch (render_model->render_type) {
 		case kVkRenderTypeSolid:
+			material_mode = MATERIAL_MODE_OPAQUE;
 			break;
 		case kVkRenderType_A_1mA_RW: // blend: scr*a + dst*(1-a), depth: RW
 		case kVkRenderType_A_1mA_R:  // blend: scr*a + dst*(1-a), depth test
+			// FIXME where is MATERIAL_MODE_TRANSLUCENT??1
 			material_mode = MATERIAL_MODE_BLEND_MIX;
 			break;
 		case kVkRenderType_A_1:   // blend: scr*a + dst, no depth test or write; sprite:kRenderGlow only