bring back ray traced transparency

1. Probabilistically let ray through material or reflect it based on
   transparency probability (which is essentially just alpha for now).
2. Introduce opaque flag to be able to treat transparent objects as not
   casting shadows. This is not very realistic, but oh well.

ref_vk/shaders/ray.rchit

@@ -75,9 +75,6 @@ void main() {
     const mat3 matWorld = mat3(gl_ObjectToWorldEXT);
     const vec3 normal = normalize(transpose(inverse(matWorld)) * (n1 * (1. - bary.x - bary.y) + n2 * bary.x + n3 * bary.y));
 
-	// FIXME there's a better way to do this
-    hit_pos += normal * normal_offset_fudge;
-
     const vec2 uvs[3] = {
         vertices[vi1].gl_tc,
         vertices[vi2].gl_tc,
@@ -93,14 +90,18 @@ void main() {
 
     const float ray_cone_width = payload.pixel_cone_spread_angle * payload.t_offset;
     const vec4 uv_lods = UVDerivsFromRayCone(gl_WorldRayDirectionEXT, normal, ray_cone_width, uvs, pos, matWorld);
-    const vec3 base_color = pow(textureGrad(textures[nonuniformEXT(tex_index)], texture_uv, uv_lods.xy, uv_lods.zw).rgb, vec3(2.));
+    const vec4 tex_color = textureGrad(textures[nonuniformEXT(tex_index)], texture_uv, uv_lods.xy, uv_lods.zw);
+    const vec3 base_color = pow(tex_color.rgb, vec3(2.));
+    /* tex_color = pow(tex_color, vec4(2.)); */
+    /* const vec3 base_color = tex_color.rgb; */
 
 	// FIXME read alpha from texture
 
     payload.hit_pos_t = vec4(hit_pos, gl_HitTEXT);
     payload.base_color = base_color * kusochki[kusok_index].color.rgb;
-	payload.alpha = kusochki[kusok_index].color.a;
-    payload.normal = normal;
+	payload.reflection = tex_color.a * kusochki[kusok_index].color.a;
+    payload.normal = normal * sign(dot(normal, -gl_WorldRayDirectionEXT));
+
     payload.emissive = kusochki[kusok_index].emissive * base_color; // TODO emissive should have a special texture
     payload.roughness = kusochki[kusok_index].roughness;
     payload.kusok_index = kusok_index;

ref_vk/shaders/ray.rgen

@@ -92,7 +92,7 @@ bool shadowed(vec3 pos, vec3 dir, float dist) {
 		;
 	traceRayEXT(tlas,
 		flags,
-		0xff,
+		GEOMETRY_BIT_OPAQUE,
 		0, 0, 1 /* miss index */,
 		pos, 0., dir, dist, 1 /* payload location */);
 	return shadow;
@@ -134,7 +134,7 @@ vec3 sampleSurfaceTriangle(vec3 view_dir, MaterialProperties material, mat4x3 em
 	if (light_dir_normal_dot <= 0.)
 		return vec3(0.);
 
-	if (shadowed(payload.hit_pos_t.xyz, light_dir, light_dist - shadow_offset_fudge))
+	if (shadowed(payload.hit_pos_t.xyz + payload.normal * shadow_offset_fudge, light_dir, light_dist - shadow_offset_fudge))
 		return vec3(0.);
 
 	light_dist *= meters_per_unit;
@@ -216,7 +216,7 @@ vec3 computeLighting(vec3 view_dir, MaterialProperties material) {
 			const float d2 = dot(light_dir, light_dir);
 			const float light_dist = sqrt(d2);
 
-			if (shadowed(payload.hit_pos_t.xyz, light_dir_norm, light_dist + shadow_offset_fudge))
+			if (shadowed(payload.hit_pos_t.xyz + payload.normal * shadow_offset_fudge, light_dir_norm, light_dist + shadow_offset_fudge))
 				continue;
 
 			const float r2 = light_pos_r.w * light_pos_r.w;
@@ -244,6 +244,8 @@ void main() {
 	payload.pixel_cone_spread_angle = push_constants.pixel_cone_spread_angle;
 
 	for (int bounce = 0; bounce < push_constants.bounces; ++bounce) {
+		// TODO early exit based on throughput being too small
+
 		const uint flags = 0
 			//| gl_RayFlagsCullFrontFacingTrianglesEXT
 			//| gl_RayFlagsOpaqueEXT
@@ -253,7 +255,7 @@ void main() {
 		const uint miss_index = 0;
 		const int ray_payload_loc = 0;
 		const float L = 10000.;
-		traceRayEXT(tlas, flags, 0xff,
+		traceRayEXT(tlas, flags, GEOMETRY_BIT_ANY,
 			sbt_offset, sbt_stride, miss_index,
 			origin, 0., direction, L,
 			ray_payload_loc);
@@ -264,21 +266,30 @@ void main() {
 			break;
 		}
 
+		//C = vec3(payload.reflection); break;
+
 		MaterialProperties material;
 		material.baseColor = payload.base_color;
 		material.metalness = 0.f;
 		material.emissive = payload.emissive;
-		material.roughness = .1f; // FIXME payload.roughness;
+		material.roughness = payload.roughness;
 
 		C += throughput * payload.emissive;
 
-		if ((payload.material_flags & kXVkMaterialFlagLighting) != 0) {
-			C += throughput * computeLighting(-direction, material);
+		// Decide whether ray continues through, or relfects
+		if (rand01() > payload.reflection) {
+			origin = payload.hit_pos_t.xyz - normal_offset_fudge * payload.normal;
+			//throughput *= 1. - payload.reflection;
+			throughput *= payload.base_color;
+			continue;
 		}
 
+		C += throughput * computeLighting(-direction, material);
+
 		if (bounce == push_constants.bounces - 1)
 			break;
 
+
 		vec3 shadingNormal = payload.normal;
 		vec3 geometryNormal = payload.normal;
 		vec3 V = -direction;

ref_vk/shaders/ray.rmiss

@@ -7,7 +7,7 @@ layout(location = 0) rayPayloadInEXT RayPayload payload;
 void main() {
     payload.hit_pos_t = vec4(-1.);
     payload.normal = vec3(0., 1., 0.);
-	payload.alpha = 0.;
+	payload.reflection = 0.;
     payload.roughness = 0.;
     payload.base_color = vec3(1., 0., 1.);
     payload.kusok_index = -1;

ref_vk/shaders/ray_common.glsl

@@ -4,7 +4,7 @@ struct RayPayload {
     vec4 hit_pos_t;
     vec3 normal;
     vec3 base_color;
-		float alpha;
+		float reflection;
 		vec3 emissive;
     float roughness;
     int kusok_index;

ref_vk/shaders/ray_interop.h

@@ -11,6 +11,9 @@
 #define PAD(x)
 #endif
 
+#define GEOMETRY_BIT_ANY 0x01
+#define GEOMETRY_BIT_OPAQUE 0x02
+
 struct Kusok {
 	uint index_offset;
 	uint vertex_offset;

ref_vk/vk_ray_internal.h

@@ -42,6 +42,7 @@ typedef struct {
 	vk_ray_model_t *model;
 	int render_mode;
 	qboolean alpha_test;
+	qboolean translucent;
 } vk_ray_draw_model_t;
 
 typedef struct {

ref_vk/vk_ray_model.c

@@ -341,6 +341,11 @@ void VK_RayFrameAddModel( vk_ray_model_t *model, const vk_render_model_t *render
 			gEngine.Host_Error("Unexpected render mode %d\n", render_model->render_mode);
 	}
 
+	draw_model->translucent = false;
+	if (additive || color[3] < 1.f) {
+		draw_model->translucent = true;
+	}
+
 	for (int i = 0; i < render_model->num_geometries; ++i) {
 		const vk_render_geometry_t *geom = render_model->geometries + i;
 		const vk_emissive_surface_t *esurf = VK_LightsAddEmissiveSurface( geom, transform_row );

ref_vk/vk_rtx.c

@@ -471,7 +471,7 @@ static void prepareTlas( VkCommandBuffer cmdbuf ) {
 			ASSERT(model->model->as != VK_NULL_HANDLE);
 			inst[i] = (VkAccelerationStructureInstanceKHR){
 				.instanceCustomIndex = model->model->kusochki_offset,
-				.mask = 0xff,
+				.mask = (model->translucent ? 0 : GEOMETRY_BIT_OPAQUE) | GEOMETRY_BIT_ANY,
 				.instanceShaderBindingTableRecordOffset = model->alpha_test ? 1 : 0,
 				.flags = model->render_mode == kRenderNormal ? VK_GEOMETRY_INSTANCE_FORCE_OPAQUE_BIT_KHR : VK_GEOMETRY_INSTANCE_FORCE_NO_OPAQUE_BIT_KHR, // TODO is render_mode a good indicator of transparency in general case?
 				.accelerationStructureReference = getASAddress(model->model->as), // TODO cache this addr