rtx: slightly improve black outlines on studio models (see #48)

The core of the issue is that there are backfacing interpolated normals, NdotL and/or NdotV end up being negative for them.
Things we tried:
1. Explicitly using geometry normals for some things. Doesn't really work that well.
2. Clamping NdotX to some small value instead of just ignoring light contribution. Works better, but there are still some artifacts.

ref_vk/shaders/brdf.h

@@ -888,7 +888,7 @@ vec3 evalCombinedBRDF(vec3 N, vec3 L, vec3 V, MaterialProperties material) {
 	const BrdfData data = prepareBRDFData(N, L, V, material);
 
 	// Ignore V and L rays "below" the hemisphere
-	if (data.Vbackfacing || data.Lbackfacing) return vec3(0.0f, 0.0f, 0.0f);
+	//if (data.Vbackfacing || data.Lbackfacing) return vec3(0.0f, 0.0f, 0.0f);
 
 	// Eval specular and diffuse BRDFs
 	vec3 specular = evalSpecular(data);
@@ -954,7 +954,7 @@ bool evalIndirectCombinedBRDF(vec2 u, vec3 shadingNormal, vec3 geometryNormal, v
 
 // Calculates probability of selecting BRDF (specular or diffuse) using the approximate Fresnel term
 float getBrdfProbability(MaterialProperties material, vec3 V, vec3 shadingNormal) {
-	
+
 	// Evaluate Fresnel term using the shading normal
 	// Note: we use the shading normal instead of the microfacet normal (half-vector) for Fresnel term here. That's suboptimal for rough surfaces at grazing angles, but half-vector is yet unknown at this point
 	float specularF0 = luminance(baseColorToSpecularF0(material.baseColor, material.metalness));

ref_vk/shaders/ray.rchit

@@ -15,7 +15,8 @@ hitAttributeEXT vec2 bary;
 float hash(float f) { return fract(sin(f)*53478.4327); }
 vec3 hashUintToVec3(uint i) { return vec3(hash(float(i)), hash(float(i)+15.43), hash(float(i)+34.)); }
 
-const float normal_offset_fudge = .01;
+// FIXME implement more robust self-intersection avoidance (see chap 6 of "Ray Tracing Gems")
+const float normal_offset_fudge = .001;
 
 // Taken from Ray Tracing Gems II, Chapter 7. Texture Coordinate Gradients Estimation for Ray Cones, by Wessam Bahnassi
 // https://www.realtimerendering.com/raytracinggems/rtg2/index.html
@@ -44,29 +45,17 @@ vec4 UVDerivsFromRayCone(vec3 vRayDir, vec3 vWorldNormal, float vRayConeWidth, v
 }
 
 void main() {
-    //const float l = gl_HitTEXT;
-    //ray_result.color = vec3(fract(l / 10.));
-    //return;
-
-    //ray_result.color = vec3(.5);
-
-	// FIXME compute hit pos based on barycentric coords (better precision)
-    vec3 hit_pos = gl_WorldRayOriginEXT + gl_WorldRayDirectionEXT * gl_HitTEXT;
     payload.t_offset += gl_HitTEXT;
 
-	// ray_result.color = fract((hit_pos + .5) / 10.);
-    //ray_result.color = vec3(1.);
-    //return;
-    //ray_result.color = hashUintToVec3(gl_GeometryIndexEXT);
-
     const int instance_kusochki_offset = gl_InstanceCustomIndexEXT;
-    //ray_result.color = hashUintToVec3(uint(instance_kusochki_offset));
     const int kusok_index = instance_kusochki_offset + gl_GeometryIndexEXT;
 
     const uint first_index_offset = kusochki[kusok_index].index_offset + gl_PrimitiveID * 3;
+
     const uint vi1 = uint(indices[first_index_offset+0]) + kusochki[kusok_index].vertex_offset;
     const uint vi2 = uint(indices[first_index_offset+1]) + kusochki[kusok_index].vertex_offset;
     const uint vi3 = uint(indices[first_index_offset+2]) + kusochki[kusok_index].vertex_offset;
+
     const vec3 n1 = vertices[vi1].normal;
     const vec3 n2 = vertices[vi2].normal;
     const vec3 n3 = vertices[vi3].normal;
@@ -80,6 +69,7 @@ void main() {
         vertices[vi2].gl_tc,
         vertices[vi3].gl_tc,
     };
+
     const vec3 pos[3] = {
         vertices[vi1].pos,
         vertices[vi2].pos,
@@ -98,11 +88,16 @@ void main() {
 
 	// FIXME read alpha from texture
 
+	const vec3 real_geom_normal = normalize(transpose(inverse(matWorld)) * cross(pos[0]-pos[1], pos[2]-pos[0]));
+	const float geom_normal_sign = sign(dot(real_geom_normal, -gl_WorldRayDirectionEXT));
+	const vec3 geom_normal = geom_normal_sign * real_geom_normal;
+
+    const vec3 hit_pos = pos[0] * (1. - bary.x - bary.y) + pos[1] * bary.x + pos[2] * bary.y + geom_normal + normal_offset_fudge;
     payload.hit_pos_t = vec4(hit_pos, gl_HitTEXT);
     payload.base_color = base_color * kusochki[kusok_index].color.rgb;
 	payload.reflection = tex_color.a * kusochki[kusok_index].color.a;
-    payload.normal = normal * sign(dot(normal, -gl_WorldRayDirectionEXT));
-
+    payload.normal = normal * geom_normal_sign;
+	payload.geometry_normal = geom_normal;
     payload.emissive = kusochki[kusok_index].emissive * base_color; // TODO emissive should have a special texture
     payload.roughness = kusochki[kusok_index].roughness;
     payload.kusok_index = kusok_index;

ref_vk/shaders/ray.rgen

@@ -125,13 +125,14 @@ vec3 sampleSurfaceTriangle(vec3 view_dir, MaterialProperties material, mat4x3 em
 	if (light_dot <= 0.)
 		return vec3(0.);
 
+	// TODO emissive normals and areas can be precomputed
 	const float area = .5 * length(cross(v1 - v2, v1 - v3)) * meters_per_unit * meters_per_unit;
 	const float light_dist2 = dot(light_dir, light_dir);
 	float pdf = light_dist2 / (area * light_dot);
 
 	light_dir = normalize(light_dir);
 
-	if (shadowed(payload.hit_pos_t.xyz + payload.normal * shadow_offset_fudge, light_dir, sqrt(light_dist2) - shadow_offset_fudge * 2.))
+	if (shadowed(payload.hit_pos_t.xyz, light_dir, sqrt(light_dist2)))
 		return vec3(0.);
 
 	// TODO sample emissive texture
@@ -211,7 +212,7 @@ vec3 computeLighting(vec3 view_dir, MaterialProperties material) {
 			const float d2 = dot(light_dir, light_dir);
 			const float light_dist = sqrt(d2);
 
-			if (shadowed(payload.hit_pos_t.xyz + payload.normal * shadow_offset_fudge, light_dir_norm, light_dist + shadow_offset_fudge))
+			if (shadowed(payload.hit_pos_t.xyz, light_dir_norm, light_dist + shadow_offset_fudge))
 				continue;
 
 			const float r2 = light_pos_r.w * light_pos_r.w;
@@ -262,8 +263,6 @@ void main() {
 			break;
 		}
 
-		//C = vec3(payload.reflection); break;
-
 		MaterialProperties material;
 		material.baseColor = payload.base_color;
 		material.metalness = 0.f;
@@ -287,7 +286,7 @@ void main() {
 			break;
 
 		vec3 shadingNormal = payload.normal;
-		vec3 geometryNormal = payload.normal;
+		vec3 geometryNormal = payload.geometry_normal;
 		vec3 V = -direction;
 		if (material.metalness == 1.0f && material.roughness == 0.0f) {
 			// Fast path for mirrors
@@ -312,8 +311,7 @@ void main() {
 			break; // Ray was eaten by the surface :(
 		}
 
-		// FIXME better offset (see ray tracing gems, ch 6?)
-		origin = payload.hit_pos_t.xyz + normal_offset_fudge * payload.normal;
+		origin = payload.hit_pos_t.xyz;
 		throughput *= brdfWeight;
 	} // for all bounces
 

ref_vk/shaders/ray.rmiss

@@ -6,7 +6,7 @@ layout(location = 0) rayPayloadInEXT RayPayload payload;
 
 void main() {
     payload.hit_pos_t = vec4(-1.);
-    payload.normal = vec3(0., 1., 0.);
+    payload.geometry_normal = payload.normal = vec3(0., 1., 0.);
 	payload.reflection = 0.;
     payload.roughness = 0.;
     payload.base_color = vec3(1., 0., 1.);

ref_vk/shaders/ray_common.glsl

@@ -3,6 +3,7 @@ struct RayPayload {
     float t_offset, pixel_cone_spread_angle;
     vec4 hit_pos_t;
     vec3 normal;
+		vec3 geometry_normal;
     vec3 base_color;
 		float reflection;
 		vec3 emissive;