rtx: add refractions support, fix #181

Decide whether the ray goes through or reflects based on brdf/fresnel.
Specular always reflects. Diffuse mode chooses between reflecting (GI) and continuing through (refraction). The decision is stochastical with probability being alpha channel (color * texture). Alpha = 0 => full refraction. Alpha = 1 => full reflection.

Lighting is not computed for refractive decision. (not sure if this is fully correct though).

It doesn't look correct all of the time.
Probably we need to split base_color into different channels fo diffuse/lighting and transmissiveness.

ref_vk/shaders/ray.rchit

@@ -103,7 +103,7 @@ void main() {
 
     payload.hit_pos_t = vec4(hit_pos, gl_HitTEXT);
     payload.base_color = base_color * kusochki[kusok_index].color.rgb;
-	payload.reflection = tex_color.a * kusochki[kusok_index].color.a;
+	payload.transmissiveness = (1. - tex_color.a * kusochki[kusok_index].color.a);
     payload.normal = normal * geom_normal_sign;
 	payload.geometry_normal = geom_normal;
     payload.emissive = kusochki[kusok_index].emissive * base_color; // TODO emissive should have a special texture

ref_vk/shaders/ray.rgen

@@ -368,17 +368,16 @@ void main() {
 		if (bounce == 0) //brdfType == SPECULAR_TYPE)
 			C += throughput * payload_opaque.emissive;
 
-		// Decide whether ray continues through, or relfects
-		if (rand01() > payload_opaque.reflection) {
-			origin = payload_opaque.hit_pos_t.xyz;
-			throughput *= payload_opaque.base_color;
-			continue;
-		}
-
+ // TODO should we do this after reflect/transmit decision?
+#define SKIP_TRASMITTED_LIGHT
+#ifndef SKIP_TRASMITTED_LIGHT
 		C += computeLighting(throughput, -direction, material);
 
 		if (bounce == push_constants.bounces - 1)
 			break;
+#else
+		vec3 prev_throughput = throughput;
+#endif
 
 		vec3 shadingNormal = payload_opaque.normal;
 		vec3 geometryNormal = payload_opaque.geometry_normal;
@@ -387,19 +386,31 @@ void main() {
 			// Fast path for mirrors
 			brdfType = SPECULAR_TYPE;
 		} else {
+				// Decide whether to sample diffuse or specular BRDF (based on Fresnel term)
+				float brdfProbability = getBrdfProbability(material, V, shadingNormal);
+				if (rand01() < brdfProbability) {
+					brdfType = SPECULAR_TYPE;
+					throughput /= brdfProbability;
+				} else {
+					if (rand01() < payload_opaque.transmissiveness) {
+						throughput *= material.baseColor;
+						direction = refract(direction, payload_opaque.geometry_normal, .8);
+						origin = payload_opaque.hit_pos_t.xyz - payload_opaque.geometry_normal * shadow_offset_fudge;
+						continue;
+					}
 
-			// Decide whether to sample diffuse or specular BRDF (based on Fresnel term)
-			float brdfProbability = getBrdfProbability(material, V, shadingNormal);
-
-			if (rand01() < brdfProbability) {
-				brdfType = SPECULAR_TYPE;
-				throughput /= brdfProbability;
-			} else {
-				brdfType = DIFFUSE_TYPE;
-				throughput /= (1.0f - brdfProbability);
-			}
+					brdfType = DIFFUSE_TYPE;
+					throughput /= (1.0f - brdfProbability);
+				}
 		}
 
+#ifdef SKIP_TRASMITTED_LIGHT
+		C += computeLighting(prev_throughput, -direction, material);
+
+		if (bounce == push_constants.bounces - 1)
+			break;
+#endif
+
 		vec2 u = vec2(rand01(), rand01());
 		vec3 brdfWeight;
 		if (!evalIndirectCombinedBRDF(u, shadingNormal, geometryNormal, V, material, brdfType, direction, brdfWeight)) {

ref_vk/shaders/ray.rmiss

@@ -11,7 +11,7 @@ layout(location = PAYLOAD_LOCATION_OPAQUE) rayPayloadInEXT RayPayloadOpaque payl
 void main() {
     payload.hit_pos_t = vec4(-1.);
     payload.geometry_normal = payload.normal = vec3(0., 1., 0.);
-	payload.reflection = 0.;
+	payload.transmissiveness = 0.;
     payload.roughness = 0.;
     payload.base_color = vec3(0.);//mix(vec3(.1, .2, .7), lights.sun_color, pow(sun_dot, 100.));
 	//vec3(1., 0., 1.);

ref_vk/shaders/ray_common.glsl

@@ -10,7 +10,7 @@ struct RayPayloadOpaque {
 	vec3 normal;
 	vec3 geometry_normal;
 	vec3 base_color;
-	float reflection;
+	float transmissiveness;
 	vec3 emissive;
 	float roughness;
 	int kusok_index;