add per-vertex studio model lighting

it doesn't correctly sample lightmaps for env lighting yet. we'll figure this out some time later ....

ref_vk/TODO.md

@@ -1,5 +1,10 @@
 ## 2021-08-07, E124
 - [x] anisotropic texture sampling
+- [x] studio model lighting prep
+	- [x] copy over R_LightVec from GL renderer
+	- [x] add per-vertex color attribute
+	- [x] support per-vertex colors
+	- [x] disable lightmaps, or use white texture for it instead
 
 # Next
 - [ ] rtx: split ray tracing into modules: pipeline mgmt, buffer mgmt
@@ -10,8 +15,16 @@
 		struct LightCluster { uint16 offset, length; }
 		uint8_t data[];
 - [ ] studio models: fix lighting: should have white texture instead of lightmap OR we could write nearest surface lightmap coords to fake light
+	- [ ] make it look correct lol
 
 # Planned
+- [ ] studio model types:
+	- [x] normal
+	- [ ] float
+	- [ ] chrome
+- [ ] simplify buffer api: do alloc+lock as a single op
+- [ ] more beams types
+- [ ] more particle types
 - [ ] rtx: better mip lods: there's a weird math that operates on fov degrees (not radians) that we copypasted from ray tracing gems 2 chapter 7. When the book is available, get through the math and figure this out.
 - [ ] sane texture memory management: do not allocate VKDeviceMemory for every texture
 - [ ] rtx: transparency layering issue, possible approaches:

ref_vk/rlight.c

@@ -0,0 +1,293 @@
+/*
+gl_rlight.c - dynamic and static lights
+Copyright (C) 2010 Uncle Mike
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+*/
+
+#include "const.h"
+#include "xash3d_types.h"
+#include "com_model.h"
+#include "pm_local.h"
+#include "studio.h"
+#include "xash3d_mathlib.h"
+#include "ref_params.h"
+
+#include "vk_common.h"
+
+/*
+=======================================================================
+
+	AMBIENT LIGHTING
+
+=======================================================================
+*/
+static vec3_t	g_trace_lightspot;
+static vec3_t	g_trace_lightvec;
+static float	g_trace_fraction;
+
+/*
+=================
+R_RecursiveLightPoint
+=================
+*/
+static qboolean R_RecursiveLightPoint( model_t *model, mnode_t *node, float p1f, float p2f, colorVec *cv, const vec3_t start, const vec3_t end )
+{
+	float		front, back, frac, midf;
+	int		i, map, side, size;
+	float		ds, dt, s, t;
+	int		sample_size;
+	color24		*lm, *dm;
+	mextrasurf_t	*info;
+	msurface_t	*surf;
+	mtexinfo_t	*tex;
+	matrix3x4		tbn;
+	vec3_t		mid;
+
+	// didn't hit anything
+	if( !node || node->contents < 0 )
+	{
+		cv->r = cv->g = cv->b = cv->a = 0;
+		return false;
+	}
+
+	// calculate mid point
+	front = PlaneDiff( start, node->plane );
+	back = PlaneDiff( end, node->plane );
+
+	side = front < 0;
+	if(( back < 0 ) == side )
+		return R_RecursiveLightPoint( model, node->children[side], p1f, p2f, cv, start, end );
+
+	frac = front / ( front - back );
+
+	VectorLerp( start, frac, end, mid );
+	midf = p1f + ( p2f - p1f ) * frac;
+
+	// co down front side
+	if( R_RecursiveLightPoint( model, node->children[side], p1f, midf, cv, start, mid ))
+		return true; // hit something
+
+	if(( back < 0 ) == side )
+	{
+		cv->r = cv->g = cv->b = cv->a = 0;
+		return false; // didn't hit anything
+	}
+
+	// check for impact on this node
+	surf = model->surfaces + node->firstsurface;
+	VectorCopy( mid, g_trace_lightspot );
+
+	for( i = 0; i < node->numsurfaces; i++, surf++ )
+	{
+		int	smax, tmax;
+
+		tex = surf->texinfo;
+		info = surf->info;
+
+		if( FBitSet( surf->flags, SURF_DRAWTILED ))
+			continue;	// no lightmaps
+
+		s = DotProduct( mid, info->lmvecs[0] ) + info->lmvecs[0][3];
+		t = DotProduct( mid, info->lmvecs[1] ) + info->lmvecs[1][3];
+
+		if( s < info->lightmapmins[0] || t < info->lightmapmins[1] )
+			continue;
+
+		ds = s - info->lightmapmins[0];
+		dt = t - info->lightmapmins[1];
+
+		if ( ds > info->lightextents[0] || dt > info->lightextents[1] )
+			continue;
+
+		cv->r = cv->g = cv->b = cv->a = 0;
+
+		if( !surf->samples )
+			return true;
+
+		sample_size = gEngine.Mod_SampleSizeForFace( surf );
+		smax = (info->lightextents[0] / sample_size) + 1;
+		tmax = (info->lightextents[1] / sample_size) + 1;
+		ds /= sample_size;
+		dt /= sample_size;
+
+		lm = surf->samples + Q_rint( dt ) * smax + Q_rint( ds );
+		g_trace_fraction = midf;
+		size = smax * tmax;
+		dm = NULL;
+
+		if( surf->info->deluxemap )
+		{
+			vec3_t	faceNormal;
+
+			if( FBitSet( surf->flags, SURF_PLANEBACK ))
+				VectorNegate( surf->plane->normal, faceNormal );
+			else VectorCopy( surf->plane->normal, faceNormal );
+
+			// compute face TBN
+#if 1
+			Vector4Set( tbn[0], surf->info->lmvecs[0][0], surf->info->lmvecs[0][1], surf->info->lmvecs[0][2], 0.0f );
+			Vector4Set( tbn[1], -surf->info->lmvecs[1][0], -surf->info->lmvecs[1][1], -surf->info->lmvecs[1][2], 0.0f );
+			Vector4Set( tbn[2], faceNormal[0], faceNormal[1], faceNormal[2], 0.0f );
+#else
+			Vector4Set( tbn[0], surf->info->lmvecs[0][0], -surf->info->lmvecs[1][0], faceNormal[0], 0.0f );
+			Vector4Set( tbn[1], surf->info->lmvecs[0][1], -surf->info->lmvecs[1][1], faceNormal[1], 0.0f );
+			Vector4Set( tbn[2], surf->info->lmvecs[0][2], -surf->info->lmvecs[1][2], faceNormal[2], 0.0f );
+#endif
+			VectorNormalize( tbn[0] );
+			VectorNormalize( tbn[1] );
+			VectorNormalize( tbn[2] );
+			dm = surf->info->deluxemap + Q_rint( dt ) * smax + Q_rint( ds );
+		}
+
+		for( map = 0; map < MAXLIGHTMAPS && surf->styles[map] != 255; map++ )
+		{
+			// FIXME VK uint	scale = tr.lightstylevalue[surf->styles[map]];
+            uint scale = 255;
+
+			/* FIXME VK if( tr.ignore_lightgamma )
+			{
+				cv->r += lm->r * scale;
+				cv->g += lm->g * scale;
+				cv->b += lm->b * scale;
+			}
+			else */
+			{
+				cv->r += gEngine.LightToTexGamma( lm->r ) * scale;
+				cv->g += gEngine.LightToTexGamma( lm->g ) * scale;
+				cv->b += gEngine.LightToTexGamma( lm->b ) * scale;
+			}
+			lm += size; // skip to next lightmap
+
+			if( dm != NULL )
+			{
+				vec3_t	srcNormal, lightNormal;
+				float	f = (1.0f / 128.0f);
+
+				VectorSet( srcNormal, ((float)dm->r - 128.0f) * f, ((float)dm->g - 128.0f) * f, ((float)dm->b - 128.0f) * f );
+				Matrix3x4_VectorIRotate( tbn, srcNormal, lightNormal );		// turn to world space
+				VectorScale( lightNormal, (float)scale * -1.0f, lightNormal );	// turn direction from light
+				VectorAdd( g_trace_lightvec, lightNormal, g_trace_lightvec );
+				dm += size; // skip to next deluxmap
+			}
+		}
+
+		return true;
+	}
+
+	// go down back side
+	return R_RecursiveLightPoint( model, node->children[!side], midf, p2f, cv, mid, end );
+}
+
+/*
+=================
+R_LightVec
+
+check bspmodels to get light from
+=================
+*/
+colorVec R_LightVecInternal( const vec3_t start, const vec3_t end, vec3_t lspot, vec3_t lvec )
+{
+	float	last_fraction;
+	int	i, maxEnts = 1;
+	colorVec	light, cv;
+    const model_t* world_model = gEngine.pfnGetModelByIndex( 1 );
+
+	if( lspot ) VectorClear( lspot );
+	if( lvec ) VectorClear( lvec );
+
+	if( world_model && world_model->lightdata )
+	{
+		light.r = light.g = light.b = light.a = 0;
+		last_fraction = 1.0f;
+
+		// get light from bmodels too
+		// FIXME VK if( CVAR_TO_BOOL( r_lighting_extended ))
+        //   maxEnts = MAX_PHYSENTS;
+
+		// check all the bsp-models
+		for( i = 0; i < maxEnts; i++ )
+		{
+			physent_t	*pe = gEngine.EV_GetPhysent( i );
+			vec3_t	offset, start_l, end_l;
+			mnode_t	*pnodes;
+			matrix4x4	matrix;
+
+			if( !pe )
+				break;
+
+			if( !pe->model || pe->model->type != mod_brush )
+				continue; // skip non-bsp models
+
+			pnodes = &pe->model->nodes[pe->model->hulls[0].firstclipnode];
+			VectorSubtract( pe->model->hulls[0].clip_mins, vec3_origin, offset );
+			VectorAdd( offset, pe->origin, offset );
+			VectorSubtract( start, offset, start_l );
+			VectorSubtract( end, offset, end_l );
+
+			// rotate start and end into the models frame of reference
+			if( !VectorIsNull( pe->angles ))
+			{
+				Matrix4x4_CreateFromEntity( matrix, pe->angles, offset, 1.0f );
+				Matrix4x4_VectorITransform( matrix, start, start_l );
+				Matrix4x4_VectorITransform( matrix, end, end_l );
+			}
+
+			VectorClear( g_trace_lightspot );
+			VectorClear( g_trace_lightvec );
+			g_trace_fraction = 1.0f;
+
+			if( !R_RecursiveLightPoint( pe->model, pnodes, 0.0f, 1.0f, &cv, start_l, end_l ))
+				continue;	// didn't hit anything
+
+			if( g_trace_fraction < last_fraction )
+			{
+				if( lspot ) VectorCopy( g_trace_lightspot, lspot );
+				if( lvec ) VectorNormalize2( g_trace_lightvec, lvec );
+				light.r = Q_min(( cv.r >> 7 ), 255 );
+				light.g = Q_min(( cv.g >> 7 ), 255 );
+				light.b = Q_min(( cv.b >> 7 ), 255 );
+				last_fraction = g_trace_fraction;
+
+				if(( light.r + light.g + light.b ) != 0 )
+					break; // we get light now
+			}
+		}
+	}
+	else
+	{
+		light.r = light.g = light.b = 255;
+		light.a = 0;
+	}
+
+	return light;
+}
+
+/*
+=================
+R_LightVec
+
+check bspmodels to get light from
+=================
+*/
+colorVec R_LightVec( const vec3_t start, const vec3_t end, vec3_t lspot, vec3_t lvec )
+{
+	colorVec	light = R_LightVecInternal( start, end, lspot, lvec );
+
+	if( /* FIXME VK CVAR_TO_BOOL( r_lighting_extended ) &&*/ lspot != NULL && lvec != NULL )
+	{
+		// trying to get light from ceiling (but ignore gradient analyze)
+		if(( light.r + light.g + light.b ) == 0 )
+			return R_LightVecInternal( end, start, lspot, lvec );
+	}
+
+	return light;
+}

ref_vk/shaders/brush.frag

@@ -21,22 +21,28 @@ layout(location=1) in vec3 vNormal;
 layout(location=2) in vec2 vTexture0;
 layout(location=3) in vec2 vLightmapUV;
 layout(location=4) in vec4 vColor;
+layout(location=5) flat in uint vFlags;
 
 layout(location=0) out vec4 outColor;
 
 // FIXME what should this be?
 const float dlight_attenuation_const = 5000.;
 
+#define FLAG_VERTEX_LIGHTING 1
+
 void main() {
 	outColor = vec4(0.);
-	const vec4 baseColor = texture(sTexture0, vTexture0) * vColor;
+	const vec4 baseColor = vColor * texture(sTexture0, vTexture0);
 
 	if (baseColor.a < alpha_test_threshold)
 		discard;
 
 	outColor.a = baseColor.a;
 
-	outColor.rgb += baseColor.rgb * texture(sLightmap, vLightmapUV).rgb;
+	if ((vFlags & FLAG_VERTEX_LIGHTING) == 0)
+		outColor.rgb += baseColor.rgb * texture(sLightmap, vLightmapUV).rgb;
+	else
+		outColor.rgb += baseColor.rgb;
 
 	for (uint i = 0; i < ubo.num_lights; ++i) {
 		const vec4 light_pos_r = ubo.lights[i].pos_r;

ref_vk/shaders/brush.vert

@@ -9,12 +9,17 @@ layout(location=0) in vec3 aPos;
 layout(location=1) in vec3 aNormal;
 layout(location=2) in vec2 aTexture0;
 layout(location=3) in vec2 aLightmapUV;
+layout(location=4) in vec4 aLightColor;
+layout(location=5) in uint aFlags;
 
 layout(location=0) out vec3 vPos;
 layout(location=1) out vec3 vNormal;
 layout(location=2) out vec2 vTexture0;
 layout(location=3) out vec2 vLightmapUV;
 layout(location=4) out vec4 vColor;
+layout(location=5) flat out uint vFlags;
+
+#define FLAG_VERTEX_LIGHTING 1
 
 void main() {
 	vPos = aPos.xyz;
@@ -22,5 +27,10 @@ void main() {
 	vTexture0 = aTexture0;
 	vLightmapUV = aLightmapUV;
 	vColor = ubo.color;
+
+	if ((aFlags & FLAG_VERTEX_LIGHTING) != 0)
+		vColor *= aLightColor;
+
+	vFlags = aFlags;
 	gl_Position = ubo.mvp * vec4(aPos.xyz, 1.);
 }

ref_vk/shaders/ray_kusochki.glsl

@@ -1,4 +1,5 @@
 #extension GL_EXT_shader_16bit_storage : require
+//#extension GL_EXT_shader_8bit_storage : require
 
 struct Kusok {
 	uint index_offset;
@@ -15,7 +16,11 @@ struct Vertex {
 	vec3 pos;
 	vec3 normal;
 	vec2 gl_tc;
-	vec2 lm_tc;
+	vec2 _unused_lm_tc;
+
+	//float padding;
+	//uint8_t color[4];
+	uint _unused_color_u8_4;
 };
 
 layout(std430, binding = 3, set = 0) readonly buffer Kusochki { Kusok kusochki[]; };

ref_vk/vk_render.c

@@ -105,6 +105,8 @@ static qboolean createPipelines( void )
 			{.binding = 0, .location = 1, .format = VK_FORMAT_R32G32B32_SFLOAT, .offset = offsetof(vk_vertex_t, normal)},
 			{.binding = 0, .location = 2, .format = VK_FORMAT_R32G32_SFLOAT, .offset = offsetof(vk_vertex_t, gl_tc)},
 			{.binding = 0, .location = 3, .format = VK_FORMAT_R32G32_SFLOAT, .offset = offsetof(vk_vertex_t, lm_tc)},
+			{.binding = 0, .location = 4, .format = VK_FORMAT_R8G8B8A8_UNORM, .offset = offsetof(vk_vertex_t, color)},
+			{.binding = 0, .location = 5, .format = VK_FORMAT_R32_UINT, .offset = offsetof(vk_vertex_t, flags)},
 		};
 
 		const vk_shader_stage_t shader_stages[] = {

ref_vk/vk_render.h

@@ -59,9 +59,12 @@ void VK_RenderAddStaticLight(vec3_t origin, vec3_t color);
 typedef struct vk_vertex_s {
 	// TODO padding needed for storage buffer reading, figure out how to fix in GLSL/SPV side
 	vec3_t pos; float p0_;
-	vec3_t normal; float p1_;
+	vec3_t normal; uint32_t flags;
 	vec2_t gl_tc; //float p2_[2];
 	vec2_t lm_tc; //float p3_[2];
+
+	rgba_t color; // per-vertex (non-rt lighting) color, color[3] == 1(255) => use color, discard lightmap; color[3] == 0 => use lightmap, discard color
+	float _padding[3];
 } vk_vertex_t;
 
 // TODO not sure how to do materials yet. Figure this out

ref_vk/vk_rmain.c

@@ -357,11 +357,7 @@ static void		GL_DrawParticles( const struct ref_viewpass_s *rvp, qboolean trans_
 	PRINT_NOT_IMPLEMENTED();
 }
 
-colorVec		R_LightVec( const float *start, const float *end, float *lightspot, float *lightvec )
-{
-	PRINT_NOT_IMPLEMENTED();
-	return (colorVec){255};
-}
+colorVec		R_LightVec( const float *start, const float *end, float *lightspot, float *lightvec );
 
 static struct mstudiotex_s *R_StudioGetTexture( struct cl_entity_s *e )
 {

ref_vk/vk_studio.c

@@ -1663,13 +1663,7 @@ void R_StudioLighting( float *lv, int bone, int flags, vec3_t normal )
 	*lv = illum * (1.0f / 255.0f);
 }
 
-/*
-====================
-R_LightLambert
-
-====================
-*/
-void R_LightLambert( vec4_t light[MAX_LOCALLIGHTS], vec3_t normal, vec3_t color, byte *out )
+void R_LightLambert( vec4_t light[MAX_LOCALLIGHTS], const vec3_t normal, vec3_t color, byte *out )
 {
 	vec3_t	finalLight;
 	vec3_t	localLight;
@@ -1712,30 +1706,33 @@ void R_LightLambert( vec4_t light[MAX_LOCALLIGHTS], vec3_t normal, vec3_t color,
 	out[2] = finalLight[2] * 255;
 }
 
-/* FIXME VK
-static void R_StudioSetColorBegin(short *ptricmds, vec3_t *pstudionorms )
+static void R_StudioSetColorBegin(const short *ptricmds, const vec3_t *pstudionorms, rgba_t out_color )
 {
 	float	*lv = (float *)g_studio.lightvalues[ptricmds[1]];
-	rgba_t color;
 
 	if( g_studio.numlocallights )
 	{
-		color[3] = tr.blend * 255;
-		R_LightLambert( g_studio.lightpos[ptricmds[0]], pstudionorms[ptricmds[1]], lv, color );
-		pglColor4ubv( color );
+		// FIXME VK color[3] = tr.blend * 255;
+		out_color[3] = 255;
+		R_LightLambert( g_studio.lightpos[ptricmds[0]], pstudionorms[ptricmds[1]], lv, out_color );
 	}
 	else
 	{
 		if( RI.currententity->curstate.rendermode == kRenderTransColor )
 		{
-			color[3] = tr.blend * 255;
-			VectorCopy( (byte*)&RI.currententity->curstate.rendercolor, color );
-			pglColor4ubv( color );
+			// FIXME VK color[3] = tr.blend * 255;
+			out_color[3] = 255;
+			VectorCopy( (byte*)&RI.currententity->curstate.rendercolor, out_color );
+		}
+		else
+		{
+			out_color[0] = lv[0] * 255.0f;
+			out_color[1] = lv[1] * 255.0f;
+			out_color[2] = lv[2] * 255.0f;
+			out_color[3] = 255; // FIXME VK tr.blend / 255.0f;
 		}
-		else pglColor4f( lv[0], lv[1], lv[2], tr.blend );
 	}
 }
-*/
 
 static void R_StudioSetColorArray(short *ptricmds, vec3_t *pstudionorms, byte *color )
 {
@@ -1758,12 +1755,6 @@ static void R_StudioSetColorArray(short *ptricmds, vec3_t *pstudionorms, byte *c
 	}
 }
 
-/*
-====================
-R_LightStrength
-
-====================
-*/
 void R_LightStrength( int bone, vec3_t localpos, vec4_t light[MAX_LOCALLIGHTS] )
 {
 	int	i;
@@ -1961,11 +1952,13 @@ static void R_StudioDrawNormalMesh( short *ptricmds, vec3_t *pstudionorms, float
 
 			VectorCopy(g_studio.verts[ptricmds[0]], dst_vtx->pos);
 			VectorCopy(g_studio.norms[ptricmds[0]], dst_vtx->normal);
-			dst_vtx->lm_tc[0] = dst_vtx->lm_tc[1] = mode == FAN ? .5f : 0.f;
-			// FIXME VK R_StudioSetColorBegin( ptricmds, pstudionorms );
+			dst_vtx->lm_tc[0] = dst_vtx->lm_tc[1] = 0.f;
 			dst_vtx->gl_tc[0] = ptricmds[2] * s;
 			dst_vtx->gl_tc[1] = ptricmds[3] * t;
 
+			dst_vtx->flags = 1; // vertex lighting instead of lightmap lighting
+			R_StudioSetColorBegin( ptricmds, pstudionorms, dst_vtx->color );
+
 			if (j > 1) {
 				switch (mode) {
 					case FAN: