diff --git a/engine/common/net_http.c b/engine/common/net_http.c index 8926a88a..f15abbd7 100644 --- a/engine/common/net_http.c +++ b/engine/common/net_http.c @@ -20,6 +20,7 @@ GNU General Public License for more details. #include "xash3d_mathlib.h" #include "ipv6text.h" #include "net_ws_private.h" +#include "miniz.h" /* ================================================= @@ -29,30 +30,18 @@ HTTP downloader ================================================= */ -#define MAX_HTTP_BUFFER_SIZE (BIT( 13 )) +#define MAX_HTTP_BUFFER_SIZE (BIT( 16 )) typedef struct httpserver_s { char host[256]; int port; char path[MAX_SYSPATH]; - qboolean needfree; struct httpserver_s *next; - } httpserver_t; -enum connectionstate -{ - HTTP_QUEUE = 0, - HTTP_OPENED, - HTTP_SOCKET, - HTTP_NS_RESOLVED, - HTTP_CONNECTED, - HTTP_REQUEST, - HTTP_REQUEST_SENT, - HTTP_RESPONSE_RECEIVED, - HTTP_FREE -}; +typedef struct httpfile_s httpfile_t; +typedef int (*http_process_fn_t)( httpfile_t *file ); typedef struct httpfile_s { @@ -66,12 +55,18 @@ typedef struct httpfile_s int lastchecksize; float checktime; float blocktime; - int id; - enum connectionstate state; + const char *blockreason; qboolean process; + qboolean got_response; + qboolean success; + qboolean compressed; + qboolean chunked; + int chunksize; resource_t *resource; + http_process_fn_t pfn_process; + struct sockaddr_storage addr; - string query_backup; + char query_backup[1024]; // query or response char buf[MAX_HTTP_BUFFER_SIZE+1]; @@ -81,8 +76,12 @@ typedef struct httpfile_s static struct http_static_s { // file and server lists - httpfile_t *first_file, *last_file; - httpserver_t *first_server, *last_server; + httpfile_t *first_file; + httpserver_t *first_server; + + int active_count, progress_count; + float progress; + qboolean resolving; } http; @@ -90,7 +89,420 @@ static CVAR_DEFINE_AUTO( http_useragent, "", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, " static CVAR_DEFINE_AUTO( http_autoremove, "1", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, "remove broken files" ); static CVAR_DEFINE_AUTO( http_timeout, "45", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, "timeout for http downloader" ); static CVAR_DEFINE_AUTO( http_maxconnections, "2", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, "maximum http connection number" ); -static CVAR_DEFINE_AUTO( http_show_request_header, "0", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, "show headers of the HTTP request" ); +static CVAR_DEFINE_AUTO( http_show_headers, "0", FCVAR_ARCHIVE | FCVAR_PRIVILEGED, "show HTTP headers (request and response)" ); + +static int HTTP_FileFree( httpfile_t *file ); +static int HTTP_FileConnect( httpfile_t *file ); +static int HTTP_FileCreateSocket( httpfile_t *file ); +static int HTTP_FileProcessStream( httpfile_t *file ); +static int HTTP_FileQueue( httpfile_t *file ); +static int HTTP_FileResolveNS( httpfile_t *file ); +static int HTTP_FileSendRequest( httpfile_t *file ); +static int HTTP_FileDecompress( httpfile_t *file ); + +/* +============== +HTTP_FreeFile + +Skip to next server/file +============== +*/ +static void HTTP_FreeFile( httpfile_t *file, qboolean error ) +{ + char incname[MAX_SYSPATH + 64]; // plus downloaded/ plus .incomplete + qboolean was_open = false; + + file->blocktime = 0; + + // Allways close file and socket + if( file->file ) + { + FS_Close( file->file ); + was_open = true; + } + + file->file = NULL; + + if( file->socket != -1 ) + { + closesocket( file->socket ); + http.active_count--; + } + + file->socket = -1; + + Q_snprintf( incname, sizeof( incname ), DEFAULT_DOWNLOADED_DIRECTORY "%s.incomplete", file->path ); + + if( error ) + { + // switch to next fastdl server if present + if( file->server && was_open ) + { + file->server = file->server->next; + + file->pfn_process = HTTP_FileQueue; // Reset download state, HTTP_Run() will open file again + return; + } + + // Called because there was no servers to download, free file now + if( http_autoremove.value == 1 ) // remove broken file + { + Con_Printf( S_ERROR "no servers to download %s\n", file->path ); + FS_Delete( incname ); + } + else // autoremove disabled, keep file + { + // warn about trash file + Con_Printf( "no servers to download %s. You may remove %s now\n", file->path, incname ); + } + } + else + { + if( file->compressed ) + { + FS_Delete( incname ); + } + else + { + // Success, rename and process file + char name[MAX_SYSPATH]; + Q_snprintf( name, sizeof( name ), DEFAULT_DOWNLOADED_DIRECTORY "%s", file->path ); + FS_Rename( incname, name ); + } + } + + file->pfn_process = HTTP_FileFree; + file->success = !error; +} + +static int HTTP_FileFree( httpfile_t *file ) +{ + return 0; // do nothing, wait for memory clean up +} + +static int HTTP_FileQueue( httpfile_t *file ) +{ + char name[MAX_SYSPATH]; + + if( http.active_count > http_maxconnections.value ) + return 0; + + if( !file->server ) + { + HTTP_FreeFile( file, true ); + return 0; + } + + Con_Reportf( "HTTP: Starting download %s from %s:%d\n", file->path, file->server->host, file->server->port ); + Q_snprintf( name, sizeof( name ), DEFAULT_DOWNLOADED_DIRECTORY "%s.incomplete", file->path ); + + if( !( file->file = FS_Open( name, "wb+", true ))) + { + Con_Printf( S_ERROR "HTTP: cannot open %s!", name ); + HTTP_FreeFile( file, true ); + return 0; + } + + file->pfn_process = HTTP_FileResolveNS; + file->blocktime = file->downloaded = file->lastchecksize = file->checktime = 0; + return 1; +} + +static int HTTP_FileResolveNS( httpfile_t *file ) +{ + net_gai_state_t res; + + if( http.resolving ) + return 0; + + memset( &file->addr, 0, sizeof( file->addr )); + + res = NET_StringToSockaddr( file->server->host, &file->addr, true, AF_UNSPEC ); + + switch( file->addr.ss_family ) + { + case AF_INET: + ((struct sockaddr_in *)&file->addr)->sin_port = MSG_BigShort( file->server->port ); + break; + case AF_INET6: + ((struct sockaddr_in6 *)&file->addr)->sin6_port = MSG_BigShort( file->server->port ); + break; + } + + if( res == NET_EAI_AGAIN ) + { + http.resolving = true; + return 0; + } + + if( res == NET_EAI_NONAME ) + { + Con_Printf( S_ERROR "failed to resolve server address for %s!\n", file->server->host ); + HTTP_FreeFile( file, true ); + return 0; + } + + file->pfn_process = HTTP_FileCreateSocket; + return 1; +} + +static int HTTP_FileCreateSocket( httpfile_t *file ) +{ + uint mode = 1; + int res; + + file->socket = socket( file->addr.ss_family, SOCK_STREAM, IPPROTO_TCP ); + + if( file->socket < 0 ) + { + Con_Printf( S_ERROR "%s: socket() returned %s\n", __func__, NET_ErrorString()); + HTTP_FreeFile( file, true ); + return 0; + } + + if( ioctlsocket( file->socket, FIONBIO, (void *)&mode ) < 0 ) + { + Con_Printf( S_ERROR "%s: ioctl() returned %s\n", __func__, NET_ErrorString()); + HTTP_FreeFile( file, true ); + return 0; + } + +#if XASH_LINUX + + res = fcntl( file->socket, F_GETFL, 0 ); + + if( res < 0 ) + { + Con_Printf( S_ERROR "%s: fcntl( F_GETFL ) returned %s\n", __func__, NET_ErrorString()); + HTTP_FreeFile( file, true ); + return 0; + } + + // SOCK_NONBLOCK is not portable, so use fcntl + if( fcntl( file->socket, F_SETFL, res | O_NONBLOCK ) < 0 ) + { + Con_Printf( S_ERROR "%s: fcntl( F_SETFL ) returned %s\n", __func__, NET_ErrorString()); + HTTP_FreeFile( file, true ); + return 0; + } +#endif + + http.active_count++; + file->pfn_process = HTTP_FileConnect; + return 1; +} + +static int HTTP_FileConnect( httpfile_t *file ) +{ + string useragent; + int res = connect( file->socket, (struct sockaddr *)&file->addr, NET_SockAddrLen( &file->addr )); + + if( res < 0 ) + { + int err = WSAGetLastError(); + if( err != WSAEWOULDBLOCK && err != WSAEINPROGRESS ) + { + Con_Printf( S_ERROR "cannot connect to server: %s\n", NET_ErrorString( )); + HTTP_FreeFile( file, true ); + return 0; + } + + file->blocktime += host.frametime; + file->blockreason = "request send"; + return 0; + } + + file->blocktime = 0; + + if( !COM_CheckStringEmpty( http_useragent.string ) || !Q_strcmp( http_useragent.string, "xash3d" )) + { + Q_snprintf( useragent, sizeof( useragent ), "%s/%s (%s-%s; build %d; %s)", + XASH_ENGINE_NAME, XASH_VERSION, Q_buildos( ), Q_buildarch( ), Q_buildnum( ), Q_buildcommit( )); + } + else Q_strncpy( useragent, http_useragent.string, sizeof( useragent )); + + file->query_length = Q_snprintf( file->buf, sizeof( file->buf ), + "GET %s%s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "User-Agent: %s\r\n" + "Accept-Encoding: gzip, deflate\r\n" + "Accept: */*\r\n\r\n", + file->server->path, file->path, + file->server->host, file->server->port, + useragent ); + Q_strncpy( file->query_backup, file->buf, sizeof( file->query_backup )); + file->bytes_sent = 0; + file->header_size = 0; + file->pfn_process = HTTP_FileSendRequest; + return 1; +} + +static int HTTP_FileSendRequest( httpfile_t *file ) +{ + int res = -1; + + res = send( file->socket, file->buf + file->bytes_sent, file->query_length - file->bytes_sent, 0 ); + + if( res >= 0 ) + { + file->bytes_sent += res; + file->blocktime = 0; + + if( file->bytes_sent >= file->query_length ) + { + if( http_show_headers.value ) + Con_Reportf( "HTTP: Request sent! (size %d data %s)\n", file->bytes_sent, file->buf ); + else + Con_Reportf( "HTTP: Request sent!\n" ); + memset( file->buf, 0, sizeof( file->buf )); + file->pfn_process = HTTP_FileProcessStream; + return 1; + } + } + else + { + int err = WSAGetLastError(); + if( err != WSAEWOULDBLOCK && err != WSAENOTCONN ) + { + Con_Printf( S_ERROR "failed to send request: %s\n", NET_ErrorString( )); + HTTP_FreeFile( file, true ); + return 0; + } + + file->blocktime += host.frametime; + file->blockreason = "request send"; + } + + return 0; +} + +static int HTTP_FileDecompress( httpfile_t *file ) +{ + fs_offset_t len; +#pragma pack( push, 1 ) + struct + { + byte magic[2]; + byte method; + byte flags; + uint32_t mtime; // ignored + byte xfl; + byte os; // can be ignored + } hdr; +#pragma pack( pop ) + + enum + { + GZFLG_FTEXT = BIT( 0 ), // can be ignored + GZFLG_FHCRC = BIT( 1 ), + GZFLG_FEXTRA = BIT( 2 ), + GZFLG_FNAME = BIT( 3 ), + GZFLG_FCOMMENT = BIT( 4 ) + }; + + z_stream decompress_stream; + char name[MAX_SYSPATH]; + fs_offset_t deflate_pos; + size_t compressed_len, decompressed_len; + byte *data_in, *data_out; + int zlib_result; + + g_fsapi.Seek( file->file, 0, SEEK_END ); + len = g_fsapi.Tell( file->file ); + + g_fsapi.Seek( file->file, 0, SEEK_SET ); + if( g_fsapi.Read( file->file, &hdr, sizeof( hdr )) != sizeof( hdr )) + { + HTTP_FreeFile( file, true ); + return 0; + } + + if( hdr.magic[0] != 0x1f && hdr.magic[1] != 0x8b && hdr.method != 0x08 ) + { + HTTP_FreeFile( file, true ); + return 0; + } + + if( FBitSet( hdr.flags, GZFLG_FEXTRA )) + { + byte res[2]; + uint16_t xlen; + + g_fsapi.Read( file->file, res, sizeof( res )); + xlen = res[0] | res[1] << 16; + g_fsapi.Seek( file->file, xlen, SEEK_CUR ); + } + + if( FBitSet( hdr.flags, GZFLG_FNAME )) + { + byte ch; + do + { + g_fsapi.Read( file->file, &ch, sizeof( ch )); + } while( ch != 0 ); + } + + if( FBitSet( hdr.flags, GZFLG_FCOMMENT )) + { + byte ch; + do + { + g_fsapi.Read( file->file, &ch, sizeof( ch )); + } while( ch != 0 ); + } + + if( FBitSet( hdr.flags, GZFLG_FHCRC )) + g_fsapi.Seek( file->file, 2, SEEK_CUR ); + + deflate_pos = g_fsapi.Tell( file->file ); + compressed_len = len - deflate_pos; + + { + byte data[4]; + + g_fsapi.Seek( file->file, -4, SEEK_END ); + g_fsapi.Read( file->file, data, sizeof( data )); + + // FIXME: this isn't correct, as this size might be modulo 2^32 + // but we probably won't decompress files this big + decompressed_len = data[0] | data[1] << 8 | data[2] << 16 | data[3] << 24; + } + + data_in = Mem_Malloc( host.mempool, compressed_len + 1 ); + data_out = Mem_Malloc( host.mempool, decompressed_len + 1 ); + + Q_snprintf( name, sizeof( name ), DEFAULT_DOWNLOADED_DIRECTORY "%s", file->path ); + + memset( &decompress_stream, 0, sizeof( decompress_stream )); + decompress_stream.total_in = decompress_stream.avail_in = compressed_len; + decompress_stream.next_in = data_in; + decompress_stream.total_out = decompress_stream.avail_out = decompressed_len; + decompress_stream.next_out = data_out; + + g_fsapi.Seek( file->file, deflate_pos, SEEK_SET ); + g_fsapi.Read( file->file, data_in, compressed_len ); + + if( inflateInit2( &decompress_stream, -MAX_WBITS ) != Z_OK ) + { + Con_Printf( S_ERROR "%s: inflateInit2 failed\n", __func__ ); + Mem_Free( data_in ); + Mem_Free( data_out ); + HTTP_FreeFile( file, true ); + return 0; + } + + zlib_result = inflate( &decompress_stream, Z_NO_FLUSH ); + inflateEnd( &decompress_stream ); + + if( zlib_result == Z_OK || zlib_result == Z_STREAM_END ) + { + Mem_Free( data_in ); + } + + g_fsapi.WriteFile( name, data_out, decompressed_len ); + HTTP_FreeFile( file, false ); + return 1; +} /* ======================== @@ -102,7 +514,7 @@ void HTTP_ClearCustomServers( void ) if( http.first_file ) return; // may be referenced - while( http.first_server && http.first_server->needfree ) + while( http.first_server ) { httpserver_t *tmp = http.first_server; @@ -111,88 +523,6 @@ void HTTP_ClearCustomServers( void ) } } -/* -============== -HTTP_FreeFile - -Skip to next server/file -============== -*/ -static void HTTP_FreeFile( httpfile_t *file, qboolean error ) -{ - char incname[256]; - - // Allways close file and socket - if( file->file ) - FS_Close( file->file ); - - file->file = NULL; - - if( file->socket != -1 ) - closesocket( file->socket ); - - file->socket = -1; - - Q_snprintf( incname, sizeof( incname ), DEFAULT_DOWNLOADED_DIRECTORY "%s.incomplete", file->path ); - if( error ) - { - // Switch to next fastdl server if present - if( file->server && ( file->state > HTTP_QUEUE ) && ( file->state != HTTP_FREE )) - { - file->server = file->server->next; - file->state = HTTP_QUEUE; // Reset download state, HTTP_Run() will open file again - return; - } - - // Called because there was no servers to download, free file now - if( http_autoremove.value == 1 ) // remove broken file - FS_Delete( incname ); - else // autoremove disabled, keep file - Con_Printf( "cannot download %s from any server. " - "You may remove %s now\n", file->path, incname ); // Warn about trash file - -#if !XASH_DEDICATED - if( file->process ) - { - if( file->resource ) - { - char buf[1024]; - sizebuf_t msg; - - MSG_Init( &msg, "DlFile", buf, sizeof( buf )); - MSG_BeginClientCmd( &msg, clc_stringcmd ); - MSG_WriteStringf( &msg, "dlfile %s", file->path ); - - Netchan_CreateFragments( &cls.netchan, &msg ); - Netchan_FragSend( &cls.netchan ); - } - else - { - CL_ProcessFile( false, file->path ); // Process file, increase counter - } - } -#endif // !XASH_DEDICATED - } - else - { - // Success, rename and process file - char name[256]; - - Q_snprintf( name, sizeof( name ), DEFAULT_DOWNLOADED_DIRECTORY "%s", file->path ); - FS_Rename( incname, name ); - -#if !XASH_DEDICATED - if( file->process ) - CL_ProcessFile( true, name ); - else -#endif - { - Con_Printf( "successfully downloaded %s, processing disabled!\n", name ); - } - } - - file->state = HTTP_FREE; -} /* =================== @@ -203,35 +533,132 @@ remove files with HTTP_FREE state from list */ static void HTTP_AutoClean( void ) { - httpfile_t *curfile, *prevfile = 0; + char buf[1024]; + sizebuf_t msg; + httpfile_t *cur, **prev = &http.first_file; + + MSG_Init( &msg, "DlFile", buf, sizeof( buf )); // clean all files marked to free - for( curfile = http.first_file; curfile; curfile = curfile->next ) + while( 1 ) { - if( curfile->state != HTTP_FREE ) - { - prevfile = curfile; - continue; - } + cur = *prev; - if( curfile == http.first_file ) - { - http.first_file = http.first_file->next; - Mem_Free( curfile ); - curfile = http.first_file; - if( !curfile ) - break; - continue; - } - - if( prevfile ) - prevfile->next = curfile->next; - Mem_Free( curfile ); - curfile = prevfile; - if( !curfile ) + if( !cur ) break; + + if( cur->pfn_process != HTTP_FileFree ) + { + prev = &cur->next; + continue; + } + +#if !XASH_DEDICATED + if( cur->process ) + { + if( cur->resource && !cur->success ) + { + MSG_BeginClientCmd( &msg, clc_stringcmd ); + MSG_WriteStringf( &msg, "dlfile %s", cur->path ); + } + else CL_ProcessFile( cur->success, cur->path ); + } + else +#endif + { + if( cur->success ) + Con_Printf( "successfully downloaded %s!\n", cur->path ); + } + + *prev = cur->next; + Mem_Free( cur ); } - http.last_file = prevfile; + + if( MSG_GetNumBytesWritten( &msg ) > 0 ) + { + // it's expected to be on fragments channel + Netchan_CreateFragments( &cls.netchan, &msg ); + Netchan_FragSend( &cls.netchan ); + } +} + +static int HTTP_FileSaveReceivedData( httpfile_t *file, int pos, int length ) +{ + while( length > 0 ) + { + int oldpos = pos; + int ret; + int len_to_write; + + if( file->chunked && file->chunksize <= 0 ) + { + char *begin = &file->buf[pos]; + + if( begin[0] == '\r' && begin[1] == '\r' ) + begin += 2; + + file->chunksize = Q_atoi_hex( 1, begin ); + + if( !file->chunksize && begin[0] == '0' ) // actually an end, not Q_atoi being stupid + { + if( file->compressed ) + { + file->blocktime = 0; + file->pfn_process = HTTP_FileDecompress; + return 1; + } + else + { + HTTP_FreeFile( file, false ); // success + return 1; + } + } + + begin = Q_strstr( begin, "\r\n" ); + if( !begin ) + { + Con_Printf( S_ERROR "can't parse chunked transfer encoding header for %s\n", file->path ); + if( http_show_headers.value ) + Con_Reportf( "Request headers: %s", file->query_backup ); + HTTP_FreeFile( file, true ); + return 0; + } + + pos = ( begin + 2 ) - file->buf; + length -= pos - oldpos; + + if( length < 0 ) + { + Con_Printf( S_ERROR "can't parse chunked transfer encoding header 2 for %s\n", file->path ); + if( http_show_headers.value ) + Con_Reportf( "Request headers: %s", file->query_backup ); + HTTP_FreeFile( file, true ); + return 0; + } + } + + if( file->chunked ) + len_to_write = Q_min( length, file->chunksize ); + else len_to_write = length; + + ret = FS_Write( file->file, &file->buf[pos], len_to_write ); + if( ret != len_to_write ) + { + // close it and go to next + Con_Printf( S_ERROR "write failed for %s!\n", file->path ); + HTTP_FreeFile( file, true ); + return 0; + } + + length -= len_to_write; + file->chunksize -= len_to_write; + + pos += ret; + file->downloaded += ret; + file->lastchecksize += ret; + } + + return 1; } /* @@ -241,93 +668,136 @@ HTTP_ProcessStream process incoming data =================== */ -static qboolean HTTP_ProcessStream( httpfile_t *curfile ) +static int HTTP_FileProcessStream( httpfile_t *curfile ) { char buf[sizeof( curfile->buf )]; char *begin = 0; int res; - if( curfile->header_size >= sizeof( buf )) - { - Con_Reportf( S_ERROR "Header too big, the size is %d\n", curfile->header_size ); - HTTP_FreeFile( curfile, true ); - return false; - } - - while( ( res = recv( curfile->socket, buf, sizeof( buf ) - curfile->header_size, 0 )) > 0) // if we got there, we are receiving data + // if we got there, we are receiving data + while(( res = recv( curfile->socket, buf, sizeof( buf ) - curfile->header_size - 1, 0 )) > 0 ) { curfile->blocktime = 0; - if( curfile->state < HTTP_RESPONSE_RECEIVED ) // Response still not received + if( !curfile->got_response ) // Response still not received { + if( curfile->header_size + res + 1 > sizeof( buf )) + { + Con_Reportf( S_ERROR "Header too big, the size is %d\n", curfile->header_size ); + HTTP_FreeFile( curfile, true ); + return 0; + } + memcpy( curfile->buf + curfile->header_size, buf, res ); curfile->buf[curfile->header_size + res] = 0; begin = Q_strstr( curfile->buf, "\r\n\r\n" ); if( begin ) // Got full header { - int cutheadersize = begin - curfile->buf + 4; // after that begin of data - char *content_length_line; + char *content_length; + char *content_encoding; + char *transfer_encoding; + + *begin = 0; // cut string to print out response if( !Q_strstr( curfile->buf, "200 OK" )) { - *begin = 0; // cut string to print out response - begin = Q_strchr( curfile->buf, '\r' ); + char *p; - if( !begin ) begin = Q_strchr( curfile->buf, '\n' ); - if( begin ) - *begin = 0; + int num = -1; - Con_Printf( S_ERROR "%s: bad response: %s\n", curfile->path, curfile->buf ); + p = Q_strchr( curfile->buf, '\r' ); + if( !p ) p = Q_strchr( curfile->buf, '\n' ); + if( p ) *p = 0; + + // extract the error code, don't assume the response is valid HTTP + if( !Q_strncmp( curfile->buf, "HTTP/1.", 7 )) + { + char tmp[4]; + + Q_strncpy( tmp, curfile->buf + 9, sizeof( tmp )); + if( Q_isdigit( tmp )) + num = Q_atoi( tmp ); + } + + switch( num ) + { + // TODO: handle redirects + case 404: + Con_Printf( S_ERROR "%s: file not found\n", curfile->path ); + break; + default: + Con_Printf( S_ERROR "%s: bad response: %s\n", curfile->path, curfile->buf ); + if( http_show_headers.value ) + Con_Printf( "Request headers: %s", curfile->query_backup ); + break; + } - if( http_show_request_header.value ) - Con_Printf( "Request headers: %s", curfile->query_backup ); HTTP_FreeFile( curfile, true ); - return false; + return 0; } - // print size - content_length_line = Q_stristr( curfile->buf, "Content-Length: " ); - if( content_length_line ) + content_encoding = Q_stristr( curfile->buf, "Content-Encoding" ); + if( content_encoding ) // fetch compressed status + { + content_encoding += sizeof( "Content-Encoding: " ) - 1; + + if( !Q_strnicmp( content_encoding, "gzip", 4 ) && ( content_encoding[4] == '\0' || content_encoding[4] == '\n' || content_encoding[4] == '\r' )) + curfile->compressed = true; + else + { + Con_Printf( S_ERROR "%s: bad Content-Encoding: %s\n", curfile->path, content_encoding ); + if( http_show_headers.value ) + Con_Printf( "Request headers: %s", curfile->query_backup ); + HTTP_FreeFile( curfile, true ); + return 0; + } + } + + if(( transfer_encoding = Q_stristr( curfile->buf, "Transfer-Encoding: chunked" ))) + { + curfile->size = -1; + curfile->chunked = true; + + Con_Reportf( "HTTP: Got 200 OK! Chunked transfer encoding%s\n", curfile->compressed ? ", compressed" : "" ); + } + else if(( content_length = Q_stristr( curfile->buf, "Content-Length: " ) )) { int size; - content_length_line += sizeof( "Content-Length: " ) - 1; - size = Q_atoi( content_length_line ); + content_length += sizeof( "Content-Length: " ) - 1; + size = Q_atoi( content_length ); - Con_Reportf( "HTTP: Got 200 OK! File size is %d\n", size ); + Con_Reportf( "HTTP: Got 200 OK! File size is %d%s\n", curfile->size, curfile->compressed ? ", compressed" : "" ); - if( ( curfile->size != -1 ) && ( curfile->size != size )) // check size if specified, not used - Con_Reportf( S_WARN "Server reports wrong file size for %s!\n", curfile->path ); + if( !curfile->compressed ) + { + if( ( curfile->size != -1 ) && ( curfile->size != size )) // check size if specified, not used + Con_Reportf( S_WARN "Server reports wrong file size for %s!\n", curfile->path ); + } curfile->size = size; curfile->header_size = 0; } - if( curfile->size == -1 ) + if( curfile->size == -1 && !curfile->chunked ) { // Usually fastdl's reports file size if link is correct Con_Printf( S_ERROR "file size is unknown, refusing download!\n" ); HTTP_FreeFile( curfile, true ); - return false; + return 0; } - curfile->state = HTTP_RESPONSE_RECEIVED; // got response, let's start download + if( http_show_headers.value ) + Con_Reportf( "Response headers: %s\n", curfile->buf ); + + curfile->got_response = true; // got response, let's start download begin += 4; - // Write remaining message part - if( res - cutheadersize - curfile->header_size > 0 ) + if( res - ( begin - curfile->buf ) > 0 ) { - int ret = FS_Write( curfile->file, begin, res - cutheadersize - curfile->header_size ); - - if( ret != res - cutheadersize - curfile->header_size ) // could not write file - { - // close it and go to next - Con_Printf( S_ERROR "write failed for %s!\n", curfile->path ); - HTTP_FreeFile( curfile, true ); - return false; - } - curfile->downloaded += ret; + if( !HTTP_FileSaveReceivedData( curfile, begin - curfile->buf, res - ( begin - curfile->buf ))) + return 0; } } else @@ -335,20 +805,11 @@ static qboolean HTTP_ProcessStream( httpfile_t *curfile ) } else if( res > 0 ) { + memcpy( curfile->buf, buf, res ); + // data download - int ret = FS_Write( curfile->file, buf, res ); - - if ( ret != res ) - { - // close it and go to next - Con_Printf( S_ERROR "write failed for %s!\n", curfile->path ); - curfile->state = HTTP_FREE; - HTTP_FreeFile( curfile, true ); - return false; - } - - curfile->downloaded += ret; - curfile->lastchecksize += ret; + if( !HTTP_FileSaveReceivedData( curfile, 0, res )) + return 0; // as after it will run in same frame if( curfile->checktime > 5 ) @@ -361,9 +822,46 @@ static qboolean HTTP_ProcessStream( httpfile_t *curfile ) } } } - curfile->checktime += host.frametime; - return true; + if( curfile->size > 0 ) + { + http.progress += (float)curfile->downloaded / curfile->size; + http.progress_count++; + + if( curfile->downloaded >= curfile->size ) + { + HTTP_FreeFile( curfile, false ); // success + return 0; + } + } + + if( res == 0 ) + { + curfile->blocktime += host.frametime; + curfile->blockreason = "waiting for data"; + } + + if( res < 0 ) + { + int err = WSAGetLastError(); + + if( err != WSAEWOULDBLOCK && err != WSAEINPROGRESS ) + { + Con_Reportf( "problem downloading %s: %s\n", curfile->path, NET_ErrorString( )); + HTTP_FreeFile( curfile, true ); + return 0; + } + + curfile->blocktime += host.frametime; + + if( !curfile->got_response ) + curfile->blockreason = "receiving header"; + else curfile->blockreason = "receiving data"; + return 0; + } + + curfile->checktime += host.frametime; + return 0; // don't block } /* @@ -377,218 +875,28 @@ Call every frame void HTTP_Run( void ) { httpfile_t *curfile; - int iActiveCount = 0; - int iProgressCount = 0; - float flProgress = 0; - qboolean fResolving = false; + + http.resolving = false; + http.progress_count = 0; + http.progress = 0; for( curfile = http.first_file; curfile; curfile = curfile->next ) { - struct sockaddr_storage addr; + int move_next = 1; - if( curfile->state == HTTP_FREE ) - continue; - - if( curfile->state == HTTP_QUEUE ) - { - char name[MAX_SYSPATH]; - - if( iActiveCount > http_maxconnections.value ) - continue; - - if( !curfile->server ) - { - Con_Printf( S_ERROR "no servers to download %s!\n", curfile->path ); - HTTP_FreeFile( curfile, true ); - break; - } - - Con_Reportf( "HTTP: Starting download %s from %s\n", curfile->path, curfile->server->host ); - Q_snprintf( name, sizeof( name ), DEFAULT_DOWNLOADED_DIRECTORY "%s.incomplete", curfile->path ); - - curfile->file = FS_Open( name, "wb", true ); - - if( !curfile->file ) - { - Con_Printf( S_ERROR "HTTP: cannot open %s!\n", name ); - HTTP_FreeFile( curfile, true ); - break; - } - - curfile->state = HTTP_OPENED; - curfile->blocktime = 0; - curfile->downloaded = 0; - curfile->lastchecksize = 0; - curfile->checktime = 0; - } - - iActiveCount++; - - if( curfile->state < HTTP_SOCKET ) // Socket is not created - { - dword mode; - - curfile->socket = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP ); - - // Now set non-blocking mode - // You may skip this if not supported by system, - // but download will lock engine, maybe you will need to add manual returns - mode = 1; - ioctlsocket( curfile->socket, FIONBIO, (void*)&mode ); -#if XASH_LINUX - // SOCK_NONBLOCK is not portable, so use fcntl - fcntl( curfile->socket, F_SETFL, fcntl( curfile->socket, F_GETFL, 0 ) | O_NONBLOCK ); -#endif - curfile->state = HTTP_SOCKET; - } - - if( curfile->state < HTTP_NS_RESOLVED ) - { - net_gai_state_t res; - char hostport[MAX_VA_STRING]; - - if( fResolving ) - continue; - - Q_snprintf( hostport, sizeof( hostport ), "%s:%d", curfile->server->host, curfile->server->port ); - - res = NET_StringToSockaddr( hostport, &addr, true, AF_INET ); - - if( res == NET_EAI_AGAIN ) - { - fResolving = true; - continue; - } - - if( res == NET_EAI_NONAME ) - { - Con_Printf( S_ERROR "failed to resolve server address for %s!\n", curfile->server->host ); - HTTP_FreeFile( curfile, true ); // Cannot connect - break; - } - curfile->state = HTTP_NS_RESOLVED; - } - - if( curfile->state < HTTP_CONNECTED ) // Connection not enstabilished - { - int res = connect( curfile->socket, (struct sockaddr*)&addr, NET_SockAddrLen( &addr ) ); - - if( res ) - { - if( WSAGetLastError() == WSAEINPROGRESS || WSAGetLastError() == WSAEWOULDBLOCK ) // Should give EWOOLDBLOCK if try recv too soon - curfile->state = HTTP_CONNECTED; - else - { - Con_Printf( S_ERROR "cannot connect to server: %s\n", NET_ErrorString( )); - HTTP_FreeFile( curfile, true ); // Cannot connect - break; - } - continue; // skip to next file - } - curfile->state = HTTP_CONNECTED; - } - - if( curfile->state < HTTP_REQUEST ) // Request not formatted - { - string useragent; - - if( !COM_CheckStringEmpty( http_useragent.string ) || !Q_strcmp( http_useragent.string, "xash3d" )) - { - Q_snprintf( useragent, sizeof( useragent ), "%s/%s (%s-%s; build %d; %s)", - XASH_ENGINE_NAME, XASH_VERSION, Q_buildos( ), Q_buildarch( ), Q_buildnum( ), Q_buildcommit( )); - } - else - { - Q_strncpy( useragent, http_useragent.string, sizeof( useragent )); - } - - curfile->query_length = Q_snprintf( curfile->buf, sizeof( curfile->buf ), - "GET %s%s HTTP/1.0\r\n" - "Host: %s:%d\r\n" - "User-Agent: %s\r\n" - "Accept: */*\r\n\r\n", curfile->server->path, - curfile->path, curfile->server->host, curfile->server->port, useragent ); - Q_strncpy( curfile->query_backup, curfile->buf, sizeof( curfile->query_backup )); - curfile->header_size = 0; - curfile->bytes_sent = 0; - curfile->state = HTTP_REQUEST; - } - - if( curfile->state < HTTP_REQUEST_SENT ) // Request not sent - { - qboolean wait = false; - - while( curfile->bytes_sent < curfile->query_length ) - { - int res = send( curfile->socket, curfile->buf + curfile->bytes_sent, curfile->query_length - curfile->bytes_sent, 0 ); - - if( res < 0 ) - { - if( WSAGetLastError() != WSAEWOULDBLOCK && WSAGetLastError() != WSAENOTCONN ) - { - Con_Printf( S_ERROR "failed to send request: %s\n", NET_ErrorString( )); - HTTP_FreeFile( curfile, true ); - wait = true; - break; - } - // blocking while waiting connection - // increase counter when blocking - curfile->blocktime += host.frametime; - wait = true; - - if( curfile->blocktime > http_timeout.value ) - { - Con_Printf( S_ERROR "timeout on request send:\n%s\n", curfile->buf ); - HTTP_FreeFile( curfile, true ); - break; - } - break; - } - else - { - curfile->bytes_sent += res; - curfile->blocktime = 0; - } - } - - if( wait ) - continue; - - Con_Reportf( "HTTP: Request sent!\n"); - memset( curfile->buf, 0, sizeof( curfile->buf )); - curfile->state = HTTP_REQUEST_SENT; - } - - if( !HTTP_ProcessStream( curfile )) - break; - - if( curfile->size > 0 ) - { - flProgress += (float)curfile->downloaded / curfile->size; - iProgressCount++; - } - - if( curfile->size > 0 && curfile->downloaded >= curfile->size ) - { - HTTP_FreeFile( curfile, false ); // success - break; - } - else if(( WSAGetLastError( ) != WSAEWOULDBLOCK ) && ( WSAGetLastError( ) != WSAEINPROGRESS )) - Con_Reportf( "problem downloading %s:\n%s\n", curfile->path, NET_ErrorString( )); - else - curfile->blocktime += host.frametime; + while( move_next > 0 ) + move_next = curfile->pfn_process( curfile ); if( curfile->blocktime > http_timeout.value ) { - Con_Printf( S_ERROR "timeout on receiving data!\n"); + Con_Printf( S_ERROR "timeout on %s (file: %s)\n", curfile->blockreason, curfile->path ); HTTP_FreeFile( curfile, true ); - break; } } // update progress - if( !Host_IsDedicated() && iProgressCount != 0 ) - Cvar_SetValue( "scr_download", flProgress/iProgressCount * 100 ); + if( !Host_IsDedicated() && http.progress_count != 0 ) + Cvar_SetValue( "scr_download", http.progress/http.progress_count * 100 ); HTTP_AutoClean(); } @@ -602,35 +910,29 @@ Add new download to end of queue */ void HTTP_AddDownload( const char *path, int size, qboolean process, resource_t *res ) { - httpfile_t *httpfile = Z_Calloc( sizeof( httpfile_t )); + httpfile_t *httpfile; + + if( !http.first_server ) + { + Con_Printf( S_ERROR "no servers to download %s\n", path ); + return; + } + + httpfile = Z_Calloc( sizeof( *httpfile )); Con_Reportf( "File %s queued to download\n", path ); httpfile->resource = res; httpfile->size = size; - httpfile->downloaded = 0; httpfile->socket = -1; - Q_strncpy ( httpfile->path, path, sizeof( httpfile->path )); + Q_strncpy( httpfile->path, path, sizeof( httpfile->path )); - if( http.last_file ) - { - // Add next to last download - httpfile->id = http.last_file->id + 1; - http.last_file->next= httpfile; - http.last_file = httpfile; - } - else - { - // It will be the only download - httpfile->id = 0; - http.last_file = http.first_file = httpfile; - } - - httpfile->file = NULL; - httpfile->next = NULL; - httpfile->state = HTTP_QUEUE; + httpfile->pfn_process = HTTP_FileQueue; httpfile->server = http.first_server; httpfile->process = process; + + httpfile->next = http.first_file; + http.first_file = httpfile; } /* @@ -704,7 +1006,6 @@ static httpserver_t *HTTP_ParseURL( const char *url ) server->path[i++] = '/'; server->path[i] = 0; server->next = NULL; - server->needfree = false; return server; } @@ -724,7 +1025,6 @@ void HTTP_AddCustomServer( const char *url ) return; } - server->needfree = true; server->next = http.first_server; http.first_server = server; } @@ -755,8 +1055,6 @@ Clear all queue */ static void HTTP_Clear_f( void ) { - http.last_file = NULL; - while( http.first_file ) { httpfile_t *file = http.first_file; @@ -785,7 +1083,7 @@ static void HTTP_Cancel_f( void ) if( !http.first_file ) return; - http.first_file->state = HTTP_FREE; + http.first_file->server = NULL; HTTP_FreeFile( http.first_file, true ); } @@ -811,18 +1109,25 @@ Print all pending downloads to console */ static void HTTP_List_f( void ) { - httpfile_t *file = http.first_file; + int i = 0; + httpfile_t *file; - while( file ) + if( !http.first_file ) + Con_Printf( "no downloads queued\n" ); + + for( file = http.first_file; file; file = file->next ) { - if ( file->server ) - Con_Printf ( "\t%d %d http://%s:%d/%s%s %d\n", file->id, file->state, - file->server->host, file->server->port, file->server->path, - file->path, file->downloaded ); - else - Con_Printf ( "\t%d %d (no server) %s\n", file->id, file->state, file->path ); + Con_Printf( "%d. %s (%d of %d)\n", i++, file->path, file->downloaded, file->size ); - file = file->next; + if( file->server ) + { + httpserver_t *server; + for( server = file->server; server; server = server->next ) + { + Con_Printf( "\thttp://%s:%d/%s%s\n", file->server->host, file->server->port, + file->server->path, file->path ); + } + } } } @@ -835,13 +1140,10 @@ When connected to new server, all old files should not increase counter */ void HTTP_ResetProcessState( void ) { - httpfile_t *file = http.first_file; + httpfile_t *file; - while( file ) - { + for( file = http.first_file; file; file = file->next ) file->process = false; - file = file->next; - } } /* @@ -851,11 +1153,7 @@ HTTP_Init */ void HTTP_Init( void ) { - char *serverfile, *line, token[1024]; - - http.last_server = NULL; - - http.first_file = http.last_file = NULL; + http.first_file = NULL; Cmd_AddRestrictedCommand( "http_download", HTTP_Download_f, "add file to download queue" ); Cmd_AddRestrictedCommand( "http_skip", HTTP_Skip_f, "skip current download server" ); @@ -868,31 +1166,7 @@ void HTTP_Init( void ) Cvar_RegisterVariable( &http_autoremove ); Cvar_RegisterVariable( &http_timeout ); Cvar_RegisterVariable( &http_maxconnections ); - Cvar_RegisterVariable( &http_show_request_header ); - - // Read servers from fastdl.txt - line = serverfile = (char *)FS_LoadFile( "fastdl.txt", 0, false ); - - if( serverfile ) - { - while(( line = COM_ParseFile( line, token, sizeof( token )))) - { - httpserver_t *server = HTTP_ParseURL( token ); - - if( !server ) - continue; - - if( !http.last_server ) - http.last_server = http.first_server = server; - else - { - http.last_server->next = server; - http.last_server = server; - } - } - - Mem_Free( serverfile ); - } + Cvar_RegisterVariable( &http_show_headers ); } /* @@ -911,6 +1185,4 @@ void HTTP_Shutdown( void ) http.first_server = http.first_server->next; Mem_Free( tmp ); } - - http.last_server = NULL; }