diff --git a/Game.Client/C_Soundscape.cs b/Game.Client/C_Soundscape.cs index 91661d8c0..dc834c7c0 100644 --- a/Game.Client/C_Soundscape.cs +++ b/Game.Client/C_Soundscape.cs @@ -712,13 +712,32 @@ void AddSoundScapeFile(ReadOnlySpan filename) { } } - void TouchPlayLooping(KeyValues? ambient) => throw new NotImplementedException(); + void TouchPlayLooping(KeyValues? ambient) { + for (KeyValues? key = ambient?.GetFirstSubKey(); key != null; key = key.GetNextKey()) { + if (stricmp(key.Name, "wave") == 0) + TouchSoundFile(key.GetString()); + } + } - void TouchPlayRandom(KeyValues? playRandom) => throw new NotImplementedException(); + void TouchPlayRandom(KeyValues? playRandom) { + for (KeyValues? key = playRandom?.GetFirstSubKey(); key != null; key = key.GetNextKey()) { + if (stricmp(key.Name, "rndwave") == 0) { + for (KeyValues? waves = key.GetFirstSubKey(); waves != null; waves = waves.GetNextKey()) + TouchSoundFile(waves.GetString()); + } + } + } - void TouchWaveFiles(KeyValues? soundScape) => throw new NotImplementedException(); + void TouchWaveFiles(KeyValues? soundScape) { + for (KeyValues? key = soundScape?.GetFirstSubKey(); key != null; key = key.GetNextKey()) { + if (stricmp(key.Name, "playlooping") == 0) + TouchPlayLooping(key); + else if (stricmp(key.Name, "playrandom") == 0) + TouchPlayRandom(key); + } + } - void TouchSoundFile(ReadOnlySpan wavefile) => throw new NotImplementedException(); + void TouchSoundFile(ReadOnlySpan wavefile) => filesystem.GetFileTime($"sound/{SoundCharsUtils.SkipSoundChars(wavefile)}", "GAME"); void TouchSoundFiles() { if (commandLine.FindParm("-makereslists") == 0) diff --git a/Game.Lua/LuaSharedImpl.cs b/Game.Lua/LuaSharedImpl.cs index ac26e39d6..10ca1c15a 100644 --- a/Game.Lua/LuaSharedImpl.cs +++ b/Game.Lua/LuaSharedImpl.cs @@ -133,9 +133,9 @@ public void CloseLuaInterface(ILuaInterface iface) { if (handle == null) return null; - using MemoryStream buffer = new(); - handle.Stream.CopyTo(buffer); - return buffer.ToArray(); + byte[] data = new byte[handle.Stream.Length]; + handle.Stream.ReadExactly(data); + return data; } LuaFile? LoadFile_FromDataTable(ReadOnlySpan path, ReadOnlySpan pathId, bool unk) { diff --git a/Game.Server/AI_NetworkManager.cs b/Game.Server/AI_NetworkManager.cs index 458bfb078..e6c56175c 100644 --- a/Game.Server/AI_NetworkManager.cs +++ b/Game.Server/AI_NetworkManager.cs @@ -127,13 +127,19 @@ public void LoadNetworkGraph() { string nrpFilename = $"maps/graphs/{gpGlobals.MapName}.ain"; - using IFileHandle? handle = filesystem.Open(nrpFilename, FileOpenOptions.Read | FileOpenOptions.Binary, "GAME"); - if (handle == null) { - DevWarning(2, $"Couldn't read {nrpFilename}!\n"); - return; + byte[] buf; + using (IFileHandle? handle = filesystem.Open(nrpFilename, FileOpenOptions.Read | FileOpenOptions.Binary, "GAME")) { + if (handle == null) { + DevWarning(2, $"Couldn't read {nrpFilename}!\n"); + return; + } + + buf = new byte[handle.Stream.Length]; + handle.Stream.ReadExactly(buf); } - LoadNetworkGraph(handle.Stream, nrpFilename, gpGlobals.MapVersion); + using MemoryStream stream = new(buf, false); + LoadNetworkGraph(stream, nrpFilename, gpGlobals.MapVersion); } public void LoadNetworkGraph(Stream stream, ReadOnlySpan nrpFilename, int currentMapVersion) { diff --git a/Source.Common/Filesystem/IFileSystem.cs b/Source.Common/Filesystem/IFileSystem.cs index 41f6f2459..dd23bfe06 100644 --- a/Source.Common/Filesystem/IFileSystem.cs +++ b/Source.Common/Filesystem/IFileSystem.cs @@ -75,6 +75,20 @@ public enum FSAsyncFlags /// public delegate void FSAsyncCallbackFunc(in FileAsyncRequest request, int bytesRead, FSAsyncStatus err); public delegate byte[] FSAllocFunc(string fileName, int bytes); +public delegate void FileSystemLoggingFunc(ReadOnlySpan fileName, ReadOnlySpan accessType); +public delegate void FileSystemWarningFunc(ReadOnlySpan fmt, ReadOnlySpan args); + +public enum FileWarningLevel +{ + Warning = -1, + Quiet = 0, + ReportUnclosed, + ReportUsage, + ReportAllAccesses, + ReportAllAccessesRead, + ReportAllAccessesReadWrite, + ReportAllAccessesAsync, +} /// /// Description of an async request @@ -300,7 +314,12 @@ public interface IFileSystem : IBaseFileSystem bool WriteFile(ReadOnlySpan fileName, ReadOnlySpan pathID, ReadOnlySpan buf); void MarkAllCRCsUnverified(); ReadOnlySpan WhereIsFile(ReadOnlySpan relativePath, ReadOnlySpan pathID = default); + void PrintOpenedFiles(); void PrintSearchPaths(); + void SetWarningFunc(FileSystemWarningFunc? warning); + void SetWarningLevel(FileWarningLevel level); + void AddLoggingFunc(FileSystemLoggingFunc logFunc); + void RemoveLoggingFunc(FileSystemLoggingFunc logFunc); /// /// FileNameHandle_t's are case-insensitive and slash-insensitive. diff --git a/Source.Common/Formats/Keyvalues/KeyValues.cs b/Source.Common/Formats/Keyvalues/KeyValues.cs index 1de5600d7..84835d415 100644 --- a/Source.Common/Formats/Keyvalues/KeyValues.cs +++ b/Source.Common/Formats/Keyvalues/KeyValues.cs @@ -102,6 +102,13 @@ public bool LoadFromStream(Stream? stream) { // Clear(); if (stream == null) return false; + if (stream.CanSeek) { + byte[] data = new byte[stream.Length - stream.Position]; + stream.ReadExactly(data); + stream.Dispose(); + stream = new MemoryStream(data, false); + } + // Size the buffer to the stream when we can, so the text is decoded in one piece. int bufferSize = stream.CanSeek ? (int)Math.Clamp(stream.Length - stream.Position, 1024, 1 << 20) : 1024; using StreamReader streamReader = new(stream, System.Text.Encoding.UTF8, true, bufferSize); diff --git a/Source.Engine/ClientLauncherAPI.cs b/Source.Engine/ClientLauncherAPI.cs index 5933a527f..a754f168e 100644 --- a/Source.Engine/ClientLauncherAPI.cs +++ b/Source.Engine/ClientLauncherAPI.cs @@ -85,8 +85,11 @@ public IClientLauncherAPI.Result RunListenServer() { public void SetStartupInfo(in StartupInfo info) { - startupInfo = info; + g_pFileSystem.SetWarningFunc(Warning); + Sys.TextMode = info.TextMode; + host_parms.BaseDir = info.BaseDirectory; + startupInfo = info; COM.InitFilesystem(info.InitialMod); } diff --git a/Source.Engine/Cmd.cs b/Source.Engine/Cmd.cs index 805b8084e..cb701b3c9 100644 --- a/Source.Engine/Cmd.cs +++ b/Source.Engine/Cmd.cs @@ -314,8 +314,18 @@ void exec(in TokenizedCommand args) { } } - using var execFile = fileSystem.Open(file, FileOpenOptions.Read, pathID); - using StreamReader reader = new StreamReader(execFile!.Stream); + byte[] buf; + using (IFileHandle? execFile = fileSystem.Open(file, FileOpenOptions.Read, pathID)) { + if (execFile == null) { + Dbg.ConMsg($"exec: couldn't exec {args[1]}\n"); + return; + } + + buf = new byte[execFile.Stream.Length]; + execFile.Stream.ReadExactly(buf); + } + + using StreamReader reader = new StreamReader(new MemoryStream(buf, false)); Dbg.ConDMsg($"execing {file}\n"); LinkedListNode? hCommand = Cbuf.Buffer.GetNextCommandHandle(); // check to make sure we're not going to overflow later diff --git a/Source.Engine/DedicatedServerAPI.cs b/Source.Engine/DedicatedServerAPI.cs index 303828a1b..684983a6a 100644 --- a/Source.Engine/DedicatedServerAPI.cs +++ b/Source.Engine/DedicatedServerAPI.cs @@ -38,6 +38,8 @@ public bool ModInit(in StartupInfo info) { eng ??= services.GetRequiredService(); eng.SetQuitting(IEngine.Quit.NotQuitting); + g_pFileSystem.SetWarningFunc(Warning); + host_parms.BaseDir = info.BaseDirectory; host_parms.Mod = GetModDirFromPath(info.InitialMod); host_parms.Game = info.InitialGame; diff --git a/Source.Engine/FileSystem.cs b/Source.Engine/FileSystem.cs index 03f379d19..2afc0b1cc 100644 --- a/Source.Engine/FileSystem.cs +++ b/Source.Engine/FileSystem.cs @@ -164,4 +164,54 @@ void path(in TokenizedCommand args) { fileSystem.PrintSearchPaths(); } + [ConCommand(helpText: "Show all files currently opened by the engine.")] + void fs_printopenfiles() { + fileSystem.PrintOpenedFiles(); + } + + [ConCommand(helpText: "Set the filesystem warning level.")] + void fs_warning_level(in TokenizedCommand args) { + if (args.ArgC() != 2) { + Warning("\"fs_warning_level n\" where n is one of:\n"); + Warning("\t0:\tFILESYSTEM_WARNING_QUIET\n"); + Warning("\t1:\tFILESYSTEM_WARNING_REPORTUNCLOSED\n"); + Warning("\t2:\tFILESYSTEM_WARNING_REPORTUSAGE\n"); + Warning("\t3:\tFILESYSTEM_WARNING_REPORTALLACCESSES\n"); + Warning("\t4:\tFILESYSTEM_WARNING_REPORTALLACCESSES_READ\n"); + Warning("\t5:\tFILESYSTEM_WARNING_REPORTALLACCESSES_READWRITE\n"); + Warning("\t6:\tFILESYSTEM_WARNING_REPORTALLACCESSES_ASYNC\n"); + return; + } + + int level = atoi(args[1]); + switch ((FileWarningLevel)level) { + case FileWarningLevel.Quiet: + Warning("fs_warning_level = FILESYSTEM_WARNING_QUIET\n"); + break; + case FileWarningLevel.ReportUnclosed: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTUNCLOSED\n"); + break; + case FileWarningLevel.ReportUsage: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTUSAGE\n"); + break; + case FileWarningLevel.ReportAllAccesses: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTALLACCESSES\n"); + break; + case FileWarningLevel.ReportAllAccessesRead: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTALLACCESSES_READ\n"); + break; + case FileWarningLevel.ReportAllAccessesReadWrite: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTALLACCESSES_READWRITE\n"); + break; + case FileWarningLevel.ReportAllAccessesAsync: + Warning("fs_warning_level = FILESYSTEM_WARNING_REPORTALLACCESSES_ASYNC\n"); + break; + default: + Warning("fs_warning_level = UNKNOWN!!!!!!!\n"); + return; + } + + fileSystem.SetWarningLevel((FileWarningLevel)level); + } + } diff --git a/Source.Filesystem/BaseFileSystem.cs b/Source.Filesystem/BaseFileSystem.cs index e75d60cb2..e5a8fc139 100644 --- a/Source.Filesystem/BaseFileSystem.cs +++ b/Source.Filesystem/BaseFileSystem.cs @@ -3,6 +3,8 @@ using CommunityToolkit.HighPerformance; +using Source.Common; +using Source.Common.Commands; using Source.Common.Filesystem; using Source.Common.Formats.BSP; using Source.Common.Formats.Keyvalues; @@ -12,6 +14,7 @@ using Source.Filesystem.GarrysMod; using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; @@ -86,7 +89,7 @@ private void AddMapPackFile(ReadOnlySpan path, ReadOnlySpan pathID, using IFileHandle? file = Open(path, FileOpenOptions.Read | FileOpenOptions.Binary, "GAME"); if (file == null) { - Warning("Couldn't open BSP for embedded pack file\n"); + Warning(FileWarningLevel.Warning, $"Couldn't open bsp {newPath} for embedded pack file check\n"); return; } @@ -101,7 +104,7 @@ private void AddMapPackFile(ReadOnlySpan path, ReadOnlySpan pathID, ZipPackFileSearchPath zip = new(this, new(newPath), file.Stream, in pakfile); if (!zip.IsValid()) { - Warning("ZipPackFileSearchPath not valid\n"); + Dbg.Warning("ZipPackFileSearchPath not valid\n"); return; } @@ -410,14 +413,14 @@ public bool IsDirectory(ReadOnlySpan fileName, ReadOnlySpan pathID) return FirstToThePost(name, id, new IsDirectory_Op(), false, out _); } - static void ParsePathID(scoped ref ReadOnlySpan fileName, scoped ref ReadOnlySpan pathID, Span tempPathID) { + void ParsePathID(scoped ref ReadOnlySpan fileName, scoped ref ReadOnlySpan pathID, Span tempPathID) { tempPathID[0] = '\0'; if (fileName.Length < 2 || fileName[0] != '/' || fileName[1] != '/') return; if (!pathID.IsEmpty) - Warning($"FS: Specified two path IDs ({fileName}, {pathID}).\n"); + Warning(FileWarningLevel.Warning, $"FS: Specified two path IDs ({fileName}, {pathID}).\n"); int read = 2; int written = 0; @@ -665,7 +668,7 @@ private ReadOnlySpan GetWritePath(ReadOnlySpan filename, ReadOnlySpa if (searchPath != null) return searchPath.GetPathString(); - Warning("Requested non-existent write path %s!\n", new string(pathID)); + Warning(FileWarningLevel.Warning, $"Requested non-existent write path {pathID}!\n"); } searchPath = FindWritePath(filename, "DEFAULT_WRITE_PATH"); @@ -713,6 +716,92 @@ public ReadOnlySpan WhereIsFile(ReadOnlySpan fileName, ReadOnlySpan< return null; } + static readonly ConVar fs_warning_mode = new("fs_warning_mode", "0", 0, "0:Off, 1:Warn main thread, 2:Warn other threads"); + + FileWarningLevel WarningLevel = FileWarningLevel.ReportUnclosed; + FileSystemWarningFunc? WarningFunc; + + int OpenedFiles; + internal readonly List LogFuncs = []; + + public void SetWarningFunc(FileSystemWarningFunc? warning) => WarningFunc = warning; + + public void SetWarningLevel(FileWarningLevel level) => WarningLevel = level; + + internal void Warning(FileWarningLevel level, ReadOnlySpan text) { + if (level > WarningLevel) + return; + + if ((fs_warning_mode.GetInt() == 1 && !ThreadInMainThread()) || (fs_warning_mode.GetInt() == 2 && ThreadInMainThread())) + return; + + string warningText = new(text.Length > 4095 ? text[..4095] : text); + + Console.Write(warningText); + + WarningFunc?.Invoke("%s", [warningText]); + } + + static string TraceThreadTag() => ThreadInMainThread() ? "" : "[a]"; + static string TraceTime() => Platform.Time.ToString("F3", CultureInfo.InvariantCulture); +#if WIN32 + static string TracePointer(object file) => RuntimeHelpers.GetHashCode(file).ToString("X16", CultureInfo.InvariantCulture); +#else + static string TracePointer(object file) => "0x" + RuntimeHelpers.GetHashCode(file).ToString("x", CultureInfo.InvariantCulture); +#endif + + internal void LogAccessToFile(ReadOnlySpan accesstype, ReadOnlySpan fullpath, ReadOnlySpan options) { + lock (LogFuncs) { + if (WarningLevel >= FileWarningLevel.ReportAllAccesses && !fullpath.Contains("console.log", StringComparison.OrdinalIgnoreCase)) + Warning(FileWarningLevel.ReportAllAccesses, $"---FS{TraceThreadTag()}: {accesstype} {fullpath} ({TraceTime()})\n"); + + for (int i = 0; i < LogFuncs.Count; ++i) + LogFuncs[i](fullpath, options); + } + } + + internal void Trace_FOpen(string filename, ReadOnlySpan options) { + Interlocked.Increment(ref OpenedFiles); + + LogAccessToFile("open", filename, options); + } + + internal void Trace_FClose(object file, string name) { + if (WarningLevel >= FileWarningLevel.ReportAllAccesses && !name.Contains("console.log", StringComparison.OrdinalIgnoreCase)) + Warning(FileWarningLevel.ReportAllAccesses, $"---FS{TraceThreadTag()}: close {name} {TracePointer(file)} {OpenedFiles} ({TraceTime()})\n"); + + Interlocked.Decrement(ref OpenedFiles); + } + + internal void Trace_FRead(int size, object file, string name) { + if (WarningLevel < FileWarningLevel.ReportAllAccessesRead) + return; + + Warning(FileWarningLevel.ReportAllAccessesRead, $"---FS{TraceThreadTag()}: read {name} {size} {TracePointer(file)} ({TraceTime()})\n"); + } + + internal void Trace_FWrite(int size, object file, string name) { + if (WarningLevel < FileWarningLevel.ReportAllAccessesReadWrite) + return; + + Warning(FileWarningLevel.ReportAllAccessesReadWrite, $"---FS{TraceThreadTag()}: write {name} {size} {TracePointer(file)}\n"); + } + + public void PrintOpenedFiles() { + } + + public void AddLoggingFunc(FileSystemLoggingFunc logFunc) { + lock (LogFuncs) { + Assert(!LogFuncs.Contains(logFunc)); + LogFuncs.Add(logFunc); + } + } + + public void RemoveLoggingFunc(FileSystemLoggingFunc logFunc) { + lock (LogFuncs) + LogFuncs.Remove(logFunc); + } + public void PrintSearchPaths() { Msg("Paths:\n"); @@ -959,7 +1048,7 @@ void PrepareFinds(ISearchPath path) { public void Close() { if (Locked == 0) { - Warning("Tried to unlock a file handle that was already unlocked!!!\n"); + Dbg.Warning("Tried to unlock a file handle that was already unlocked!!!\n"); Assert(false); return; } diff --git a/Source.Filesystem/BaseFileSystemAsync.cs b/Source.Filesystem/BaseFileSystemAsync.cs index 84f279c16..589080974 100644 --- a/Source.Filesystem/BaseFileSystemAsync.cs +++ b/Source.Filesystem/BaseFileSystemAsync.cs @@ -172,7 +172,7 @@ FSAsyncStatus SyncRead(in FileAsyncRequest request) { } if (request.Offset + bytesToRead > fileSize) - Warning($"Failed to read {bytesToRead} bytes @ {request.Offset} (end offst = {request.Offset + bytesToRead}) from \"{request.FileName}\" - file only has {fileSize} bytes\n"); + Dbg.Warning($"Failed to read {bytesToRead} bytes @ {request.Offset} (end offst = {request.Offset + bytesToRead}) from \"{request.FileName}\" - file only has {fileSize} bytes\n"); if (request.Offset > 0) stream.Seek(request.Offset, SeekOrigin.Begin); @@ -193,7 +193,9 @@ FSAsyncStatus SyncRead(in FileAsyncRequest request) { DoAsyncCallback(request, dest, Math.Min(bytesRead, bytesToRead), result); } - // todo: fs_log async + if (WarningLevel >= FileWarningLevel.ReportAllAccessesAsync) + LogAccessToFile("async", request.FileName, ""); + return result; } diff --git a/Source.Filesystem/DiskFileHandle.cs b/Source.Filesystem/DiskFileHandle.cs index 17c108a5c..920f5a756 100644 --- a/Source.Filesystem/DiskFileHandle.cs +++ b/Source.Filesystem/DiskFileHandle.cs @@ -3,8 +3,58 @@ using Source.Common.Filesystem; +using System.Buffers; + namespace Source.FileSystem; +public sealed class DiskFileStream(BaseFileSystem? filesystem, string path, string traceName, FileMode mode, FileAccess access, FileShare share) : FileStream(path, mode, access, share) +{ + bool closed; + + public override int Read(byte[] buffer, int offset, int count) { + int bytesRead = base.Read(buffer, offset, count); + filesystem?.Trace_FRead(bytesRead, this, traceName); + return bytesRead; + } + + public override int ReadByte() { + int value = base.ReadByte(); + filesystem?.Trace_FRead(value < 0 ? 0 : 1, this, traceName); + return value; + } + + public override void CopyTo(Stream destination, int bufferSize) { + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try { + int bytesRead; + while ((bytesRead = Read(buffer, 0, buffer.Length)) != 0) + destination.Write(buffer, 0, bytesRead); + } + finally { + ArrayPool.Shared.Return(buffer); + } + } + + public override void Write(byte[] buffer, int offset, int count) { + base.Write(buffer, offset, count); + filesystem?.Trace_FWrite(count, this, traceName); + } + + public override void WriteByte(byte value) { + base.WriteByte(value); + filesystem?.Trace_FWrite(1, this, traceName); + } + + protected override void Dispose(bool disposing) { + if (disposing && !closed) { + closed = true; + filesystem?.Trace_FClose(this, traceName); + } + + base.Dispose(disposing); + } +} + public class DiskFileHandle(IFileSystem filesystem, FileStream data, FileNameHandle_t fileName) : IFileHandle, IDisposable { private bool disposedValue; @@ -15,8 +65,10 @@ public class DiskFileHandle(IFileSystem filesystem, FileStream data, FileNameHan protected virtual void Dispose(bool disposing) { - if (!disposedValue && disposing) + if (!disposedValue && disposing) { data.Dispose(); + disposedValue = true; + } } public void Dispose() { diff --git a/Source.Filesystem/DiskSearchPath.cs b/Source.Filesystem/DiskSearchPath.cs index d53724021..f6709048f 100644 --- a/Source.Filesystem/DiskSearchPath.cs +++ b/Source.Filesystem/DiskSearchPath.cs @@ -120,7 +120,18 @@ public override bool IsFileWritable(ReadOnlySpan path) { FileShare share = access == FileAccess.Read ? FileShare.ReadWrite | FileShare.Delete : FileShare.Read; try { - return new DiskFileHandle(parent, info.Open(mode, access, share), parent.FindOrAddFileName(path)); +#if WIN32 + string traceName = absPath.Replace('/', '\\'); +#else + string traceName = absPath; +#endif + DiskFileHandle handle = new(parent, new DiskFileStream(parent as BaseFileSystem, info.FullName, traceName, mode, access, share), parent.FindOrAddFileName(path)); + (parent as BaseFileSystem)?.Trace_FOpen(traceName, operation switch { + FileOpenOptions.Write => options.Extended() ? "w+" : "w", + FileOpenOptions.Append => options.Extended() ? "a+" : "a", + _ => options.Extended() ? "r+" : "r" + }); + return handle; } catch { return null; @@ -176,6 +187,8 @@ public override DateTime Time(ReadOnlySpan path) { if (!info.Exists) return BaseFileSystem.g_AddonFileSystem.GetFileSize(absPath) >= 0 ? DateTime.UnixEpoch.AddSeconds(1) : DateTime.UnixEpoch; + if (parent is BaseFileSystem fs && fs.LogFuncs.Count != 0) + fs.LogAccessToFile("filetime", absPath, ""); return info.LastWriteTimeUtc; }