From 33f9f5465baaaf25f33318df936341ba51cbeaac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=B6eh=20Matt?= <5415177+ZehMatt@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:14:52 +0300 Subject: [PATCH] Store VPK entries as structs indexing the tree buffer --- Source.Filesystem/PackStoreSearchPath.cs | 83 ++++++++------ Source.VPK/VpkArchive.cs | 132 +++++++++++++++++++++-- Source.VPK/VpkDirectory.cs | 20 ++-- Source.VPK/VpkEntry.cs | 104 ++++-------------- Source.VPK/VpkReaderBase.cs | 116 ++------------------ 5 files changed, 206 insertions(+), 249 deletions(-) diff --git a/Source.Filesystem/PackStoreSearchPath.cs b/Source.Filesystem/PackStoreSearchPath.cs index bef6f48d1..a6ff372ee 100644 --- a/Source.Filesystem/PackStoreSearchPath.cs +++ b/Source.Filesystem/PackStoreSearchPath.cs @@ -2,6 +2,9 @@ using Source.FileSystem; using Source.VPK; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; namespace Source.Filesystem; @@ -40,34 +43,32 @@ public PackStore(string key, string vpkPath) { Key = key; Vpk.Load(vpkPath); - int entryCount = 0; - foreach (var dir in Vpk.Directories) - entryCount += dir.Entries.Count; - DirectoryLookups.EnsureCapacity(Vpk.Directories.Count); - EntryLookups.EnsureCapacity(entryCount); + EntryLookups.EnsureCapacity(Vpk.EntryCount); Span buildPath = stackalloc char[260]; - foreach (var dir in Vpk.Directories) { - DirectoryLookups[dir.Path.Hash()] = dir; - - foreach (var entry in dir.Entries) { - var path = entry.Path.Replace('\\', '/'); - var filename = entry.Filename; - var ext = entry.Extension; - - int strlen = 0; - if (path.Length > 0 && path[0] != ' ') { - path.CopyTo(buildPath[strlen..]); strlen += path.Length; - buildPath[strlen] = '/'; strlen += 1; - } - filename.CopyTo(buildPath[strlen..]); strlen += filename.Length; - buildPath[strlen] = '.'; strlen += 1; - ext.CopyTo(buildPath[strlen..]); strlen += ext.Length; - - ReadOnlySpan finalSpan = buildPath[..strlen]; - EntryLookups[finalSpan.Hash()] = entry; + foreach (VpkDirectory dir in Vpk.Directories) { + int pathLength = Encoding.Latin1.GetChars(Vpk.GetPath(in dir), buildPath); + DirectoryLookups[((ReadOnlySpan)buildPath[..pathLength]).Hash()] = dir; + } + + foreach (VpkEntry entry in Vpk.EnumerateEntries()) { + ReadOnlySpan path = Vpk.GetPath(in entry); + ReadOnlySpan filename = Vpk.GetFilename(in entry); + ReadOnlySpan ext = Vpk.GetExtension(in entry); + + int strlen = 0; + if (path.Length > 0 && path[0] != ' ') { + strlen += Encoding.Latin1.GetChars(path, buildPath[strlen..]); + buildPath[..strlen].Replace('\\', '/'); + buildPath[strlen] = '/'; strlen += 1; } + strlen += Encoding.Latin1.GetChars(filename, buildPath[strlen..]); + buildPath[strlen] = '.'; strlen += 1; + strlen += Encoding.Latin1.GetChars(ext, buildPath[strlen..]); + + ReadOnlySpan finalSpan = buildPath[..strlen]; + EntryLookups[finalSpan.Hash()] = entry; } } } @@ -141,8 +142,9 @@ public override bool IsFileWritable(ReadOnlySpan path) { } public override IFileHandle? Open(ReadOnlySpan path, FileOpenOptions options) { - if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry)) - return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(entry.Data)); + ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash()); + if (!Unsafe.IsNullRef(ref entry)) + return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(store!.Vpk.ReadData(in entry))); return null; } @@ -153,7 +155,8 @@ public override bool IsFileWritable(ReadOnlySpan path) { public override bool SetFileWritable(ReadOnlySpan path, bool writable) => false; public override long Size(ReadOnlySpan path) { - if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry)) + ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash()); + if (!Unsafe.IsNullRef(ref entry)) return entry.EntryLength; return -1; } @@ -184,9 +187,14 @@ public override void PrepareFinds(List files, List dirs, string? int wildcardAsteriskAtFile = wildcardFile.IndexOf('*'); int wildcardAsteriskAtExt = wildcardExt.IndexOf('*'); + VpkArchive vpk = store!.Vpk; + Span pathBuffer = stackalloc char[260]; + Span filenameBuffer = stackalloc char[260]; + Span extBuffer = stackalloc char[260]; + foreach (var directory in vpkDirectoryLookups) { VpkDirectory dir = directory.Value; - string path = dir.Path; + ReadOnlySpan path = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in dir), pathBuffer)]; path.FileInfo(null, out var baseDirectory, out var baseName, out _); if (wildcardAsteriskAtDir == -1) { @@ -206,35 +214,40 @@ public override void PrepareFinds(List files, List dirs, string? foreach (var entryKVP in vpkEntryLookups) { VpkEntry entry = entryKVP.Value; + ReadOnlySpan entryPath = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in entry), pathBuffer)]; if (wildcardAsteriskAtDir == -1) { - if (!entry.Path.PathEquals(wildcardDir)) + if (!entryPath.PathEquals(wildcardDir)) continue; } else { - if (wildcardDir != "*" && !entry.Path.PathStartsWith(wildcardDir)) + if (wildcardDir != "*" && !entryPath.PathStartsWith(wildcardDir)) continue; } + ReadOnlySpan entryFilename = filenameBuffer[..Encoding.Latin1.GetChars(vpk.GetFilename(in entry), filenameBuffer)]; + if (wildcardAsteriskAtFile == -1) { - if (!entry.Filename.PathEquals(wildcardFile)) + if (!entryFilename.PathEquals(wildcardFile)) continue; } else { - if (wildcardFile[0] != '*' && !entry.Filename.PathStartsWith(wildcardFile)) + if (wildcardFile[0] != '*' && !entryFilename.PathStartsWith(wildcardFile)) continue; } + ReadOnlySpan entryExt = extBuffer[..Encoding.Latin1.GetChars(vpk.GetExtension(in entry), extBuffer)]; + if (wildcardAsteriskAtExt == -1) { - if (!entry.Extension.PathEquals(wildcardExt)) + if (!entryExt.PathEquals(wildcardExt)) continue; } else { - if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entry.Extension.PathStartsWith(wildcardExt)) + if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entryExt.PathStartsWith(wildcardExt)) continue; } - files.Add(entry.FilenameAndExtension); + files.Add(string.Concat(entryFilename, ".", entryExt)); } } } diff --git a/Source.VPK/VpkArchive.cs b/Source.VPK/VpkArchive.cs index 0438b8697..09d57be81 100644 --- a/Source.VPK/VpkArchive.cs +++ b/Source.VPK/VpkArchive.cs @@ -1,5 +1,8 @@ using Microsoft.Win32.SafeHandles; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + using Source.VPK.Exceptions; using Source.VPK.V1; @@ -13,6 +16,125 @@ public class VpkArchive : IDisposable internal List Parts { get; set; } internal string ArchivePath { get; set; } public readonly SafeFileHandle FileHandle; + public byte[] Tree { get; private set; } = []; + public int EntryCount { get; private set; } + + const int EntryDataSize = sizeof(uint) + sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(uint) + sizeof(ushort); + + struct TreeReader(byte[] tree) + { + readonly byte[] Tree = tree; + int Position; + + public bool ReadString(out int offset, out ushort length) { + ReadOnlySpan remaining = Tree.AsSpan(Position); + int end = remaining.IndexOf((byte)0); + offset = Position; + if (end == -1) { + end = remaining.Length; + Position += end; + } + else + Position += end + 1; + + length = (ushort)end; + return end != 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + T Read() where T : unmanaged { + T value = MemoryMarshal.Read(Tree.AsSpan(Position)); + Position += Unsafe.SizeOf(); + return value; + } + + public bool ReadEntry(int pathOffset, ushort pathLength, int extensionOffset, ushort extensionLength, out VpkEntry entry) { + entry = default; + if (!ReadString(out int filenameOffset, out ushort filenameLength)) + return false; + + if (Tree.Length - Position < EntryDataSize) + return false; + + uint crc = Read(); + ushort preloadBytes = Read(); + ushort archiveIndex = Read(); + uint entryOffset = Read(); + uint entryLength = Read(); + // skip terminator + Read(); + int preloadTreeOffset = Position; + Position = Math.Min(Position + preloadBytes, Tree.Length); + + entry = new VpkEntry(pathOffset, pathLength, filenameOffset, filenameLength, extensionOffset, extensionLength, + crc, preloadBytes, preloadTreeOffset, archiveIndex, entryOffset, entryLength); + return true; + } + } + + static void LowerAscii(Span text) { + for (int i = 0; i < text.Length; i++) { + if ((uint)(text[i] - 'A') <= 'Z' - 'A') + text[i] |= 0x20; + } + } + + void ParseTree() { + TreeReader reader = new(Tree); + while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) { + LowerAscii(Tree.AsSpan(extensionOffset, extensionLength)); // once per extension block + while (reader.ReadString(out int pathOffset, out ushort pathLength)) { + LowerAscii(Tree.AsSpan(pathOffset, pathLength)); + Directories.Add(new VpkDirectory(pathOffset, pathLength)); + while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry)) { + LowerAscii(Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength)); + EntryCount++; + } + } + } + } + + public IEnumerable EnumerateEntries() { + TreeReader reader = new(Tree); + while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) { + while (reader.ReadString(out int pathOffset, out ushort pathLength)) { + while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry)) + yield return entry; + } + } + } + + public ReadOnlySpan GetPath(in VpkDirectory directory) => Tree.AsSpan(directory.PathOffset, directory.PathLength); + public ReadOnlySpan GetPath(in VpkEntry entry) => Tree.AsSpan(entry.PathOffset, entry.PathLength); + public ReadOnlySpan GetFilename(in VpkEntry entry) => Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength); + public ReadOnlySpan GetExtension(in VpkEntry entry) => Tree.AsSpan(entry.ExtensionOffset, entry.ExtensionLength); + public ReadOnlySpan GetPreloadData(in VpkEntry entry) => Tree.AsSpan(entry.PreloadTreeOffset, Math.Min(entry.PreloadBytes, Tree.Length - entry.PreloadTreeOffset)); + + public byte[]? ReadPreloadData(in VpkEntry entry) => entry.HasPreloadData ? GetPreloadData(in entry).ToArray() : null; + public byte[]? ReadAnyData(in VpkEntry entry) => entry.HasPreloadData ? ReadPreloadData(in entry) : ReadData(in entry); + + ArchivePart? FindPart(ushort archiveIndex) { + if (Parts == null) + return null; + + for (int i = 0, c = Parts.Count; i < c; i++) { + ArchivePart part = Parts[i]; + if (part.Index == archiveIndex) + return part; + } + return null; + } + + public byte[] ReadData(in VpkEntry entry) { + ArchivePart? part = FindPart(entry.ArchiveIndex); + if (part == null) + return []; + + byte[] data = new byte[entry.PreloadBytes + entry.EntryLength]; + GetPreloadData(in entry).CopyTo(data); + RandomAccess.Read(part.FileHandle, data.AsSpan(entry.PreloadBytes), entry.EntryOffset); + return data; + } public override string ToString() => $"VpkArchive '{ArchivePath}' [{Directories.Count} directories, {Parts.Count} parts]"; @@ -66,9 +188,8 @@ public void Load(string filename, VpkVersions.Versions version = VpkVersions.Ver break; } - var reader = _reader.ReadDirectories(this, header.TreeLength); - while (reader.MoveNext()) - Directories.Add(reader.Current); + Tree = _reader.ReadTree(header.TreeLength); + ParseTree(); } finally { _reader?.Dispose(); @@ -109,9 +230,8 @@ public void Load(byte[] file, VpkVersions.Versions version = VpkVersions.Version break; } - var reader = _reader.ReadDirectories(this, header.TreeLength); - while (reader.MoveNext()) - Directories.Add(reader.Current); + Tree = _reader.ReadTree(header.TreeLength); + ParseTree(); } finally { _reader?.Dispose(); diff --git a/Source.VPK/VpkDirectory.cs b/Source.VPK/VpkDirectory.cs index 64835c2c1..cb3d0c0d0 100644 --- a/Source.VPK/VpkDirectory.cs +++ b/Source.VPK/VpkDirectory.cs @@ -1,18 +1,10 @@ namespace Source.VPK { - public class VpkDirectory - { - public List Entries { get; set; } - public string Path { get; set; } - internal VpkArchive ParentArchive { get; set; } + public readonly struct VpkDirectory(int pathOffset, ushort pathLength) + { + public readonly int PathOffset = pathOffset; + public readonly ushort PathLength = pathLength; - public override string ToString() => $"VpkDirectory '{Path}' [{Entries.Count} entries]"; - - internal VpkDirectory(VpkArchive parentArchive, string path, List entries) - { - ParentArchive = parentArchive; - Path = path.ToLower(); - Entries = entries; - } - } + public override string ToString() => $"VpkDirectory [path<{PathOffset}-{PathLength}>]"; + } } diff --git a/Source.VPK/VpkEntry.cs b/Source.VPK/VpkEntry.cs index 69fcc5172..6df4eb44c 100644 --- a/Source.VPK/VpkEntry.cs +++ b/Source.VPK/VpkEntry.cs @@ -1,90 +1,24 @@ namespace Source.VPK { - public class VpkEntry + public readonly struct VpkEntry(int pathOffset, ushort pathLength, int filenameOffset, ushort filenameLength, int extensionOffset, ushort extensionLength, + uint crc, ushort preloadBytes, int preloadTreeOffset, ushort archiveIndex, uint entryOffset, uint entryLength) { - public string Extension { get; set; } - public string Path { get; set; } - public string Filename { get; set; } - public string FilenameAndExtension { get => field ??= Filename + "." + Extension; set; } - public byte[] PreloadData { get { return ReadPreloadData(); } } - public byte[] Data { get { return ReadData(); } } - public bool HasPreloadData { get; set; } - - public readonly uint CRC; - public readonly ushort PreloadBytes; - public readonly nuint PreloadDataOffset; - public readonly ushort ArchiveIndex; - public readonly uint EntryOffset; - public readonly uint EntryLength; - public readonly VpkArchive ParentArchive; - - public override string ToString() => $"VpkEntry '{Path}/{Filename}.{Extension}' [crc {CRC}, entry<{EntryOffset}-{EntryLength}>]"; - - internal VpkEntry(VpkArchive parentArchive, uint crc, ushort preloadBytes, nuint preloadDataOffset, ushort archiveIndex, uint entryOffset, - uint entryLength, string extension, string path, string filename) { - ParentArchive = parentArchive; - CRC = crc; - PreloadBytes = preloadBytes; - PreloadDataOffset = preloadDataOffset; - ArchiveIndex = archiveIndex; - EntryOffset = entryOffset; - EntryLength = entryLength; - Extension = extension; - Path = path; - Filename = filename; - HasPreloadData = preloadBytes > 0; - - } - - private byte[] ReadPreloadData() { - if (PreloadBytes > 0) { - var buff = new byte[PreloadBytes]; - using (var fs = new FileStream(ParentArchive.ArchivePath, FileMode.Open, FileAccess.Read)) { - buff = new byte[PreloadBytes]; - fs.Seek((long)PreloadDataOffset, SeekOrigin.Begin); - fs.Read(buff, 0, buff.Length); - } - return buff; - } - return null; - } - - private byte[]? dataCache; - ArchivePart? partFile; - - private byte[] ReadData() { - if (dataCache != null) - return dataCache; - - if (partFile == null) { - List parentParts = ParentArchive.Parts; - for (int i = 0, c = parentParts.Count; i < c; i++) { - ArchivePart part = parentParts[i]; - if (part.Index == ArchiveIndex) { - partFile = part; - break; - } - } - } - - if (partFile == null) - return []; - //throw new Exception("Part file was null!"); - - if (HasPreloadData) { - dataCache = new byte[PreloadBytes + EntryLength]; - - RandomAccess.Read(ParentArchive.FileHandle, dataCache.AsSpan()[..PreloadBytes], (long)PreloadDataOffset); - RandomAccess.Read(ParentArchive.FileHandle, dataCache.AsSpan()[PreloadBytes..][..(int)EntryLength], (long)EntryOffset); - } - else { - dataCache = new byte[EntryLength]; - RandomAccess.Read(partFile.FileHandle, dataCache.AsSpan(), (long)EntryOffset); - } - - return dataCache; - } - - public byte[] AnyData { get { if (this.HasPreloadData) return this.ReadPreloadData(); else return this.ReadData(); } } + public readonly int PathOffset = pathOffset; + public readonly int FilenameOffset = filenameOffset; + public readonly int ExtensionOffset = extensionOffset; + public readonly ushort PathLength = pathLength; + public readonly ushort FilenameLength = filenameLength; + public readonly ushort ExtensionLength = extensionLength; + + public readonly uint CRC = crc; + public readonly ushort PreloadBytes = preloadBytes; + public readonly ushort ArchiveIndex = archiveIndex; + public readonly int PreloadTreeOffset = preloadTreeOffset; + public readonly uint EntryOffset = entryOffset; + public readonly uint EntryLength = entryLength; + + public bool HasPreloadData => PreloadBytes > 0; + + public override string ToString() => $"VpkEntry [crc {CRC}, entry<{EntryOffset}-{EntryLength}>]"; } } diff --git a/Source.VPK/VpkReaderBase.cs b/Source.VPK/VpkReaderBase.cs index b0f185611..6abb67faa 100644 --- a/Source.VPK/VpkReaderBase.cs +++ b/Source.VPK/VpkReaderBase.cs @@ -1,18 +1,11 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Text; namespace Source.VPK { internal abstract class VpkReaderBase : IDisposable { - const int EntryDataSize = sizeof(uint) + sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(uint) + sizeof(ushort); - readonly Stream stream; - readonly List entryBuffer = []; - byte[] tree = []; - int treePosition; - long treeOffset; protected VpkReaderBase(string filename) => stream = File.OpenRead(filename); protected VpkReaderBase(byte[] file) => stream = new MemoryStream(file); @@ -21,115 +14,20 @@ internal abstract class VpkReaderBase : IDisposable public abstract IVpkArchiveHeader ReadArchiveHeader(); - void LoadTree(uint treeLength) { - treeOffset = stream.Position; - long remaining = Math.Max(stream.Length - treeOffset, 0); - tree = new byte[Math.Min(treeLength, remaining)]; - int read = stream.ReadAtLeast(tree, tree.Length, false); - if (read != tree.Length) - Array.Resize(ref tree, read); - treePosition = 0; - } - - string ReadNullTerminatedString() { - ReadOnlySpan remaining = tree.AsSpan(treePosition); - int length = remaining.IndexOf((byte)0); - if (length == -1) { - length = remaining.Length; - treePosition += length; - } - else - treePosition += length + 1; - - return length == 0 ? string.Empty : Encoding.Latin1.GetString(remaining[..length]); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - T ReadTree() where T : unmanaged { - T value = MemoryMarshal.Read(tree.AsSpan(treePosition)); - treePosition += Unsafe.SizeOf(); - return value; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] protected static ref readonly T BytesToStructure(ReadOnlySpan bytearray) where T : unmanaged { return ref MemoryMarshal.Cast(bytearray[..Unsafe.SizeOf()])[0]; } - public struct DirectoryReader(VpkReaderBase reader, VpkArchive parentArchive) - { - readonly VpkReaderBase Reader = reader; - readonly VpkArchive ParentArchive = parentArchive; - string? ext = null; - public VpkDirectory Current = null!; - - public bool MoveNext() { - while (true) { - if (ext == null) { - ext = Reader.ReadNullTerminatedString(); - if (string.IsNullOrEmpty(ext)) - return false; // end of tree - ext = ext.ToLowerInvariant(); // once per extension block - } - - var path = Reader.ReadNullTerminatedString(); - if (string.IsNullOrEmpty(path)) { - ext = null; // end of this ext's paths, get the next one - continue; - } - - Current = new VpkDirectory(ParentArchive, path, - Reader.ReadEntries(ParentArchive, ext, path).ToList()); - return true; - } - } - } - #region default - public DirectoryReader ReadDirectories(VpkArchive parentArchive, uint treeLength) { - LoadTree(treeLength); - return new(this, parentArchive); - } - - public struct EntryReader(VpkReaderBase reader, VpkArchive parentArchive, string ext, string path) - { - readonly VpkReaderBase Reader = reader; - readonly VpkArchive ParentArchive = parentArchive; - readonly string Ext = ext; - readonly string Path = path; - public bool MoveNext() { - var fileName = Reader.ReadNullTerminatedString(); - if (string.IsNullOrEmpty(fileName)) - return false; - - if (Reader.tree.Length - Reader.treePosition < EntryDataSize) - return false; - - var crc = Reader.ReadTree(); - var preloadBytes = Reader.ReadTree(); - var archiveIdx = Reader.ReadTree(); - var entryOffset = Reader.ReadTree(); - var entryLen = Reader.ReadTree(); - // skip terminator - Reader.ReadTree(); - nuint preloadDataOffset = (nuint)(Reader.treeOffset + Reader.treePosition); - Reader.treePosition = Math.Min(Reader.treePosition + preloadBytes, Reader.tree.Length); - - Current = new VpkEntry(ParentArchive, crc, preloadBytes, preloadDataOffset, archiveIdx, entryOffset, entryLen, Ext, Path, fileName.ToLowerInvariant()); - return true; - } - - public List ToList() { - List buffer = Reader.entryBuffer; - buffer.Clear(); - while (MoveNext()) - buffer.Add(Current); - return new(buffer); - } - - public VpkEntry Current = null!; + public byte[] ReadTree(uint treeLength) { + long remaining = Math.Max(stream.Length - stream.Position, 0); + byte[] tree = new byte[Math.Min(treeLength, remaining)]; + int read = stream.ReadAtLeast(tree, tree.Length, false); + if (read != tree.Length) + Array.Resize(ref tree, read); + return tree; } - public EntryReader ReadEntries(VpkArchive parentArchive, string ext, string path) => new(this, parentArchive, ext, path.ToLowerInvariant()); [MethodImpl(MethodImplOptions.AggressiveInlining)] protected T Read() where T : unmanaged {