Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 48 additions & 35 deletions Source.Filesystem/PackStoreSearchPath.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
using Source.FileSystem;
using Source.VPK;

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;

namespace Source.Filesystem;

Expand Down Expand Up @@ -40,34 +43,32 @@ public PackStore(string key, string vpkPath) {
Key = key;
Vpk.Load(vpkPath);

int entryCount = 0;
foreach (var dir in Vpk.Directories)
entryCount += dir.Entries.Count;

DirectoryLookups.EnsureCapacity(Vpk.Directories.Count);
EntryLookups.EnsureCapacity(entryCount);
EntryLookups.EnsureCapacity(Vpk.EntryCount);

Span<char> buildPath = stackalloc char[260];
foreach (var dir in Vpk.Directories) {
DirectoryLookups[dir.Path.Hash()] = dir;

foreach (var entry in dir.Entries) {
var path = entry.Path.Replace('\\', '/');
var filename = entry.Filename;
var ext = entry.Extension;

int strlen = 0;
if (path.Length > 0 && path[0] != ' ') {
path.CopyTo(buildPath[strlen..]); strlen += path.Length;
buildPath[strlen] = '/'; strlen += 1;
}
filename.CopyTo(buildPath[strlen..]); strlen += filename.Length;
buildPath[strlen] = '.'; strlen += 1;
ext.CopyTo(buildPath[strlen..]); strlen += ext.Length;

ReadOnlySpan<char> finalSpan = buildPath[..strlen];
EntryLookups[finalSpan.Hash()] = entry;
foreach (VpkDirectory dir in Vpk.Directories) {
int pathLength = Encoding.Latin1.GetChars(Vpk.GetPath(in dir), buildPath);
DirectoryLookups[((ReadOnlySpan<char>)buildPath[..pathLength]).Hash()] = dir;
}

foreach (VpkEntry entry in Vpk.EnumerateEntries()) {
ReadOnlySpan<byte> path = Vpk.GetPath(in entry);
ReadOnlySpan<byte> filename = Vpk.GetFilename(in entry);
ReadOnlySpan<byte> ext = Vpk.GetExtension(in entry);

int strlen = 0;
if (path.Length > 0 && path[0] != ' ') {
strlen += Encoding.Latin1.GetChars(path, buildPath[strlen..]);
buildPath[..strlen].Replace('\\', '/');
buildPath[strlen] = '/'; strlen += 1;
}
strlen += Encoding.Latin1.GetChars(filename, buildPath[strlen..]);
buildPath[strlen] = '.'; strlen += 1;
strlen += Encoding.Latin1.GetChars(ext, buildPath[strlen..]);

ReadOnlySpan<char> finalSpan = buildPath[..strlen];
EntryLookups[finalSpan.Hash()] = entry;
}
}
}
Expand Down Expand Up @@ -141,8 +142,9 @@ public override bool IsFileWritable(ReadOnlySpan<char> path) {
}

public override IFileHandle? Open(ReadOnlySpan<char> path, FileOpenOptions options) {
if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry))
return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(entry.Data));
ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash());
if (!Unsafe.IsNullRef(ref entry))
return new VpkFileHandle(parent, parent.FindOrAddFileName(path), new MemoryStream(store!.Vpk.ReadData(in entry)));

return null;
}
Expand All @@ -153,7 +155,8 @@ public override bool IsFileWritable(ReadOnlySpan<char> path) {
public override bool SetFileWritable(ReadOnlySpan<char> path, bool writable) => false;

public override long Size(ReadOnlySpan<char> path) {
if (vpkEntryLookups.TryGetValue(path.Hash(), out VpkEntry? entry))
ref VpkEntry entry = ref CollectionsMarshal.GetValueRefOrNullRef(vpkEntryLookups, path.Hash());
if (!Unsafe.IsNullRef(ref entry))
return entry.EntryLength;
return -1;
}
Expand Down Expand Up @@ -184,9 +187,14 @@ public override void PrepareFinds(List<string> files, List<string> dirs, string?
int wildcardAsteriskAtFile = wildcardFile.IndexOf('*');
int wildcardAsteriskAtExt = wildcardExt.IndexOf('*');

VpkArchive vpk = store!.Vpk;
Span<char> pathBuffer = stackalloc char[260];
Span<char> filenameBuffer = stackalloc char[260];
Span<char> extBuffer = stackalloc char[260];

foreach (var directory in vpkDirectoryLookups) {
VpkDirectory dir = directory.Value;
string path = dir.Path;
ReadOnlySpan<char> path = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in dir), pathBuffer)];
path.FileInfo(null, out var baseDirectory, out var baseName, out _);

if (wildcardAsteriskAtDir == -1) {
Expand All @@ -206,35 +214,40 @@ public override void PrepareFinds(List<string> files, List<string> dirs, string?

foreach (var entryKVP in vpkEntryLookups) {
VpkEntry entry = entryKVP.Value;
ReadOnlySpan<char> entryPath = pathBuffer[..Encoding.Latin1.GetChars(vpk.GetPath(in entry), pathBuffer)];

if (wildcardAsteriskAtDir == -1) {
if (!entry.Path.PathEquals(wildcardDir))
if (!entryPath.PathEquals(wildcardDir))
continue;
}
else {
if (wildcardDir != "*" && !entry.Path.PathStartsWith(wildcardDir))
if (wildcardDir != "*" && !entryPath.PathStartsWith(wildcardDir))
continue;
}

ReadOnlySpan<char> entryFilename = filenameBuffer[..Encoding.Latin1.GetChars(vpk.GetFilename(in entry), filenameBuffer)];

if (wildcardAsteriskAtFile == -1) {
if (!entry.Filename.PathEquals(wildcardFile))
if (!entryFilename.PathEquals(wildcardFile))
continue;
}
else {
if (wildcardFile[0] != '*' && !entry.Filename.PathStartsWith(wildcardFile))
if (wildcardFile[0] != '*' && !entryFilename.PathStartsWith(wildcardFile))
continue;
}

ReadOnlySpan<char> entryExt = extBuffer[..Encoding.Latin1.GetChars(vpk.GetExtension(in entry), extBuffer)];

if (wildcardAsteriskAtExt == -1) {
if (!entry.Extension.PathEquals(wildcardExt))
if (!entryExt.PathEquals(wildcardExt))
continue;
}
else {
if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entry.Extension.PathStartsWith(wildcardExt))
if (!wildcardExt.IsEmpty && wildcardExt[0] != '*' && !entryExt.PathStartsWith(wildcardExt))
continue;
}

files.Add(entry.FilenameAndExtension);
files.Add(string.Concat(entryFilename, ".", entryExt));
}
}
}
132 changes: 126 additions & 6 deletions Source.VPK/VpkArchive.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
using Microsoft.Win32.SafeHandles;

using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

using Source.VPK.Exceptions;
using Source.VPK.V1;

Expand All @@ -13,6 +16,125 @@ public class VpkArchive : IDisposable
internal List<ArchivePart> Parts { get; set; }
internal string ArchivePath { get; set; }
public readonly SafeFileHandle FileHandle;
public byte[] Tree { get; private set; } = [];
public int EntryCount { get; private set; }

const int EntryDataSize = sizeof(uint) + sizeof(ushort) + sizeof(ushort) + sizeof(uint) + sizeof(uint) + sizeof(ushort);

struct TreeReader(byte[] tree)
{
readonly byte[] Tree = tree;
int Position;

public bool ReadString(out int offset, out ushort length) {
ReadOnlySpan<byte> remaining = Tree.AsSpan(Position);
int end = remaining.IndexOf((byte)0);
offset = Position;
if (end == -1) {
end = remaining.Length;
Position += end;
}
else
Position += end + 1;

length = (ushort)end;
return end != 0;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
T Read<T>() where T : unmanaged {
T value = MemoryMarshal.Read<T>(Tree.AsSpan(Position));
Position += Unsafe.SizeOf<T>();
return value;
}

public bool ReadEntry(int pathOffset, ushort pathLength, int extensionOffset, ushort extensionLength, out VpkEntry entry) {
entry = default;
if (!ReadString(out int filenameOffset, out ushort filenameLength))
return false;

if (Tree.Length - Position < EntryDataSize)
return false;

uint crc = Read<uint>();
ushort preloadBytes = Read<ushort>();
ushort archiveIndex = Read<ushort>();
uint entryOffset = Read<uint>();
uint entryLength = Read<uint>();
// skip terminator
Read<ushort>();
int preloadTreeOffset = Position;
Position = Math.Min(Position + preloadBytes, Tree.Length);

entry = new VpkEntry(pathOffset, pathLength, filenameOffset, filenameLength, extensionOffset, extensionLength,
crc, preloadBytes, preloadTreeOffset, archiveIndex, entryOffset, entryLength);
return true;
}
}

static void LowerAscii(Span<byte> text) {
for (int i = 0; i < text.Length; i++) {
if ((uint)(text[i] - 'A') <= 'Z' - 'A')
text[i] |= 0x20;
}
}

void ParseTree() {
TreeReader reader = new(Tree);
while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) {
LowerAscii(Tree.AsSpan(extensionOffset, extensionLength)); // once per extension block
while (reader.ReadString(out int pathOffset, out ushort pathLength)) {
LowerAscii(Tree.AsSpan(pathOffset, pathLength));
Directories.Add(new VpkDirectory(pathOffset, pathLength));
while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry)) {
LowerAscii(Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength));
EntryCount++;
}
}
}
}

public IEnumerable<VpkEntry> EnumerateEntries() {
TreeReader reader = new(Tree);
while (reader.ReadString(out int extensionOffset, out ushort extensionLength)) {
while (reader.ReadString(out int pathOffset, out ushort pathLength)) {
while (reader.ReadEntry(pathOffset, pathLength, extensionOffset, extensionLength, out VpkEntry entry))
yield return entry;
}
}
}

public ReadOnlySpan<byte> GetPath(in VpkDirectory directory) => Tree.AsSpan(directory.PathOffset, directory.PathLength);
public ReadOnlySpan<byte> GetPath(in VpkEntry entry) => Tree.AsSpan(entry.PathOffset, entry.PathLength);
public ReadOnlySpan<byte> GetFilename(in VpkEntry entry) => Tree.AsSpan(entry.FilenameOffset, entry.FilenameLength);
public ReadOnlySpan<byte> GetExtension(in VpkEntry entry) => Tree.AsSpan(entry.ExtensionOffset, entry.ExtensionLength);
public ReadOnlySpan<byte> GetPreloadData(in VpkEntry entry) => Tree.AsSpan(entry.PreloadTreeOffset, Math.Min(entry.PreloadBytes, Tree.Length - entry.PreloadTreeOffset));

public byte[]? ReadPreloadData(in VpkEntry entry) => entry.HasPreloadData ? GetPreloadData(in entry).ToArray() : null;
public byte[]? ReadAnyData(in VpkEntry entry) => entry.HasPreloadData ? ReadPreloadData(in entry) : ReadData(in entry);

ArchivePart? FindPart(ushort archiveIndex) {
if (Parts == null)
return null;

for (int i = 0, c = Parts.Count; i < c; i++) {
ArchivePart part = Parts[i];
if (part.Index == archiveIndex)
return part;
}
return null;
}

public byte[] ReadData(in VpkEntry entry) {
ArchivePart? part = FindPart(entry.ArchiveIndex);
if (part == null)
return [];

byte[] data = new byte[entry.PreloadBytes + entry.EntryLength];
GetPreloadData(in entry).CopyTo(data);
RandomAccess.Read(part.FileHandle, data.AsSpan(entry.PreloadBytes), entry.EntryOffset);
return data;
}

public override string ToString() => $"VpkArchive '{ArchivePath}' [{Directories.Count} directories, {Parts.Count} parts]";

Expand Down Expand Up @@ -66,9 +188,8 @@ public void Load(string filename, VpkVersions.Versions version = VpkVersions.Ver
break;
}

var reader = _reader.ReadDirectories(this, header.TreeLength);
while (reader.MoveNext())
Directories.Add(reader.Current);
Tree = _reader.ReadTree(header.TreeLength);
ParseTree();
}
finally {
_reader?.Dispose();
Expand Down Expand Up @@ -109,9 +230,8 @@ public void Load(byte[] file, VpkVersions.Versions version = VpkVersions.Version
break;
}

var reader = _reader.ReadDirectories(this, header.TreeLength);
while (reader.MoveNext())
Directories.Add(reader.Current);
Tree = _reader.ReadTree(header.TreeLength);
ParseTree();
}
finally {
_reader?.Dispose();
Expand Down
20 changes: 6 additions & 14 deletions Source.VPK/VpkDirectory.cs
Original file line number Diff line number Diff line change
@@ -1,18 +1,10 @@
namespace Source.VPK
{
public class VpkDirectory
{
public List<VpkEntry> Entries { get; set; }
public string Path { get; set; }
internal VpkArchive ParentArchive { get; set; }
public readonly struct VpkDirectory(int pathOffset, ushort pathLength)
{
public readonly int PathOffset = pathOffset;
public readonly ushort PathLength = pathLength;

public override string ToString() => $"VpkDirectory '{Path}' [{Entries.Count} entries]";

internal VpkDirectory(VpkArchive parentArchive, string path, List<VpkEntry> entries)
{
ParentArchive = parentArchive;
Path = path.ToLower();
Entries = entries;
}
}
public override string ToString() => $"VpkDirectory [path<{PathOffset}-{PathLength}>]";
}
}
Loading
Loading