cardidi/Flawless-Version-Control
1
0
Fork 0
Code

feat: Add standard ata extractor and adjust format

Browse Source
This commit is contained in:
Ca2didi 2025-03-21 00:05:18 +08:00
parent 733e2b7a63
commit c332d15db6
8 changed files with 228 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
Flawless.Core/BinaryDataFormat/DataExtractor.cs Normal file
View File

@ -0,0 +1,170 @@
using System.IO.Hashing;
using System.Text;
using Flawless.Core.Modal;
using Nerdbank.Streams;
namespace Flawless.Core.BinaryDataFormat;
public static class DataExtractor
{
public static byte GuessStandardDepotHeaderVersion(Stream depotStream)
{
depotStream.Seek(8, SeekOrigin.Current);
var val = depotStream.ReadByte();
depotStream.Seek(-9, SeekOrigin.Current);
if (val < 0) ;
return (byte)val;
}
public static byte GuessNetworkDepotHeaderVersion(Stream depotStream)
{
var val = depotStream.ReadByte();
depotStream.Seek(-1, SeekOrigin.Current);
if (val < 0) ;
return (byte)val;
}
public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream)
{
StandardDepotHeaderV1 r = default;
_ = depotStream.Position; // Test if can seek
try
{
using var reader = new BinaryReader(depotStream, Encoding.ASCII, true);
// Magic number check
r.MagicNumber = reader.ReadUInt32();
if (r.MagicNumber != StandardDepotHeaderV1.FormatMagicNumber)
throw new InvalidDataException("File may not a depot or broken.");
// Do Crc checking
r.HeaderCRCChecksum = reader.ReadUInt32();
Span<byte> crcArea = stackalloc byte[64 - 8];
if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!");
if (Crc32.HashToUInt32(crcArea) != r.HeaderCRCChecksum)
throw new InvalidDataException("Header CRC check is failed!");
depotStream.Seek(8 - 64, SeekOrigin.Current);
// Let's read file
r.Version = reader.ReadByte();
if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}");
r.CompressType = reader.ReadByte();
reader.ReadInt16(); // Preserved
reader.ReadInt32(); // Preserved
r.DepotMd5ChecksumLower = reader.ReadUInt64();
r.DepotMd5ChecksumUpper = reader.ReadUInt64();
r.GenerateTime = reader.ReadUInt64();
r.FileMapSize = reader.ReadUInt64();
r.PayloadSize = reader.ReadUInt64();
reader.ReadUInt64(); // Preserved
}
catch (EndOfStreamException e)
{
throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
}
return r;
}
public static NetworkDepotHeaderV1 ExtractNetworkDepotHeaderV1(Stream depotStream)
{
NetworkDepotHeaderV1 r;
_ = depotStream.Position; // Test if can seek
try
{
using var reader = new BinaryReader(depotStream, Encoding.ASCII, true);
// Let's read file
r.Version = reader.ReadByte();
if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}");
r.NetworkTransmissionFeature = reader.ReadByte();
reader.ReadUInt16(); // Preserve
reader.ReadUInt32(); // Preserve
r.Md5ChecksumLower = reader.ReadUInt64();
r.Md5ChecksumUpper = reader.ReadUInt64();
r.GenerateTime = reader.ReadUInt64();
r.FileMapSize = reader.ReadUInt64();
r.PayloadSize = reader.ReadUInt64();
}
catch (EndOfStreamException e)
{
throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
}
return r;
}
public static Stream ExtractStandardDepotFile(Stream depotStream, DepotFileInfo fileInfo)
{
return ExtractStandardDepotFile(depotStream, fileInfo.Size, fileInfo.Offset);
}
public static Stream ExtractStandardDepotFile(Stream depotStream, ulong size, ulong offset)
{
depotStream.Seek((long)offset + 64, SeekOrigin.Begin);
return depotStream.ReadSlice((long) size);
}
public static async IAsyncEnumerable<DepotFileInfo> ExtractDepotFileInfoMapAsync(ulong fileMapSize, Stream depotStream)
{
var splitor = '$';
depotStream.Seek((long) fileMapSize, SeekOrigin.End);
var state = -1;
var buffer = new char[64];
var builder = new StringBuilder();
using var reader = new StreamReader(depotStream, Encoding.UTF8, leaveOpen: true);
var path = string.Empty;
var size = 0UL;
var offset = 0UL;
// Read loop
while (true)
{
try
{
var length = await reader.ReadBlockAsync(buffer, 0, 64);
if (length == 0) break;
for (int i = 0; i < length; i++)
{
var c = buffer[i];
if (c != splitor) builder.Append(c);
else
{
switch ((state = (state + 1) % 3))
{
case 0: path = builder.ToString(); break;
case 1: size = ulong.Parse(builder.ToString()); break;
case 2: offset = ulong.Parse(builder.ToString()); break;
}
builder.Clear();
}
}
}
catch (EndOfStreamException e)
{
throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
}
// Do output at here.
yield return new DepotFileInfo(offset, size, path);
}
// Check is this the real ending
if (builder.Length > 0 || state != 0)
throw new InvalidDataException("Stream is too small! Maybe file is broken.");
}
}

36
Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs
View File

@ -1,12 +1,13 @@
using System.Runtime.InteropServices;
using Flawless.Abstraction;
namespace Flawless.Core.BinaryDataFormat;
/* Depot Transmission Format Design - Version 1
*
* We have shrink some layout design and remap fields in order to optimize for networking transmission. You may noticed
* that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk,
* when using network transmission, we may compress it from outside. So we let compressing to go.
* that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk.
* When using network transmission, we may compress it from outside. So we let compressing to go.
*
* Notice that we assume that all data are represent as LITTLE ENDIAN.
*
@ -23,7 +24,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 6 : (Preserve)
* 7 : (Preserve)
* ------------------------------------------------------------
* 8 : File Map String Size
* 8 : MD5 Checksum (Standard DepotMD5Checksum)
* 9 : ~
* 10 : ~
* 11 : ~
@ -33,7 +34,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 14 : ~
* 15 : ~
* ------------------------------------------------------------
* 16 : MD5 Checksum (Standard DepotMD5Checksum)
* 16 : ~
* 17 : ~
* 18 : ~
* 19 : ~
@ -43,7 +44,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 22 : ~
* 23 : ~
* ------------------------------------------------------------
* 24 :
* 24 : Depot Generate Time
* 25 : ~
* 26 : ~
* 27 : ~
@ -53,7 +54,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 30 : ~
* 31 : ~
* ------------------------------------------------------------
* 32 : Depot Generate Time
* 32 : File Map Size
* 33 : ~
* 34 : ~
* 35 : ~
@ -73,19 +74,17 @@ namespace Flawless.Core.BinaryDataFormat;
* 46 : ~
* 47 : ~
* ------------------------------------------------------------
* PAYLOAD (OPTIONAL)
* PAYLOAD (Optional, Binary, Uncompress)
* ------------------------------------------------------------
* FILE NAME MAP (OPTIONAL)
* FILE MAP (Optional, UTF-8, Uncompress)
* ------------------------------------------------------------
*/
[Flags]
public enum NetworkTransmissionFeatureFlag: byte
{
FileMapIsJson = 1 << 0,
WithPayload = 1 << 0,
WithFileMap = 1 << 1,
WithPayload = 1 << 2,
CompressFileMap = 1 << 7,
}
[Serializable, StructLayout(LayoutKind.Explicit, CharSet = CharSet.Ansi, Pack = 4, Size = 48)]
@ -93,16 +92,15 @@ public struct NetworkDepotHeaderV1
{
[FieldOffset(0)] public byte Version;
[FieldOffset(1)] public NetworkTransmissionFeatureFlag NetworkTransmissionFeature;
[FieldOffset(8)] public ulong FileMapStringSize;
[FieldOffset(1)] public byte NetworkTransmissionFeature;
[FieldOffset(16)] public ulong Md5ChecksumLower;
[FieldOffset(8)] public ulong Md5ChecksumLower;
[FieldOffset(24)] public ulong Md5ChecksumUpper;
[FieldOffset(16)] public ulong Md5ChecksumUpper;
[FieldOffset(32)] public ulong GenerateTime;
[FieldOffset(24)] public ulong GenerateTime;
[FieldOffset(32)] public ulong FileMapSize;
[FieldOffset(40)] public ulong PayloadSize;
}
}

57
Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs
View File

@ -1,13 +1,12 @@
using System.Runtime.InteropServices;
using Flawless.Abstraction;
using Flawless.Core.Modal;
namespace Flawless.Core.BinaryDataFormat;
/* Depot File System Format Design - Version 1
*
* For best accessing performance, consider use checksum as filename. Binary info did not contains file name mapping, so
* use another file to get the file map of this depot. The structure to describe a map has already defined below and you
* can choose binary or text to store them. Advice to use JSON as text-based solution.
* For best accessing performance, consider use checksum as filename.
*
* Consider of compability when depot format was updated, we have configure a lots of area as empty.
*
@ -34,7 +33,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 14 : (Preserve)
* 15 : (Preserve)
* ------------------------------------------------------------
* 16 : File Map MD5 Checksum (From extern map data)
* 16 : Depot MD5 Checksum (From 48 to end, uncompressed)
* 17 : ~
* 18 : ~
* 19 : ~
@ -43,7 +42,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 22 : ~
* 23 : ~
* ------------------------------------------------------------
* 24 :
* 24 : ~
* 25 : ~
* 26 : ~
* 27 : ~
@ -52,7 +51,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 30 : ~
* 31 : ~
* ------------------------------------------------------------
* 32 : Depot MD5 Checksum (From 48 to end, uncompressed)
* 32 : Depot Generate Time
* 33 : ~
* 34 : ~
* 35 : ~
@ -61,7 +60,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 38 : ~
* 39 : ~
* ------------------------------------------------------------
* 40 : ~
* 40 : File Map Size
* 41 : ~
* 42 : ~
* 43 : ~
@ -70,7 +69,7 @@ namespace Flawless.Core.BinaryDataFormat;
* 46 : ~
* 47 : ~
* ------------------------------------------------------------
* 48 : Depot Generate Time
* 48 : Payload Size
* 49 : ~
* 50 : ~
* 51 : ~
@ -79,16 +78,18 @@ namespace Flawless.Core.BinaryDataFormat;
* 54 : ~
* 55 : ~
* ------------------------------------------------------------
* 56 : Payload Size
* 57 : ~
* 58 : ~
* 59 : ~
* 60 : ~
* 61 : ~
* 62 : ~
* 63 : ~
* 56 : (Preserve)
* 57 : (Preserve)
* 58 : (Preserve)
* 59 : (Preserve)
* 60 : (Preserve)
* 61 : (Preserve)
* 62 : (Preserve)
* 63 : (Preserve)
* ------------------------------------------------------------
* PAYLOAD
* PAYLOAD (Binary, Compressed)
* ------------------------------------------------------------
* FILE MAP (UTF-8, Compressed)
* ------------------------------------------------------------
*/
@ -107,23 +108,13 @@ public struct StandardDepotHeaderV1
[FieldOffset(9)] public byte CompressType;
[FieldOffset(16)] public ulong FileMapMd5ChecksumLower;
[FieldOffset(16)] public ulong DepotMd5ChecksumLower;
[FieldOffset(24)] public ulong FileMapMd5ChecksumUpper;
[FieldOffset(32)] public ulong DepotMd5ChecksumLower;
[FieldOffset(40)] public ulong DepotMd5ChecksumUpper;
[FieldOffset(24)] public ulong DepotMd5ChecksumUpper;
[FieldOffset(48)] public ulong GenerateTime;
[FieldOffset(32)] public ulong GenerateTime;
[FieldOffset(56)] public ulong PayloadSize;
}
[Serializable]
public struct StandardDepotMapV1
{
public uint FileCount;
public DepotFileInfo[] Files;
[FieldOffset(40)] public ulong FileMapSize;
[FieldOffset(48)] public ulong PayloadSize;
}

7
Flawless.Core/Flawless.Core.csproj
View File

@ -10,4 +10,11 @@
<ProjectReference Include="..\Flawless.Abstract\Flawless.Abstract.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Nerdbank.Streams" Version="2.11.86" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.IO.Compression" Version="4.3.0" />
<PackageReference Include="System.IO.Hashing" Version="9.0.3" />
</ItemGroup>
</Project>

4
Flawless.Core/Interface/IRepositoryCommit.cs
View File

@ -13,10 +13,10 @@ public interface IRepositoryCommit
public DateTime CommitTime { get; }
public string Message { get; }
public ulong ManifestId { get; }
public IRepositoryCommit? GetParentCommit();
public IRepositoryCommit? GetChildCommit();
public ValueTask<CommitManifest> GetManifest(CancellationToken cancellationToken = default);
}

3
Flawless.Core/Modal/CommitManifest.cs
View File

@ -1,3 +1,4 @@
namespace Flawless.Core.Modal;
public record class CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths);
[Serializable]
public record struct CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths);

18
Flawless.Core/Modal/Depot.cs
View File

@ -4,21 +4,13 @@ namespace Flawless.Core.Modal;
public class Depot
{
public string RawDataPath { get; }
public HashId DepotHash { get; }
public byte Version { get; init; }
public byte Version { get; }
public byte ChecksumConfuser { get; }
public HashId Hash { get; init; }
public DateTime GenerateTime { get; }
public CompressType CompressType { get; init; }
public CompressType CompressType { get; }
public ulong PayloadSize { get; }
public uint FileCount { get; }
public DateTime GenerateTime { get; init; }
public DepotFileInfo[] Files { get; }
public ulong PayloadSize { get; init; }
}

2
Flawless.Core/Modal/DepotFileInfo.cs
View File

@ -1,4 +1,4 @@
namespace Flawless.Core.Modal;
[Serializable]
public record struct DepotFileInfo(ulong Size, ulong Offset, string Path);
public record struct DepotFileInfo(ulong Offset, ulong Size, string Path);