feat: Add standard ata extractor and adjust format

Flawless.Core/BinaryDataFormat/DataExtractor.cs

@@ -0,0 +1,170 @@
+using System.IO.Hashing;
+using System.Text;
+using Flawless.Core.Modal;
+using Nerdbank.Streams;
+
+namespace Flawless.Core.BinaryDataFormat;
+
+public static class DataExtractor
+{
+    public static byte GuessStandardDepotHeaderVersion(Stream depotStream)
+    {
+        depotStream.Seek(8, SeekOrigin.Current);
+        var val = depotStream.ReadByte();
+        depotStream.Seek(-9, SeekOrigin.Current);
+
+        if (val < 0) ;
+
+        return (byte)val;
+    }
+
+    public static byte GuessNetworkDepotHeaderVersion(Stream depotStream)
+    {
+        var val = depotStream.ReadByte();
+        depotStream.Seek(-1, SeekOrigin.Current);
+        
+        if (val < 0) ;
+
+        return (byte)val;
+    }
+
+    public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream)
+    {
+        StandardDepotHeaderV1 r = default;
+        _ = depotStream.Position; // Test if can seek
+        
+        try
+        {
+            using var reader = new BinaryReader(depotStream, Encoding.ASCII, true);
+            
+            // Magic number check
+            r.MagicNumber = reader.ReadUInt32();
+            if (r.MagicNumber != StandardDepotHeaderV1.FormatMagicNumber)
+                throw new InvalidDataException("File may not a depot or broken.");
+
+            // Do Crc checking
+            r.HeaderCRCChecksum = reader.ReadUInt32();
+            Span<byte> crcArea = stackalloc byte[64 - 8];
+            if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!");
+            if (Crc32.HashToUInt32(crcArea) != r.HeaderCRCChecksum)
+                throw new InvalidDataException("Header CRC check is failed!");
+            depotStream.Seek(8 - 64, SeekOrigin.Current);
+
+            // Let's read file
+            r.Version = reader.ReadByte();
+            if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}");
+            r.CompressType = reader.ReadByte();
+            reader.ReadInt16(); // Preserved
+            reader.ReadInt32(); // Preserved
+            r.DepotMd5ChecksumLower = reader.ReadUInt64();
+            r.DepotMd5ChecksumUpper = reader.ReadUInt64();
+            r.GenerateTime = reader.ReadUInt64();
+            r.FileMapSize = reader.ReadUInt64();
+            r.PayloadSize = reader.ReadUInt64();
+            reader.ReadUInt64(); // Preserved
+
+        }
+        catch (EndOfStreamException e)
+        {
+            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
+        }
+        
+        return r;
+    }
+    
+    
+    public static NetworkDepotHeaderV1 ExtractNetworkDepotHeaderV1(Stream depotStream)
+    {
+        NetworkDepotHeaderV1 r;
+        _ = depotStream.Position; // Test if can seek
+        
+        try
+        {
+            using var reader = new BinaryReader(depotStream, Encoding.ASCII, true);
+            
+            // Let's read file
+            r.Version = reader.ReadByte();
+            if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}");
+            r.NetworkTransmissionFeature = reader.ReadByte();
+            reader.ReadUInt16(); // Preserve
+            reader.ReadUInt32(); // Preserve
+            r.Md5ChecksumLower = reader.ReadUInt64();
+            r.Md5ChecksumUpper = reader.ReadUInt64();
+            r.GenerateTime = reader.ReadUInt64();
+            r.FileMapSize = reader.ReadUInt64();
+            r.PayloadSize = reader.ReadUInt64();
+            
+        }
+        catch (EndOfStreamException e)
+        {
+            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
+        }
+
+        return r;
+    }
+
+    public static Stream ExtractStandardDepotFile(Stream depotStream, DepotFileInfo fileInfo)
+    {
+        return ExtractStandardDepotFile(depotStream, fileInfo.Size, fileInfo.Offset);
+    }
+
+    public static Stream ExtractStandardDepotFile(Stream depotStream, ulong size, ulong offset)
+    {
+        depotStream.Seek((long)offset + 64, SeekOrigin.Begin);
+        return depotStream.ReadSlice((long) size);
+    }
+
+    public static async IAsyncEnumerable<DepotFileInfo> ExtractDepotFileInfoMapAsync(ulong fileMapSize, Stream depotStream)
+    {
+        var splitor = '$';
+        depotStream.Seek((long) fileMapSize, SeekOrigin.End);
+
+        var state = -1;
+        var buffer = new char[64];
+        var builder = new StringBuilder();
+        using var reader = new StreamReader(depotStream, Encoding.UTF8, leaveOpen: true);
+
+        var path = string.Empty;
+        var size = 0UL;
+        var offset = 0UL;
+
+        // Read loop
+        while (true)
+        {
+            try
+            {
+                var length = await reader.ReadBlockAsync(buffer, 0, 64);
+                if (length == 0) break;
+                for (int i = 0; i < length; i++)
+                {
+                    var c = buffer[i];
+                    if (c != splitor) builder.Append(c);
+                    else
+                    {
+                        switch ((state = (state + 1) % 3))
+                        {
+                            case 0: path = builder.ToString(); break;
+                            case 1: size = ulong.Parse(builder.ToString()); break;
+                            case 2: offset = ulong.Parse(builder.ToString()); break;
+                        }
+                        
+                        builder.Clear();
+                    }
+                }
+            }
+            catch (EndOfStreamException e)
+            {
+                throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
+            }
+                
+                
+            // Do output at here.
+            yield return new DepotFileInfo(offset, size, path);
+
+        }
+            
+        // Check is this the real ending
+        if (builder.Length > 0 || state != 0) 
+            throw new InvalidDataException("Stream is too small! Maybe file is broken.");
+    }
+}

Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs

@@ -1,12 +1,13 @@
 using System.Runtime.InteropServices;
+using Flawless.Abstraction;
 
 namespace Flawless.Core.BinaryDataFormat;
 
 /* Depot Transmission Format Design - Version 1
  *
  * We have shrink some layout design and remap fields in order to optimize for networking transmission. You may noticed
- * that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk,
+ * that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk.
- * when using network transmission, we may compress it from outside. So we let compressing to go.
+ * When using network transmission, we may compress it from outside. So we let compressing to go.
  *
  * Notice that we assume that all data are represent as LITTLE ENDIAN.
  *
@@ -23,7 +24,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 6       : (Preserve)
  * 7       : (Preserve)
  * ------------------------------------------------------------
- * 8       : File Map String Size
+ * 8       : MD5 Checksum (Standard DepotMD5Checksum)
  * 9       : ~
  * 10      : ~
  * 11      : ~
@@ -33,7 +34,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 14      : ~
  * 15      : ~
  * ------------------------------------------------------------
- * 16      : MD5 Checksum (Standard DepotMD5Checksum)
+ * 16      : ~
  * 17      : ~
  * 18      : ~
  * 19      : ~
@@ -43,7 +44,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 22      : ~
  * 23      : ~
  * ------------------------------------------------------------
- * 24      : 
+ * 24      : Depot Generate Time
  * 25      : ~
  * 26      : ~
  * 27      : ~
@@ -53,7 +54,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 30      : ~
  * 31      : ~
  * ------------------------------------------------------------
- * 32      : Depot Generate Time
+ * 32      : File Map Size
  * 33      : ~
  * 34      : ~
  * 35      : ~
@@ -73,19 +74,17 @@ namespace Flawless.Core.BinaryDataFormat;
  * 46      : ~
  * 47      : ~
  * ------------------------------------------------------------
- * PAYLOAD (OPTIONAL)
+ * PAYLOAD (Optional, Binary, Uncompress)
  * ------------------------------------------------------------
- * FILE NAME MAP (OPTIONAL)
+ * FILE MAP (Optional, UTF-8, Uncompress)
  * ------------------------------------------------------------
  */
 
 [Flags]
 public enum NetworkTransmissionFeatureFlag: byte
 {
-    FileMapIsJson = 1 << 0,
+    WithPayload = 1 << 0,
     WithFileMap = 1 << 1,
-    WithPayload = 1 << 2,
-    CompressFileMap = 1 << 7,
 }
 
 [Serializable, StructLayout(LayoutKind.Explicit, CharSet = CharSet.Ansi, Pack = 4, Size = 48)]
@@ -93,16 +92,15 @@ public struct NetworkDepotHeaderV1
 {
     [FieldOffset(0)] public byte Version;
     
-    [FieldOffset(1)] public NetworkTransmissionFeatureFlag NetworkTransmissionFeature;
+    [FieldOffset(1)] public byte NetworkTransmissionFeature;
-    
-    [FieldOffset(8)] public ulong FileMapStringSize;
 
-    [FieldOffset(16)] public ulong Md5ChecksumLower;
+    [FieldOffset(8)] public ulong Md5ChecksumLower;
     
-    [FieldOffset(24)] public ulong Md5ChecksumUpper;
+    [FieldOffset(16)] public ulong Md5ChecksumUpper;
     
-    [FieldOffset(32)] public ulong GenerateTime;
+    [FieldOffset(24)] public ulong GenerateTime;
+    
+    [FieldOffset(32)] public ulong FileMapSize;
     
     [FieldOffset(40)] public ulong PayloadSize;
-
+}
-}

Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs

@@ -1,13 +1,12 @@
 using System.Runtime.InteropServices;
+using Flawless.Abstraction;
 using Flawless.Core.Modal;
 
 namespace Flawless.Core.BinaryDataFormat;
 
 /* Depot File System Format Design - Version 1
  *
- * For best accessing performance, consider use checksum as filename. Binary info did not contains file name mapping, so
+ * For best accessing performance, consider use checksum as filename.
- * use another file to get the file map of this depot. The structure to describe a map has already defined below and you
- * can choose binary or text to store them. Advice to use JSON as text-based solution.
  *
  * Consider of compability when depot format was updated, we have configure a lots of area as empty.
  *
@@ -34,7 +33,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 14      : (Preserve)
  * 15      : (Preserve)
  * ------------------------------------------------------------
- * 16      : File Map MD5 Checksum (From extern map data)
+ * 16      : Depot MD5 Checksum (From 48 to end, uncompressed)
  * 17      : ~
  * 18      : ~
  * 19      : ~
@@ -43,7 +42,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 22      : ~
  * 23      : ~
  * ------------------------------------------------------------
- * 24      : 
+ * 24      : ~
  * 25      : ~
  * 26      : ~
  * 27      : ~
@@ -52,7 +51,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 30      : ~
  * 31      : ~
  * ------------------------------------------------------------
- * 32      : Depot MD5 Checksum (From 48 to end, uncompressed)
+ * 32      : Depot Generate Time
  * 33      : ~
  * 34      : ~
  * 35      : ~
@@ -61,7 +60,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 38      : ~
  * 39      : ~
  * ------------------------------------------------------------
- * 40      : ~
+ * 40      : File Map Size
  * 41      : ~
  * 42      : ~
  * 43      : ~
@@ -70,7 +69,7 @@ namespace Flawless.Core.BinaryDataFormat;
  * 46      : ~
  * 47      : ~
  * ------------------------------------------------------------
- * 48      : Depot Generate Time
+ * 48      : Payload Size
  * 49      : ~
  * 50      : ~
  * 51      : ~
@@ -79,16 +78,18 @@ namespace Flawless.Core.BinaryDataFormat;
  * 54      : ~
  * 55      : ~
  * ------------------------------------------------------------
- * 56      : Payload Size
+ * 56      : (Preserve)
- * 57      : ~
+ * 57      : (Preserve)
- * 58      : ~
+ * 58      : (Preserve)
- * 59      : ~
+ * 59      : (Preserve)
- * 60      : ~
+ * 60      : (Preserve)
- * 61      : ~
+ * 61      : (Preserve)
- * 62      : ~
+ * 62      : (Preserve)
- * 63      : ~
+ * 63      : (Preserve)
  * ------------------------------------------------------------
- * PAYLOAD
+ * PAYLOAD (Binary, Compressed)
+ * ------------------------------------------------------------
+ * FILE MAP (UTF-8, Compressed)
  * ------------------------------------------------------------
  */
 
@@ -107,23 +108,13 @@ public struct StandardDepotHeaderV1
 
     [FieldOffset(9)] public byte CompressType;
     
-    [FieldOffset(16)] public ulong FileMapMd5ChecksumLower;
+    [FieldOffset(16)] public ulong DepotMd5ChecksumLower;
     
-    [FieldOffset(24)] public ulong FileMapMd5ChecksumUpper;
+    [FieldOffset(24)] public ulong DepotMd5ChecksumUpper;
-    
-    [FieldOffset(32)] public ulong DepotMd5ChecksumLower;
-    
-    [FieldOffset(40)] public ulong DepotMd5ChecksumUpper;
 
-    [FieldOffset(48)] public ulong GenerateTime;
+    [FieldOffset(32)] public ulong GenerateTime;
     
-    [FieldOffset(56)] public ulong PayloadSize;
+    [FieldOffset(40)] public ulong FileMapSize;
-}
+    
-
+    [FieldOffset(48)] public ulong PayloadSize;
-[Serializable]
-public struct StandardDepotMapV1
-{
-    public uint FileCount;
-
-    public DepotFileInfo[] Files;
 }

Flawless.Core/Flawless.Core.csproj

@@ -10,4 +10,11 @@
       <ProjectReference Include="..\Flawless.Abstract\Flawless.Abstract.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="Nerdbank.Streams" Version="2.11.86" />
+      <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+      <PackageReference Include="System.IO.Compression" Version="4.3.0" />
+      <PackageReference Include="System.IO.Hashing" Version="9.0.3" />
+    </ItemGroup>
+
 </Project>

Flawless.Core/Interface/IRepositoryCommit.cs

@@ -13,10 +13,10 @@ public interface IRepositoryCommit
     public DateTime CommitTime { get; }
 
     public string Message { get; }
+    
+    public ulong ManifestId { get; }
 
     public IRepositoryCommit? GetParentCommit();
     
     public IRepositoryCommit? GetChildCommit();
-    
-    public ValueTask<CommitManifest> GetManifest(CancellationToken cancellationToken = default);
 }

Flawless.Core/Modal/CommitManifest.cs

@@ -1,3 +1,4 @@
 namespace Flawless.Core.Modal;
 
-public record class CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths);
+[Serializable]
+public record struct CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths);

Flawless.Core/Modal/Depot.cs

@@ -4,21 +4,13 @@ namespace Flawless.Core.Modal;
 
 public class Depot
 {
-    public string RawDataPath { get; }
+    public byte Version { get; init; }
-
-    public HashId DepotHash { get; }
     
-    public byte Version { get; }
+    public HashId Hash { get; init; }
-
-    public byte ChecksumConfuser { get; }
     
-    public DateTime GenerateTime { get; }
+    public CompressType CompressType { get; init; }
 
-    public CompressType CompressType { get; }
+    public DateTime GenerateTime { get; init; }
-    
-    public ulong PayloadSize { get; }
-    
-    public uint FileCount { get; }
 
-    public DepotFileInfo[] Files { get; }
+    public ulong PayloadSize { get; init; }
 }

Flawless.Core/Modal/DepotFileInfo.cs

@@ -1,4 +1,4 @@
 namespace Flawless.Core.Modal;
 
 [Serializable]
-public record struct DepotFileInfo(ulong Size, ulong Offset, string Path);
+public record struct DepotFileInfo(ulong Offset, ulong Size, string Path);