diff --git a/Flawless.Core/BinaryDataFormat/DataExtractor.cs b/Flawless.Core/BinaryDataFormat/DataExtractor.cs new file mode 100644 index 0000000..612a2f9 --- /dev/null +++ b/Flawless.Core/BinaryDataFormat/DataExtractor.cs @@ -0,0 +1,170 @@ +using System.IO.Hashing; +using System.Text; +using Flawless.Core.Modal; +using Nerdbank.Streams; + +namespace Flawless.Core.BinaryDataFormat; + +public static class DataExtractor +{ + public static byte GuessStandardDepotHeaderVersion(Stream depotStream) + { + depotStream.Seek(8, SeekOrigin.Current); + var val = depotStream.ReadByte(); + depotStream.Seek(-9, SeekOrigin.Current); + + if (val < 0) ; + + return (byte)val; + } + + public static byte GuessNetworkDepotHeaderVersion(Stream depotStream) + { + var val = depotStream.ReadByte(); + depotStream.Seek(-1, SeekOrigin.Current); + + if (val < 0) ; + + return (byte)val; + } + + public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream) + { + StandardDepotHeaderV1 r = default; + _ = depotStream.Position; // Test if can seek + + try + { + using var reader = new BinaryReader(depotStream, Encoding.ASCII, true); + + // Magic number check + r.MagicNumber = reader.ReadUInt32(); + if (r.MagicNumber != StandardDepotHeaderV1.FormatMagicNumber) + throw new InvalidDataException("File may not a depot or broken."); + + // Do Crc checking + r.HeaderCRCChecksum = reader.ReadUInt32(); + Span crcArea = stackalloc byte[64 - 8]; + if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!"); + if (Crc32.HashToUInt32(crcArea) != r.HeaderCRCChecksum) + throw new InvalidDataException("Header CRC check is failed!"); + depotStream.Seek(8 - 64, SeekOrigin.Current); + + // Let's read file + r.Version = reader.ReadByte(); + if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}"); + r.CompressType = reader.ReadByte(); + reader.ReadInt16(); // Preserved + reader.ReadInt32(); // Preserved + r.DepotMd5ChecksumLower = reader.ReadUInt64(); + r.DepotMd5ChecksumUpper = reader.ReadUInt64(); + r.GenerateTime = reader.ReadUInt64(); + r.FileMapSize = reader.ReadUInt64(); + r.PayloadSize = reader.ReadUInt64(); + reader.ReadUInt64(); // Preserved + + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + return r; + } + + + public static NetworkDepotHeaderV1 ExtractNetworkDepotHeaderV1(Stream depotStream) + { + NetworkDepotHeaderV1 r; + _ = depotStream.Position; // Test if can seek + + try + { + using var reader = new BinaryReader(depotStream, Encoding.ASCII, true); + + // Let's read file + r.Version = reader.ReadByte(); + if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}"); + r.NetworkTransmissionFeature = reader.ReadByte(); + reader.ReadUInt16(); // Preserve + reader.ReadUInt32(); // Preserve + r.Md5ChecksumLower = reader.ReadUInt64(); + r.Md5ChecksumUpper = reader.ReadUInt64(); + r.GenerateTime = reader.ReadUInt64(); + r.FileMapSize = reader.ReadUInt64(); + r.PayloadSize = reader.ReadUInt64(); + + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + return r; + } + + public static Stream ExtractStandardDepotFile(Stream depotStream, DepotFileInfo fileInfo) + { + return ExtractStandardDepotFile(depotStream, fileInfo.Size, fileInfo.Offset); + } + + public static Stream ExtractStandardDepotFile(Stream depotStream, ulong size, ulong offset) + { + depotStream.Seek((long)offset + 64, SeekOrigin.Begin); + return depotStream.ReadSlice((long) size); + } + + public static async IAsyncEnumerable ExtractDepotFileInfoMapAsync(ulong fileMapSize, Stream depotStream) + { + var splitor = '$'; + depotStream.Seek((long) fileMapSize, SeekOrigin.End); + + var state = -1; + var buffer = new char[64]; + var builder = new StringBuilder(); + using var reader = new StreamReader(depotStream, Encoding.UTF8, leaveOpen: true); + + var path = string.Empty; + var size = 0UL; + var offset = 0UL; + + // Read loop + while (true) + { + try + { + var length = await reader.ReadBlockAsync(buffer, 0, 64); + if (length == 0) break; + for (int i = 0; i < length; i++) + { + var c = buffer[i]; + if (c != splitor) builder.Append(c); + else + { + switch ((state = (state + 1) % 3)) + { + case 0: path = builder.ToString(); break; + case 1: size = ulong.Parse(builder.ToString()); break; + case 2: offset = ulong.Parse(builder.ToString()); break; + } + + builder.Clear(); + } + } + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + + // Do output at here. + yield return new DepotFileInfo(offset, size, path); + + } + + // Check is this the real ending + if (builder.Length > 0 || state != 0) + throw new InvalidDataException("Stream is too small! Maybe file is broken."); + } +} \ No newline at end of file diff --git a/Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs b/Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs index 27f3e48..76fcff0 100644 --- a/Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs +++ b/Flawless.Core/BinaryDataFormat/NetworkDepotObjectV1.cs @@ -1,12 +1,13 @@ using System.Runtime.InteropServices; +using Flawless.Abstraction; namespace Flawless.Core.BinaryDataFormat; /* Depot Transmission Format Design - Version 1 * * We have shrink some layout design and remap fields in order to optimize for networking transmission. You may noticed - * that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk, - * when using network transmission, we may compress it from outside. So we let compressing to go. + * that we don't have a compressing info, this is due to compressing is mainly about how did depot stored in local disk. + * When using network transmission, we may compress it from outside. So we let compressing to go. * * Notice that we assume that all data are represent as LITTLE ENDIAN. * @@ -23,7 +24,7 @@ namespace Flawless.Core.BinaryDataFormat; * 6 : (Preserve) * 7 : (Preserve) * ------------------------------------------------------------ - * 8 : File Map String Size + * 8 : MD5 Checksum (Standard DepotMD5Checksum) * 9 : ~ * 10 : ~ * 11 : ~ @@ -33,7 +34,7 @@ namespace Flawless.Core.BinaryDataFormat; * 14 : ~ * 15 : ~ * ------------------------------------------------------------ - * 16 : MD5 Checksum (Standard DepotMD5Checksum) + * 16 : ~ * 17 : ~ * 18 : ~ * 19 : ~ @@ -43,7 +44,7 @@ namespace Flawless.Core.BinaryDataFormat; * 22 : ~ * 23 : ~ * ------------------------------------------------------------ - * 24 : + * 24 : Depot Generate Time * 25 : ~ * 26 : ~ * 27 : ~ @@ -53,7 +54,7 @@ namespace Flawless.Core.BinaryDataFormat; * 30 : ~ * 31 : ~ * ------------------------------------------------------------ - * 32 : Depot Generate Time + * 32 : File Map Size * 33 : ~ * 34 : ~ * 35 : ~ @@ -73,19 +74,17 @@ namespace Flawless.Core.BinaryDataFormat; * 46 : ~ * 47 : ~ * ------------------------------------------------------------ - * PAYLOAD (OPTIONAL) + * PAYLOAD (Optional, Binary, Uncompress) * ------------------------------------------------------------ - * FILE NAME MAP (OPTIONAL) + * FILE MAP (Optional, UTF-8, Uncompress) * ------------------------------------------------------------ */ [Flags] public enum NetworkTransmissionFeatureFlag: byte { - FileMapIsJson = 1 << 0, + WithPayload = 1 << 0, WithFileMap = 1 << 1, - WithPayload = 1 << 2, - CompressFileMap = 1 << 7, } [Serializable, StructLayout(LayoutKind.Explicit, CharSet = CharSet.Ansi, Pack = 4, Size = 48)] @@ -93,16 +92,15 @@ public struct NetworkDepotHeaderV1 { [FieldOffset(0)] public byte Version; - [FieldOffset(1)] public NetworkTransmissionFeatureFlag NetworkTransmissionFeature; - - [FieldOffset(8)] public ulong FileMapStringSize; + [FieldOffset(1)] public byte NetworkTransmissionFeature; - [FieldOffset(16)] public ulong Md5ChecksumLower; + [FieldOffset(8)] public ulong Md5ChecksumLower; - [FieldOffset(24)] public ulong Md5ChecksumUpper; + [FieldOffset(16)] public ulong Md5ChecksumUpper; - [FieldOffset(32)] public ulong GenerateTime; + [FieldOffset(24)] public ulong GenerateTime; + + [FieldOffset(32)] public ulong FileMapSize; [FieldOffset(40)] public ulong PayloadSize; - -} +} \ No newline at end of file diff --git a/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs b/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs index af77df2..e5f3bb0 100644 --- a/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs +++ b/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs @@ -1,13 +1,12 @@ using System.Runtime.InteropServices; +using Flawless.Abstraction; using Flawless.Core.Modal; namespace Flawless.Core.BinaryDataFormat; /* Depot File System Format Design - Version 1 * - * For best accessing performance, consider use checksum as filename. Binary info did not contains file name mapping, so - * use another file to get the file map of this depot. The structure to describe a map has already defined below and you - * can choose binary or text to store them. Advice to use JSON as text-based solution. + * For best accessing performance, consider use checksum as filename. * * Consider of compability when depot format was updated, we have configure a lots of area as empty. * @@ -34,7 +33,7 @@ namespace Flawless.Core.BinaryDataFormat; * 14 : (Preserve) * 15 : (Preserve) * ------------------------------------------------------------ - * 16 : File Map MD5 Checksum (From extern map data) + * 16 : Depot MD5 Checksum (From 48 to end, uncompressed) * 17 : ~ * 18 : ~ * 19 : ~ @@ -43,7 +42,7 @@ namespace Flawless.Core.BinaryDataFormat; * 22 : ~ * 23 : ~ * ------------------------------------------------------------ - * 24 : + * 24 : ~ * 25 : ~ * 26 : ~ * 27 : ~ @@ -52,7 +51,7 @@ namespace Flawless.Core.BinaryDataFormat; * 30 : ~ * 31 : ~ * ------------------------------------------------------------ - * 32 : Depot MD5 Checksum (From 48 to end, uncompressed) + * 32 : Depot Generate Time * 33 : ~ * 34 : ~ * 35 : ~ @@ -61,7 +60,7 @@ namespace Flawless.Core.BinaryDataFormat; * 38 : ~ * 39 : ~ * ------------------------------------------------------------ - * 40 : ~ + * 40 : File Map Size * 41 : ~ * 42 : ~ * 43 : ~ @@ -70,7 +69,7 @@ namespace Flawless.Core.BinaryDataFormat; * 46 : ~ * 47 : ~ * ------------------------------------------------------------ - * 48 : Depot Generate Time + * 48 : Payload Size * 49 : ~ * 50 : ~ * 51 : ~ @@ -79,16 +78,18 @@ namespace Flawless.Core.BinaryDataFormat; * 54 : ~ * 55 : ~ * ------------------------------------------------------------ - * 56 : Payload Size - * 57 : ~ - * 58 : ~ - * 59 : ~ - * 60 : ~ - * 61 : ~ - * 62 : ~ - * 63 : ~ + * 56 : (Preserve) + * 57 : (Preserve) + * 58 : (Preserve) + * 59 : (Preserve) + * 60 : (Preserve) + * 61 : (Preserve) + * 62 : (Preserve) + * 63 : (Preserve) * ------------------------------------------------------------ - * PAYLOAD + * PAYLOAD (Binary, Compressed) + * ------------------------------------------------------------ + * FILE MAP (UTF-8, Compressed) * ------------------------------------------------------------ */ @@ -107,23 +108,13 @@ public struct StandardDepotHeaderV1 [FieldOffset(9)] public byte CompressType; - [FieldOffset(16)] public ulong FileMapMd5ChecksumLower; + [FieldOffset(16)] public ulong DepotMd5ChecksumLower; - [FieldOffset(24)] public ulong FileMapMd5ChecksumUpper; - - [FieldOffset(32)] public ulong DepotMd5ChecksumLower; - - [FieldOffset(40)] public ulong DepotMd5ChecksumUpper; + [FieldOffset(24)] public ulong DepotMd5ChecksumUpper; - [FieldOffset(48)] public ulong GenerateTime; + [FieldOffset(32)] public ulong GenerateTime; - [FieldOffset(56)] public ulong PayloadSize; -} - -[Serializable] -public struct StandardDepotMapV1 -{ - public uint FileCount; - - public DepotFileInfo[] Files; + [FieldOffset(40)] public ulong FileMapSize; + + [FieldOffset(48)] public ulong PayloadSize; } \ No newline at end of file diff --git a/Flawless.Core/Flawless.Core.csproj b/Flawless.Core/Flawless.Core.csproj index 8f02b57..dc796d3 100644 --- a/Flawless.Core/Flawless.Core.csproj +++ b/Flawless.Core/Flawless.Core.csproj @@ -10,4 +10,11 @@ + + + + + + + diff --git a/Flawless.Core/Interface/IRepositoryCommit.cs b/Flawless.Core/Interface/IRepositoryCommit.cs index 694017b..606d461 100644 --- a/Flawless.Core/Interface/IRepositoryCommit.cs +++ b/Flawless.Core/Interface/IRepositoryCommit.cs @@ -13,10 +13,10 @@ public interface IRepositoryCommit public DateTime CommitTime { get; } public string Message { get; } + + public ulong ManifestId { get; } public IRepositoryCommit? GetParentCommit(); public IRepositoryCommit? GetChildCommit(); - - public ValueTask GetManifest(CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/Flawless.Core/Modal/CommitManifest.cs b/Flawless.Core/Modal/CommitManifest.cs index b6c2a02..e7fd338 100644 --- a/Flawless.Core/Modal/CommitManifest.cs +++ b/Flawless.Core/Modal/CommitManifest.cs @@ -1,3 +1,4 @@ namespace Flawless.Core.Modal; -public record class CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths); \ No newline at end of file +[Serializable] +public record struct CommitManifest(ulong ManifestId, DepotLabel Depot, string[] FilePaths); \ No newline at end of file diff --git a/Flawless.Core/Modal/Depot.cs b/Flawless.Core/Modal/Depot.cs index b342aec..a0911b9 100644 --- a/Flawless.Core/Modal/Depot.cs +++ b/Flawless.Core/Modal/Depot.cs @@ -4,21 +4,13 @@ namespace Flawless.Core.Modal; public class Depot { - public string RawDataPath { get; } - - public HashId DepotHash { get; } + public byte Version { get; init; } - public byte Version { get; } - - public byte ChecksumConfuser { get; } + public HashId Hash { get; init; } - public DateTime GenerateTime { get; } + public CompressType CompressType { get; init; } - public CompressType CompressType { get; } - - public ulong PayloadSize { get; } - - public uint FileCount { get; } + public DateTime GenerateTime { get; init; } - public DepotFileInfo[] Files { get; } + public ulong PayloadSize { get; init; } } \ No newline at end of file diff --git a/Flawless.Core/Modal/DepotFileInfo.cs b/Flawless.Core/Modal/DepotFileInfo.cs index eb9831b..1f2b261 100644 --- a/Flawless.Core/Modal/DepotFileInfo.cs +++ b/Flawless.Core/Modal/DepotFileInfo.cs @@ -1,4 +1,4 @@ namespace Flawless.Core.Modal; [Serializable] -public record struct DepotFileInfo(ulong Size, ulong Offset, string Path); \ No newline at end of file +public record struct DepotFileInfo(ulong Offset, ulong Size, string Path); \ No newline at end of file