using System.IO.Hashing; using System.Text; using Flawless.Core.Modal; using Nerdbank.Streams; namespace Flawless.Core.BinaryDataFormat; public static class DataTransformer { public static byte GuessStandardDepotHeaderVersion(Stream depotStream) { depotStream.Seek(8, SeekOrigin.Current); var val = depotStream.ReadByte(); if (val < 0) ; return (byte)val; } public static void CreateAndInsertStandardDepotFile (Stream depotStream, IEnumerable workfiles, Func payloadLocator) { if (!depotStream.CanWrite || !depotStream.CanRead) throw new IOException("Depot stream is not writable/readable."); if (payloadLocator == null) throw new ArgumentNullException(nameof(payloadLocator)); long headerStart = depotStream.Position; long payloadStart = 0; long payloadEnd = 0; long fileMapStart = 0; long fileMapEnd = 0; try { using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true); writer.Write(StandardDepotHeaderV1.FormatMagicNumber); writer.Write((uint) 0); writer.Write((byte) 1); // Crc of header - later writer.Write((byte) CompressType.Raw); writer.Write((ushort) 0); // Preserved writer.Write((uint) 0); // Preserved writer.Write((ulong) 0); // Hash of this file - later writer.Write((ulong) 0); writer.Write(DateTime.UtcNow.ToBinary()); writer.Write((ulong) 0); // Filemap Size - later writer.Write((ulong) 0); // Payload Size - later writer.Write((ulong) 0); // Preserved writer.Flush(); } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } // Write files into binary List fileinfos = new List(); try { // Payload start at here payloadStart = depotStream.Position; foreach (var wf in workfiles) { var startPos = depotStream.Position; using var rs = payloadLocator(wf); rs.CopyTo(depotStream); fileinfos.Add(new DepotFileInfo( (ulong)(startPos - payloadStart), (ulong)(depotStream.Position - startPos), wf.ModifyTime, wf.WorkPath)); depotStream.Flush(); } payloadEnd = depotStream.Position; // Filemap start at here var splitor = '$'; fileMapStart = depotStream.Position; using (var writer = new StreamWriter(depotStream, Encoding.UTF8, 1024, true)) { // Format: {Path}${Size}${ModifyTime}${Offset}$ foreach (var df in fileinfos) { writer.Write(df.Path); writer.Write(splitor); writer.Write(df.Size.ToString()); writer.Write(splitor); writer.Write(df.ModifyTime.ToBinary().ToString()); writer.Write(splitor); writer.Write(df.Offset.ToString()); writer.Write(splitor); } } depotStream.Flush(); fileMapEnd = depotStream.Position; } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } // Write rest part of header and calculate crc try { using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true); ulong payloadSize = (ulong)(payloadEnd - payloadStart); ulong filemapSize = (ulong)(fileMapEnd - fileMapStart); // Write fs size depotStream.Seek(headerStart + 40, SeekOrigin.Begin); writer.Write(filemapSize); writer.Write(payloadSize); writer.Flush(); // Calculate CRC depotStream.Seek(headerStart + 8, SeekOrigin.Begin); Span crcArea = stackalloc byte[64 - 8]; if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!"); var crc = Crc32.HashToUInt32(crcArea); // Write CRC depotStream.Seek(headerStart + 4, SeekOrigin.Begin); writer.Write(crc); writer.Flush(); } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } } public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream) { StandardDepotHeaderV1 r = default; _ = depotStream.Position; // Test if can seek try { using var reader = new BinaryReader(depotStream, Encoding.ASCII, true); // Magic number check r.MagicNumber = reader.ReadUInt32(); if (r.MagicNumber != StandardDepotHeaderV1.FormatMagicNumber) throw new InvalidDataException("File may not a depot or broken."); // Do Crc checking r.HeaderCRCChecksum = reader.ReadUInt32(); Span crcArea = stackalloc byte[64 - 8]; if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!"); if (Crc32.HashToUInt32(crcArea) != r.HeaderCRCChecksum) throw new InvalidDataException("Header CRC check is failed!"); depotStream.Seek(8 - 64, SeekOrigin.Current); // Let's read file r.Version = reader.ReadByte(); if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}"); r.CompressType = reader.ReadByte(); reader.ReadInt16(); // Preserved reader.ReadInt32(); // Preserved r.DepotMd5ChecksumLower = reader.ReadUInt64(); r.DepotMd5ChecksumUpper = reader.ReadUInt64(); r.GenerateTime = reader.ReadUInt64(); r.FileMapSize = reader.ReadUInt64(); r.PayloadSize = reader.ReadUInt64(); reader.ReadUInt64(); // Preserved } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } return r; } public static Stream ExtractStandardDepotFile(Stream depotStream, DepotFileInfo fileInfo) { return ExtractStandardDepotFile(depotStream, fileInfo.Size, fileInfo.Offset); } public static Stream ExtractStandardDepotFile(Stream depotStream, ulong size, ulong offset) { depotStream.Seek((long)offset + 64, SeekOrigin.Begin); return depotStream.ReadSlice((long) size); } public static async IAsyncEnumerable ExtractDepotFileInfoMapAsync(Stream depotStream, ulong fileMapSize) { var splitor = '$'; depotStream.Seek(-(long)fileMapSize, SeekOrigin.End); var state = -1; var buffer = new char[64]; var builder = new StringBuilder(); using var reader = new StreamReader(depotStream, Encoding.UTF8, leaveOpen: true); var path = string.Empty; var size = 0UL; var offset = 0UL; DateTime modifyTime = default; // Read loop while (true) { int length = 0; try { length = await reader.ReadBlockAsync(buffer, 0, 64); if (length == 0) break; } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } for (int i = 0; i < length; i++) { var c = buffer[i]; if (c != splitor) builder.Append(c); else { switch ((state = (state + 1) % 4)) { case 0: path = builder.ToString(); break; case 1: size = ulong.Parse(builder.ToString()); break; case 2: modifyTime = DateTime.FromBinary(long.Parse(builder.ToString())); break; case 3: { offset = ulong.Parse(builder.ToString()); yield return new DepotFileInfo(offset, size, modifyTime, path); } break; } builder.Clear(); } } } // Check is this the real ending if (builder.Length > 0 || (state > 0 && state < 3)) throw new InvalidDataException("Stream is too small! Maybe file is broken."); } }