Flawless-Version-Control/Flawless.Core/BinaryDataFormat/DataTransformer.cs

using System.IO.Hashing;
using System.Text;
using Flawless.Core.Modal;
using Nerdbank.Streams;

namespace Flawless.Core.BinaryDataFormat;

public static class DataTransformer
{
    public static byte GuessStandardDepotHeaderVersion(Stream depotStream)
    {
        depotStream.Seek(8, SeekOrigin.Current);
        var val = depotStream.ReadByte();

        if (val < 0) ;

        return (byte)val;
    }

    public static void CreateAndInsertStandardDepotFile
        (Stream depotStream, IEnumerable<WorkspaceFile> workfiles, Func<WorkspaceFile, Stream> payloadLocator)
    {
        if (!depotStream.CanWrite || !depotStream.CanRead) throw new IOException("Depot stream is not writable/readable.");
        if (payloadLocator == null) throw new ArgumentNullException(nameof(payloadLocator));

        long headerStart = depotStream.Position;
        long payloadStart = 0;
        long payloadEnd = 0;
        long fileMapStart = 0;
        long fileMapEnd = 0;

        try
        {
            using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true);

            writer.Write(StandardDepotHeaderV1.FormatMagicNumber);
            writer.Write((uint) 0);
            writer.Write((byte) 1); // Crc of header - later
            writer.Write((byte) CompressType.Raw);
            writer.Write((ushort) 0); // Preserved
            writer.Write((uint) 0); // Preserved
            writer.Write((ulong) 0); // Hash of this file - later
            writer.Write((ulong) 0);
            writer.Write(DateTime.UtcNow.ToBinary());
            writer.Write((ulong) 0); // Filemap Size - later
            writer.Write((ulong) 0); // Payload Size - later
            writer.Write((ulong) 0); // Preserved
            
            writer.Flush();
        }
        catch (EndOfStreamException e)
        {
            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
        }
        
        
        // Write files into binary
        List<DepotFileInfo> fileinfos = new List<DepotFileInfo>();
        try
        {
            // Payload start at here
            payloadStart = depotStream.Position;
            
            foreach (var wf in workfiles)
            {
                var startPos = depotStream.Position;
                using var rs = payloadLocator(wf);
                
                rs.CopyTo(depotStream);
                fileinfos.Add(new DepotFileInfo(
                    (ulong)(startPos - payloadStart),
                    (ulong)(depotStream.Position - startPos),
                    wf.ModifyTime, wf.WorkPath));
                
                depotStream.Flush();
            }
            
            payloadEnd = depotStream.Position;
            
            // Filemap start at here
            var splitor = '$';
            fileMapStart = depotStream.Position;
            
            using (var writer = new StreamWriter(depotStream, Encoding.UTF8, 1024, true))
            {
                // Format: {Path}${Size}${ModifyTime}${Offset}$
                foreach (var df in fileinfos)
                {
                    writer.Write(df.Path);
                    writer.Write(splitor);
                    
                    writer.Write(df.Size.ToString());
                    writer.Write(splitor);
                    
                    writer.Write(df.ModifyTime.ToBinary().ToString());
                    writer.Write(splitor);
                    
                    writer.Write(df.Offset.ToString());
                    writer.Write(splitor);
                }
            }
            
            depotStream.Flush();
            
            fileMapEnd = depotStream.Position;
            
        }
        catch (EndOfStreamException e)
        {
            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
        }

        // Write rest part of header and calculate crc
        try
        {
            using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true);

            ulong payloadSize = (ulong)(payloadEnd - payloadStart);
            ulong filemapSize = (ulong)(fileMapEnd - fileMapStart);
            
            // Write fs size
            depotStream.Seek(headerStart + 40, SeekOrigin.Begin);
            writer.Write(filemapSize);
            writer.Write(payloadSize);
            
            writer.Flush();
            
            // Calculate CRC
            depotStream.Seek(headerStart + 8, SeekOrigin.Begin);
            Span<byte> crcArea = stackalloc byte[64 - 8];
            if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!");
            var crc = Crc32.HashToUInt32(crcArea);
            
            // Write CRC
            depotStream.Seek(headerStart + 4, SeekOrigin.Begin);
            writer.Write(crc);
            
            writer.Flush();
        }
        catch (EndOfStreamException e)
        {
            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
        }
        
    }

    public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream)
    {
        StandardDepotHeaderV1 r = default;
        _ = depotStream.Position; // Test if can seek
        
        try
        {
            using var reader = new BinaryReader(depotStream, Encoding.ASCII, true);
            
            // Magic number check
            r.MagicNumber = reader.ReadUInt32();
            if (r.MagicNumber != StandardDepotHeaderV1.FormatMagicNumber)
                throw new InvalidDataException("File may not a depot or broken.");

            // Do Crc checking
            r.HeaderCRCChecksum = reader.ReadUInt32();
            Span<byte> crcArea = stackalloc byte[64 - 8];
            if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!");
            if (Crc32.HashToUInt32(crcArea) != r.HeaderCRCChecksum)
                throw new InvalidDataException("Header CRC check is failed!");
            depotStream.Seek(8 - 64, SeekOrigin.Current);

            // Let's read file
            r.Version = reader.ReadByte();
            if (r.Version != 1) throw new InvalidDataException($"Version is mismatch! Current version is {r.Version}");
            r.CompressType = reader.ReadByte();
            reader.ReadInt16(); // Preserved
            reader.ReadInt32(); // Preserved
            r.DepotMd5ChecksumLower = reader.ReadUInt64();
            r.DepotMd5ChecksumUpper = reader.ReadUInt64();
            r.GenerateTime = reader.ReadUInt64();
            r.FileMapSize = reader.ReadUInt64();
            r.PayloadSize = reader.ReadUInt64();
            reader.ReadUInt64(); // Preserved

        }
        catch (EndOfStreamException e)
        {
            throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
        }
        
        return r;
    }

    public static Stream ExtractStandardDepotFile(Stream depotStream, DepotFileInfo fileInfo)
    {
        return ExtractStandardDepotFile(depotStream, fileInfo.Size, fileInfo.Offset);
    }

    public static Stream ExtractStandardDepotFile(Stream depotStream, ulong size, ulong offset)
    {
        depotStream.Seek((long)offset + 64, SeekOrigin.Begin);
        return depotStream.ReadSlice((long) size);
    }

    public static async IAsyncEnumerable<DepotFileInfo> ExtractDepotFileInfoMapAsync(Stream depotStream, ulong fileMapSize)
    {
        var splitor = '$';
        depotStream.Seek(-(long)fileMapSize, SeekOrigin.End);

        var state = -1;
        var buffer = new char[64];
        var builder = new StringBuilder();
        using var reader = new StreamReader(depotStream, Encoding.UTF8, leaveOpen: true);

        var path = string.Empty;
        var size = 0UL;
        var offset = 0UL;
        DateTime modifyTime = default;

        // Read loop
        while (true)
        {
            int length = 0;
            
            try
            {
                length = await reader.ReadBlockAsync(buffer, 0, 64);
                if (length == 0) break;
            }
            catch (EndOfStreamException e)
            {
                throw new InvalidDataException("Stream is too small! Maybe file is broken.", e);
            }
                
            for (int i = 0; i < length; i++)
            {
                var c = buffer[i];
                if (c != splitor) builder.Append(c);
                else
                {
                    switch ((state = (state + 1) % 4))
                    {
                        case 0: path = builder.ToString(); break;
                        case 1: size = ulong.Parse(builder.ToString()); break;
                        case 2: modifyTime = DateTime.FromBinary(long.Parse(builder.ToString())); break;
                        case 3:
                        {
                            offset = ulong.Parse(builder.ToString());
                            yield return new DepotFileInfo(offset, size, modifyTime, path);
                        } break;
                    }
                        
                    builder.Clear();
                }
            }
            

        }
            
        // Check is this the real ending
        if (builder.Length > 0 || (state > 0 && state < 3)) 
            throw new InvalidDataException("Stream is too small! Maybe file is broken.");
    }
}