From c80b2759a9c402d8cea012f2dbee843ab551ccdb Mon Sep 17 00:00:00 2001 From: Cardidi Date: Wed, 2 Apr 2025 03:37:39 +0800 Subject: [PATCH] feat: Finish depot creation and transmission parts. --- Flawless.Client/AppDefaultValues.cs | 2 + .../Models/RepositoryLocalDatabaseModel.cs | 9 +- .../Service/LocalFileTreeAccessor.cs | 35 ++- Flawless.Client/Service/Remote_Generated.cs | 51 +++- .../Service/RepositoryFileTreeAccessor.cs | 35 ++- Flawless.Client/Service/RepositoryService.cs | 283 ++++++++++++++---- Flawless.Client/ViewModels/HomeViewModel.cs | 2 +- .../ViewModels/RepositoryViewModel.cs | 111 ++++++- .../RepositoryPage/RepoCommitPageView.axaml | 3 +- .../RepositoryPage/RepoFileTreePageView.axaml | 14 +- .../RepoWorkspacePageView.axaml | 6 +- .../Request/CommitRequest.cs | 4 +- .../Response/CommitSuccessResponse.cs | 2 +- .../BinaryDataFormat/DataTransformer.cs | 178 +++++++++-- .../BinaryDataFormat/StandardDepotObjectV1.cs | 2 +- Flawless.Core/Modal/WorkspaceFile.cs | 11 +- .../Controllers/RepositoryInnieController.cs | 95 +++--- Flawless.Server/Flawless.Server.csproj | 2 + Flawless.Server/Models/Repository.cs | 2 +- Flawless.Server/Models/RepositoryCommit.cs | 10 +- Flawless.Server/Models/RepositoryDepot.cs | 2 +- Flawless.Server/Models/RepositoryMember.cs | 6 +- Flawless.Server/Program.cs | 6 + 23 files changed, 676 insertions(+), 195 deletions(-) diff --git a/Flawless.Client/AppDefaultValues.cs b/Flawless.Client/AppDefaultValues.cs index 273c5bc..9a6c03e 100644 --- a/Flawless.Client/AppDefaultValues.cs +++ b/Flawless.Client/AppDefaultValues.cs @@ -13,6 +13,8 @@ public static class AppDefaultValues public const string RepoLocalStorageDepotFolder = "depots"; + public const string RepoLocalStorageTempFolder = "temp"; + public static string ProgramDataDirectory { get; } = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Ca2dWorks", "FlawlessClient"); diff --git a/Flawless.Client/Models/RepositoryLocalDatabaseModel.cs b/Flawless.Client/Models/RepositoryLocalDatabaseModel.cs index eb249cf..f356388 100644 --- a/Flawless.Client/Models/RepositoryLocalDatabaseModel.cs +++ b/Flawless.Client/Models/RepositoryLocalDatabaseModel.cs @@ -1,7 +1,7 @@ using System; using System.Collections.ObjectModel; -using System.Text.Json.Serialization; using Flawless.Client.Service; +using Newtonsoft.Json; using ReactiveUI.SourceGenerators; namespace Flawless.Client.Models; @@ -18,8 +18,13 @@ public partial class RepositoryLocalDatabaseModel : ReactiveModel [JsonIgnore] public RepositoryFileTreeAccessor? RepoAccessor { get; set; } + [JsonProperty("currentCommit")] [Reactive] private Guid? _currentCommit; + [JsonProperty("commitMessage")] [Reactive] private string? _commitMessage; - + + [JsonProperty("lastOperationTime")] + [Reactive] private DateTime _lastOprationTime; + } \ No newline at end of file diff --git a/Flawless.Client/Service/LocalFileTreeAccessor.cs b/Flawless.Client/Service/LocalFileTreeAccessor.cs index 81e1748..869c908 100644 --- a/Flawless.Client/Service/LocalFileTreeAccessor.cs +++ b/Flawless.Client/Service/LocalFileTreeAccessor.cs @@ -44,20 +44,25 @@ public class LocalFileTreeAccessor private IReadOnlyDictionary _baseline; - private Dictionary _difference; + private Dictionary _changes = new(); + + private Dictionary _currentFiles = new(); private object _optLock = new(); + public DateTime LastScanTimeUtc { get; private set; } + public string WorkingDirectory => _rootDirectory; public IReadOnlyDictionary BaselineFiles => _baseline; - public IReadOnlyDictionary Changes => _difference; + public IReadOnlyDictionary Changes => _changes; + + public IReadOnlyDictionary CurrentFiles => _currentFiles; public LocalFileTreeAccessor(RepositoryModel repo, IEnumerable baselines) { _repo = repo; - _difference = new Dictionary(); _baseline = baselines.ToImmutableDictionary(b => b.WorkPath); _rootDirectory = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); } @@ -67,7 +72,7 @@ public class LocalFileTreeAccessor lock (_optLock) { _baseline = baselines.ToImmutableDictionary(b => b.WorkPath); - _difference.Clear(); + _changes.Clear(); RefreshInternal(); } @@ -77,7 +82,7 @@ public class LocalFileTreeAccessor { lock (_optLock) { - _difference.Clear(); + _changes.Clear(); RefreshInternal(); } @@ -85,7 +90,7 @@ public class LocalFileTreeAccessor private void RefreshInternal() { - Dictionary currentFiles = new(); + _currentFiles.Clear(); foreach (var f in Directory.GetFiles(_rootDirectory, "*", SearchOption.AllDirectories)) { var workPath = WorkPath.FromPlatformPath(f, _rootDirectory); @@ -93,16 +98,18 @@ public class LocalFileTreeAccessor continue; var modifyTime = File.GetLastWriteTimeUtc(f); - currentFiles.Add(workPath, new WorkspaceFile { WorkPath = workPath, ModifyTime = modifyTime }); + _currentFiles.Add(workPath, new WorkspaceFile { WorkPath = workPath, ModifyTime = modifyTime }); } - // Find those are changed - var changes = currentFiles.Values.Where(v => _baseline.TryGetValue(v.WorkPath, out var fi) && fi.ModifyTime != v.ModifyTime); - var news = currentFiles.Values.Where(v => !_baseline.ContainsKey(v.WorkPath)); - var removed = _baseline.Values.Where(v => !currentFiles.ContainsKey(v.WorkPath)); + LastScanTimeUtc = DateTime.UtcNow; - foreach (var f in changes) _difference.Add(f.WorkPath, new ChangeRecord(ChangeType.Modify, f)); - foreach (var f in news) _difference.Add(f.WorkPath, new ChangeRecord(ChangeType.Add, f)); - foreach (var f in removed) _difference.Add(f.WorkPath, new ChangeRecord(ChangeType.Remove, f)); + // Find those are changed + var changes = _currentFiles.Values.Where(v => _baseline.TryGetValue(v.WorkPath, out var fi) && fi.ModifyTime != v.ModifyTime); + var news = _currentFiles.Values.Where(v => !_baseline.ContainsKey(v.WorkPath)); + var removed = _baseline.Values.Where(v => !_currentFiles.ContainsKey(v.WorkPath)); + + foreach (var f in changes) _changes.Add(f.WorkPath, new ChangeRecord(ChangeType.Modify, f)); + foreach (var f in news) _changes.Add(f.WorkPath, new ChangeRecord(ChangeType.Add, f)); + foreach (var f in removed) _changes.Add(f.WorkPath, new ChangeRecord(ChangeType.Remove, f)); } } \ No newline at end of file diff --git a/Flawless.Client/Service/Remote_Generated.cs b/Flawless.Client/Service/Remote_Generated.cs index 5db99a8..3d9b6be 100644 --- a/Flawless.Client/Service/Remote_Generated.cs +++ b/Flawless.Client/Service/Remote_Generated.cs @@ -5,6 +5,7 @@ using Refit; using System.Collections.Generic; +using System.IO; using System.Text.Json.Serialization; using System.Threading.Tasks; @@ -113,25 +114,25 @@ namespace Flawless.Client.Remote /// OK /// Thrown when the request returns a non-success status code. [Headers("Accept: text/plain, application/json, text/json")] - [Get("/api/repo/{userName}/{repositoryName}/fetch_manifest")] - Task FetchManifest(string userName, string repositoryName, [Query] string commitId); + [Post("/api/repo/{userName}/{repositoryName}/fetch_manifest")] + Task FetchManifest(string userName, string repositoryName, [Query] string commitId); /// OK /// Thrown when the request returns a non-success status code. [Headers("Accept: text/plain, application/json, text/json")] - [Get("/api/repo/{userName}/{repositoryName}/fetch_depot")] - Task FetchDepot(string userName, string repositoryName, [Query] string depotId); + [Post("/api/repo/{userName}/{repositoryName}/fetch_depot")] + Task> FetchDepot(string userName, string repositoryName, [Query] string depotId); /// OK /// Thrown when the request returns a non-success status code. [Headers("Accept: text/plain, application/json, text/json")] - [Get("/api/repo/{userName}/{repositoryName}/list_commit")] + [Post("/api/repo/{userName}/{repositoryName}/list_commit")] Task ListCommit(string userName, string repositoryName); /// OK /// Thrown when the request returns a non-success status code. [Headers("Accept: text/plain, application/json, text/json")] - [Get("/api/repo/{userName}/{repositoryName}/list_locked_files")] + [Post("/api/repo/{userName}/{repositoryName}/list_locked_files")] Task ListLockedFiles(string userName, string repositoryName); /// OK @@ -155,7 +156,7 @@ namespace Flawless.Client.Remote [Multipart] [Headers("Accept: text/plain, application/json, text/json")] [Post("/api/repo/{userName}/{repositoryName}/create_commit")] - Task CreateCommit(string userName, string repositoryName, StreamPart depot, string message, IEnumerable workspaceSnapshot, IEnumerable requiredDepots, string mainDepotId); + Task CreateCommit(string userName, string repositoryName, StreamPart depot, string message, IEnumerable workspaceSnapshot, IEnumerable requiredDepots, string mainDepotId); /// OK /// Thrown when the request returns a non-success status code. @@ -232,6 +233,21 @@ namespace Flawless.Client.Remote + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.2.0.0 (NJsonSchema v11.1.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class CommitManifest + { + + [JsonPropertyName("manifestId")] + public System.Guid ManifestId { get; set; } + + [JsonPropertyName("depot")] + public ICollection Depot { get; set; } + + [JsonPropertyName("filePaths")] + public ICollection FilePaths { get; set; } + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.2.0.0 (NJsonSchema v11.1.0.0 (Newtonsoft.Json v13.0.0.0))")] public partial class CommitSuccessResponse { @@ -242,6 +258,21 @@ namespace Flawless.Client.Remote [JsonPropertyName("commitId")] public System.Guid CommitId { get; set; } + [JsonPropertyName("mainDepotId")] + public System.Guid MainDepotId { get; set; } + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.2.0.0 (NJsonSchema v11.1.0.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class DepotLabel + { + + [JsonPropertyName("id")] + public System.Guid Id { get; set; } + + [JsonPropertyName("length")] + public long Length { get; set; } + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.2.0.0 (NJsonSchema v11.1.0.0 (Newtonsoft.Json v13.0.0.0))")] @@ -535,6 +566,12 @@ namespace Flawless.Client.Remote public partial class WorkspaceFile { + [JsonPropertyName("modifyTime")] + public System.DateTimeOffset ModifyTime { get; set; } + + [JsonPropertyName("workPath")] + public string WorkPath { get; set; } + } [System.CodeDom.Compiler.GeneratedCode("NSwag", "14.2.0.0 (NJsonSchema v11.1.0.0 (Newtonsoft.Json v13.0.0.0))")] diff --git a/Flawless.Client/Service/RepositoryFileTreeAccessor.cs b/Flawless.Client/Service/RepositoryFileTreeAccessor.cs index 7e76b9b..bdadbd6 100644 --- a/Flawless.Client/Service/RepositoryFileTreeAccessor.cs +++ b/Flawless.Client/Service/RepositoryFileTreeAccessor.cs @@ -1,14 +1,16 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; +using System.Linq; using System.Threading.Tasks; using Flawless.Core.BinaryDataFormat; using Flawless.Core.Modal; namespace Flawless.Client.Service; -public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable +public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable, IEnumerable { private struct FileReadInfoCache @@ -77,6 +79,7 @@ public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable foreach (var (id, stream) in _mappings) { + stream.Seek(0, SeekOrigin.Begin); var header = DataTransformer.ExtractStandardDepotHeaderV1(stream); _headers.Add(id, header); await foreach (var inf in DataTransformer.ExtractDepotFileInfoMapAsync(stream, header.FileMapSize)) @@ -109,7 +112,7 @@ public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable return false; } - public bool TryWriteDataIntoStream(string workPath, Stream stream) + public bool TryWriteDataIntoStream(string workPath, Stream stream, out DateTime modifyTime) { DisposeCheck(); if (stream == null || !stream.CanWrite) throw new ArgumentException("Stream is not writable!"); @@ -118,6 +121,24 @@ public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable { var baseStream = DataTransformer.ExtractStandardDepotFile(_mappings[r.DepotId], r.FileInfo); baseStream.CopyTo(stream); + modifyTime = r.FileInfo.ModifyTime; + return true; + } + + modifyTime = DateTime.MinValue; + return false; + } + + public bool TryWriteDataIntoStream(string workPath, string destinationPath) + { + DisposeCheck(); + + if (_cached.TryGetValue(workPath, out var r)) + { + using var ws = new FileStream(destinationPath, FileMode.OpenOrCreate, FileAccess.Write); + var baseStream = DataTransformer.ExtractStandardDepotFile(_mappings[r.DepotId], r.FileInfo); + baseStream.CopyTo(ws); + File.SetLastWriteTimeUtc(destinationPath, r.FileInfo.ModifyTime); return true; } @@ -129,4 +150,14 @@ public class RepositoryFileTreeAccessor : IDisposable, IAsyncDisposable if (_disposed) throw new ObjectDisposedException("Accessor has already been disposed"); } + public IEnumerator GetEnumerator() + { + return _cached.Values + .Select(cv => new WorkspaceFile(cv.FileInfo.ModifyTime, cv.FileInfo.Path)).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } } \ No newline at end of file diff --git a/Flawless.Client/Service/RepositoryService.cs b/Flawless.Client/Service/RepositoryService.cs index 9aff3a5..8d383bd 100644 --- a/Flawless.Client/Service/RepositoryService.cs +++ b/Flawless.Client/Service/RepositoryService.cs @@ -3,12 +3,14 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; +using System.Net; using System.Threading.Tasks; using Flawless.Abstraction; using Flawless.Client.Models; +using Flawless.Core.BinaryDataFormat; using Flawless.Core.Modal; using Newtonsoft.Json; -using ValueTaskSupplement; +using Refit; namespace Flawless.Client.Service; @@ -42,18 +44,16 @@ public class RepositoryService : BaseService public bool SaveRepositoryLocalDatabaseChanges(RepositoryModel repo) { + var localRepo = GetRepositoryLocalDatabase(repo); + localRepo.LastOprationTime = DateTime.Now; + var dbPath = PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name); - if (File.Exists(dbPath)) return false; + Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); - if (!_localRepoDbModel.TryGetValue(repo, out var localRepo)) - { - using var writeFs = new StreamWriter(new FileStream(dbPath, FileMode.Truncate)); - JsonSerializer.CreateDefault().Serialize(writeFs, localRepo); - - return true; - } - - return false; + using var writeFs = new StreamWriter(new FileStream(dbPath, FileMode.OpenOrCreate)); + JsonSerializer.CreateDefault().Serialize(writeFs, localRepo); + + return true; } public RepositoryLocalDatabaseModel GetRepositoryLocalDatabase(RepositoryModel repo) @@ -183,16 +183,6 @@ public class RepositoryService : BaseService return true; } - public async ValueTask UpdateDownloadedStatusFromDiskAsync(RepositoryModel repo) - { - var isFolderExists = Directory.Exists(PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name)); - var isDbFileExists = File.Exists(PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name)); - - repo.IsDownloaded = isFolderExists && isDbFileExists; - - return true; - } - public async ValueTask UpdateMembersFromServerAsync(RepositoryModel repo) { var api = Api.C; @@ -267,17 +257,22 @@ public class RepositoryService : BaseService public async ValueTask OpenRepositoryOnStorageAsync(RepositoryModel repo) { - if (!await UpdateDownloadedStatusFromDiskAsync(repo) || repo.IsDownloaded == false) return false; + if (!await UpdateRepositoriesDownloadedStatusFromDiskAsync() || repo.IsDownloaded == false) return false; if (!await UpdateCommitsFromServerAsync(repo)) return false; var ls = GetRepositoryLocalDatabase(repo); if (ls.CurrentCommit != null) { - var accessor = await DownloadDepotsToGetCommitFileTreeFromServerAsync(repo, ls.CurrentCommit.Value); + var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, ls.CurrentCommit.Value); if (accessor == null) return false; ls.RepoAccessor = accessor; + + // Remember to cache accessor everytime it will being used. + await accessor.CreateCacheAsync(); + ls.LocalAccessor.SetBaseline(accessor); } + SaveRepositoryLocalDatabaseChanges(repo); _openedRepos.Add(repo); return true; } @@ -287,6 +282,7 @@ public class RepositoryService : BaseService // Create basic structures. if (!TryCreateRepositoryBaseStorageStructure(repo)) return false; + SaveRepositoryLocalDatabaseChanges(repo); repo.IsDownloaded = true; _openedRepos.Add(repo); return true; @@ -301,20 +297,21 @@ public class RepositoryService : BaseService var peekCommit = repo.Commits.MaxBy(sl => sl.CommittedOn); if (peekCommit == null) return false; // Should not use this function! - // Create basic structures. - if (!TryCreateRepositoryBaseStorageStructure(repo)) return false; - // Download base repo info - var accessor = await DownloadDepotsToGetCommitFileTreeFromServerAsync(repo, peekCommit.CommitId); + var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, peekCommit.CommitId); if (accessor == null) { await DeleteFromDiskAsync(repo); return false; }; + // Remember to cache accessor everytime it will being used. + await accessor.CreateCacheAsync(); + var ls = GetRepositoryLocalDatabase(repo); ls.CurrentCommit = peekCommit.CommitId; ls.RepoAccessor = accessor; + ls.LocalAccessor.SetBaseline(accessor); try { @@ -325,8 +322,8 @@ public class RepositoryService : BaseService // Write into fs if (directory != null) Directory.CreateDirectory(directory); - await using var write = File.Create(pfs); - accessor.TryWriteDataIntoStream(f.WorkPath, write); + if (!accessor.TryWriteDataIntoStream(f.WorkPath, pfs)) + throw new InvalidDataException($"Can not write {f.WorkPath} into repository."); } } catch (Exception e) @@ -336,6 +333,7 @@ public class RepositoryService : BaseService return false; } + SaveRepositoryLocalDatabaseChanges(repo); repo.IsDownloaded = true; _openedRepos.Add(repo); return true; @@ -426,7 +424,8 @@ public class RepositoryService : BaseService return true; } - public async ValueTask DownloadDepotsToGetCommitFileTreeFromServerAsync + public async ValueTask + DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync (RepositoryModel repo, Guid commit, bool storeDownloadedDepots = true) { if (commit == Guid.Empty) return null; @@ -438,7 +437,7 @@ public class RepositoryService : BaseService // Prepare folders var path = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); var depotsRoot = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); - Directory.CreateDirectory(path); + Directory.CreateDirectory(depotsRoot); // Generate download depots list var mainDepotLabel = manifest.Value.Depot; @@ -452,12 +451,10 @@ public class RepositoryService : BaseService // Download them var downloadedDepots = await DownloadDepotsAndCopyNetworkStreamIntoNewMemoryStreamFromServerAsync(repo, willDownload); - + if (downloadedDepots == null) return null; + try { - // Check if anyone download failed - if (downloadedDepots == null || downloadedDepots.Any(dl => dl.Item2 == null)) - throw new Exception("Some depots are not able to be downloaded."); if (storeDownloadedDepots) { @@ -494,21 +491,22 @@ public class RepositoryService : BaseService } public async ValueTask WriteDownloadedDepotsFromServerToStorageAsync - (RepositoryModel repo, IEnumerable<(Guid, Stream)> depots) + (RepositoryModel repo, IEnumerable<(Guid id, Stream stream)> depots) { var depotsRoot = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); - var tasks = depots.Select(async d => + foreach (var d in depots) { - var dst = Path.Combine(depotsRoot, d.Item1.ToString()); + var dst = Path.Combine(depotsRoot, d.id.ToString()); await using var ws = new FileStream(dst, FileMode.Create); - await d.Item2.CopyToAsync(ws); - d.Item2.Seek(0, SeekOrigin.Begin); - }); - - await Task.WhenAll(tasks); + + // Make sure always to be at begin. + d.stream.Seek(0, SeekOrigin.Begin); + await d.stream.CopyToAsync(ws); + d.stream.Seek(0, SeekOrigin.Begin); + } } - public async ValueTask<(Guid, Stream?)[]?> DownloadDepotsAndCopyNetworkStreamIntoNewMemoryStreamFromServerAsync + public async ValueTask<(Guid, Stream)[]?> DownloadDepotsAndCopyNetworkStreamIntoNewMemoryStreamFromServerAsync (RepositoryModel repo, IEnumerable depotsId) { try @@ -520,24 +518,28 @@ public class RepositoryService : BaseService return null; } - return await ValueTaskEx.WhenAll(depotsId.Select(p => DownloadDepotInternalAsync(p.Id, p.Length))); + var result = new List<(Guid, Stream)>(); + foreach (var dl in depotsId) + { + using var rsp = await Api.C.Gateway.FetchDepot(repo.OwnerName, repo.Name, dl.Id.ToString()); + if (rsp.StatusCode != HttpStatusCode.OK) + { + Console.WriteLine($"Failed to fetch depot {dl.Id}"); + return null; + } + + var memoryStream = new MemoryStream(new byte[dl.Length]); + await rsp.Content!.CopyToAsync(memoryStream); + result.Add((dl.Id, memoryStream)); + } + + return result.ToArray(); } catch (Exception e) { Console.WriteLine(e); return null; } - - async ValueTask<(Guid, Stream?)> DownloadDepotInternalAsync(Guid depotId, long length) - { - using var rsp = await Api.C.Gateway.FetchDepot(repo.OwnerName, repo.Name, depotId.ToString()); - if (rsp.StatusCode != 200) return (depotId, null); - if (rsp.Stream.Length != length) return (depotId, null); - - var memoryStream = new MemoryStream(new byte[rsp.Stream.Length]); - await rsp.Stream.CopyToAsync(memoryStream); - return (depotId, memoryStream); - } } public async ValueTask DownloadManifestFromServerAsync(RepositoryModel repo, Guid manifestId) @@ -551,10 +553,11 @@ public class RepositoryService : BaseService return null; } - using var manifestResponse = await api.Gateway.FetchManifest(repo.OwnerName, repo.Name, manifestId.ToString()); - if (manifestResponse.StatusCode != 200) return null; - - return await System.Text.Json.JsonSerializer.DeserializeAsync(manifestResponse.Stream); + var rsp = await api.Gateway.FetchManifest(repo.OwnerName, repo.Name, manifestId.ToString()); + return new( + rsp.ManifestId, + rsp.Depot.Select(x => new DepotLabel(x.Id, x.Length)).ToArray(), + rsp.FilePaths.Select(x => new WorkspaceFile(x.ModifyTime.UtcDateTime, x.WorkPath)).ToArray()); } catch (Exception e) { @@ -579,4 +582,164 @@ public class RepositoryService : BaseService repo.IsDownloaded = false; return true; } + + public async ValueTask CommitWorkspaceAsBaselineAsync + (RepositoryModel repo, IEnumerable changes, string message) + { + var localDb = GetRepositoryLocalDatabase(repo); + var manifestList = CreateCommitManifestByCurrentBaselineAndChanges(localDb.LocalAccessor, changes); + var api = Api.C; + + try + { + + // Renew for once. + if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) + { + api.ClearGateway(); + return null; + } + + // Generate depot + var tempDepotPath = await CreateDepotIntoTempFileAsync(repo, manifestList); + if (tempDepotPath == null) return null; + + // Upload and create commit + await using var str = File.OpenRead(tempDepotPath); + var snapshot = manifestList.Select(l => $"{l.ModifyTime.ToBinary()}${l.WorkPath}"); + if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) + { + api.ClearGateway(); + return null; + } + + var rsp = await api.Gateway.CreateCommit(repo.OwnerName, repo.Name, + new StreamPart(str, Path.GetFileName(tempDepotPath)), message, snapshot, null!, null!); + + // Move depot file to destination + var depotsPath = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); + var finalPath = Path.Combine(depotsPath, rsp.MainDepotId.ToString()); + Directory.CreateDirectory(depotsPath); + File.Move(tempDepotPath, finalPath, true); + + // Fetch mapped manifest + var manifest = await DownloadManifestFromServerAsync(repo, rsp.CommitId); + if (manifest == null) return null; + + var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, rsp.CommitId); + if (accessor == null) return null; //todo this is a really fatal issue... + if (localDb.RepoAccessor != null) + { + try { await localDb.RepoAccessor.DisposeAsync(); } + catch (Exception e) { Console.WriteLine(e); } + } + + // Point to newest state. + localDb.RepoAccessor = accessor; + localDb.CurrentCommit = rsp.CommitId; + localDb.LocalAccessor.SetBaseline(accessor); + SaveRepositoryLocalDatabaseChanges(repo); + + return manifest; + } + catch (Exception e) + { + Console.WriteLine(e); + return null; + } + } + + public List CreateCommitManifestByCurrentBaselineAndChanges + (LocalFileTreeAccessor accessor, IEnumerable changes, bool hard = false) + { + // Create a new depot file manifest. + var files = accessor.BaselineFiles.Values.ToList(); + foreach (var c in changes) + { + switch (c.Type) + { + case LocalFileTreeAccessor.ChangeType.Folder: + { + if (hard) throw new InvalidProgramException( + $"Can not commit folder into version control: {c.File.WorkPath}"); + + Console.WriteLine($"Can not commit folder into version control...Ignored: {c.File.WorkPath}"); + continue; + } + case LocalFileTreeAccessor.ChangeType.Add: + { + if (files.Any(f => f.WorkPath == c.File.WorkPath)) + { + if (hard) throw new InvalidProgramException( + $"Can not create an existed record into version control: {c.File.WorkPath}"); + + Console.WriteLine($"Can not create an existed record into version control...Ignored: {c.File.WorkPath}"); + continue; + } + + files.Add(c.File); + break; + } + case LocalFileTreeAccessor.ChangeType.Remove: + { + var idx = files.FindIndex(f => f.WorkPath == c.File.WorkPath); + if (idx < 0) + { + if (hard) throw new InvalidProgramException( + $"Can not delete a missed record into version control: {c.File.WorkPath}"); + + Console.WriteLine($"Can not delete a missed record into version control...Ignored: {c.File.WorkPath}"); + continue; + } + + files.RemoveAt(idx); + break; + } + case LocalFileTreeAccessor.ChangeType.Modify: + { + var idx = files.FindIndex(f => f.WorkPath == c.File.WorkPath); + if (idx < 0) + { + if (hard) throw new InvalidProgramException( + $"Can not modify a missed record into version control: {c.File.WorkPath}"); + + Console.WriteLine($"Can not modify a missed record into version control...Ignored: {c.File.WorkPath}"); + continue; + } + + files[idx] = c.File; + break; + } + } + } + + return files; + } + + public async ValueTask CreateDepotIntoTempFileAsync(RepositoryModel repo, IEnumerable depotFiles) + { + var repoWs = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); + var commitTempFolder = Directory.CreateTempSubdirectory("FlawlessDepot_"); + var depotFile = Path.Combine(commitTempFolder.FullName, "depot.bin"); + + try + { + // No UI thread blocked + await Task.Run(async () => + { + await using var fs = new FileStream(depotFile, FileMode.Create); + DataTransformer.CreateAndInsertStandardDepotFile(fs, depotFiles, + wf => File.OpenRead(WorkPath.ToPlatformPath(wf.WorkPath, repoWs))); + + }); + } + catch (Exception e) + { + Directory.Delete(repoWs, true); + Console.WriteLine(e); + return null; + } + + return depotFile; + } } \ No newline at end of file diff --git a/Flawless.Client/ViewModels/HomeViewModel.cs b/Flawless.Client/ViewModels/HomeViewModel.cs index e85c79c..02a8ae1 100644 --- a/Flawless.Client/ViewModels/HomeViewModel.cs +++ b/Flawless.Client/ViewModels/HomeViewModel.cs @@ -109,7 +109,7 @@ public partial class HomeViewModel : ViewModelBase, IRoutableViewModel if (mr == DialogResult.Yes) { if (await RepositoryService.C.DeleteFromDiskAsync(_selectedRepository)) - await RepositoryService.C.UpdateDownloadedStatusFromDiskAsync(_selectedRepository); + await RepositoryService.C.UpdateRepositoriesDownloadedStatusFromDiskAsync(); } } diff --git a/Flawless.Client/ViewModels/RepositoryViewModel.cs b/Flawless.Client/ViewModels/RepositoryViewModel.cs index 84bd57e..a1b38cd 100644 --- a/Flawless.Client/ViewModels/RepositoryViewModel.cs +++ b/Flawless.Client/ViewModels/RepositoryViewModel.cs @@ -4,6 +4,7 @@ using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Threading.Tasks; using Avalonia.Controls; using Avalonia.Controls.Models.TreeDataGrid; @@ -11,6 +12,7 @@ using DynamicData; using DynamicData.Binding; using Flawless.Client.Models; using Flawless.Client.Service; +using Flawless.Core.Modal; using ReactiveUI; using ReactiveUI.SourceGenerators; using ChangeType = Flawless.Client.Service.LocalFileTreeAccessor.ChangeType; @@ -64,6 +66,38 @@ public class LocalChangesNode } } +public class CommitTransitNode +{ + public required string Guid { get; set; } + + public required string Author { get; set; } + + public required string Message { get; set; } + + public required DateTime? CommitAt { get; set; } + + public static CommitTransitNode FromCommit(RepositoryModel.Commit cm) + { + string msg; + if (cm.Message.Length > 28) + { + msg = cm.Message.Substring(0, 28) + "..."; + } + else + { + msg = cm.Message; + } + + return new CommitTransitNode + { + Guid = cm.CommitId.ToString(), + Author = cm.Author, + CommitAt = cm.CommittedOn.ToLocalTime(), + Message = msg, + }; + } +} + public partial class RepositoryViewModel : RoutableViewModelBase { public RepositoryModel Repository { get; } @@ -72,6 +106,11 @@ public partial class RepositoryViewModel : RoutableViewModelBase public HierarchicalTreeDataGridSource LocalChange { get; } + + public HierarchicalTreeDataGridSource FileTree { get; } + + public FlatTreeDataGridSource Commits { get; } + public ObservableCollection LocalChangeSetRaw { get; } = new(); public UserModel User { get; } @@ -91,12 +130,6 @@ public partial class RepositoryViewModel : RoutableViewModelBase Repository.Members.ObserveCollectionChanges().Subscribe(_ => RefreshRepositoryRoleInfo()); // Setup local change set - LocalChangeSetRaw.Add(new LocalChangesNode - { - Type = "Add", - FullPath = "test.md", - ModifiedTime = DateTime.Now, - }); LocalChange = new HierarchicalTreeDataGridSource(LocalChangeSetRaw) { Columns = @@ -104,15 +137,15 @@ public partial class RepositoryViewModel : RoutableViewModelBase new CheckBoxColumn( string.Empty, n => n.Included, (n, v) => n.Included = v), - new TextColumn( - "Change", - n => n.Contents != null ? String.Empty : n.Type), - new HierarchicalExpanderColumn( new TextColumn( "Name", n => Path.GetFileName(n.FullPath)), n => n.Contents), + + new TextColumn( + "Change", + n => n.Contents != null ? String.Empty : n.Type.ToString()), new TextColumn( "File Type", @@ -126,9 +159,63 @@ public partial class RepositoryViewModel : RoutableViewModelBase "ModifiedTime", n => n.ModifiedTime.HasValue ? n.ModifiedTime.Value.ToLocalTime() : null), } }; + + Commits = new FlatTreeDataGridSource(Repository.Commits.Select(CommitTransitNode.FromCommit)) + { + Columns = + { + new TextColumn( + string.Empty, n => n.Guid == LocalDatabase.CurrentCommit.ToString() ? "*" : String.Empty), + + new TextColumn( + "Message", x => x.Message), + + new TextColumn( + "Author", x => x.Author), + + new TextColumn( + "Time", x => x.CommitAt!.Value), + + new TextColumn( + "Id", x => x.Guid.Substring(0, 13)), + } + }; + + DetectLocalChangesAsyncCommand.Execute(); + } + + + private void CollectChanges(List store, IEnumerable changesNode) + { + foreach (var n in changesNode) + { + if (n.Contents != null) CollectChanges(store, n.Contents); + else if (n.Included) + { + store.Add(new LocalFileTreeAccessor.ChangeRecord(Enum.Parse(n.Type), new WorkspaceFile + { + WorkPath = n.FullPath, + ModifyTime = n.ModifiedTime!.Value + })); + } + } + } + + [ReactiveCommand] + private async Task CommitSelectedChangesAsync() + { + var changes = new List(); + CollectChanges(changes, LocalChangeSetRaw); + + if (changes.Count == 0) return; + var manifest = await RepositoryService.C.CommitWorkspaceAsBaselineAsync(Repository, changes, + LocalDatabase.CommitMessage ?? string.Empty); - // Do refresh when entered - // DetectLocalChangesAsyncCommand.Execute(); + if (manifest == null) return; + + LocalDatabase.LocalAccessor.SetBaseline(manifest.Value.FilePaths); + LocalDatabase.CommitMessage = string.Empty; + await DetectLocalChangesAsyncCommand.Execute(); } [ReactiveCommand] diff --git a/Flawless.Client/Views/RepositoryPage/RepoCommitPageView.axaml b/Flawless.Client/Views/RepositoryPage/RepoCommitPageView.axaml index 58bacfd..8dcb700 100644 --- a/Flawless.Client/Views/RepositoryPage/RepoCommitPageView.axaml +++ b/Flawless.Client/Views/RepositoryPage/RepoCommitPageView.axaml @@ -7,8 +7,7 @@ mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450" x:Class="Flawless.Client.Views.RepositoryPage.RepoCommitPageView"> - - + diff --git a/Flawless.Client/Views/RepositoryPage/RepoFileTreePageView.axaml b/Flawless.Client/Views/RepositoryPage/RepoFileTreePageView.axaml index 049e09a..abe3352 100644 --- a/Flawless.Client/Views/RepositoryPage/RepoFileTreePageView.axaml +++ b/Flawless.Client/Views/RepositoryPage/RepoFileTreePageView.axaml @@ -9,12 +9,12 @@ - - - - - - + + + + + + + diff --git a/Flawless.Client/Views/RepositoryPage/RepoWorkspacePageView.axaml b/Flawless.Client/Views/RepositoryPage/RepoWorkspacePageView.axaml index 0211cb7..2d2b108 100644 --- a/Flawless.Client/Views/RepositoryPage/RepoWorkspacePageView.axaml +++ b/Flawless.Client/Views/RepositoryPage/RepoWorkspacePageView.axaml @@ -12,7 +12,7 @@ - + @@ -20,10 +20,10 @@ - + - + diff --git a/Flawless.Communication/Request/CommitRequest.cs b/Flawless.Communication/Request/CommitRequest.cs index 03d238f..74042d2 100644 --- a/Flawless.Communication/Request/CommitRequest.cs +++ b/Flawless.Communication/Request/CommitRequest.cs @@ -8,9 +8,9 @@ public class CommitRequest public required string Message { get; set; } - public required WorkspaceFile[] WorkspaceSnapshot { get; set; } + public required List WorkspaceSnapshot { get; set; } - public string[]? RequiredDepots { get; set; } + public List? RequiredDepots { get; set; } public string? MainDepotId { get; set; } // If commit is not modify files, but changes workspace files (Delete) We will not require a main depot id. diff --git a/Flawless.Communication/Response/CommitSuccessResponse.cs b/Flawless.Communication/Response/CommitSuccessResponse.cs index a3dd975..a34eec1 100644 --- a/Flawless.Communication/Response/CommitSuccessResponse.cs +++ b/Flawless.Communication/Response/CommitSuccessResponse.cs @@ -1,3 +1,3 @@ namespace Flawless.Communication.Response; -public record CommitSuccessResponse(DateTime CommittedOn, Guid CommitId); \ No newline at end of file +public record CommitSuccessResponse(DateTime CommittedOn, Guid CommitId, Guid MainDepotId); \ No newline at end of file diff --git a/Flawless.Core/BinaryDataFormat/DataTransformer.cs b/Flawless.Core/BinaryDataFormat/DataTransformer.cs index 638b3f2..43e9d39 100644 --- a/Flawless.Core/BinaryDataFormat/DataTransformer.cs +++ b/Flawless.Core/BinaryDataFormat/DataTransformer.cs @@ -16,7 +16,133 @@ public static class DataTransformer return (byte)val; } - + + public static void CreateAndInsertStandardDepotFile + (Stream depotStream, IEnumerable workfiles, Func payloadLocator) + { + if (!depotStream.CanWrite || !depotStream.CanRead) throw new IOException("Depot stream is not writable/readable."); + if (payloadLocator == null) throw new ArgumentNullException(nameof(payloadLocator)); + + long headerStart = depotStream.Position; + long payloadStart = 0; + long payloadEnd = 0; + long fileMapStart = 0; + long fileMapEnd = 0; + + try + { + using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true); + + writer.Write(StandardDepotHeaderV1.FormatMagicNumber); + writer.Write((uint) 0); + writer.Write((byte) 1); // Crc of header - later + writer.Write((byte) CompressType.Raw); + writer.Write((ushort) 0); // Preserved + writer.Write((uint) 0); // Preserved + writer.Write((ulong) 0); // Hash of this file - later + writer.Write((ulong) 0); + writer.Write(DateTime.UtcNow.ToBinary()); + writer.Write((ulong) 0); // Filemap Size - later + writer.Write((ulong) 0); // Payload Size - later + writer.Write((ulong) 0); // Preserved + + writer.Flush(); + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + + // Write files into binary + List fileinfos = new List(); + try + { + // Payload start at here + payloadStart = depotStream.Position; + + foreach (var wf in workfiles) + { + var startPos = depotStream.Position; + using var rs = payloadLocator(wf); + + rs.CopyTo(depotStream); + fileinfos.Add(new DepotFileInfo( + (ulong)(startPos - payloadStart), + (ulong)(depotStream.Position - startPos), + wf.ModifyTime, wf.WorkPath)); + + depotStream.Flush(); + } + + payloadEnd = depotStream.Position; + + // Filemap start at here + var splitor = '$'; + fileMapStart = depotStream.Position; + + using (var writer = new StreamWriter(depotStream, Encoding.UTF8, 1024, true)) + { + // Format: {Path}${Size}${ModifyTime}${Offset}$ + foreach (var df in fileinfos) + { + writer.Write(df.Path); + writer.Write(splitor); + + writer.Write(df.Size.ToString()); + writer.Write(splitor); + + writer.Write(df.ModifyTime.ToBinary().ToString()); + writer.Write(splitor); + + writer.Write(df.Offset.ToString()); + writer.Write(splitor); + } + } + + depotStream.Flush(); + + fileMapEnd = depotStream.Position; + + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + // Write rest part of header and calculate crc + try + { + using var writer = new BinaryWriter(depotStream, Encoding.ASCII, true); + + ulong payloadSize = (ulong)(payloadEnd - payloadStart); + ulong filemapSize = (ulong)(fileMapEnd - fileMapStart); + + // Write fs size + depotStream.Seek(headerStart + 40, SeekOrigin.Begin); + writer.Write(filemapSize); + writer.Write(payloadSize); + + writer.Flush(); + + // Calculate CRC + depotStream.Seek(headerStart + 8, SeekOrigin.Begin); + Span crcArea = stackalloc byte[64 - 8]; + if (depotStream.Read(crcArea) != 64 - 8) throw new InvalidDataException("Stream is too short!"); + var crc = Crc32.HashToUInt32(crcArea); + + // Write CRC + depotStream.Seek(headerStart + 4, SeekOrigin.Begin); + writer.Write(crc); + + writer.Flush(); + } + catch (EndOfStreamException e) + { + throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); + } + + } public static StandardDepotHeaderV1 ExtractStandardDepotHeaderV1(Stream depotStream) { @@ -76,7 +202,7 @@ public static class DataTransformer public static async IAsyncEnumerable ExtractDepotFileInfoMapAsync(Stream depotStream, ulong fileMapSize) { var splitor = '$'; - depotStream.Seek((long) fileMapSize, SeekOrigin.End); + depotStream.Seek(-(long)fileMapSize, SeekOrigin.End); var state = -1; var buffer = new char[64]; @@ -91,41 +217,45 @@ public static class DataTransformer // Read loop while (true) { + int length = 0; + try { - var length = await reader.ReadBlockAsync(buffer, 0, 64); + length = await reader.ReadBlockAsync(buffer, 0, 64); if (length == 0) break; - for (int i = 0; i < length; i++) - { - var c = buffer[i]; - if (c != splitor) builder.Append(c); - else - { - switch ((state = (state + 1) % 4)) - { - case 0: path = builder.ToString(); break; - case 1: size = ulong.Parse(builder.ToString()); break; - case 2: modifyTime = DateTime.FromBinary(long.Parse(builder.ToString())); break; - case 3: offset = ulong.Parse(builder.ToString()); break; - } - - builder.Clear(); - } - } } catch (EndOfStreamException e) { throw new InvalidDataException("Stream is too small! Maybe file is broken.", e); } - - // Do output at here. - yield return new DepotFileInfo(offset, size, modifyTime, path); + for (int i = 0; i < length; i++) + { + var c = buffer[i]; + if (c != splitor) builder.Append(c); + else + { + switch ((state = (state + 1) % 4)) + { + case 0: path = builder.ToString(); break; + case 1: size = ulong.Parse(builder.ToString()); break; + case 2: modifyTime = DateTime.FromBinary(long.Parse(builder.ToString())); break; + case 3: + { + offset = ulong.Parse(builder.ToString()); + yield return new DepotFileInfo(offset, size, modifyTime, path); + } break; + } + + builder.Clear(); + } + } + } // Check is this the real ending - if (builder.Length > 0 || state != 0) + if (builder.Length > 0 || (state > 0 && state < 3)) throw new InvalidDataException("Stream is too small! Maybe file is broken."); } } \ No newline at end of file diff --git a/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs b/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs index e5f3bb0..64b1688 100644 --- a/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs +++ b/Flawless.Core/BinaryDataFormat/StandardDepotObjectV1.cs @@ -33,7 +33,7 @@ namespace Flawless.Core.BinaryDataFormat; * 14 : (Preserve) * 15 : (Preserve) * ------------------------------------------------------------ - * 16 : Depot MD5 Checksum (From 48 to end, uncompressed) + * 16 : Depot MD5 Checksum (From 64 to end, uncompressed) * 17 : ~ * 18 : ~ * 19 : ~ diff --git a/Flawless.Core/Modal/WorkspaceFile.cs b/Flawless.Core/Modal/WorkspaceFile.cs index 76a6e26..b42de09 100644 --- a/Flawless.Core/Modal/WorkspaceFile.cs +++ b/Flawless.Core/Modal/WorkspaceFile.cs @@ -1,11 +1,18 @@ namespace Flawless.Core.Modal; +[Serializable] public struct WorkspaceFile : IEquatable { - public required DateTime ModifyTime; + public DateTime ModifyTime { get; set; } - public required string WorkPath; + public string WorkPath { get; set; } + public WorkspaceFile(DateTime modifyTime, string workPath) + { + ModifyTime = modifyTime; + WorkPath = workPath; + } + public bool Equals(WorkspaceFile other) { return ModifyTime.Equals(other.ModifyTime) && WorkPath == other.WorkPath; diff --git a/Flawless.Server/Controllers/RepositoryInnieController.cs b/Flawless.Server/Controllers/RepositoryInnieController.cs index 7ddb8ee..5232ebf 100644 --- a/Flawless.Server/Controllers/RepositoryInnieController.cs +++ b/Flawless.Server/Controllers/RepositoryInnieController.cs @@ -1,6 +1,4 @@ -using System.Collections.Concurrent; -using System.Text.Json; -using System.Text.Json.Serialization; +using System.Text.Json; using Flawless.Communication.Request; using Flawless.Communication.Response; using Flawless.Communication.Shared; @@ -9,15 +7,13 @@ using Flawless.Core.Modal; using Flawless.Server.Models; using Flawless.Server.Services; using Flawless.Server.Utility; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Microsoft.OpenApi.Validations.Rules; namespace Flawless.Server.Controllers; -[ApiController, Authorize, Route("api/repo/{userName}/{repositoryName}")] +[ApiController, Microsoft.AspNetCore.Authorization.Authorize, Route("api/repo/{userName}/{repositoryName}")] public class RepositoryInnieController( UserManager userManager, AppDbContext dbContext, @@ -223,30 +219,30 @@ public class RepositoryInnieController( } - [HttpGet("fetch_manifest")] - [ProducesResponseType(200)] - public async Task DownloadManifestAsync(string userName, string repositoryName, [FromQuery] string commitId) + [HttpPost("fetch_manifest")] + public async Task> DownloadManifestAsync(string userName, string repositoryName, [FromQuery] string commitId) { if (!Guid.TryParse(commitId, out var commitGuid)) return BadRequest(new FailedResponse("Invalid commit id")); var user = (await userManager.GetUserAsync(HttpContext.User))!; var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); - if (grantIssue is not Repository rp) return (IActionResult) grantIssue; + if (grantIssue is not Repository rp) return (ActionResult) grantIssue; // Start checkout file. var target = transformer.GetCommitManifestPath(rp.Id, commitGuid); - if (System.IO.File.Exists(target)) return File(System.IO.File.OpenRead(target), "application/octet-stream"); - return NotFound(new FailedResponse($"Could not find commit manifest {target}")); + if (!System.IO.File.Exists(target)) return NotFound(new FailedResponse($"Could not find commit manifest {target}")); + + await using var manifest = System.IO.File.OpenRead(target); + return await JsonSerializer.DeserializeAsync(manifest); } - [HttpGet("fetch_depot")] - [ProducesResponseType(200)] - public async Task DownloadDepotAsync(string userName, string repositoryName, [FromQuery] string depotId) + [HttpPost("fetch_depot")] + public async Task> DownloadDepotAsync(string userName, string repositoryName, [FromQuery] string depotId) { if (!Guid.TryParse(depotId, out var depotGuid)) return BadRequest(new FailedResponse("Invalid depot id")); var user = (await userManager.GetUserAsync(HttpContext.User))!; var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); - if (grantIssue is not Repository rp) return (IActionResult) grantIssue; + if (grantIssue is not Repository rp) return (ActionResult) grantIssue; // Start checkout file. var target = transformer.GetDepotPath(rp.Id, depotGuid); @@ -255,7 +251,7 @@ public class RepositoryInnieController( } - [HttpGet("list_commit")] + [HttpPost("list_commit")] public async Task>> ListCommitsAsync(string userName, string repositoryName) { var user = (await userManager.GetUserAsync(HttpContext.User))!; @@ -277,7 +273,7 @@ public class RepositoryInnieController( return Ok(new ListingResponse(r.ToArray())); } - [HttpGet("list_locked_files")] + [HttpPost("list_locked_files")] public async Task>> ListLocksAsync(string userName, string repositoryName) { var user = (await userManager.GetUserAsync(HttpContext.User))!; @@ -393,6 +389,7 @@ public class RepositoryInnieController( var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Developer); if (grantIssue is not Repository rp) return (IActionResult) grantIssue; + // Appoint or upload a commit - two in one choice. if (req.Depot != null ^ req.MainDepotId != null) return await CommitInternalAsync(rp, user, req); // Not valid response @@ -402,7 +399,17 @@ public class RepositoryInnieController( private async Task CommitInternalAsync(Repository rp, AppUser user, FormCommitRequest req) { - var test = new HashSet(req.WorkspaceSnapshot); + var actualFs = req.WorkspaceSnapshot.Select(s => + { + var idx = s.IndexOf('$'); + if (idx < 0) throw new InvalidDataException($"WorkPath '{s}' is invalid!"); + var dateTimeStr = s.Substring(0, idx); + var pathStr = s.Substring(idx + 1); + + return new WorkspaceFile(DateTime.FromBinary(long.Parse(dateTimeStr)), pathStr); + }).ToArray(); + + var test = new HashSet(actualFs); var createNewDepot = false; RepositoryDepot mainDepot; List depotLabels = new(); @@ -481,8 +488,13 @@ public class RepositoryInnieController( .Where(cm => actualRequiredDepots.Contains(cm.DepotId)) .ToArrayAsync()); + // Create commit and let it alloc a main key + rp.Depots.Add(mainDepot); + await dbContext.SaveChangesAsync(); + // Then write depot into disk var depotPath = transformer.GetDepotPath(rp.Id, mainDepot.DepotId); + Directory.CreateDirectory(Path.GetDirectoryName(depotPath)!); await using (var depotStream = System.IO.File.Create(depotPath)) { cacheStream.Seek(0, SeekOrigin.Begin); @@ -492,56 +504,51 @@ public class RepositoryInnieController( // Everything alright, so make response depotLabels.Add(new DepotLabel(mainDepot.DepotId, mainDepot.Length)); depotLabels.AddRange(mainDepot.Dependencies.Select(d => new DepotLabel(d.DepotId, d.Length))); - rp.Depots.Add(mainDepot); - createNewDepot = true; } - // Create manifest file and write to disk - var commitId = Guid.NewGuid(); - var manifestPath = transformer.GetCommitManifestPath(rp.Id, commitId); - var manifest = new CommitManifest - { - ManifestId = commitId, - Depot = depotLabels.ToArray(), - FilePaths = req.WorkspaceSnapshot - }; - - await using (var manifestStream = System.IO.File.Create(manifestPath)) - await JsonSerializer.SerializeAsync(manifestStream, manifest); - // Create commit info var commit = new RepositoryCommit { - Id = commitId, Author = user, CommittedOn = DateTime.UtcNow, MainDepot = mainDepot, - Message = manifestPath + Message = req.Message, }; - rp.Commits.Add(commit); - try { // Write changes into db. + rp.Commits.Add(commit); await dbContext.SaveChangesAsync(); } catch (Exception) { - // Revert manifest create operation - if (createNewDepot) System.IO.File.Delete(transformer.GetDepotPath(rp.Id, mainDepot.DepotId)); - System.IO.File.Delete(manifestPath); - + // Revert depot create operation + System.IO.File.Delete(transformer.GetDepotPath(rp.Id, mainDepot.DepotId)); throw; } - - return Ok(new CommitSuccessResponse(commit.CommittedOn, commit.Id)); + + // Create manifest file and write to disk + var manifestPath = transformer.GetCommitManifestPath(rp.Id, commit.Id); + var manifest = new CommitManifest + { + ManifestId = commit.Id, + Depot = depotLabels.ToArray(), + FilePaths = actualFs + }; + + Directory.CreateDirectory(Path.GetDirectoryName(manifestPath)!); + await using (var manifestStream = System.IO.File.Create(manifestPath)) + await JsonSerializer.SerializeAsync(manifestStream, manifest); + + return Ok(new CommitSuccessResponse(commit.CommittedOn, commit.Id, mainDepot.DepotId)); } private static async ValueTask StencilWorkspaceSnapshotAsync(Stream depotStream, HashSet unresolved) { // Get version + depotStream.Seek(0, SeekOrigin.Begin); // Fix: Get an invalid version code due to offset is bad. var version = DataTransformer.GuessStandardDepotHeaderVersion(depotStream); if (version != 1) throw new InvalidDataException($"Unable to get depot header version, feedback is {version}."); diff --git a/Flawless.Server/Flawless.Server.csproj b/Flawless.Server/Flawless.Server.csproj index a8990ac..ad3ce90 100644 --- a/Flawless.Server/Flawless.Server.csproj +++ b/Flawless.Server/Flawless.Server.csproj @@ -38,6 +38,8 @@ + + diff --git a/Flawless.Server/Models/Repository.cs b/Flawless.Server/Models/Repository.cs index 0fa0d4c..f12d5c3 100644 --- a/Flawless.Server/Models/Repository.cs +++ b/Flawless.Server/Models/Repository.cs @@ -7,7 +7,7 @@ public class Repository { [Key] [Required] - public Guid Id { get; set; } = Guid.NewGuid(); + public Guid Id { get; set; } [Required] public required AppUser Owner { get; set; } diff --git a/Flawless.Server/Models/RepositoryCommit.cs b/Flawless.Server/Models/RepositoryCommit.cs index e3f472a..aa7e85a 100644 --- a/Flawless.Server/Models/RepositoryCommit.cs +++ b/Flawless.Server/Models/RepositoryCommit.cs @@ -5,17 +5,17 @@ namespace Flawless.Server.Models; public class RepositoryCommit { [Required] - public required Guid Id { get; set; } = Guid.NewGuid(); + public Guid Id { get; set; } [Required] - public required AppUser Author { get; set; } + public AppUser Author { get; set; } [Required] - public required DateTime CommittedOn { get; set; } + public DateTime CommittedOn { get; set; } [Required] - public required string Message { get; set; } = String.Empty; + public string Message { get; set; } = String.Empty; [Required] - public required RepositoryDepot MainDepot { get; set; } + public RepositoryDepot MainDepot { get; set; } } \ No newline at end of file diff --git a/Flawless.Server/Models/RepositoryDepot.cs b/Flawless.Server/Models/RepositoryDepot.cs index ab0d9b2..8e23a76 100644 --- a/Flawless.Server/Models/RepositoryDepot.cs +++ b/Flawless.Server/Models/RepositoryDepot.cs @@ -6,7 +6,7 @@ public class RepositoryDepot { [Key] [Required] - public Guid DepotId { get; set; } = Guid.NewGuid(); + public Guid DepotId { get; set; } [Required] public long Length { get; set; } diff --git a/Flawless.Server/Models/RepositoryMember.cs b/Flawless.Server/Models/RepositoryMember.cs index 0612bf0..d580719 100644 --- a/Flawless.Server/Models/RepositoryMember.cs +++ b/Flawless.Server/Models/RepositoryMember.cs @@ -5,10 +5,8 @@ namespace Flawless.Server.Models; public class RepositoryMember { - [Key] - [Required] - public Guid Id { get; set; } = Guid.NewGuid(); - + [Key] [Required] public Guid Id { get; set; } + [Required] public required AppUser User { get; set; } diff --git a/Flawless.Server/Program.cs b/Flawless.Server/Program.cs index 0731f82..fcf5ec5 100644 --- a/Flawless.Server/Program.cs +++ b/Flawless.Server/Program.cs @@ -34,6 +34,12 @@ public static class Program private static void ConfigAppService(WebApplicationBuilder builder) { + // Set size limit + builder.WebHost.ConfigureKestrel(opt => + { + opt.Limits.MaxRequestBodySize = long.MaxValue; // As big as possible... + }); + // Api related builder.Services.AddSingleton(); builder.Services.AddOpenApi();