diff --git a/Flawless.Communication/Flawless.Communication.csproj b/Flawless.Communication/Flawless.Communication.csproj index 40b98ef..5db5616 100644 --- a/Flawless.Communication/Flawless.Communication.csproj +++ b/Flawless.Communication/Flawless.Communication.csproj @@ -6,4 +6,8 @@ net9.0 + + + + diff --git a/Flawless.Communication/Request/CommitRequest.cs b/Flawless.Communication/Request/CommitRequest.cs index 23b2122..4d21065 100644 --- a/Flawless.Communication/Request/CommitRequest.cs +++ b/Flawless.Communication/Request/CommitRequest.cs @@ -1,10 +1,13 @@ +using Flawless.Core.Modal; + namespace Flawless.Communication.Request; public class CommitRequest { + public required string Message { get; set; } - public required string[] WorkspaceSnapshot { get; set; } + public required WorkspaceFile[] WorkspaceSnapshot { get; set; } public string[]? RequiredDepots { get; set; } diff --git a/Flawless.Core/BinaryDataFormat/DataTransformer.cs b/Flawless.Core/BinaryDataFormat/DataTransformer.cs index 0574e6e..638b3f2 100644 --- a/Flawless.Core/BinaryDataFormat/DataTransformer.cs +++ b/Flawless.Core/BinaryDataFormat/DataTransformer.cs @@ -11,7 +11,6 @@ public static class DataTransformer { depotStream.Seek(8, SeekOrigin.Current); var val = depotStream.ReadByte(); - depotStream.Seek(-9, SeekOrigin.Current); if (val < 0) ; @@ -74,7 +73,7 @@ public static class DataTransformer return depotStream.ReadSlice((long) size); } - public static async IAsyncEnumerable ExtractDepotFileInfoMapAsync(ulong fileMapSize, Stream depotStream) + public static async IAsyncEnumerable ExtractDepotFileInfoMapAsync(Stream depotStream, ulong fileMapSize) { var splitor = '$'; depotStream.Seek((long) fileMapSize, SeekOrigin.End); @@ -87,6 +86,7 @@ public static class DataTransformer var path = string.Empty; var size = 0UL; var offset = 0UL; + DateTime modifyTime = default; // Read loop while (true) @@ -101,11 +101,12 @@ public static class DataTransformer if (c != splitor) builder.Append(c); else { - switch ((state = (state + 1) % 3)) + switch ((state = (state + 1) % 4)) { case 0: path = builder.ToString(); break; case 1: size = ulong.Parse(builder.ToString()); break; - case 2: offset = ulong.Parse(builder.ToString()); break; + case 2: modifyTime = DateTime.FromBinary(long.Parse(builder.ToString())); break; + case 3: offset = ulong.Parse(builder.ToString()); break; } builder.Clear(); @@ -119,7 +120,7 @@ public static class DataTransformer // Do output at here. - yield return new DepotFileInfo(offset, size, path); + yield return new DepotFileInfo(offset, size, modifyTime, path); } diff --git a/Flawless.Core/Modal/CommitManifest.cs b/Flawless.Core/Modal/CommitManifest.cs index e172b58..ea0db22 100644 --- a/Flawless.Core/Modal/CommitManifest.cs +++ b/Flawless.Core/Modal/CommitManifest.cs @@ -1,4 +1,4 @@ namespace Flawless.Core.Modal; [Serializable] -public record struct CommitManifest(Guid ManifestId, DepotLabel Depot, string[] FilePaths); \ No newline at end of file +public record struct CommitManifest(Guid ManifestId, DepotLabel Depot, WorkspaceFile[] FilePaths); \ No newline at end of file diff --git a/Flawless.Core/Modal/DepotFileInfo.cs b/Flawless.Core/Modal/DepotFileInfo.cs index 1f2b261..d2ea03e 100644 --- a/Flawless.Core/Modal/DepotFileInfo.cs +++ b/Flawless.Core/Modal/DepotFileInfo.cs @@ -1,4 +1,4 @@ namespace Flawless.Core.Modal; [Serializable] -public record struct DepotFileInfo(ulong Offset, ulong Size, string Path); \ No newline at end of file +public record struct DepotFileInfo(ulong Offset, ulong Size, DateTime ModifyTime, string Path); \ No newline at end of file diff --git a/Flawless.Core/Modal/DepotLabel.cs b/Flawless.Core/Modal/DepotLabel.cs index 5c4866d..d4a94f4 100644 --- a/Flawless.Core/Modal/DepotLabel.cs +++ b/Flawless.Core/Modal/DepotLabel.cs @@ -3,4 +3,4 @@ namespace Flawless.Core.Modal; [Serializable] -public record struct DepotLabel(HashId Id, HashId[] Dependency); \ No newline at end of file +public record struct DepotLabel(Guid Id, long Length, Guid[] Dependency); \ No newline at end of file diff --git a/Flawless.Core/Modal/WorkspaceFile.cs b/Flawless.Core/Modal/WorkspaceFile.cs new file mode 100644 index 0000000..76a6e26 --- /dev/null +++ b/Flawless.Core/Modal/WorkspaceFile.cs @@ -0,0 +1,33 @@ +namespace Flawless.Core.Modal; + +public struct WorkspaceFile : IEquatable +{ + public required DateTime ModifyTime; + + public required string WorkPath; + + public bool Equals(WorkspaceFile other) + { + return ModifyTime.Equals(other.ModifyTime) && WorkPath == other.WorkPath; + } + + public override bool Equals(object? obj) + { + return obj is WorkspaceFile other && Equals(other); + } + + public override int GetHashCode() + { + return HashCode.Combine(ModifyTime, WorkPath); + } + + public static bool operator ==(WorkspaceFile left, WorkspaceFile right) + { + return left.Equals(right); + } + + public static bool operator !=(WorkspaceFile left, WorkspaceFile right) + { + return !left.Equals(right); + } +} \ No newline at end of file diff --git a/Flawless.Server/Controllers/RepositoryControl.cs b/Flawless.Server/Controllers/RepositoryControl.cs index cf8bdbf..4fbc99c 100644 --- a/Flawless.Server/Controllers/RepositoryControl.cs +++ b/Flawless.Server/Controllers/RepositoryControl.cs @@ -1,6 +1,11 @@ -using Flawless.Communication.Request; +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; +using Flawless.Communication.Request; using Flawless.Communication.Response; using Flawless.Communication.Shared; +using Flawless.Core.BinaryDataFormat; +using Flawless.Core.Modal; using Flawless.Server.Models; using Flawless.Server.Services; using Flawless.Server.Utility; @@ -15,12 +20,13 @@ namespace Flawless.Server.Controllers; public class RepositoryControl( UserManager userManager, AppDbContext dbContext, - PathTransformer transformer) : ControllerBase + PathTransformer transformer, + JsonSerializerOptions serializerOpt) : ControllerBase { public class FormCommitRequest : CommitRequest { - public IFormFile Depot { get; set; } + public IFormFile? Depot { get; set; } } @@ -36,22 +42,31 @@ public class RepositoryControl( return true; } - - [HttpGet("fetch/manifest/{commitId}")] - public async Task DownloadManifestAsync(string userName, string repositoryName, string commitId) + private async ValueTask ValidateRepositoryAsync( + string userName, string repositoryName, AppUser user, RepositoryRole role) { - if (!Guid.TryParse(commitId, out var commitGuid)) return BadRequest(new FailedResponse("Invalid commit id")); var rp = await dbContext.Repositories .Include(r => r.Owner) .Include(r => r.Members) .FirstOrDefaultAsync(r => r.Owner.UserName == userName && r.Name == repositoryName); - if (rp == null) return NotFound(new FailedResponse($"Could not find repository {repositoryName}")); - var u = (await userManager.FindByNameAsync(userName))!; - if (UserNotGranted(out var rsp, u, rp, RepositoryRole.Guest)) return rsp!; + if (rp == null) return NotFound(new FailedResponse($"Could not find repository {userName}:{repositoryName}")); + if (UserNotGranted(out var rsp, user, rp, role)) return rsp!; + + return rp; + } + + + [HttpGet("fetch/manifest/{commitId}")] + public async Task DownloadManifestAsync(string userName, string repositoryName, string commitId) + { + if (!Guid.TryParse(commitId, out var commitGuid)) return BadRequest(new FailedResponse("Invalid commit id")); + var user = (await userManager.GetUserAsync(HttpContext.User))!; + var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); + if (grantIssue is not Repository rp) return (IActionResult) grantIssue; // Start checkout file. - var target = transformer.GetCommitManifest(rp.Id, commitGuid); + var target = transformer.GetCommitManifestPath(rp.Id, commitGuid); if (System.IO.File.Exists(target)) return File(System.IO.File.OpenRead(target), "application/octet-stream"); return NotFound(new FailedResponse($"Could not find commit manifest {target}")); } @@ -61,42 +76,258 @@ public class RepositoryControl( public async Task DownloadDepotAsync(string userName, string repositoryName, string depotId) { if (!Guid.TryParse(depotId, out var depotGuid)) return BadRequest(new FailedResponse("Invalid depot id")); - var rp = await dbContext.Repositories - .Include(r => r.Owner) - .Include(r => r.Members) - .FirstOrDefaultAsync(r => r.Owner.UserName == userName && r.Name == repositoryName); - - if (rp == null) return NotFound(new FailedResponse($"Could not find repository {repositoryName}")); - var u = (await userManager.FindByNameAsync(userName))!; - if (UserNotGranted(out var rsp, u, rp, RepositoryRole.Guest)) return rsp!; + var user = (await userManager.GetUserAsync(HttpContext.User))!; + var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); + if (grantIssue is not Repository rp) return (IActionResult) grantIssue; // Start checkout file. - var target = transformer.GetDepotManifest(rp.Id, depotGuid); + var target = transformer.GetDepotPath(rp.Id, depotGuid); if (System.IO.File.Exists(target)) return File(System.IO.File.OpenRead(target), "application/octet-stream"); return NotFound(new FailedResponse($"Could not find depot {target}")); } + + [HttpGet("list/commit")] + public async Task ListCommitsAsync(string userName, string repositoryName) + { + var user = (await userManager.GetUserAsync(HttpContext.User))!; + var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); + if (grantIssue is not Repository) return (IActionResult) grantIssue; + + var commit = await dbContext.Repositories + .Where(r => r.Owner.UserName == userName && r.Name == repositoryName) + .SelectMany(r => r.Commits) + .Include(r => r.Author) + .OrderByDescending(r => r.CommittedOn) + .ToArrayAsync(); - // [HttpPost("commit")] - // public async Task CommitAsync([FromForm] FormCommitRequest request) - // { - // } - // - // [HttpGet("list/commit")] - // public async Task ListCommitsAsync(string userName, string repositoryName) - // { - // - // } - // - // [HttpGet("query/commit/{keyword}")] - // public async Task QueryCommitsAsync(string userName, string repositoryName, string keyword) - // { - // - // } - // - // [HttpGet("get/commit/{commitId}")] - // public async Task GetCommitAsync(string userName, string repositoryName, string commitId) - // { - // - // } + return Ok(new + { + Commits = commit + }); + } + + + [HttpGet("peek/commit")] + public async Task PeekCommitAsync(string userName, string repositoryName) + { + var user = (await userManager.GetUserAsync(HttpContext.User))!; + var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); + if (grantIssue is not Repository rp) return (IActionResult) grantIssue; + + var commit = await dbContext.Repositories + .Where(r => r.Owner.UserName == userName && r.Name == repositoryName) + .SelectMany(r => r.Commits) + .Include(r => r.Author) + .OrderByDescending(r => r.CommittedOn) + .FirstOrDefaultAsync(); + + return Ok(new + { + Commit = commit?.Id.ToString() ?? string.Empty + }); + } + + [HttpPost("commit")] + public async Task CommitAsync(string userName, string repositoryName, [FromForm] FormCommitRequest req) + { + var user = (await userManager.GetUserAsync(HttpContext.User))!; + var grantIssue = await ValidateRepositoryAsync(userName, repositoryName, user, RepositoryRole.Guest); + if (grantIssue is not Repository rp) return (IActionResult) grantIssue; + + if (req.Depot != null ^ req.MainDepotId != null) return await CommitInternalAsync(rp, user, req); + + // Not valid response + if (req.Depot != null) return BadRequest(new FailedResponse("You can not choose a existed depot while upload a new depot!")); + return BadRequest(new FailedResponse("You must choose a existed depot upload a new depot as baseline!")); + } + + private async Task CommitInternalAsync(Repository rp, AppUser user, FormCommitRequest req) + { + var test = new HashSet(req.WorkspaceSnapshot); + var createNewDepot = false; + RepositoryDepot mainDepot; + DepotLabel depotLabel; + + // Judge if receive new depot or use old one... + if (req.Depot == null) + { + var mainDepotId = Guid.Parse(req.MainDepotId!); + var preDepot = await StencilWorkspaceSnapshotViaDatabaseAsync(rp.Id, mainDepotId, test); + + if (preDepot == null) + return BadRequest(new FailedResponse("Not a valid main depot!", "NoMainDepot")); + + // If still not able to resolve + if (test.Count > 0) + return BadRequest(new FailedResponse(test, "DependencyUnsolved")); + + // Set this depot label... + mainDepot = preDepot; + var deps = mainDepot.Dependencies.Select(d => d.DepotId).ToArray(); + depotLabel = new DepotLabel(mainDepot.DepotId, mainDepot.Length, deps); + } + else + { + var actualRequiredDepots = new HashSet(); + + // Read and create a new depot + using var cacheStream = new MemoryStream(new byte[req.Depot.Length], true); + await req.Depot.CopyToAsync(cacheStream, HttpContext.RequestAborted); + + // Look if main depot can do this... + await StencilWorkspaceSnapshotAsync(cacheStream, test); + + // Oh no, will we need to load their parents? + var unresolvedCount = test.Count; + if (unresolvedCount != 0) + { + // Yes + foreach (var subDepot in req.RequiredDepots?.Select(Guid.Parse) ?? []) + { + await StencilWorkspaceSnapshotViaDatabaseAsync(rp.Id, subDepot, test); + + var rest = test.Count; + if (rest == 0) + { + actualRequiredDepots.Add(subDepot); + break; + } + + // If test is changed? + if (unresolvedCount != rest) + { + actualRequiredDepots.Add(subDepot); + unresolvedCount = rest; + } + } + + // If still not able to resolve + if (unresolvedCount > 0) + return BadRequest(new + { + Reason = "Unable to resolve workspace snapshot! May lost some essential dependency.", + Unresolved = test + }); + } + + // Great, a valid depot, so create it in db. + mainDepot = new RepositoryDepot { Length = req.Depot.Length }; + mainDepot.Dependencies.AddRange( + await dbContext.Repositories + .Where(r => r.Id == rp.Id) + .SelectMany(r => r.Depots) + .Where(cm => actualRequiredDepots.Contains(cm.DepotId)) + .ToArrayAsync()); + + depotLabel = new DepotLabel(mainDepot.DepotId, mainDepot.Length, actualRequiredDepots.ToArray()); + rp.Depots.Add(mainDepot); + + // Then write depot into disk + var depotPath = transformer.GetDepotPath(rp.Id, mainDepot.DepotId); + await using (var depotStream = System.IO.File.Create(depotPath)) + { + cacheStream.Seek(0, SeekOrigin.Begin); + await cacheStream.CopyToAsync(depotStream, HttpContext.RequestAborted); + } + + createNewDepot = true; + } + + // Create manifest file and write to disk + var commitId = Guid.NewGuid(); + var manifestPath = transformer.GetCommitManifestPath(rp.Id, commitId); + var manifest = new CommitManifest + { + ManifestId = commitId, + Depot = depotLabel, + FilePaths = req.WorkspaceSnapshot + }; + + await using (var manifestStream = System.IO.File.Create(manifestPath)) + await JsonSerializer.SerializeAsync(manifestStream, manifest, serializerOpt); + + // Create commit info + var commit = new RepositoryCommit + { + Id = commitId, + Author = user, + CommittedOn = DateTime.UtcNow, + MainDepot = mainDepot, + Message = manifestPath + }; + + rp.Commits.Add(commit); + + try + { + // Write changes into db. + await dbContext.SaveChangesAsync(); + } + catch (Exception) + { + // Revert manifest create operation + if (createNewDepot) System.IO.File.Delete(transformer.GetDepotPath(rp.Id, mainDepot.DepotId)); + System.IO.File.Delete(manifestPath); + + throw; + } + + return Ok(new + { + CreatedAt = commit.CommittedOn, + CommitId = commit.Id + }); + } + + private static async ValueTask StencilWorkspaceSnapshotAsync(Stream depotStream, HashSet unresolved) + { + // Get version + var version = DataTransformer.GuessStandardDepotHeaderVersion(depotStream); + if (version != 1) throw new InvalidDataException($"Unable to get depot header version, feedback is {version}."); + + // Get header + depotStream.Seek(0, SeekOrigin.Begin); + var header = DataTransformer.ExtractStandardDepotHeaderV1(depotStream); + + // Start validation + depotStream.Seek(0, SeekOrigin.Begin); + await foreach (var inf in DataTransformer.ExtractDepotFileInfoMapAsync(depotStream, header.FileMapSize)) + { + var test = new WorkspaceFile + { + ModifyTime = inf.ModifyTime, + WorkPath = inf.Path, + }; + + unresolved.Remove(test); + if (unresolved.Count == 0) break; // Early quit to avoid extra performance loss. + } + } + + private async Task StencilWorkspaceSnapshotViaDatabaseAsync(Guid rpId, Guid depotId, HashSet unresolved) + { + var depot = await dbContext.Repositories + .SelectMany(r => r.Depots) + .Include(r => r.Dependencies) + .ThenInclude(repositoryDepot => repositoryDepot.Dependencies) + .FirstOrDefaultAsync(r => r.DepotId == depotId); + + await RecurringAsync(transformer, rpId, depot, unresolved); + return depot; + + static async ValueTask RecurringAsync(PathTransformer transformer, Guid rpId, RepositoryDepot? depot, HashSet unresolved) + { + if (depot == null) return; + var path = transformer.GetDepotPath(rpId, depot.DepotId); + + // require do not extend lifetime scope. + await using (var fs = new BufferedStream(new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))) + await StencilWorkspaceSnapshotAsync(fs, unresolved); + + for (var i = 0; i < depot.Dependencies.Count && unresolved.Count > 0; i++) + await RecurringAsync(transformer, rpId, depot.Dependencies[i], unresolved); + + } + } } \ No newline at end of file diff --git a/Flawless.Server/Models/Repository.cs b/Flawless.Server/Models/Repository.cs index 433ed22..8c2c90a 100644 --- a/Flawless.Server/Models/Repository.cs +++ b/Flawless.Server/Models/Repository.cs @@ -22,4 +22,6 @@ public class Repository public List Members { get; set; } = new(); public List Commits { get; set; } = new(); + + public List Depots { get; set; } = new(); } \ No newline at end of file diff --git a/Flawless.Server/Models/RepositoryCommit.cs b/Flawless.Server/Models/RepositoryCommit.cs index 7490dbd..e3f472a 100644 --- a/Flawless.Server/Models/RepositoryCommit.cs +++ b/Flawless.Server/Models/RepositoryCommit.cs @@ -5,16 +5,17 @@ namespace Flawless.Server.Models; public class RepositoryCommit { [Required] - public Guid Id { get; set; } = Guid.NewGuid(); + public required Guid Id { get; set; } = Guid.NewGuid(); + + [Required] + public required AppUser Author { get; set; } [Required] public required DateTime CommittedOn { get; set; } [Required] - public string Message { get; set; } = String.Empty; + public required string Message { get; set; } = String.Empty; [Required] public required RepositoryDepot MainDepot { get; set; } - - public RepositoryCommit? Parent { get; set; } } \ No newline at end of file diff --git a/Flawless.Server/Models/RepositoryDepot.cs b/Flawless.Server/Models/RepositoryDepot.cs index 53c55e5..ab0d9b2 100644 --- a/Flawless.Server/Models/RepositoryDepot.cs +++ b/Flawless.Server/Models/RepositoryDepot.cs @@ -6,8 +6,10 @@ public class RepositoryDepot { [Key] [Required] - public Guid DepotId { get; set; } + public Guid DepotId { get; set; } = Guid.NewGuid(); + + [Required] + public long Length { get; set; } - [Required] public List Dependencies { get; set; } = new(); } \ No newline at end of file diff --git a/Flawless.Server/Utility/PathTransformer.cs b/Flawless.Server/Utility/PathTransformer.cs index e16bb46..e0ad15b 100644 --- a/Flawless.Server/Utility/PathTransformer.cs +++ b/Flawless.Server/Utility/PathTransformer.cs @@ -16,7 +16,7 @@ public class PathTransformer(IConfiguration config) return Path.Combine(_rootPath, repositoryId.ToString()); } - public string GetCommitManifest(Guid repositoryId, Guid commitId) + public string GetCommitManifestPath(Guid repositoryId, Guid commitId) { if (repositoryId == Guid.Empty) throw new ArgumentException(nameof(repositoryId)); if (commitId == Guid.Empty) throw new ArgumentException(nameof(commitId)); @@ -24,7 +24,7 @@ public class PathTransformer(IConfiguration config) return Path.Combine(_rootPath, repositoryId.ToString(), "Manifests", commitId.ToString()); } - public string GetDepotManifest(Guid repositoryId, Guid depotId) + public string GetDepotPath(Guid repositoryId, Guid depotId) { if (repositoryId == Guid.Empty) throw new ArgumentException(nameof(repositoryId)); if (depotId == Guid.Empty) throw new ArgumentException(nameof(depotId));