using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Net; using System.Threading.Tasks; using Flawless.Abstraction; using Flawless.Client.Models; using Flawless.Core.BinaryDataFormat; using Flawless.Core.Modal; using Newtonsoft.Json; using Refit; namespace Flawless.Client.Service; public class RepositoryService : BaseService { public ObservableCollection Repositories => _repositories; private readonly ObservableCollection _repositories = new(); private readonly Dictionary _localRepoDbModel = new(); private readonly HashSet _openedRepos = new(); private bool TryCreateRepositoryBaseStorageStructure(RepositoryModel repo) { var dbPath = PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name); if (File.Exists(dbPath)) return false; // Get directories var localRepoDb = GetRepositoryLocalDatabase(repo); var folderPath = PathUtility.GetWorkspaceManagerPath(repo.OwnerName, repo.Name); // Create initial data. Directory.CreateDirectory(folderPath); using var writeFs = new StreamWriter(new FileStream(dbPath, FileMode.Create)); JsonSerializer.CreateDefault().Serialize(writeFs, localRepoDb); return true; } public bool SaveRepositoryLocalDatabaseChanges(RepositoryModel repo) { var localRepo = GetRepositoryLocalDatabase(repo); localRepo.LastOprationTime = DateTime.Now; var dbPath = PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name); Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!); using var writeFs = new StreamWriter(new FileStream(dbPath, FileMode.OpenOrCreate)); JsonSerializer.CreateDefault().Serialize(writeFs, localRepo); return true; } public RepositoryLocalDatabaseModel GetRepositoryLocalDatabase(RepositoryModel repo) { if (!_localRepoDbModel.TryGetValue(repo, out var localRepo)) { var dbPath = PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name); if (File.Exists(dbPath)) { // Use existed target using var readFs = new StreamReader(new FileStream(dbPath, FileMode.Open)); localRepo = (JsonSerializer.CreateDefault().Deserialize(readFs, typeof(RepositoryLocalDatabaseModel)) as RepositoryLocalDatabaseModel)!; // todo add broken test. localRepo.RootModal = repo; localRepo.LocalAccessor = new LocalFileTreeAccessor(repo, []); } else { // Create new one. localRepo = new RepositoryLocalDatabaseModel { RootModal = repo, LocalAccessor = new LocalFileTreeAccessor(repo, []) }; } _localRepoDbModel.Add(repo, localRepo); } return localRepo; } public async ValueTask CreateRepositoryOnServerAsync(string repositoryName, string description) { RepositoryModel repo; var api = Api.C; try { if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return null; } var r = await api.Gateway.RepoCreate(repositoryName, description); repo = new RepositoryModel { Name = r.RepositoryName, OwnerName = r.OwnerUsername, StandaloneName = RepositoryModel.GetStandaloneName(r.RepositoryName, r.OwnerUsername), Description = r.Description, Archived = r.IsArchived, OwnByCurrentUser = (int) r.Role == (int) RepositoryModel.RepositoryRole.Owner }; Repositories.Insert(0, repo); } catch (Exception e) { Console.WriteLine(e); return null; } return repo; } public async ValueTask UpdateRepositoriesFromServerAsync() { var api = Api.C; try { if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return false; } var result = (await api.Gateway.RepoList()).Result; var dict = result.ToDictionary(rp => RepositoryModel.GetStandaloneName(rp.RepositoryName, rp.OwnerUsername)); for (var i = 0; i < Repositories.Count; i++) { var ele = Repositories[i]; if (!dict.Remove(ele.StandaloneName, out var role)) { Repositories.RemoveAt(i); i -= 1; continue; } ele.Archived = ele.Archived; ele.Description = ele.Description; } foreach (var (repoStandaloneName, rsp) in dict) { var repo = new RepositoryModel(); repo.Name = rsp.RepositoryName; repo.OwnerName = rsp.OwnerUsername; repo.StandaloneName = repoStandaloneName; repo.Description = rsp.Description; repo.Archived = rsp.IsArchived; repo.OwnByCurrentUser = (int) rsp.Role == (int) RepositoryModel.RepositoryRole.Owner; Repositories.Add(repo); } } catch (Exception e) { Console.WriteLine(e); return false; } return true; } public async ValueTask UpdateRepositoriesDownloadedStatusFromDiskAsync() { foreach (var repo in _repositories) { var isFolderExists = Directory.Exists(PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name)); var isDbFileExists = File.Exists(PathUtility.GetWorkspaceDbPath(repo.OwnerName, repo.Name)); repo.IsDownloaded = isFolderExists && isDbFileExists; } return true; } public async ValueTask UpdateMembersFromServerAsync(RepositoryModel repo) { var api = Api.C; try { if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return false; } var members = await api.Gateway.GetUsers(repo.Name, repo.OwnerName); // Update existed var dict = members.Result.ToDictionary(m => m.Username); for (var i = 0; i < repo.Members.Count; i++) { var ele = repo.Members[i]; if (!dict.Remove(ele.Username, out var role)) { repo.Members.RemoveAt(i); i -= 1; continue; } ele.Username = role.Username; ele.Role = (RepositoryModel.RepositoryRole) role.Role; } // Add missing foreach (var role in dict.Values) { var r = new RepositoryModel.Member { Username = role.Username, Role = (RepositoryModel.RepositoryRole) role.Role }; repo.Members.Add(r); } } catch (Exception e) { Console.WriteLine(e); return false; } return true; } public bool IsRepositoryOpened(RepositoryModel repo) { return _openedRepos.Any(r => r == repo); } public async ValueTask CloseRepositoryAsync(RepositoryModel repo) { if (_openedRepos.Contains(repo)) { var ls = GetRepositoryLocalDatabase(repo); if (ls.RepoAccessor != null) await ls.RepoAccessor!.DisposeAsync(); ls.RepoAccessor = null; if (!SaveRepositoryLocalDatabaseChanges(repo)) return false; _openedRepos.Remove(repo); return true; } return false; } public async ValueTask OpenRepositoryOnStorageAsync(RepositoryModel repo) { if (!await UpdateRepositoriesDownloadedStatusFromDiskAsync() || repo.IsDownloaded == false) return false; if (!await UpdateCommitsFromServerAsync(repo)) return false; var ls = GetRepositoryLocalDatabase(repo); if (ls.CurrentCommit != null) { var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, ls.CurrentCommit.Value); if (accessor == null) return false; ls.RepoAccessor = accessor; // Remember to cache accessor everytime it will being used. await accessor.CreateCacheAsync(); ls.LocalAccessor.SetBaseline(accessor); } SaveRepositoryLocalDatabaseChanges(repo); _openedRepos.Add(repo); return true; } public async ValueTask CreateNewRepositoryOnStorageAsync(RepositoryModel repo) { // Create basic structures. if (!TryCreateRepositoryBaseStorageStructure(repo)) return false; SaveRepositoryLocalDatabaseChanges(repo); repo.IsDownloaded = true; _openedRepos.Add(repo); return true; } public async ValueTask CloneRepositoryFromRemoteAsync(RepositoryModel repo) { // Create basic structures. if (!TryCreateRepositoryBaseStorageStructure(repo)) return false; if (!await UpdateCommitsFromServerAsync(repo)) return false; var peekCommit = repo.Commits.MaxBy(sl => sl.CommittedOn); if (peekCommit == null) return false; // Should not use this function! // Download base repo info var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, peekCommit.CommitId); if (accessor == null) { await DeleteFromDiskAsync(repo); return false; }; // Remember to cache accessor everytime it will being used. await accessor.CreateCacheAsync(); var ls = GetRepositoryLocalDatabase(repo); ls.CurrentCommit = peekCommit.CommitId; ls.RepoAccessor = accessor; ls.LocalAccessor.SetBaseline(accessor); try { foreach (var f in accessor.Manifest.FilePaths) { var pfs = WorkPath.ToPlatformPath(f.WorkPath, ls.LocalAccessor.WorkingDirectory); var directory = Path.GetDirectoryName(pfs); // Write into fs if (directory != null) Directory.CreateDirectory(directory); if (!accessor.TryWriteDataIntoStream(f.WorkPath, pfs)) throw new InvalidDataException($"Can not write {f.WorkPath} into repository."); } } catch (Exception e) { Console.WriteLine(e); await DeleteFromDiskAsync(repo); return false; } SaveRepositoryLocalDatabaseChanges(repo); repo.IsDownloaded = true; _openedRepos.Add(repo); return true; } public async ValueTask ShouldUpdateLocalCommitsCacheFromServerAsync(RepositoryModel repo) { var api = Api.C; try { if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return false; } var rsp = await api.Gateway.PeekCommit(repo.OwnerName, repo.Name); var emptyRepo = repo.Commits.Count == 0; // If they both empty if ((rsp.Result == Guid.Empty) == emptyRepo) return false; if (emptyRepo) return true; return rsp.Result == repo.Commits.MaxBy(cm => cm.CommittedOn)!.CommitId; } catch (Exception e) { Console.WriteLine(e); return false; } } public async ValueTask UpdateCommitsFromServerAsync(RepositoryModel repo) { var api = Api.C; try { if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return false; } var rsp = await api.Gateway.ListCommit(repo.OwnerName, repo.Name); // Update existed var dict = rsp.Result.ToDictionary(m => m.Id); for (var i = 0; i < repo.Commits.Count; i++) { var ele = repo.Commits[i]; if (!dict.Remove(ele.CommitId, out var cm)) { repo.Members.RemoveAt(i); i -= 1; continue; } ele.Message = cm.Message; ele.DepotId = cm.MainDepotId; ele.CommittedOn = cm.CommitedOn.UtcDateTime; ele.Author = cm.Author; } // Add missing foreach (var cm in dict.Values) { var r = new RepositoryModel.Commit { CommitId = cm.Id, Message = cm.Message, DepotId = cm.MainDepotId, CommittedOn = cm.CommitedOn.UtcDateTime, Author = cm.Author, }; repo.Commits.Add(r); } // Resort them again repo.Commits.Sort((l, r) => r.CommittedOn.CompareTo(l.CommittedOn)); } catch (Exception e) { Console.WriteLine(e); return false; } return true; } public async ValueTask DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync (RepositoryModel repo, Guid commit, bool storeDownloadedDepots = true) { if (commit == Guid.Empty) return null; // Get depots list and manifest var manifest = await DownloadManifestFromServerAsync(repo, commit); if (manifest == null) return null; // Prepare folders var path = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); var depotsRoot = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); Directory.CreateDirectory(depotsRoot); // Generate download depots list var mainDepotLabel = manifest.Value.Depot; var willDownload = mainDepotLabel.Where(label => { var dpPath = Path.Combine(depotsRoot, label.Id.ToString()); if (!File.Exists(dpPath)) return true; // todo Needs a way to check if that valid. return false; }); // Download them var downloadedDepots = await DownloadDepotsAndCopyNetworkStreamIntoNewMemoryStreamFromServerAsync(repo, willDownload); if (downloadedDepots == null) return null; try { if (storeDownloadedDepots) { // Write new downloaded depots into disk var transform = downloadedDepots.Select(dl => (dl.Item1, dl.Item2!)); await WriteDownloadedDepotsFromServerToStorageAsync(repo, transform); } // Create mapping dictionary var mappingDict = downloadedDepots.ToDictionary(i => i.Item1, i => i.Item2!); foreach (var dl in mainDepotLabel) { if (mappingDict.ContainsKey(dl.Id)) continue; var dst = Path.Combine(depotsRoot, dl.Id.ToString()); mappingDict.Add(dl.Id, new FileStream(dst, FileMode.Create)); } return new RepositoryFileTreeAccessor(mappingDict, manifest.Value); } catch (Exception e) { if (downloadedDepots != null) foreach (var t in downloadedDepots) { if (t.Item2 == null) continue; try { await t.Item2.DisposeAsync(); } catch (Exception ex) { Console.WriteLine(ex); } } Console.WriteLine(e); return null; } } public async ValueTask WriteDownloadedDepotsFromServerToStorageAsync (RepositoryModel repo, IEnumerable<(Guid id, Stream stream)> depots) { var depotsRoot = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); foreach (var d in depots) { var dst = Path.Combine(depotsRoot, d.id.ToString()); await using var ws = new FileStream(dst, FileMode.Create); // Make sure always to be at begin. d.stream.Seek(0, SeekOrigin.Begin); await d.stream.CopyToAsync(ws); d.stream.Seek(0, SeekOrigin.Begin); } } public async ValueTask<(Guid, Stream)[]?> DownloadDepotsAndCopyNetworkStreamIntoNewMemoryStreamFromServerAsync (RepositoryModel repo, IEnumerable depotsId) { try { var api = Api.C; if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return null; } var result = new List<(Guid, Stream)>(); foreach (var dl in depotsId) { using var rsp = await Api.C.Gateway.FetchDepot(repo.OwnerName, repo.Name, dl.Id.ToString()); if (rsp.StatusCode != HttpStatusCode.OK) { Console.WriteLine($"Failed to fetch depot {dl.Id}"); return null; } var memoryStream = new MemoryStream(new byte[dl.Length]); await rsp.Content!.CopyToAsync(memoryStream); result.Add((dl.Id, memoryStream)); } return result.ToArray(); } catch (Exception e) { Console.WriteLine(e); return null; } } public async ValueTask DownloadManifestFromServerAsync(RepositoryModel repo, Guid manifestId) { try { var api = Api.C; if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return null; } var rsp = await api.Gateway.FetchManifest(repo.OwnerName, repo.Name, manifestId.ToString()); return new( rsp.ManifestId, rsp.Depot.Select(x => new DepotLabel(x.Id, x.Length)).ToArray(), rsp.FilePaths.Select(x => new WorkspaceFile(x.ModifyTime.UtcDateTime, x.WorkPath)).ToArray()); } catch (Exception e) { Console.WriteLine(e); return null; } } public async ValueTask DeleteFromDiskAsync(RepositoryModel repo) { try { var path = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); if (Directory.Exists(path)) Directory.Delete(path, true); } catch (Exception e) { Console.WriteLine(e); return false; } repo.IsDownloaded = false; return true; } public async ValueTask CommitWorkspaceAsBaselineAsync (RepositoryModel repo, IEnumerable changes, string message) { var localDb = GetRepositoryLocalDatabase(repo); var manifestList = CreateCommitManifestByCurrentBaselineAndChanges(localDb.LocalAccessor, changes); var api = Api.C; try { // Renew for once. if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return null; } // Generate depot var tempDepotPath = await CreateDepotIntoTempFileAsync(repo, manifestList); if (tempDepotPath == null) return null; // Upload and create commit await using var str = File.OpenRead(tempDepotPath); var snapshot = manifestList.Select(l => $"{l.ModifyTime.ToBinary()}${l.WorkPath}"); if (api.RequireRefreshToken() && !(await api.TryRefreshTokenAsync())) { api.ClearGateway(); return null; } var rsp = await api.Gateway.CreateCommit(repo.OwnerName, repo.Name, new StreamPart(str, Path.GetFileName(tempDepotPath)), message, snapshot, null!, null!); // Move depot file to destination var depotsPath = PathUtility.GetWorkspaceDepotCachePath(repo.OwnerName, repo.Name); var finalPath = Path.Combine(depotsPath, rsp.MainDepotId.ToString()); Directory.CreateDirectory(depotsPath); File.Move(tempDepotPath, finalPath, true); // Fetch mapped manifest var manifest = await DownloadManifestFromServerAsync(repo, rsp.CommitId); if (manifest == null) return null; var accessor = await DownloadDepotsAndUseLocalCachesToGenerateRepositoryFileTreeAccessorFromServerAsync(repo, rsp.CommitId); if (accessor == null) return null; //todo this is a really fatal issue... if (localDb.RepoAccessor != null) { try { await localDb.RepoAccessor.DisposeAsync(); } catch (Exception e) { Console.WriteLine(e); } } // Point to newest state. localDb.RepoAccessor = accessor; localDb.CurrentCommit = rsp.CommitId; localDb.LocalAccessor.SetBaseline(accessor); SaveRepositoryLocalDatabaseChanges(repo); return manifest; } catch (Exception e) { Console.WriteLine(e); return null; } } public List CreateCommitManifestByCurrentBaselineAndChanges (LocalFileTreeAccessor accessor, IEnumerable changes, bool hard = false) { // Create a new depot file manifest. var files = accessor.BaselineFiles.Values.ToList(); foreach (var c in changes) { switch (c.Type) { case LocalFileTreeAccessor.ChangeType.Folder: { if (hard) throw new InvalidProgramException( $"Can not commit folder into version control: {c.File.WorkPath}"); Console.WriteLine($"Can not commit folder into version control...Ignored: {c.File.WorkPath}"); continue; } case LocalFileTreeAccessor.ChangeType.Add: { if (files.Any(f => f.WorkPath == c.File.WorkPath)) { if (hard) throw new InvalidProgramException( $"Can not create an existed record into version control: {c.File.WorkPath}"); Console.WriteLine($"Can not create an existed record into version control...Ignored: {c.File.WorkPath}"); continue; } files.Add(c.File); break; } case LocalFileTreeAccessor.ChangeType.Remove: { var idx = files.FindIndex(f => f.WorkPath == c.File.WorkPath); if (idx < 0) { if (hard) throw new InvalidProgramException( $"Can not delete a missed record into version control: {c.File.WorkPath}"); Console.WriteLine($"Can not delete a missed record into version control...Ignored: {c.File.WorkPath}"); continue; } files.RemoveAt(idx); break; } case LocalFileTreeAccessor.ChangeType.Modify: { var idx = files.FindIndex(f => f.WorkPath == c.File.WorkPath); if (idx < 0) { if (hard) throw new InvalidProgramException( $"Can not modify a missed record into version control: {c.File.WorkPath}"); Console.WriteLine($"Can not modify a missed record into version control...Ignored: {c.File.WorkPath}"); continue; } files[idx] = c.File; break; } } } return files; } public async ValueTask CreateDepotIntoTempFileAsync(RepositoryModel repo, IEnumerable depotFiles) { var repoWs = PathUtility.GetWorkspacePath(repo.OwnerName, repo.Name); var commitTempFolder = Directory.CreateTempSubdirectory("FlawlessDepot_"); var depotFile = Path.Combine(commitTempFolder.FullName, "depot.bin"); try { // No UI thread blocked await Task.Run(async () => { await using var fs = new FileStream(depotFile, FileMode.Create); DataTransformer.CreateAndInsertStandardDepotFile(fs, depotFiles, wf => File.OpenRead(WorkPath.ToPlatformPath(wf.WorkPath, repoWs))); }); } catch (Exception e) { Directory.Delete(repoWs, true); Console.WriteLine(e); return null; } return depotFile; } }