From 4efe4afa952843abffab700830555736cd38afb7 Mon Sep 17 00:00:00 2001 From: yahavi Date: Mon, 16 Oct 2023 15:59:40 +0300 Subject: [PATCH] Transfer - Show visited dirs counter --- .../transferfiles/delayedartifactshandler.go | 2 +- .../commands/transferfiles/fulltransfer.go | 7 +++++ .../commands/transferfiles/state/runstatus.go | 11 ++++---- .../transferfiles/state/statemanager.go | 14 ++++++++-- .../transferfiles/state/statemanager_test.go | 28 +++++++++++++++++++ artifactory/commands/transferfiles/status.go | 25 +++++++++-------- .../commands/transferfiles/transfer.go | 3 +- .../transferfiles/transferfileprogress.go | 13 +++++++-- .../progressbar/transferprogressbarmanager.go | 12 ++++++++ 9 files changed, 92 insertions(+), 23 deletions(-) diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 0292e51fc..693babc20 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -177,7 +177,7 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon if base.progressBar != nil { base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts)) } - if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil { + if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil { log.Warn("Couldn't decrease the delayed files counter", err.Error()) } diff --git a/artifactory/commands/transferfiles/fulltransfer.go b/artifactory/commands/transferfiles/fulltransfer.go index 4e90d7564..1d9ba5909 100644 --- a/artifactory/commands/transferfiles/fulltransfer.go +++ b/artifactory/commands/transferfiles/fulltransfer.go @@ -217,6 +217,13 @@ func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, p func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrapper producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng, item servicesUtils.ResultItem) (err error) { + if m.progressBar != nil { + m.progressBar.incNumberOfVisitedDirs() + } + if err = m.stateManager.IncVisitedDirectories(); err != nil { + return + } + newRelativePath := getFolderRelativePath(item.Name, params.relativePath) folderHandler := m.createFolderFullTransferHandlerFunc(pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng) diff --git a/artifactory/commands/transferfiles/state/runstatus.go b/artifactory/commands/transferfiles/state/runstatus.go index d9d48e032..3cfd7e347 100644 --- a/artifactory/commands/transferfiles/state/runstatus.go +++ b/artifactory/commands/transferfiles/state/runstatus.go @@ -33,11 +33,12 @@ type TransferRunStatus struct { Version int `json:"version,omitempty"` CurrentRepoKey string `json:"current_repo,omitempty"` // True if currently transferring a build info repository. - BuildInfoRepo bool `json:"build_info_repo,omitempty"` - CurrentRepoPhase int `json:"current_repo_phase,omitempty"` - WorkingThreads int `json:"working_threads,omitempty"` - DelayedFiles uint `json:"delayed_files,omitempty"` - TransferFailures uint `json:"transfer_failures,omitempty"` + BuildInfoRepo bool `json:"build_info_repo,omitempty"` + CurrentRepoPhase int `json:"current_repo_phase,omitempty"` + WorkingThreads int `json:"working_threads,omitempty"` + VisitedDirectories uint64 `json:"visited_dirs,omitempty"` + DelayedFiles uint64 `json:"delayed_files,omitempty"` + TransferFailures uint `json:"transfer_failures,omitempty"` TimeEstimationManager `json:"time_estimation,omitempty"` StaleChunks []StaleChunks `json:"stale_chunks,omitempty"` } diff --git a/artifactory/commands/transferfiles/state/statemanager.go b/artifactory/commands/transferfiles/state/statemanager.go index 82adbc92e..7829305ea 100644 --- a/artifactory/commands/transferfiles/state/statemanager.go +++ b/artifactory/commands/transferfiles/state/statemanager.go @@ -67,7 +67,7 @@ func (ts *TransferStateManager) UnlockTransferStateManager() error { // buildInfoRepo - True if build info repository // reset - Delete the repository's previous transfer info func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, totalFiles int64, buildInfoRepo, reset bool) error { - err := ts.TransferState.Action(func(state *TransferState) error { + err := ts.Action(func(*TransferState) error { transferState, repoTransferSnapshot, err := getTransferStateAndSnapshot(repoKey, reset) if err != nil { return err @@ -82,9 +82,10 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot if err != nil { return err } - return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { + return ts.action(func(transferRunStatus *TransferRunStatus) error { transferRunStatus.CurrentRepoKey = repoKey transferRunStatus.BuildInfoRepo = buildInfoRepo + transferRunStatus.VisitedDirectories = 0 transferRunStatus.OverallTransfer.TransferredUnits += ts.CurrentRepo.Phase1Info.TransferredUnits transferRunStatus.OverallTransfer.TransferredSizeBytes += ts.CurrentRepo.Phase1Info.TransferredSizeBytes @@ -263,7 +264,14 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er }) } -func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error { +func (ts *TransferStateManager) IncVisitedDirectories() error { + return ts.action(func(transferRunStatus *TransferRunStatus) error { + transferRunStatus.VisitedDirectories++ + return nil + }) +} + +func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint64, increase bool) error { return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { if increase { transferRunStatus.DelayedFiles += count diff --git a/artifactory/commands/transferfiles/state/statemanager_test.go b/artifactory/commands/transferfiles/state/statemanager_test.go index 18e92adf0..d4bffb323 100644 --- a/artifactory/commands/transferfiles/state/statemanager_test.go +++ b/artifactory/commands/transferfiles/state/statemanager_test.go @@ -146,6 +146,34 @@ func TestReposOverallBiFiles(t *testing.T) { assert.Equal(t, int64(8), stateManager.OverallBiFiles.TransferredUnits) } +func TestDelayedFiles(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + + // Increase delayed files count + assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(2, true)) + assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(4, true)) + assert.Equal(t, uint64(6), stateManager.DelayedFiles) + + // Decrease delayed files count + assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(3, false)) + assert.Equal(t, uint64(3), stateManager.DelayedFiles) +} + +func TestVisitedDirectories(t *testing.T) { + stateManager, cleanUp := InitStateTest(t) + defer cleanUp() + + // Increase visited directories count + assert.NoError(t, stateManager.IncVisitedDirectories()) + assert.NoError(t, stateManager.IncVisitedDirectories()) + assert.Equal(t, uint64(2), stateManager.VisitedDirectories) + + // Set repository state and ensure the visited directories became 0 + assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, true, true)) + assert.Zero(t, stateManager.VisitedDirectories) +} + func assertReposTransferredSize(t *testing.T, stateManager *TransferStateManager, expectedSize int64, repoKeys ...string) { totalTransferredSize, err := stateManager.GetReposTransferredSizeBytes(repoKeys...) assert.NoError(t, err) diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index 818c3db55..1a5f236a5 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -79,11 +79,6 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings. addString(output, "๐Ÿงต", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2) addString(output, "โšก", "Transfer speed", stateManager.GetSpeedString(), 2) addString(output, "โŒ›", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1) - delayedTxt := strconv.FormatUint(uint64(stateManager.DelayedFiles), 10) - if stateManager.DelayedFiles > 0 { - delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")" - } - addString(output, "โœ‹", "Delayed files", delayedTxt, 2) failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10) if stateManager.TransferFailures > 0 { failureTxt += " (" + progressbar.RetryFailureContentNote + ")" @@ -100,20 +95,28 @@ func calcPercentageInt64(transferred, total int64) string { func setRepositoryStatus(stateManager *state.TransferStateManager, output *strings.Builder) { addTitle(output, "Current Repository Status") - addString(output, "๐Ÿท ", "Name", stateManager.CurrentRepoKey, 2) + addString(output, "๐Ÿท ", "Name", stateManager.CurrentRepoKey, 3) currentRepo := stateManager.CurrentRepo switch stateManager.CurrentRepoPhase { case api.Phase1, api.Phase3: if stateManager.CurrentRepoPhase == api.Phase1 { - addString(output, "๐Ÿ”ข", "Phase", "Transferring all files in the repository (1/3)", 2) + addString(output, "๐Ÿ”ข", "Phase", "Transferring all files in the repository (1/3)", 3) } else { - addString(output, "๐Ÿ”ข", "Phase", "Retrying transfer failures (3/3)", 2) + addString(output, "๐Ÿ”ข", "Phase", "Retrying transfer failures (3/3)", 3) } - addString(output, "๐Ÿ—„ ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 2) - addString(output, "๐Ÿ“„", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 2) + addString(output, "๐Ÿ—„ ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3) + addString(output, "๐Ÿ“„", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3) case api.Phase2: - addString(output, "๐Ÿ”ข", "Phase", "Transferring newly created and modified files (2/3)", 2) + addString(output, "๐Ÿ”ข", "Phase", "Transferring newly created and modified files (2/3)", 3) + } + if stateManager.CurrentRepoPhase == api.Phase1 { + addString(output, "๐Ÿ“", "Visited directories", strconv.FormatUint(stateManager.VisitedDirectories, 10), 2) } + delayedTxt := strconv.FormatUint(stateManager.DelayedFiles, 10) + if stateManager.DelayedFiles > 0 { + delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")" + } + addString(output, "โœ‹", "Delayed files", delayedTxt, 2) } func addStaleChunks(stateManager *state.TransferStateManager, output *strings.Builder) { diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index 40faa5e7c..5b2cabe2e 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -233,8 +233,9 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui if e != nil { return e } - tdc.stateManager.DelayedFiles = uint(numberInitialDelays) + tdc.stateManager.DelayedFiles = uint64(numberInitialDelays) } else { + tdc.stateManager.VisitedDirectories = 0 tdc.stateManager.TransferFailures = 0 tdc.stateManager.DelayedFiles = 0 } diff --git a/artifactory/commands/transferfiles/transferfileprogress.go b/artifactory/commands/transferfiles/transferfileprogress.go index 059a37acb..88283ff76 100644 --- a/artifactory/commands/transferfiles/transferfileprogress.go +++ b/artifactory/commands/transferfiles/transferfileprogress.go @@ -31,6 +31,8 @@ type TransferProgressMng struct { speedBar *progressbar.TasksProgressBar // A bar showing the estimated remaining time for the transfer timeEstBar *progressbar.TasksProgressBar + // A bar showing the number of visited directories + visitedDirsBar *progressbar.TasksProgressBar // A bar showing the number of delayed artifacts in the process delayedBar *progressbar.TasksProgressBar // A bar showing the number of transfer failures in the process @@ -71,6 +73,7 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar() transfer.speedBar = transfer.transferMng.NewSpeedProgBar() transfer.timeEstBar = transfer.transferMng.NewTimeEstBar() + transfer.visitedDirsBar = transfer.transferMng.NewVisitedDirsBar() transfer.delayedBar = transfer.transferMng.NewDelayedBar() // Init global error count for the process transfer.errorBar = transfer.transferMng.NewErrorBar() @@ -220,10 +223,16 @@ func (t *TransferProgressMng) RemoveRepository() { time.Sleep(progressbar.ProgressRefreshRate) } +func (t *TransferProgressMng) incNumberOfVisitedDirs() { + if t.ShouldDisplay() { + t.visitedDirsBar.SetGeneralProgressTotal(t.visitedDirsBar.GetTotal() + 1) + } +} + func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) { if t.ShouldDisplay() { diff := int64(n) - t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff) + t.delayedBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff) } } @@ -251,7 +260,7 @@ func (t *TransferProgressMng) StopGracefully() { } func (t *TransferProgressMng) abortMetricsBars() { - for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} { + for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.visitedDirsBar, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} { if barPtr != nil { barPtr.GetBar().Abort(true) } diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index 676b07b6d..91a3f952f 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -25,6 +25,7 @@ type transferLabels struct { Note string TransferSpeed string EstimatedTime string + VisitedDirs string DelayedFiles string TransferFailures string WorkingThreads string @@ -53,6 +54,7 @@ func initSProgressBarLabels(windows bool) transferLabels { pbs.Note = formatString(" ๐ŸŸ ", " Note: ", windows) pbs.TransferSpeed = formatString(" โšก", " Transfer speed: ", windows) pbs.EstimatedTime = formatString(" โŒ›", " Estimated time remaining: ", windows) + pbs.VisitedDirs = formatString(" ๐Ÿ“", " Visited directories: ", windows) pbs.DelayedFiles = formatString(" โœ‹", " Delayed files: ", windows) pbs.TransferFailures = formatString(" โŒ", " Transfer failures: ", windows) pbs.WorkingThreads = formatString(" ๐Ÿงต", " Working threads: ", windows) @@ -308,6 +310,16 @@ func (tpm *TransferProgressMng) NewTimeEstBar() *TasksProgressBar { }) } +func (tpm *TransferProgressMng) NewVisitedDirsBar() *TasksProgressBar { + getVals := func() (int, error) { + if tpm.ignoreState { + return 0, nil + } + return int(tpm.stateMng.VisitedDirectories), nil + } + return tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.VisitedDirs, nil) +} + func (tpm *TransferProgressMng) NewDelayedBar() *TasksProgressBar { getVals := func() (int, error) { if tpm.ignoreState {