diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 31219f00e..d5d40e5a5 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -174,6 +174,11 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon return err } + base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts)) + if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil { + log.Warn("Couldn't decrease the delayed files counter", err.Error()) + } + // Remove the file, so it won't be consumed again. if err = os.Remove(filePath); err != nil { return errorutils.CheckError(err) @@ -203,6 +208,23 @@ func getDelayFiles(repoKeys []string) (filesPaths []string, err error) { return getErrorOrDelayFiles(repoKeys, getJfrogTransferRepoDelaysDir) } +func getDelayedFilesCount(repoKeys []string) (int, error) { + files, err := getDelayFiles(repoKeys) + if err != nil { + return -1, err + } + + count := 0 + for _, file := range files { + delayedFiles, err := readDelayFile(file) + if err != nil { + return -1, err + } + count += len(delayedFiles.DelayedArtifacts) + } + return count, nil +} + const ( maven = "Maven" gradle = "Gradle" @@ -256,6 +278,10 @@ func (delayHelper delayUploadHelper) delayUploadIfNecessary(phase phaseBase, fil if shouldDelay(file.Name) { delayed = true delayHelper.delayedArtifactsChannelMng.add(file) + phase.progressBar.changeNumberOfDelayedFiles(1) + if err := phase.stateManager.ChangeDelayedFilesCountBy(1, true); err != nil { + log.Warn("Couldn't increase the delayed files counter", err.Error()) + } } } return diff --git a/artifactory/commands/transferfiles/state/runstatus.go b/artifactory/commands/transferfiles/state/runstatus.go index 1b9ffa9d6..d9d48e032 100644 --- a/artifactory/commands/transferfiles/state/runstatus.go +++ b/artifactory/commands/transferfiles/state/runstatus.go @@ -36,6 +36,7 @@ type TransferRunStatus struct { BuildInfoRepo bool `json:"build_info_repo,omitempty"` CurrentRepoPhase int `json:"current_repo_phase,omitempty"` WorkingThreads int `json:"working_threads,omitempty"` + DelayedFiles uint `json:"delayed_files,omitempty"` TransferFailures uint `json:"transfer_failures,omitempty"` TimeEstimationManager `json:"time_estimation,omitempty"` StaleChunks []StaleChunks `json:"stale_chunks,omitempty"` diff --git a/artifactory/commands/transferfiles/state/statemanager.go b/artifactory/commands/transferfiles/state/statemanager.go index 5425832e1..82adbc92e 100644 --- a/artifactory/commands/transferfiles/state/statemanager.go +++ b/artifactory/commands/transferfiles/state/statemanager.go @@ -263,6 +263,17 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er }) } +func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error { + return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { + if increase { + transferRunStatus.DelayedFiles += count + } else { + transferRunStatus.DelayedFiles -= count + } + return nil + }) +} + func (ts *TransferStateManager) ChangeTransferFailureCountBy(count uint, increase bool) error { return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { if increase { diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index db95454cd..42b397733 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -78,6 +78,7 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings. addString(output, "๐Ÿงต", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2) addString(output, "โšก", "Transfer speed", stateManager.GetSpeedString(), 2) addString(output, "โŒ›", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1) + addString(output, "โœ‹", "Delayed files", strconv.FormatUint(uint64(stateManager.DelayedFiles), 10), 2) failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10) if stateManager.TransferFailures > 0 { failureTxt += " (" + "In Phase 3 and in subsequent executions, we'll retry transferring the failed files." + ")" diff --git a/artifactory/commands/transferfiles/status_test.go b/artifactory/commands/transferfiles/status_test.go index f1c7226ab..8c3bb0a93 100644 --- a/artifactory/commands/transferfiles/status_test.go +++ b/artifactory/commands/transferfiles/status_test.go @@ -69,6 +69,7 @@ func TestShowStatus(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") assert.Contains(t, results, "Estimated time remaining: Less than a minute") + assert.Contains(t, results, "Delayed files: 20") assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files.)") // Check repository status @@ -99,6 +100,7 @@ func TestShowStatusDiffPhase(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") assert.Contains(t, results, "Estimated time remaining: Not available in this phase") + assert.Contains(t, results, "Delayed files: 20") assert.Contains(t, results, "Transfer failures: 223") // Check repository status @@ -129,6 +131,7 @@ func TestShowBuildInfoRepo(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: Not available while transferring a build-info repository") assert.Contains(t, results, "Estimated time remaining: Less than a minute") + assert.Contains(t, results, "Delayed files: 20") assert.Contains(t, results, "Transfer failures: 223") // Check repository status @@ -174,6 +177,7 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks stateManager.TotalRepositories.TotalUnits = 1111 stateManager.TotalRepositories.TransferredUnits = 15 stateManager.WorkingThreads = 16 + stateManager.DelayedFiles = 20 stateManager.TransferFailures = 223 stateManager.TimeEstimationManager.LastSpeeds = []float64{12} diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index 218a9fb28..40faa5e7c 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -228,8 +228,15 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui return e } tdc.stateManager.TransferFailures = uint(numberInitialErrors) + + numberInitialDelays, e := getDelayedFilesCount(allSourceLocalRepos) + if e != nil { + return e + } + tdc.stateManager.DelayedFiles = uint(numberInitialDelays) } else { tdc.stateManager.TransferFailures = 0 + tdc.stateManager.DelayedFiles = 0 } return nil } diff --git a/artifactory/commands/transferfiles/transferfileprogress.go b/artifactory/commands/transferfiles/transferfileprogress.go index 4f26b2bc0..b9400f846 100644 --- a/artifactory/commands/transferfiles/transferfileprogress.go +++ b/artifactory/commands/transferfiles/transferfileprogress.go @@ -31,6 +31,8 @@ type TransferProgressMng struct { speedBar *progressbar.TasksProgressBar // A bar showing the estimated remaining time for the transfer timeEstBar *progressbar.TasksProgressBar + // A bar showing the number of delayed artifacts in the process + delayedBar *progressbar.TasksProgressBar // A bar showing the number of transfer failures in the process errorBar *progressbar.TasksProgressBar // shows a note to the user if errors exists @@ -71,6 +73,7 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar() transfer.speedBar = transfer.transferMng.NewSpeedProgBar() transfer.timeEstBar = transfer.transferMng.NewTimeEstBar() + transfer.delayedBar = transfer.transferMng.NewDelayedBar() // Init global error count for the process transfer.errorBar = transfer.transferMng.NewErrorBar() transfer.errorNote = transfer.transferMng.NewErrorNote() @@ -220,6 +223,13 @@ func (t *TransferProgressMng) RemoveRepository() { time.Sleep(progressbar.ProgressRefreshRate) } +func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) { + if t.ShouldDisplay() { + diff := int64(n) + t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff) + } +} + func (t *TransferProgressMng) changeNumberOfFailuresBy(n int) { if t.ShouldDisplay() { diff := int64(n) @@ -244,7 +254,7 @@ func (t *TransferProgressMng) StopGracefully() { } func (t *TransferProgressMng) abortMetricsBars() { - for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.errorBar, t.errorNote, t.speedBar, t.timeEstBar, t.totalSize} { + for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.errorNote, t.speedBar, t.timeEstBar, t.totalSize} { if barPtr != nil { barPtr.GetBar().Abort(true) } diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index f94a812cf..ad74f5d67 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -22,6 +22,7 @@ type transferLabels struct { RetryFailureContentNote string TransferSpeed string EstimatedTime string + DelayedFiles string TransferFailures string WorkingThreads string RunningFor string @@ -50,6 +51,7 @@ func initSProgressBarLabels(windows bool) transferLabels { pbs.Note = formatString(" ๐ŸŸ ", " Note: ", windows) pbs.TransferSpeed = formatString(" โšก", " Transfer speed: ", windows) pbs.EstimatedTime = formatString(" โŒ›", " Estimated time remaining: ", windows) + pbs.DelayedFiles = formatString(" โœ‹", " Delayed files: ", windows) pbs.TransferFailures = formatString(" โŒ", " Transfer failures: ", windows) pbs.WorkingThreads = formatString(" ๐Ÿงต", " Working threads: ", windows) pbs.RunningFor = formatString(" ๐Ÿƒ๐Ÿผ", " Running for: ", windows) @@ -307,6 +309,18 @@ func (tpm *TransferProgressMng) NewTimeEstBar() *TasksProgressBar { return pb } +func (tpm *TransferProgressMng) NewDelayedBar() *TasksProgressBar { + getVals := func() (delayedCount int, err error) { + delayedCount = 0 + if !tpm.ignoreState { + delayedCount = int(tpm.stateMng.DelayedFiles) + } + return delayedCount, err + } + pb := tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.DelayedFiles) + return pb +} + func (tpm *TransferProgressMng) NewErrorBar() *TasksProgressBar { getVals := func() (errnums int, err error) { errnums = 0