Transfer - Show visited dirs counter

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -177,7 +177,7 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 		if base.progressBar != nil {
 			base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts))
 		}
-		if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
+		if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
 			log.Warn("Couldn't decrease the delayed files counter", err.Error())
 		}
 

artifactory/commands/transferfiles/fulltransfer.go

@@ -217,6 +217,13 @@ func (m *fullTransferPhase) searchAndHandleFolderContents(params folderParams, p
 func (m *fullTransferPhase) handleFoundChildFolder(params folderParams, pcWrapper producerConsumerWrapper,
 	uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng,
 	item servicesUtils.ResultItem) (err error) {
+	if m.progressBar != nil {
+		m.progressBar.incNumberOfVisitedDirs()
+	}
+	if err = m.stateManager.IncVisitedDirectories(); err != nil {
+		return
+	}
+
 	newRelativePath := getFolderRelativePath(item.Name, params.relativePath)
 
 	folderHandler := m.createFolderFullTransferHandlerFunc(pcWrapper, uploadChunkChan, delayHelper, errorsChannelMng)

artifactory/commands/transferfiles/state/runstatus.go

@@ -33,11 +33,12 @@ type TransferRunStatus struct {
 	Version        int    `json:"version,omitempty"`
 	CurrentRepoKey string `json:"current_repo,omitempty"`
 	// True if currently transferring a build info repository.
-	BuildInfoRepo         bool `json:"build_info_repo,omitempty"`
-	CurrentRepoPhase      int  `json:"current_repo_phase,omitempty"`
-	WorkingThreads        int  `json:"working_threads,omitempty"`
-	DelayedFiles          uint `json:"delayed_files,omitempty"`
-	TransferFailures      uint `json:"transfer_failures,omitempty"`
+	BuildInfoRepo         bool   `json:"build_info_repo,omitempty"`
+	CurrentRepoPhase      int    `json:"current_repo_phase,omitempty"`
+	WorkingThreads        int    `json:"working_threads,omitempty"`
+	VisitedDirectories    uint64 `json:"visited_dirs,omitempty"`
+	DelayedFiles          uint64 `json:"delayed_files,omitempty"`
+	TransferFailures      uint   `json:"transfer_failures,omitempty"`
 	TimeEstimationManager `json:"time_estimation,omitempty"`
 	StaleChunks           []StaleChunks `json:"stale_chunks,omitempty"`
 }

artifactory/commands/transferfiles/state/statemanager.go

@@ -67,7 +67,7 @@ func (ts *TransferStateManager) UnlockTransferStateManager() error {
 // buildInfoRepo  - True if build info repository
 // reset          - Delete the repository's previous transfer info
 func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, totalFiles int64, buildInfoRepo, reset bool) error {
-	err := ts.TransferState.Action(func(state *TransferState) error {
+	err := ts.Action(func(*TransferState) error {
 		transferState, repoTransferSnapshot, err := getTransferStateAndSnapshot(repoKey, reset)
 		if err != nil {
 			return err
@@ -82,9 +82,10 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
 	if err != nil {
 		return err
 	}
-	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
 		transferRunStatus.CurrentRepoKey = repoKey
 		transferRunStatus.BuildInfoRepo = buildInfoRepo
+		transferRunStatus.VisitedDirectories = 0
 
 		transferRunStatus.OverallTransfer.TransferredUnits += ts.CurrentRepo.Phase1Info.TransferredUnits
 		transferRunStatus.OverallTransfer.TransferredSizeBytes += ts.CurrentRepo.Phase1Info.TransferredSizeBytes
@@ -263,7 +264,14 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er
 	})
 }
 
-func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error {
+func (ts *TransferStateManager) IncVisitedDirectories() error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
+		transferRunStatus.VisitedDirectories++
+		return nil
+	})
+}
+
+func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint64, increase bool) error {
 	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
 		if increase {
 			transferRunStatus.DelayedFiles += count

artifactory/commands/transferfiles/state/statemanager_test.go

@@ -146,6 +146,34 @@ func TestReposOverallBiFiles(t *testing.T) {
 	assert.Equal(t, int64(8), stateManager.OverallBiFiles.TransferredUnits)
 }
 
+func TestDelayedFiles(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(2, true))
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(4, true))
+	assert.Equal(t, uint64(6), stateManager.DelayedFiles)
+
+	// Decrease delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(3, false))
+	assert.Equal(t, uint64(3), stateManager.DelayedFiles)
+}
+
+func TestVisitedDirectories(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase visited directories count
+	assert.NoError(t, stateManager.IncVisitedDirectories())
+	assert.NoError(t, stateManager.IncVisitedDirectories())
+	assert.Equal(t, uint64(2), stateManager.VisitedDirectories)
+
+	// Set repository state and ensure the visited directories became 0
+	assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, true, true))
+	assert.Zero(t, stateManager.VisitedDirectories)
+}
+
 func assertReposTransferredSize(t *testing.T, stateManager *TransferStateManager, expectedSize int64, repoKeys ...string) {
 	totalTransferredSize, err := stateManager.GetReposTransferredSizeBytes(repoKeys...)
 	assert.NoError(t, err)

artifactory/commands/transferfiles/status.go

@@ -79,11 +79,6 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings.
 	addString(output, "🧵", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2)
 	addString(output, "⚡", "Transfer speed", stateManager.GetSpeedString(), 2)
 	addString(output, "⌛", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
-	delayedTxt := strconv.FormatUint(uint64(stateManager.DelayedFiles), 10)
-	if stateManager.DelayedFiles > 0 {
-		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
-	}
-	addString(output, "✋", "Delayed files", delayedTxt, 2)
 	failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10)
 	if stateManager.TransferFailures > 0 {
 		failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
@@ -100,20 +95,28 @@ func calcPercentageInt64(transferred, total int64) string {
 
 func setRepositoryStatus(stateManager *state.TransferStateManager, output *strings.Builder) {
 	addTitle(output, "Current Repository Status")
-	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 2)
+	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 3)
 	currentRepo := stateManager.CurrentRepo
 	switch stateManager.CurrentRepoPhase {
 	case api.Phase1, api.Phase3:
 		if stateManager.CurrentRepoPhase == api.Phase1 {
-			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 2)
+			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 3)
 		} else {
-			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 2)
+			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 3)
 		}
-		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 2)
-		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 2)
+		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3)
+		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3)
 	case api.Phase2:
-		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 2)
+		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 3)
+	}
+	if stateManager.CurrentRepoPhase == api.Phase1 {
+		addString(output, "📁", "Visited directories", strconv.FormatUint(stateManager.VisitedDirectories, 10), 2)
 	}
+	delayedTxt := strconv.FormatUint(stateManager.DelayedFiles, 10)
+	if stateManager.DelayedFiles > 0 {
+		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
+	}
+	addString(output, "✋", "Delayed files", delayedTxt, 2)
 }
 
 func addStaleChunks(stateManager *state.TransferStateManager, output *strings.Builder) {

artifactory/commands/transferfiles/transfer.go

@@ -233,8 +233,9 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui
 		if e != nil {
 			return e
 		}
-		tdc.stateManager.DelayedFiles = uint(numberInitialDelays)
+		tdc.stateManager.DelayedFiles = uint64(numberInitialDelays)
 	} else {
+		tdc.stateManager.VisitedDirectories = 0
 		tdc.stateManager.TransferFailures = 0
 		tdc.stateManager.DelayedFiles = 0
 	}

artifactory/commands/transferfiles/transferfileprogress.go

@@ -31,6 +31,8 @@ type TransferProgressMng struct {
 	speedBar *progressbar.TasksProgressBar
 	// A bar showing the estimated remaining time for the transfer
 	timeEstBar *progressbar.TasksProgressBar
+	// A bar showing the number of visited directories
+	visitedDirsBar *progressbar.TasksProgressBar
 	// A bar showing the number of delayed artifacts in the process
 	delayedBar *progressbar.TasksProgressBar
 	// A bar showing the number of transfer failures in the process
@@ -71,6 +73,7 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom
 	transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar()
 	transfer.speedBar = transfer.transferMng.NewSpeedProgBar()
 	transfer.timeEstBar = transfer.transferMng.NewTimeEstBar()
+	transfer.visitedDirsBar = transfer.transferMng.NewVisitedDirsBar()
 	transfer.delayedBar = transfer.transferMng.NewDelayedBar()
 	// Init global error count for the process
 	transfer.errorBar = transfer.transferMng.NewErrorBar()
@@ -220,10 +223,16 @@ func (t *TransferProgressMng) RemoveRepository() {
 	time.Sleep(progressbar.ProgressRefreshRate)
 }
 
+func (t *TransferProgressMng) incNumberOfVisitedDirs() {
+	if t.ShouldDisplay() {
+		t.visitedDirsBar.SetGeneralProgressTotal(t.visitedDirsBar.GetTotal() + 1)
+	}
+}
+
 func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) {
 	if t.ShouldDisplay() {
 		diff := int64(n)
-		t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff)
+		t.delayedBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff)
 	}
 }
 
@@ -251,7 +260,7 @@ func (t *TransferProgressMng) StopGracefully() {
 }
 
 func (t *TransferProgressMng) abortMetricsBars() {
-	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} {
+	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.visitedDirsBar, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} {
 		if barPtr != nil {
 			barPtr.GetBar().Abort(true)
 		}

utils/progressbar/transferprogressbarmanager.go

@@ -25,6 +25,7 @@ type transferLabels struct {
 	Note             string
 	TransferSpeed    string
 	EstimatedTime    string
+	VisitedDirs      string
 	DelayedFiles     string
 	TransferFailures string
 	WorkingThreads   string
@@ -53,6 +54,7 @@ func initSProgressBarLabels(windows bool) transferLabels {
 	pbs.Note = formatString(" 🟠", " Note: ", windows)
 	pbs.TransferSpeed = formatString(" ⚡", " Transfer speed: ", windows)
 	pbs.EstimatedTime = formatString(" ⌛", " Estimated time remaining: ", windows)
+	pbs.VisitedDirs = formatString(" 📁", " Visited directories: ", windows)
 	pbs.DelayedFiles = formatString(" ✋", " Delayed files: ", windows)
 	pbs.TransferFailures = formatString(" ❌", " Transfer failures: ", windows)
 	pbs.WorkingThreads = formatString(" 🧵", " Working threads: ", windows)
@@ -308,6 +310,16 @@ func (tpm *TransferProgressMng) NewTimeEstBar() *TasksProgressBar {
 	})
 }
 
+func (tpm *TransferProgressMng) NewVisitedDirsBar() *TasksProgressBar {
+	getVals := func() (int, error) {
+		if tpm.ignoreState {
+			return 0, nil
+		}
+		return int(tpm.stateMng.VisitedDirectories), nil
+	}
+	return tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.VisitedDirs, nil)
+}
+
 func (tpm *TransferProgressMng) NewDelayedBar() *TasksProgressBar {
 	getVals := func() (int, error) {
 		if tpm.ignoreState {