From 5bb3dd692fb05d23c4da613cd49522fbb3a71ec0 Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:15:05 -0500 Subject: [PATCH 1/6] BXC-4771 add download file limit --- .../impl/download/DownloadBulkService.java | 16 ++++++++++++++++ .../impl/download/DownloadBulkServiceTest.java | 17 +++++++++++++++++ .../src/main/webapp/WEB-INF/service-context.xml | 1 + 3 files changed, 34 insertions(+) diff --git a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java index bbb081e99d..920ef4ea99 100644 --- a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java +++ b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java @@ -28,6 +28,7 @@ public class DownloadBulkService { private AccessControlService aclService; private RepositoryObjectLoader repoObjLoader; private Path basePath; + private int fileLimit; public Path downloadBulk(DownloadBulkRequest request) { var pidString = request.getWorkPidString(); @@ -60,7 +61,13 @@ private void zipFiles(WorkObject workObject, AccessGroupSet agentPrincipals, Pat } Map duplicates = new HashMap<>(); + int count = 0; + var limit = getLoopLimit(memberObjects.size()); +// while (count < getLoopLimit(memberObjects.size())) { for (ContentObject memberObject : memberObjects ) { + if (count == limit) { + break; + } if (!(memberObject instanceof FileObject)) { continue; } @@ -84,8 +91,10 @@ private void zipFiles(WorkObject workObject, AccessGroupSet agentPrincipals, Pat IOUtils.copy(binaryStream, zipOut); } + count++; } } + } } @@ -101,6 +110,9 @@ private String formatFilename(String filename, int copyNumber) { var base = FilenameUtils.removeExtension(filename); return base + "(" + copyNumber + ")." + extension; } + private int getLoopLimit(int arraySize) { + return Math.min(fileLimit, arraySize); + } public void setAclService(AccessControlService aclService) { this.aclService = aclService; @@ -113,4 +125,8 @@ public void setRepoObjLoader(RepositoryObjectLoader repoObjLoader) { public void setBasePath(Path basePath) { this.basePath = basePath; } + + public void setFileLimit(int fileLimit) { + this.fileLimit = fileLimit; + } } diff --git a/operations/src/test/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkServiceTest.java b/operations/src/test/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkServiceTest.java index ffea60c4b7..30a98cf5b4 100644 --- a/operations/src/test/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkServiceTest.java +++ b/operations/src/test/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkServiceTest.java @@ -77,6 +77,7 @@ public void init() throws Exception { service.setAclService(aclService); service.setRepoObjLoader(repoObjLoader); service.setBasePath(zipStorageBasePath); + service.setFileLimit(5); parentPid = PIDs.get(PARENT_UUID); fileObject1Pid = PIDs.get(CHILD1_UUID); fileObject2Pid = PIDs.get(CHILD2_UUID); @@ -165,6 +166,22 @@ public void tombstoneTest() throws IOException { assertZipFiles(List.of(), List.of()); } + @Test + public void fileLimitTest() throws IOException { + when(repoObjLoader.getWorkObject(any(PID.class))).thenReturn(parentWork); + when(parentWork.getMembers()).thenReturn(List.of(fileObject1, fileObject2)); + makeBinaryObject(fileObject1, FILENAME1); + makeBinaryObject(fileObject2, FILENAME2); + when(aclService.hasAccess(eq(fileObject1Pid), any(), + eq(Permission.viewOriginal))).thenReturn(true); + when(aclService.hasAccess(eq(fileObject2Pid), any(), + eq(Permission.viewOriginal))).thenReturn(true); + service.setFileLimit(1); + service.downloadBulk(request); + // the zip file should have one entry + assertZipFiles(List.of(FILENAME1), List.of("flower")); + } + @Test public void successTest() throws IOException { when(repoObjLoader.getWorkObject(eq(parentPid))).thenReturn(parentWork); diff --git a/web-services-app/src/main/webapp/WEB-INF/service-context.xml b/web-services-app/src/main/webapp/WEB-INF/service-context.xml index 8b0e2bab89..9ff45a9a76 100644 --- a/web-services-app/src/main/webapp/WEB-INF/service-context.xml +++ b/web-services-app/src/main/webapp/WEB-INF/service-context.xml @@ -498,6 +498,7 @@ + From d6d0d99a5d4c846f0141408f39b1ad3b5a66d738 Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:17:25 -0500 Subject: [PATCH 2/6] BXC-4771 cleanup --- .../boxc/operations/impl/download/DownloadBulkService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java index 920ef4ea99..eeb40ecf84 100644 --- a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java +++ b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java @@ -62,10 +62,8 @@ private void zipFiles(WorkObject workObject, AccessGroupSet agentPrincipals, Pat Map duplicates = new HashMap<>(); int count = 0; - var limit = getLoopLimit(memberObjects.size()); -// while (count < getLoopLimit(memberObjects.size())) { for (ContentObject memberObject : memberObjects ) { - if (count == limit) { + if (count == getLoopLimit(memberObjects.size())) { break; } if (!(memberObject instanceof FileObject)) { @@ -94,7 +92,6 @@ private void zipFiles(WorkObject workObject, AccessGroupSet agentPrincipals, Pat count++; } } - } } From 8fa030333c43488c360b80d942e920661883b0b8 Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:47:05 -0500 Subject: [PATCH 3/6] BXC-4771 set file limit in test --- .../unc/lib/boxc/web/services/rest/DownloadBulkControllerIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/web-services-app/src/test/java/edu/unc/lib/boxc/web/services/rest/DownloadBulkControllerIT.java b/web-services-app/src/test/java/edu/unc/lib/boxc/web/services/rest/DownloadBulkControllerIT.java index 19f149f9fd..2ae9383c49 100644 --- a/web-services-app/src/test/java/edu/unc/lib/boxc/web/services/rest/DownloadBulkControllerIT.java +++ b/web-services-app/src/test/java/edu/unc/lib/boxc/web/services/rest/DownloadBulkControllerIT.java @@ -77,6 +77,7 @@ public void init() throws FileNotFoundException { downloadBulkService.setAclService(aclService); downloadBulkService.setBasePath(tmpFolder); downloadBulkService.setRepoObjLoader(repositoryObjectLoader); + downloadBulkService.setFileLimit(5); controller.setDownloadBulkService(downloadBulkService); fileInputStream = new FileInputStream("src/test/resources/__files/bunny.jpg"); mvc = MockMvcBuilders.standaloneSetup(controller) From e08ebc5c31ed9de4a02bbd3e91d1727e30cc8549 Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Mon, 18 Nov 2024 11:29:55 -0500 Subject: [PATCH 4/6] BXC-4771 DRY up code and update limit --- .../lib/boxc/operations/impl/download/DownloadBulkService.java | 3 ++- web-services-app/src/main/webapp/WEB-INF/service-context.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java index eeb40ecf84..e19ea83332 100644 --- a/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java +++ b/operations/src/main/java/edu/unc/lib/boxc/operations/impl/download/DownloadBulkService.java @@ -62,8 +62,9 @@ private void zipFiles(WorkObject workObject, AccessGroupSet agentPrincipals, Pat Map duplicates = new HashMap<>(); int count = 0; + var loopLimit = getLoopLimit(memberObjects.size()); for (ContentObject memberObject : memberObjects ) { - if (count == getLoopLimit(memberObjects.size())) { + if (count == loopLimit) { break; } if (!(memberObject instanceof FileObject)) { diff --git a/web-services-app/src/main/webapp/WEB-INF/service-context.xml b/web-services-app/src/main/webapp/WEB-INF/service-context.xml index 9ff45a9a76..5b5540aacb 100644 --- a/web-services-app/src/main/webapp/WEB-INF/service-context.xml +++ b/web-services-app/src/main/webapp/WEB-INF/service-context.xml @@ -498,7 +498,7 @@ - + From 62b77509bbc846e045823f3a678a4cb3b2f188ae Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:36:16 -0500 Subject: [PATCH 5/6] BXC-4771 set limit to 99 for 100 files downloaded --- web-services-app/src/main/webapp/WEB-INF/service-context.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-services-app/src/main/webapp/WEB-INF/service-context.xml b/web-services-app/src/main/webapp/WEB-INF/service-context.xml index 5b5540aacb..638c9f61b6 100644 --- a/web-services-app/src/main/webapp/WEB-INF/service-context.xml +++ b/web-services-app/src/main/webapp/WEB-INF/service-context.xml @@ -498,7 +498,7 @@ - + From 23f24fd237c090422eff7491a5698f973a05f216 Mon Sep 17 00:00:00 2001 From: sharonl <6546457+sharonluong@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:58:53 -0500 Subject: [PATCH 6/6] BXC-4771 set limit to 100 for 100 files downloaded --- web-services-app/src/main/webapp/WEB-INF/service-context.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-services-app/src/main/webapp/WEB-INF/service-context.xml b/web-services-app/src/main/webapp/WEB-INF/service-context.xml index 638c9f61b6..5b5540aacb 100644 --- a/web-services-app/src/main/webapp/WEB-INF/service-context.xml +++ b/web-services-app/src/main/webapp/WEB-INF/service-context.xml @@ -498,7 +498,7 @@ - +