From ea15cd3668f7c64d5ce417ba3665dc81c9bd2025 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 27 Oct 2023 11:54:00 -0700 Subject: [PATCH 01/23] prepare versions for next release --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 154ad58..e3b9922 100644 --- a/pom.xml +++ b/pom.xml @@ -6,8 +6,8 @@ UTF-8 metadig - 2.5.0 - 2.5.0 + 3.0.0-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 @@ -15,7 +15,7 @@ edu.ucsb.nceas metadig-webapp war - 2.5.0 + 3.0.0-SNAPSHOT metadig-webapp From accc9569471f40e1b70460d48c28a711ccfaf194 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 27 Oct 2023 11:54:13 -0700 Subject: [PATCH 02/23] add a dispatcher close method after running a single check --- src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java index c5e4db4..a5ae2c7 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java @@ -44,6 +44,7 @@ import edu.ucsb.nceas.mdqengine.store.StoreFactory; import edu.ucsb.nceas.mdqengine.serialize.JsonMarshaller; import edu.ucsb.nceas.mdqengine.serialize.XmlMarshaller; +import edu.ucsb.nceas.mdqengine.dispatch.Dispatcher; /** * Root resource (exposed at "checks" path) @@ -157,6 +158,7 @@ public Response run( Check check = store.getCheck(id); run = engine.runCheck(check, input, params, sysMeta); store.createRun(run); + Dispatcher.getDispatcher("python").close(); } catch (Exception e) { log.error(e.getMessage(), e); return Response.serverError().entity(e).build(); From 0df7d154d3cca1f5e1ca29e8a92774f9a3a771c2 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Mon, 30 Oct 2023 15:50:58 -0700 Subject: [PATCH 03/23] build docker image on gha --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34b61a0..cb58458 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: docker-publish: name: Docker Build and Publish - if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') + if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'feature') needs: maven-build runs-on: ubuntu-latest permissions: From ce6f1ee333a3757f7bdbeceb0d42f142af5e7736 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 3 Nov 2023 12:07:28 -0700 Subject: [PATCH 04/23] make sure to close dispatcher after running suite --- src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java index 92b35aa..e8b527e 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java @@ -34,6 +34,7 @@ import edu.ucsb.nceas.mdqengine.model.Suite; import edu.ucsb.nceas.mdqengine.serialize.JsonMarshaller; import edu.ucsb.nceas.mdqengine.serialize.XmlMarshaller; +import edu.ucsb.nceas.mdqengine.dispatch.Dispatcher; /** * Root resource (exposed at "suites" path) @@ -229,6 +230,7 @@ public Response run( Suite suite = store.getSuite(id); run = engine.runSuite(suite, input, params, sysMeta); store.createRun(run); + Dispatcher.getDispatcher("python").close(); } catch (Exception e) { log.error(e.getMessage(), e); return Response.serverError().entity(e).build(); From 8c9b38c08f05db9b602f0f9eda607ae7e29b0d66 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 3 Nov 2023 14:03:24 -0700 Subject: [PATCH 05/23] fix gha to build container --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb58458..80b21c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: docker-publish: name: Docker Build and Publish - if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'feature') + if: github.ref_name == 'develop' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref_name, 'feature') needs: maven-build runs-on: ubuntu-latest permissions: From bae004859f8b907a2567a916e3ba94f51567b9e6 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Wed, 6 Dec 2023 11:16:55 -0700 Subject: [PATCH 06/23] trigger a build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15116f7..fdc4594 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MetaDIG webapp +# MetaDIG webapp Web service for interacting with the [MetaDIG Engine](https://github.com/NCEAS/metadig-engine) API. This project builds a 'metadig-webapp.war' file that can be deployed in the Tomcat webapps directory. Depending on the Tomcat configuration, the war may be unpacked automatically each time a new version is copied in place. From c7d0085f96b10634c23cf66803c68bbbdd863dce Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 12 Jan 2024 15:56:12 -0700 Subject: [PATCH 07/23] use a local store to get suite resources --- .../ucsb/nceas/mdq/rest/SuitesResource.java | 185 +++++++++--------- 1 file changed, 97 insertions(+), 88 deletions(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java index e8b527e..0988e19 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java @@ -41,58 +41,55 @@ */ @Path("suites") public class SuitesResource { - - private Log log = LogFactory.getLog(this.getClass()); + + private Log log = LogFactory.getLog(this.getClass()); private static Controller metadigCtrl = null; - - public SuitesResource() {} - + + public SuitesResource() {} + /** - * Method handling HTTP GET requests. The returned object will be sent - * to the client as "text/plain" media type. + * Method handling HTTP GET requests. The returned object will be sent to the client as + * "text/plain" media type. * * @return String that will be returned as a text/plain response. */ @GET @Produces(MediaType.APPLICATION_JSON) public String listSuites() { - boolean persist = true; + boolean persist = false; MDQStore store = null; - MDQEngine engine = null; try { store = StoreFactory.getStore(persist); - engine = new MDQEngine(); - } catch (MetadigException | IOException | ConfigurationException e) { + } catch (MetadigException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } - Collection suites = store.listSuites(); - store.shutdown(); + Collection suites = store.listSuites(); + store.shutdown(); return JsonMarshaller.toJson(suites); } - + @GET @Path("/{id}") @Produces(MediaType.TEXT_XML) - public String getSuite(@PathParam("id") String id) throws UnsupportedEncodingException, JAXBException { - boolean persist = true; + public String getSuite(@PathParam("id") String id) + throws UnsupportedEncodingException, JAXBException { + boolean persist = false; MDQStore store = null; - MDQEngine engine = null; try { store = StoreFactory.getStore(persist); - engine = new MDQEngine(); - } catch (MetadigException | IOException | ConfigurationException e) { + } catch (MetadigException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } - Suite suite = store.getSuite(id); + Suite suite = store.getSuite(id); store.shutdown(); return XmlMarshaller.toXml(suite, true); } - -// @POST -// @Consumes(MediaType.MULTIPART_FORM_DATA) + + // @POST + // @Consumes(MediaType.MULTIPART_FORM_DATA) public boolean createSuite(@FormDataParam("suite") InputStream xml) { boolean persist = true; MDQStore store = null; @@ -102,25 +99,26 @@ public boolean createSuite(@FormDataParam("suite") InputStream xml) { engine = new MDQEngine(); } catch (MetadigException | IOException | ConfigurationException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } - Suite suite = null; - try { - suite = (Suite) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Suite.class); - store.createSuite(suite); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } finally { - store.shutdown(); + Suite suite = null; + try { + suite = (Suite) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Suite.class); + store.createSuite(suite); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } finally { + store.shutdown(); } return true; } - -// @PUT -// @Path("/{id}") -// @Consumes(MediaType.MULTIPART_FORM_DATA) - public boolean updateSuite(@PathParam("id") String id, @FormDataParam("suite") InputStream xml) throws JAXBException, IOException { + + // @PUT + // @Path("/{id}") + // @Consumes(MediaType.MULTIPART_FORM_DATA) + public boolean updateSuite(@PathParam("id") String id, @FormDataParam("suite") InputStream xml) + throws JAXBException, IOException { boolean persist = true; MDQStore store = null; MDQEngine engine = null; @@ -129,24 +127,24 @@ public boolean updateSuite(@PathParam("id") String id, @FormDataParam("suite") I engine = new MDQEngine(); } catch (MetadigException | IOException | ConfigurationException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } - Suite suite = null; - try { - suite = (Suite) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Suite.class); - store.updateSuite(suite); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } finally { - store.shutdown(); + Suite suite = null; + try { + suite = (Suite) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Suite.class); + store.updateSuite(suite); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } finally { + store.shutdown(); } return true; } - -// @DELETE -// @Path("/{id}") -// @Produces(MediaType.TEXT_PLAIN) + + // @DELETE + // @Path("/{id}") + // @Produces(MediaType.TEXT_PLAIN) public boolean deleteSuite(@PathParam("id") String id) { boolean persist = true; MDQStore store = null; @@ -156,33 +154,37 @@ public boolean deleteSuite(@PathParam("id") String id) { engine = new MDQEngine(); } catch (MetadigException | IOException | ConfigurationException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } - Suite suite = store.getSuite(id); - store.deleteSuite(suite); - store.shutdown(); + Suite suite = store.getSuite(id); + store.deleteSuite(suite); + store.shutdown(); return true; } - + @POST @Path("/{id}/run") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response run( - @PathParam("id") String id, // id is the metadig suite id - @FormDataParam("document") InputStream input, // the input metadata document - @FormDataParam("systemMetadata") InputStream sysMetaStream, // the system metadata for the input metadata document - @FormDataParam("priority") String priority, // the priority to enqueue the metadig engine request with ("high", "medium", "low") - @Context Request r) throws UnsupportedEncodingException, JAXBException { + public Response run(@PathParam("id") String id, // id is the metadig suite id + @FormDataParam("document") InputStream input, // the input metadata document + @FormDataParam("systemMetadata") InputStream sysMetaStream, // the system metadata for + // the input metadata + // document + @FormDataParam("priority") String priority, // the priority to enqueue the metadig + // engine request with + // ("high", "medium", "low") + @Context Request r) throws UnsupportedEncodingException, JAXBException { boolean persist = true; MDQStore store = null; MDQEngine engine = null; - if(priority == null) priority = "low"; - Run run = null; + if (priority == null) + priority = "low"; + Run run = null; String resultString = null; - // Copy the sysmeta input stream because we need to read it twice + // Copy the sysmeta input stream because we need to read it twice ByteArrayOutputStream bos = new ByteArrayOutputStream(); byte[] streamData = null; @@ -202,27 +204,29 @@ public Response run( ByteArrayInputStream sysmetaStream = new ByteArrayInputStream(streamData); - SystemMetadata sysMeta = null; - // Read sysmeta input stream to get values to log and pass to the controller - if (sysMetaStream != null) { - try { - sysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysmetaStream); - } catch (InstantiationException | IllegalAccessException - | IOException | MarshallingException e) { - log.warn("Could not unmarshall SystemMetadata from stream", e); - } - } + SystemMetadata sysMeta = null; + // Read sysmeta input stream to get values to log and pass to the controller + if (sysMetaStream != null) { + try { + sysMeta = + TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysmetaStream); + } catch (InstantiationException | IllegalAccessException | IOException + | MarshallingException e) { + log.warn("Could not unmarshall SystemMetadata from stream", e); + } + } - // If the request is identifying itself as 'high', then process it now, otherwise send it + // If the request is identifying itself as 'high', then process it now, + // otherwise send it // to the processing queue. - if(priority.equals("high")) { + if (priority.equals("high")) { try { store = StoreFactory.getStore(persist); engine = new MDQEngine(); } catch (MetadigException | IOException | ConfigurationException e) { InternalServerErrorException ise = new InternalServerErrorException(e.getMessage()); - throw(ise); + throw (ise); } try { log.info("Running suite " + id + " for pid " + sysMeta.getIdentifier().getValue()); @@ -239,8 +243,9 @@ public Response run( } // determine the format of plot to return - List vs = - Variant.mediaTypes(MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE).build(); + List vs = Variant + .mediaTypes(MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE) + .build(); Variant v = r.selectVariant(vs); if (v == null) { return Response.notAcceptable(vs).build(); @@ -254,7 +259,7 @@ public Response run( } } else { try { - if(metadigCtrl == null) { + if (metadigCtrl == null) { metadigCtrl = Controller.getInstance(); // Start the controller if it has not already been started. if (!metadigCtrl.getIsStarted()) { @@ -266,7 +271,8 @@ public Response run( return Response.serverError().entity(e).build(); } - // Check if the metadig-engine controller has been started. If not, return a message. + // Check if the metadig-engine controller has been started. If not, return a + // message. // TODO: return a properly formatted XML error message if (!metadigCtrl.getIsStarted()) { return Response.serverError().build(); @@ -278,8 +284,11 @@ public Response run( DateTime requestDateTime = new DateTime(); NodeReference dataSource = sysMeta.getOriginMemberNode(); String metadataPid = sysMeta.getIdentifier().getValue(); - log.info("Queue generation request of quality document for: " + dataSource.getValue() + ", PID: " + metadataPid + ", " + id + ", " + requestDateTime.toString()); - metadigCtrl.processQualityRequest(dataSource.getValue(), metadataPid, input, id, "", requestDateTime, sysmetaStream2); + log.info("Queue generation request of quality document for: " + + dataSource.getValue() + ", PID: " + metadataPid + ", " + id + ", " + + requestDateTime.toString()); + metadigCtrl.processQualityRequest(dataSource.getValue(), metadataPid, input, id, "", + requestDateTime, sysmetaStream2); } catch (Exception e) { log.error(e.getMessage(), e); return Response.serverError().entity(e).build(); @@ -288,5 +297,5 @@ public Response run( return Response.ok().build(); } - + } From 20778baed8895436b5ddcad94c6a89e90bd9df7a Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 12 Jan 2024 15:56:55 -0700 Subject: [PATCH 08/23] add stylesheet for formatting --- .vscode/eclipse-java-google-style.xml | 337 ++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 .vscode/eclipse-java-google-style.xml diff --git a/.vscode/eclipse-java-google-style.xml b/.vscode/eclipse-java-google-style.xml new file mode 100644 index 0000000..0dd7ed5 --- /dev/null +++ b/.vscode/eclipse-java-google-style.xml @@ -0,0 +1,337 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From beb685e37630a89b756673ac56aae64a6f2a9607 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Thu, 18 Jan 2024 15:30:21 -0700 Subject: [PATCH 09/23] setup Jep in the context listener at startup --- .../edu/ucsb/nceas/mdq/MetadigContextListener.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java index 7bb7674..7e687c9 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java +++ b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java @@ -7,15 +7,24 @@ import javax.servlet.ServletContextListener; import java.io.IOException; import java.util.concurrent.TimeoutException; +import edu.ucsb.nceas.mdqengine.dispatch.Dispatcher; import edu.ucsb.nceas.mdqengine.Controller; +import edu.ucsb.nceas.mdqengine.exception.MetadigException; +@WebListener public class MetadigContextListener implements ServletContextListener { public static Log log = LogFactory.getLog(MetadigContextListener.class); @Override public void contextInitialized(ServletContextEvent servletContextEvent) { - log.debug("Metadig 'contextInitialized' called."); + try { + Dispatcher.setupJep(); + } catch (MetadigException e) { + log.error("Error setting up Jep. Python checks may not work.", e); + } + + log.info("Metadig 'contextInitialized' called."); } @Override From 9f00a8c66b9d00172e8792b22ab9f381c9608238 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Tue, 30 Jan 2024 15:06:19 -0700 Subject: [PATCH 10/23] trigger a build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fdc4594..3ea1fce 100644 --- a/README.md +++ b/README.md @@ -58,4 +58,4 @@ Authorization for Docker Hub can be setup in several ways. One method is to crea ``` -The appropriate username and password is available from the NCEAS secure repo. +The appropriate username and password is available from the NCEAS secure repo. From 9114da22bdbcf7341fa526e554fd7220e3151d84 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Wed, 31 Jan 2024 15:47:26 -0700 Subject: [PATCH 11/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ea1fce..fdc4594 100644 --- a/README.md +++ b/README.md @@ -58,4 +58,4 @@ Authorization for Docker Hub can be setup in several ways. One method is to crea ``` -The appropriate username and password is available from the NCEAS secure repo. +The appropriate username and password is available from the NCEAS secure repo. From 2705e0d483f897a9a30c07d6f064cc58d2e19d52 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 2 Feb 2024 09:45:37 -0700 Subject: [PATCH 12/23] return the suite run result to console --- src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java index 0988e19..2e140cb 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java @@ -295,7 +295,7 @@ public Response run(@PathParam("id") String id, // id is the metadig suite id } } - return Response.ok().build(); + return Response.ok(resultString).build(); } } From 07b028c74a9184179bce397e697f4981dbeb3719 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 2 Feb 2024 09:54:07 -0700 Subject: [PATCH 13/23] add code styling --- .vscode/settings.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..aeac9eb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "java.configuration.updateBuildConfiguration": "interactive", + "java.compile.nullAnalysis.mode": "automatic", + "java.format.settings.url": ".vscode/eclipse-java-google-style.xml" +} \ No newline at end of file From 5378646f87d465a175bebf0455a6f326633ecb1c Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 2 Feb 2024 09:58:51 -0700 Subject: [PATCH 14/23] add files that are helpful for debugging direct API calls --- src/test/resources/test-api.sh | 15 + src/test/resources/testfile.txt | 1220 +++++++++++++++++++++++++++++++ 2 files changed, 1235 insertions(+) create mode 100644 src/test/resources/test-api.sh create mode 100644 src/test/resources/testfile.txt diff --git a/src/test/resources/test-api.sh b/src/test/resources/test-api.sh new file mode 100644 index 0000000..0447f9c --- /dev/null +++ b/src/test/resources/test-api.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# API endpoint URL with placeholder for {id} +url="http://localhost:8080/metadig-webapp-3.0.0-SNAPSHOT/checks/resource.keywords.controlled-2.0.0/run/" +# url="http://localhost:8080/metadig-webapp-3.0.0-SNAPSHOT/suites/FAIR-suite-0.4.0/run/" +# Headers +headers=( + "Content-Type: multipart/mixed" +) + +# Make the API request using curl +curl -X POST "$url" \ + --header 'Content-Type: multipart/form-data; boundary=---------BOUNDARY' \ + --data-binary @testfile.txt + diff --git a/src/test/resources/testfile.txt b/src/test/resources/testfile.txt new file mode 100644 index 0000000..1f751c7 --- /dev/null +++ b/src/test/resources/testfile.txt @@ -0,0 +1,1220 @@ + +-----------BOUNDARY +Content-Disposition: form-data; name="priority" +Content-Type: text/plain + +high +-----------BOUNDARY +Content-Disposition: form-data; name="document" +Content-Type: application/xml;charset=UTF-8 + + + + + Fairbanks sulfate isotope measurements during ALPACA (2022） + + + Allison + Moon + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + allimoon@uw.edu + https://orcid.org/0000-0002-1648-4869 + + + + Ursula + Jongebloed + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + + + + Kayane + Dingilian + + – School of Earth and Atmospheric Sciences, Georgia Institute of Technology, Atlanta + Postdoctoral researcher + + + + Andrew + Schauer + + Department of Earth and Space Sciences, University of Washington, Seattle + Research Scientist + + + + Yuk-Chun + Chan + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + + + + Meeta + Cesler-Maloney + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Graduate Student + + + + William + Simpson + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Professor + + + + Rodney + Weber + + – School of Earth and Atmospheric Sciences, Georgia Institute of Technology, Atlanta + Professor + + + + Ling + Tsiang + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + + + + Fouad + Yazbeck + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + + + + Shuting + Zhai + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + + + + Alanna + Wedum + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + + + + Alexander + Turner + + Department of Atmospheric Sciences, University of Washington, Seattle + Professor + + + + Sarah + Albertin + + IGE, Univ. Grenoble Alpes, CNRS, INRAE, IRD + Graduate Student + + + + Slimane + Bekki + + – LATMOS/IPSL, Sorbonne Université, UVSQ, CNRS + Research Scientist + + + + Joël + Savarino + + IGE, Univ. Grenoble Alpes, CNRS, INRAE, IRD + Senior Scientist + + + + Konstantin + Gribanov + + Climate and Environment Physics Laboratory, Ural Federal University + Senior Researcher + + + + Kerri + Pratt + + Department of Chemistry, University of Michigan, Ann Arbor + Professor + + + + Emily + Costa + + Department of Chemistry, University of Michigan, Ann Arbor + Graduate Student + + + + Cort + Anastasio + + - Department of Land, Air, & Water Resources, University of California, Davis + Professor + + + + Michael + Sunday + + - Department of Land, Air, & Water Resources, University of California, Davis + Postdoctoral researcher + + + + Laura + Heinlein + + - Department of Land, Air, & Water Resources, University of California, Davis + Graduate Student + + + + Jingqiu + Mao + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Professor + + + + Becky + Alexander + + Department of Atmospheric Sciences, University of Washington, Seattle + Professor + beckya@uw.edu + + 2023 + + Within and surrounding high-latitude cities, poor air quality disturbs Arctic ecosystems, influences climate, and harms human health. The Fairbanks North Star Borough has wintertime particulate matter (PM) concentrations that exceed the Environmental Protection Agency’s (EPA) threshold for public health. Particulate sulfate (SO42-) is the most abundant inorganic species and contributes approximately 20% of the total PM mass in Fairbanks, but air quality models underestimate observed sulfate concentrations. Here we quantify sulfate sources using size-resolved δ34S(SO42-), δ18O(SO42-), and Δ17O(SO42-) of particulate sulfate in Fairbanks from January 18th to February 25th, 2022 using a Bayesian isotope mixing model. + This data contains sulfur and oxygen isotope measurements for size-resolved filter samples collected during the Alaskan Layered Pollution and Chemical Analysis field campaign in 2022. Ambient concentrations of SO2, HMS, non-HMS S(IV), SO4, and the sulfur oxidation ratio are also included. Finally, the median fractional contributions of each oxidation pathway calculated in the Bayesian isotope mixing model are included with their 95% confidence interval. + + + sulfate + isotope + aerosol + air quality + haze + ALPACA + Fairbanks + Alaska + None + + + This work is licensed under the Creative Commons Attribution 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by/4.0/. + + + + Community Technical College in downtown Fairbanks + + -147.4919 + -147.4919 + 65.1256 + 65.1256 + + + + + + 2022-01-18 + + + + 2022-02-25 + + + + + + + http://www.w3.org/ns/dcat#theme + https://purl.dataone.org/odo/ADCAD_00034 + + + http://www.w3.org/ns/dcat#theme + https://purl.dataone.org/odo/ADCAD_00039 + + + http://purl.dataone.org/odo/SENSO_00000005 + http://purl.dataone.org/odo/SENSO_00000002 + + + + Allison + Moon + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + allimoon@uw.edu + https://orcid.org/0000-0002-1648-4869 + + + + Becky + Alexander + + Department of Atmospheric Sciences, University of Washington, Seattle + Professor + beckya@uw.edu + + + NSF Arctic Data Center + support@arcticdata.io + http://arcticdata.io + Q77285095 + + + + +
+ Ethical Research Practices + Collection of this data did not involve human subjects or animals. Throughout the field campaign, Fairbanks town hall meetings were held to communicate the research being done during ALPACA with the community. These results will be publically available both through peer-reviewed publications and community outreach materials. +
+ + + + + A Volumetric Flow Controlled Particulate Sampling System (TE-5170) with a 4-stage cascade impactor (TE-230) was used to collect 24-hr size-resolved aerosol samples at Fairbanks Community Technical College (CTC) (64.84064°N, 147.72677°W) between January 17th to February 25th, 2022. Size-resolved bins were determined by calculating the particle size cut-off (Dp,50) at 50% collection efficiency using the corrected flow rate in each sample. Two filter samples were collected for one week each at the Poker Flat Research range (65.1256° N, 147.4919° W), a relatively clean site 46 km north of Fairbanks, to represent a two-week average of background sulfate. Atmospheric particles were collected using a high-volume sampler (Digitel, DH77, TSP inlet, 1 m3 min-1) on pre-combusted quartz filters (Whatman 150nm-diameter). + + + + + https://google.com Isotope samples were prepared for silver salt pyrolysis as described in Schauer et al. and Geng et al.54,55 Briefly, the filtrate was neutralized by converting anions to sodium-form with an offline cation exchange resin (AG 50W-X8 Resin from Bio-Rad). This converts sulfuric acid (H2SO4) to sodium sulfate (Na2SO4), which prevents sample loss due to evaporation. This step is followed by the removal of soluble organics by adding 30% H2O2 and drying in a MiVAc Duo concentrator. Sulfate was separated from other ions in the sample matrix in a Dionex ICS-2000 before being converted to Ag2SO4 using Ag+-charged cation-exchange resin as described in Geng et al. + + + + + Oxygen isotope measurements were performed on a Finnegan MAT253 isotope ratio mass spectrometer using the same configuration as Geng et al.55 Oxygen isotope measurements were corrected for isotopic exchange with quartz and conversion of HMS and non-HMS S(IV) to sulfate during sample preparation (see supplement section 1.4.1). Sulfur isotope composition was measured using a separate Finnegan MAT253 isotope ratio mass spectrometer with the same configuration as Jongebloed et al.56 δ34S values were normalized to the Vienna Canyon Diablo Troilite (VCDT) scale using four in-house reference materials that are regularly calibrated against the international reference materials IAEA-S-1, IAEA-S-3, and NBS-127. A sulfur isotope correction for sulfate formed from HMS and S(IV) during sample processing was estimated as described in section 1.4.2. The δ34S composition of Fairbanks fuel oils #1 and #2 aquired during winter 2022 was measured by combusting 6 μL of fuel oil in a 50 μL tin capsule packed with tin powder. A source signature for δ34Semission was calculated using the δ34S measurements for fuel oil #1 (+3.7±0.6‰) and fuel oil #2 (+4.9±0.1‰) and weighing the values by sulfur content (896 and 2,053 ppmv, respectively) and domestic use (33% and 67%, respectively) (Table S4)19. This yielded a δ34Semission signature of +4.7 (±0.6)‰. The analytical error of the measurements (±0.8‰, ±0.2‰, and ±1.0‰ for δ18O, Δ17O, and δ34S, respectively) was estimated from duplicate sample analysis (performed on 30% of the Fairbanks samples) and replicate measurements of standards in quartz and silver capsules. The fully propagated error including isotopic corrections for the three measurements are: δ18O (±1.9‰), Δ17O (±0.4‰), and δ34S (±1.2‰). + + + + + We developed an isotope mixing model to investigate the contributions of primary sulfate and five secondary sulfate formation pathways (H2O2, O3, TMI-O2, OH, and NO2). The model inputs are the δ18O(SO42−), Δ17O(SO42−), and δ34S(SO42−) observations. We assumed the observed δ18O(SO42−), Δ17O(SO42−), and δ34S(SO42−) follow a multivariate Gaussian distribution. We used a Dirichlet distribution as the prior for the fractional contributions such that each fraction is in the interval [0,1] and all fractions sum to 1. Markov Chain Monte Carlo was used to calculate the fractional contributions of each sulfate formation pathway for each sample and estimate the uncertainty by providing a 95% confidence interval. + + + + + Schauer, A. J., Kunasek, S. A., Sofen, E. D., Erbland, J., Savarino, J., Johnson, B. W., Amos, H. M., Shaheen, R., Abaunza, M., Jackson, T. L., Thiemens, M. H., & Alexander, B. (2012). Oxygen isotope exchange with quartz during pyrolysis of silver sulfate and silver nitrate: Oxygen isotope exchange during pyrolysis of Ag 2 SO 4 and AgNO 3. Rapid Communications in Mass Spectrometry, 26(18), 2151–2157. https://doi.org/10.1002/rcm.6332 + Geng, L., Schauer, A. J., Kunasek, S. A., Sofen, E. D., Erbland, J., Savarino, J., Allman, D. J., Sletten, R. S., & Alexander, B. (2013). Analysis of oxygen-17 excess of nitrate and sulfate at sub-micromole levels using the pyrolysis method: Analysis of oxygen-17 excess of nitrate and sulfate. Rapid Communications in Mass Spectrometry, 27(21), 2411–2419. https://doi.org/10.1002/rcm.6703 + + + + + + A Volumetric Flow Controlled Particulate Sampling System (TE-5170) with a 4-stage cascade impactor (TE-230) was used to collect 24-hr size-resolved aerosol samples at Fairbanks Community Technical College (CTC) (64.84064°N, 147.72677°W) between January 17th to February 25th, 2022. Size-resolved bins were determined by calculating the particle size cut-off (Dp,50) at 50% collection efficiency using the corrected flow rate in each sample (equation S1 in the Supporting Information). For each collection period, filters were combined to form three-size bins: particle diameters < 0.7 μm (PM<0.7), 0.7-2.5 μm (PM0.7-2.5), and 2.5-10 μm (PM2.5-10). Both PM0.7 and PM0.7-2.5 fall within the EPA-regulated fine particle range deemed PM2.5, but are analyzed in separate size bins here. Filter samples were collected daily between 9:30 AM to 9:00 AM the following day except for one exceptionally polluted period between January 31st to February 3rd when the filters were changed twice per day at approximately 9:30 AM and 5 PM local time. + + + + No sampling description provided. + + + + + Fairbanks sulfate isotope measurements during ALPACA + + + Allison + Moon + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + principalInvestigator + + + + Ursula + Jongebloed + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + principalInvestigator + + + + Kayane + Dingilian + + – School of Earth and Atmospheric Sciences, Georgia Institute of Technology, Atlanta + Postdoctoral researcher + principalInvestigator + + + + Andrew + Schauer + + Department of Earth and Space Sciences, University of Washington, Seattle + Research Scientist + principalInvestigator + + + + Yuk-Chun + Chan + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + principalInvestigator + + + + Meeta + Cesler-Maloney + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Graduate Student + principalInvestigator + + + + William + Simpson + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Professor + principalInvestigator + + + + Rodney + Weber + + – School of Earth and Atmospheric Sciences, Georgia Institute of Technology, Atlanta + Professor + principalInvestigator + + + + Ling + Tsiang + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + principalInvestigator + + + + Fouad + Yazbeck + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + principalInvestigator + + + + Shuting + Zhai + + Department of Atmospheric Sciences, University of Washington, Seattle + Graduate Student + principalInvestigator + + + + Alanna + Wedum + + Department of Atmospheric Sciences, University of Washington, Seattle + Undergraduate Student + principalInvestigator + + + + Alexander + Turner + + Department of Atmospheric Sciences, University of Washington, Seattle + Professor + principalInvestigator + + + + Sarah + Albertin + + IGE, Univ. Grenoble Alpes, CNRS, INRAE, IRD + Graduate Student + principalInvestigator + + + + Slimane + Bekki + + – LATMOS/IPSL, Sorbonne Université, UVSQ, CNRS + Research Scientist + principalInvestigator + + + + Joël + Savarino + + IGE, Univ. Grenoble Alpes, CNRS, INRAE, IRD + Senior Scientist + principalInvestigator + + + + Konstantin + Gribanov + + Climate and Environment Physics Laboratory, Ural Federal University + Senior Researcher + principalInvestigator + + + + Kerri + Pratt + + Department of Chemistry, University of Michigan, Ann Arbor + Professor + principalInvestigator + + + + Emily + Costa + + Department of Chemistry, University of Michigan, Ann Arbor + Graduate Student + principalInvestigator + + + + Cort + Anastasio + + - Department of Land, Air, & Water Resources, University of California, Davis + Professor + principalInvestigator + + + + Michael + Sunday + + - Department of Land, Air, & Water Resources, University of California, Davis + Postdoctoral researcher + principalInvestigator + + + + Laura + Heinlein + + - Department of Land, Air, & Water Resources, University of California, Davis + Graduate Student + principalInvestigator + + + + Jingqiu + Mao + + Department of Chemistry, Biochemistry, and Geophysical Institute, University of Alaska Fairbanks + Professor + principalInvestigator + + + + Becky + Alexander + + Department of Atmospheric Sciences, University of Washington, Seattle + Professor + principalInvestigator + + + The National Oceanic and Atmospheric Administration (OAR)in the AC4 - Urban atmosphere in a warming climate: chemistry, carbon and composition division. + 0124733 + Fairbanks sulfate isotope measurements during ALPACA + + + + Moon_et_al_isotope_data.csv + Isotope data + + Moon_et_al_isotope_data.csv + 44286 + 629095bed85261d7587fbb79c24d98d7 + + + 1 + + + column + + , + + + + + + https://cn.dataone.org/cn/v2/resolve/urn:uuid:3a6f9c5e-3c7b-4ada-819e-e00ec19053cb + + + + + + Date + Date of measurement. Multiple date format included. + + + DD/MM and PM period + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00002043 + + + + δ34S(‰) + δ34 S is calculated using the ratio of 34S/32S in the sample over the ratio of Vienna Canyon Diablo Troilite (VCDT). The unit is permille (‰). + + + + permil + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + δ18O(‰) + δ18 O is calculated using the ratio of 18O/16O compared to the ratio found in Vienna Standard Mean Ocean water (VSMOW). The unit is permille (‰). + + + + permil + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00001677 + + + + Δ17O(‰) + ∆17 O is calculated using the following equation: +∆17 O(SO4)= δ17 O(SO4 )-0.52∙(δ18 O(SO4)) +where d17O is calculated using the ratio of 17O/16O compared to the ratio found in Vienna Standard Mean Ocean water (VSMOW). The unit is permille (‰). + + + + permil + + + real + + + + + + SO2 (umol/m3) + Ambient concentrations of Sulfur dioxide in umol/m3. + + + + millimolePerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000662 + + + + SOR (SO4/SO2+SO4+HMS+non-HMS SIV) + SOR is the sulfur oxidation ratio. It's the molar concentration of SO4 over the molar concentrations of SO2, HMS, non-HMS S(IV), and SO4. + + + + dimensionless + + + real + + + + + + SO4_PM0.7_ug/m3 + The measurement of particulate matter 0.7 of Sulfate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00002156 + + + + SO4_PM0.7-2.5_ug/m3 + The measurement of particulate matter 0.7 - 2.5 of Sulfate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00002156 + + + + SO4_PM2.5-PM10_ug/m3 + The measurement of particulate matter 2.5 - 10 of Sulfate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00002156 + + + + Non-HMS_SIV_PM0.7_ug/m3 + The concentration of PM 0.7 non- hydroxymethanesulfonate S(IV) in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + Non-HMS_SIV_PM0.7-2.5_ug/m3 + The concentration of PM 0.7-2.5 non- hydroxymethanesulfonate S(IV) in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + Non-HMS_SIV_PM2.5-PM10_ug/m3 + The concentration of PM 2.5-10 non- hydroxymethanesulfonate S(IV) in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + HMS_PM0.7_ug/m3 + The concentration of PM 0.7 hydroxymethanesulfonate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + HMS_PM0.7-2.5_ug/m3 + The concentration of PM 0.7-2.5 hydroxymethanesulfonate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + HMS_PM2.5-PM10_ug/m3 + The concentration of PM 2.5-10 hydroxymethanesulfonate in ug/m3. + + + + microgramPerMeterCubed + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000512 + + + + primary_median_fraction + The median percentage of total sulfate from the primary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + OH_median_fraction + The median percentage of total sulfate from the OH secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + H2O2_median_fraction + The median percentage of total sulfate from the H2O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + TMI_median_fraction + The median percentage of total sulfate from the TMI-O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + O3_median_fraction + The median percentage of total sulfate from the O3 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + NO2_median_fraction + The median percentage of total sulfate from the NO2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + primary_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the primary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + OH_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the OH secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + H2O2_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the H2O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + TMI_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the TMI-O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + O3_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the O3 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + NO2_97.5th_percentile_fraction + The 97.5 percentile percentage of total sulfate from the NO2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + primary_2.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the primary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + OH2.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the OH secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + H2O22.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the H2O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + TMI2.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the TMI-O2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + O32.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the O3 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + NO2_2.5th_percentile_fraction + The 2.5 percentile percentage of total sulfate from the NO2 secondary sulfate formation pathway. + + + + dimensionless + + + real + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000629 + + + + + + + + + + microgram per meter cubed + + + + + +-----------BOUNDARY +Content-Disposition: form-data; name="systemMetadata" +Content-Type: application/xml;charset=UTF-8 + + + + 0 + urn:uuid:6a04c674-61ba-4a36-bdc0-5585f1fce2d0 + https://eml.ecoinformatics.org/eml-2.2.0 + 61569 + 881bf599ca27fbf538444830408e0eab + http://orcid.org/0000-0003-4703-1974 + http://orcid.org/0000-0002-1648-4869 + + + http://orcid.org/0000-0003-4703-1974 + read + write + changePermission + + + CN=arctic-data-admins,DC=dataone,DC=org + read + write + changePermission + + + public + read + + + + false + 2023-11-22T20:51:11.883+00:00 + 2023-11-22T20:51:11.883+00:00 + urn:node:mnTestARCTIC + urn:node:mnTestARCTIC + Fairbanks_sulfate_isotope_measurements_during_ALPA.xml + +-----------BOUNDARY-- + From fcea6cbd38ae299cb15ac631924030012a6c7a05 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Wed, 7 Feb 2024 16:12:20 -0700 Subject: [PATCH 15/23] trigger a build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fdc4594..3ea1fce 100644 --- a/README.md +++ b/README.md @@ -58,4 +58,4 @@ Authorization for Docker Hub can be setup in several ways. One method is to crea ``` -The appropriate username and password is available from the NCEAS secure repo. +The appropriate username and password is available from the NCEAS secure repo. From cc0b572ad328330755a8206acda71cfbda051b44 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Mon, 14 Aug 2023 16:44:26 -0700 Subject: [PATCH 16/23] Update README.md From ea4f7a266e7e89d806ead2cd9756b7d10744a012 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Tue, 15 Aug 2023 11:19:19 -0700 Subject: [PATCH 17/23] Update README.md From ed2890ab65f8e522b29a86a4a82180f523c9d552 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Tue, 15 Aug 2023 11:53:51 -0700 Subject: [PATCH 18/23] Update README.md From d2dce26d49614d7effaef94b482ed3eec1cdc0ab Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Mon, 4 Mar 2024 14:56:39 -0700 Subject: [PATCH 19/23] remove trailing whitespace --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ea1fce..15116f7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MetaDIG webapp +# MetaDIG webapp Web service for interacting with the [MetaDIG Engine](https://github.com/NCEAS/metadig-engine) API. This project builds a 'metadig-webapp.war' file that can be deployed in the Tomcat webapps directory. Depending on the Tomcat configuration, the war may be unpacked automatically each time a new version is copied in place. @@ -58,4 +58,4 @@ Authorization for Docker Hub can be setup in several ways. One method is to crea ``` -The appropriate username and password is available from the NCEAS secure repo. +The appropriate username and password is available from the NCEAS secure repo. From f5a5910bc8a7db79e4902cccc92ea2d005684858 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Mon, 4 Mar 2024 15:10:12 -0700 Subject: [PATCH 20/23] clean up code and add some comments --- .../nceas/mdq/MetadigContextListener.java | 4 +- .../ucsb/nceas/mdq/rest/ChecksResource.java | 174 +++++++++--------- .../ucsb/nceas/mdq/rest/SuitesResource.java | 3 + src/test/resources/test-api.sh | 8 +- 4 files changed, 96 insertions(+), 93 deletions(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java index 7e687c9..b61b1a1 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java +++ b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java @@ -21,7 +21,7 @@ public void contextInitialized(ServletContextEvent servletContextEvent) { try { Dispatcher.setupJep(); } catch (MetadigException e) { - log.error("Error setting up Jep. Python checks may not work.", e); + throw new RuntimeException("Error setting up Jep. Aborting startup.", e); } log.info("Metadig 'contextInitialized' called."); @@ -35,7 +35,7 @@ public void contextDestroyed(ServletContextEvent servletContextEvent) { try { log.debug("Shutting down controller..."); controller.shutdown(); - log.info("Controller shutdonw successfully."); + log.info("Controller shutdown successfully."); } catch (IOException | TimeoutException e) { log.error("Error shutting down metadig controller."); e.printStackTrace(); diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java index a5ae2c7..e8229db 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java @@ -18,16 +18,13 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Request; import javax.ws.rs.core.Response; import javax.ws.rs.core.Variant; import javax.xml.bind.JAXBException; -import com.hp.hpl.jena.shared.ConfigException; import edu.ucsb.nceas.mdqengine.exception.MetadigException; import edu.ucsb.nceas.mdqengine.exception.MetadigStoreException; -import net.sf.saxon.functions.ConstantFunction; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; @@ -51,25 +48,25 @@ */ @Path("checks") public class ChecksResource { - - private Log log = LogFactory.getLog(this.getClass()); - - private MDQStore store = null; - - private MDQEngine engine = null; - - public ChecksResource() throws MetadigStoreException { - boolean persist = false; - this.store = StoreFactory.getStore(persist); + + private Log log = LogFactory.getLog(this.getClass()); + + private MDQStore store = null; + + private MDQEngine engine = null; + + public ChecksResource() throws MetadigStoreException { + boolean persist = false; + this.store = StoreFactory.getStore(persist); - try { - this.engine = new MDQEngine(); - this.engine.setStore(this.store); - } catch (MetadigException | IOException | ConfigurationException e) { - log.error(e.getMessage(), e); - } - } - + try { + this.engine = new MDQEngine(); + this.engine.setStore(this.store); + } catch (MetadigException | IOException | ConfigurationException e) { + log.error(e.getMessage(), e); + } + } + /** * Method handling HTTP GET requests. The returned object will be sent * to the client as "text/plain" media type. @@ -79,7 +76,7 @@ public ChecksResource() throws MetadigStoreException { @GET @Produces(MediaType.APPLICATION_JSON) public String listChecks() { - Collection checks = store.listChecks(); + Collection checks = store.listChecks(); return JsonMarshaller.toJson(checks); } @@ -87,45 +84,48 @@ public String listChecks() { @Path("/{id}") @Produces(MediaType.TEXT_XML) public String getCheck(@PathParam("id") String id) throws UnsupportedEncodingException, JAXBException { - Check check = store.getCheck(id); + Check check = store.getCheck(id); return (String) XmlMarshaller.toXml(check, true); } // @POST // @Consumes(MediaType.MULTIPART_FORM_DATA) +// not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean createCheck(@FormDataParam("check") InputStream xml) { - Check check = null; - try { - check = (Check) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Check.class); - store.createCheck(check); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } + Check check = null; + try { + check = (Check) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Check.class); + store.createCheck(check); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } return true; } // @PUT // @Path("/{id}") // @Consumes(MediaType.MULTIPART_FORM_DATA) +// not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean updateCheck(@PathParam("id") String id, @FormDataParam("check") InputStream xml) throws JAXBException, IOException { - Check check = null; - try { - check = (Check) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Check.class); - store.updateCheck(check); - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } + Check check = null; + try { + check = (Check) XmlMarshaller.fromXml(IOUtils.toString(xml, "UTF-8"), Check.class); + store.updateCheck(check); + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; + } return true; } // @DELETE // @Path("/{id}") // @Produces(MediaType.TEXT_PLAIN) +// not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean updateCheck(@PathParam("id") String id) { - Check check = store.getCheck(id); - store.deleteCheck(check); + Check check = store.getCheck(id); + store.deleteCheck(check); return true; } @@ -134,52 +134,52 @@ public boolean updateCheck(@PathParam("id") String id) { @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) public Response run( - @PathParam("id") String id, - @FormDataParam("document") InputStream input, - @FormDataParam("systemMetadata") InputStream sysMetaStream, - @Context Request r) throws UnsupportedEncodingException, JAXBException { - - Run run = null; - // include SM if it was provided - SystemMetadata sysMeta = null; - if (sysMetaStream != null) { - try { - sysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysMetaStream); - } catch (InstantiationException | IllegalAccessException - | IOException | MarshallingException e) { - log.warn("Could not unmarshall SystemMetadata from stream", e); - } - } - try { - Map params = new HashMap(); -// params.putAll(formParams); -// params.remove("id"); -// params.remove("document"); - Check check = store.getCheck(id); - run = engine.runCheck(check, input, params, sysMeta); - store.createRun(run); - Dispatcher.getDispatcher("python").close(); - } catch (Exception e) { - log.error(e.getMessage(), e); - return Response.serverError().entity(e).build(); - } - - // determine the format of plot to return + @PathParam("id") String id, + @FormDataParam("document") InputStream input, + @FormDataParam("systemMetadata") InputStream sysMetaStream, + @Context Request r) throws UnsupportedEncodingException, JAXBException { + + Run run = null; + // include SM if it was provided + SystemMetadata sysMeta = null; + if (sysMetaStream != null) { + try { + sysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysMetaStream); + } catch (InstantiationException | IllegalAccessException + | IOException | MarshallingException e) { + log.warn("Could not unmarshall SystemMetadata from stream", e); + } + } + try { + Map params = new HashMap(); +// params.putAll(formParams); +// params.remove("id"); +// params.remove("document"); + Check check = store.getCheck(id); + run = engine.runCheck(check, input, params, sysMeta); + store.createRun(run); + Dispatcher.getDispatcher("python").close(); + } catch (Exception e) { + log.error(e.getMessage(), e); + return Response.serverError().entity(e).build(); + } + + // determine the format of plot to return String resultString = null; - List vs = - Variant.mediaTypes(MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE).build(); - Variant v = r.selectVariant(vs); - if (v == null) { - return Response.notAcceptable(vs).build(); - } else { - MediaType mt = v.getMediaType(); - if (mt.equals(MediaType.APPLICATION_XML_TYPE)) { - resultString = XmlMarshaller.toXml(run, true); - } else { - resultString = JsonMarshaller.toJson(run); - } - } - - return Response.ok(resultString).build(); + List vs = + Variant.mediaTypes(MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE).build(); + Variant v = r.selectVariant(vs); + if (v == null) { + return Response.notAcceptable(vs).build(); + } else { + MediaType mt = v.getMediaType(); + if (mt.equals(MediaType.APPLICATION_XML_TYPE)) { + resultString = XmlMarshaller.toXml(run, true); + } else { + resultString = JsonMarshaller.toJson(run); + } + } + + return Response.ok(resultString).build(); } } diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java index 2e140cb..573db49 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/SuitesResource.java @@ -90,6 +90,7 @@ public String getSuite(@PathParam("id") String id) // @POST // @Consumes(MediaType.MULTIPART_FORM_DATA) + // not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean createSuite(@FormDataParam("suite") InputStream xml) { boolean persist = true; MDQStore store = null; @@ -117,6 +118,7 @@ public boolean createSuite(@FormDataParam("suite") InputStream xml) { // @PUT // @Path("/{id}") // @Consumes(MediaType.MULTIPART_FORM_DATA) + // not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean updateSuite(@PathParam("id") String id, @FormDataParam("suite") InputStream xml) throws JAXBException, IOException { boolean persist = true; @@ -145,6 +147,7 @@ public boolean updateSuite(@PathParam("id") String id, @FormDataParam("suite") I // @DELETE // @Path("/{id}") // @Produces(MediaType.TEXT_PLAIN) + // not enabled for security reasons, see: https://github.com/NCEAS/metadig-webapp/issues/21 public boolean deleteSuite(@PathParam("id") String id) { boolean persist = true; MDQStore store = null; diff --git a/src/test/resources/test-api.sh b/src/test/resources/test-api.sh index 0447f9c..405fe2a 100644 --- a/src/test/resources/test-api.sh +++ b/src/test/resources/test-api.sh @@ -1,15 +1,15 @@ #!/bin/bash -# API endpoint URL with placeholder for {id} -url="http://localhost:8080/metadig-webapp-3.0.0-SNAPSHOT/checks/resource.keywords.controlled-2.0.0/run/" -# url="http://localhost:8080/metadig-webapp-3.0.0-SNAPSHOT/suites/FAIR-suite-0.4.0/run/" +# API endpoint URL for testing a check or suite +check_url="http://localhost:8080/metadig-webapp-3.0.0/checks/resource.keywords.controlled-2.0.0/run/" +suite_url="http://localhost:8080/metadig-webapp-3.0.0/suites/FAIR-suite-0.4.0/run/" # Headers headers=( "Content-Type: multipart/mixed" ) # Make the API request using curl -curl -X POST "$url" \ +curl -X POST "$suite_url" \ --header 'Content-Type: multipart/form-data; boundary=---------BOUNDARY' \ --data-binary @testfile.txt From 1f681153e80ff5abe1fd0dd9fc7a0cd2f1e6259d Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Mon, 4 Mar 2024 15:10:58 -0700 Subject: [PATCH 21/23] remove commented out code --- src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java index e8229db..bddede7 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java +++ b/src/main/java/edu/ucsb/nceas/mdq/rest/ChecksResource.java @@ -152,9 +152,6 @@ public Response run( } try { Map params = new HashMap(); -// params.putAll(formParams); -// params.remove("id"); -// params.remove("document"); Check check = store.getCheck(id); run = engine.runCheck(check, input, params, sysMeta); store.createRun(run); From f83de96def35337a56d87866c84908be52e58fdd Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Wed, 6 Mar 2024 11:55:59 -0700 Subject: [PATCH 22/23] fix a typo --- src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java index b61b1a1..dd3dcef 100644 --- a/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java +++ b/src/main/java/edu/ucsb/nceas/mdq/MetadigContextListener.java @@ -35,7 +35,7 @@ public void contextDestroyed(ServletContextEvent servletContextEvent) { try { log.debug("Shutting down controller..."); controller.shutdown(); - log.info("Controller shutdown successfully."); + log.info("Controller shut down successfully."); } catch (IOException | TimeoutException e) { log.error("Error shutting down metadig controller."); e.printStackTrace(); From 15a589917068a14277b54d72d71eb5cf47cda449 Mon Sep 17 00:00:00 2001 From: Jeanette Clark Date: Fri, 8 Mar 2024 10:13:11 -0700 Subject: [PATCH 23/23] prep for release --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e3b9922..3a3fb18 100644 --- a/pom.xml +++ b/pom.xml @@ -6,8 +6,8 @@ UTF-8 metadig - 3.0.0-SNAPSHOT - 3.0.0-SNAPSHOT + 3.0.0 + 3.0.0 4.0.0