diff --git a/src/dcat-us/compile-dcat-feed.test.ts b/src/dcat-us/compile-dcat-feed.test.ts index adc14ee..29ffcb3 100644 --- a/src/dcat-us/compile-dcat-feed.test.ts +++ b/src/dcat-us/compile-dcat-feed.test.ts @@ -2,7 +2,82 @@ import { compileDcatFeedEntry } from './compile-dcat-feed'; import * as datasetFromApi from '../test-helpers/mock-dataset.json'; import { DcatUsError } from './dcat-us-error'; +describe('generating DCAT-US 1.0 feed', () => { + const version = '1.1'; + it('should throw 400 DcatUs error if template contains transformer that is not defined', async function () { + const dcatTemplate = { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + issued: '{{created:toISO}}' + } + + try { + compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version); + } catch (error) { + expect(error).toBeInstanceOf(DcatUsError); + expect(error).toHaveProperty('statusCode', 400); + } + }); + + it('show return distribution in a single array', async function () { + const dcatTemplate = { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + distribution: [ + 'distro1', + 'distro2', + ['distro3', 'distro4'] + ] + } + + const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version)); + expect(dcatDataset.distribution).toBeDefined(); + expect(dcatDataset.distribution).toStrictEqual(['distro1', 'distro2', 'distro3', 'distro4']); + }); + + it('show not return uninterpolated distribution in dataset', async function () { + const dcatTemplate = { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + distribution: ['distro1', '{{distroname}}'] + } + const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version)); + expect(dcatDataset.distribution).toStrictEqual(['distro1']); + }); + + it('should contain default theme if spatial key exists in dataset', async function () { + const distribution = ['distro1', '{{distroname}}']; + const dcatTemplate = { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + distribution, + spatial: '{{extent}}' + } + const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version)); + expect(dcatDataset.theme).toBeDefined(); + expect(dcatDataset.theme).toStrictEqual(['geospatial']); + }); + + it('should throw error if geojson from provider is missing', async function () { + const dcatTemplate = { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + issued: '{{created:toISO}}' + }; + + expect(() => { + compileDcatFeedEntry(undefined, dcatTemplate, {}, version); + }).toThrow(DcatUsError); + }); +}); + describe('generating DCAT-US 3.0 feed', () => { + const version = '3.0'; it('should throw 400 DcatUs error if template contains transformer that is not defined', async function () { const dcatTemplate = { title: '{{name}}', @@ -12,7 +87,7 @@ describe('generating DCAT-US 3.0 feed', () => { } try { - compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}); + compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version); } catch (error) { expect(error).toBeInstanceOf(DcatUsError); expect(error).toHaveProperty('statusCode', 400); @@ -31,7 +106,7 @@ describe('generating DCAT-US 3.0 feed', () => { ] } - const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {})); + const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version)); expect(dcatDataset['dcat:distribution']).toBeDefined(); expect(dcatDataset['dcat:distribution']).toStrictEqual(['distro1', 'distro2', 'distro3', 'distro4']); }); @@ -43,7 +118,7 @@ describe('generating DCAT-US 3.0 feed', () => { keyword: '{{tags}}', 'dcat:distribution': ['distro1', '{{distroname}}'] } - const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {})); + const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version)); expect(dcatDataset['dcat:distribution']).toStrictEqual(['distro1']); }); @@ -56,9 +131,7 @@ describe('generating DCAT-US 3.0 feed', () => { }; expect(() => { - compileDcatFeedEntry(undefined, dcatTemplate, {}); + compileDcatFeedEntry(undefined, dcatTemplate, {}, version); }).toThrow(DcatUsError); - }); - }); \ No newline at end of file diff --git a/src/dcat-us/compile-dcat-feed.ts b/src/dcat-us/compile-dcat-feed.ts index b439410..42dea26 100644 --- a/src/dcat-us/compile-dcat-feed.ts +++ b/src/dcat-us/compile-dcat-feed.ts @@ -9,15 +9,33 @@ type Feature = { properties: Record }; -export function compileDcatFeedEntry(geojsonFeature: Feature | undefined, feedTemplate: DcatDatasetTemplate, feedTemplateTransforms: TransformsList): string { +export function compileDcatFeedEntry( + geojsonFeature: Feature | undefined, + feedTemplate: DcatDatasetTemplate, + feedTemplateTransforms: TransformsList, + version: string +): string { try { const dcatFeedItem = generateDcatItem(feedTemplate, feedTemplateTransforms, geojsonFeature); - return indent(JSON.stringify({ - ...dcatFeedItem, - 'dcat:distribution': - Array.isArray(dcatFeedItem['dcat:distribution']) && - removeUninterpolatedDistributions(_.flatten(dcatFeedItem['dcat:distribution'])), - }, null, '\t'), 2); + let feedEntry: Record; + if (version === '1.1') { + feedEntry = { + ...dcatFeedItem, + distribution: Array.isArray(dcatFeedItem.distribution) && removeUninterpolatedDistributions(_.flatten(dcatFeedItem.distribution)), + theme: dcatFeedItem.spatial && ['geospatial'] + }; + } + + if (version === '3.0') { + feedEntry = { + ...dcatFeedItem, + 'dcat:distribution': + Array.isArray(dcatFeedItem['dcat:distribution']) && + removeUninterpolatedDistributions(_.flatten(dcatFeedItem['dcat:distribution'])), + }; + } + + return indent(JSON.stringify(feedEntry, null, '\t'), 2); } catch (err) { throw new DcatUsError(err.message, 400); } diff --git a/src/dcat-us/constants/contexts.ts b/src/dcat-us/constants/contexts.ts index bd9766b..6e80744 100644 --- a/src/dcat-us/constants/contexts.ts +++ b/src/dcat-us/constants/contexts.ts @@ -1,4 +1,15 @@ -export const HEADER_V3 = { +// Context header for DCAT US 1.1 +export const HEADER_V_1_1 = { + '@context': + 'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld', + '@type': 'dcat:Catalog', + conformsTo: 'https://project-open-data.cio.gov/v1.1/schema', + describedBy: 'https://project-open-data.cio.gov/v1.1/schema/catalog.json', +}; + +// Context header for DCAT US 3.0 +// source: https://raw.githubusercontent.com/DOI-DO/dcat-us/refs/heads/main/context/dcat-us-3.0.jsonld +export const HEADER_V_3 = { '@context': { '@version': 1.1, '@protected': true, diff --git a/src/dcat-us/index.test.ts b/src/dcat-us/index.test.ts index 152bf80..7b6c986 100644 --- a/src/dcat-us/index.test.ts +++ b/src/dcat-us/index.test.ts @@ -1,21 +1,74 @@ import { readableFromArray, streamToString } from '../test-helpers/stream-utils'; -import { getDataStreamDcatUs11 } from './'; +import { getDataStreamDcatUs } from './'; import * as datasetFromApi from '../test-helpers/mock-dataset.json'; -import { HEADER_V3 } from './constants/contexts'; +import { HEADER_V_3 } from './constants/contexts'; -async function generateDcatFeed(dataset, template, templateTransforms) { - const { stream: dcatStream } = getDataStreamDcatUs11(template, templateTransforms); +async function generateDcatFeed(dataset, template, templateTransforms, version) { + const { stream: dcatStream } = getDataStreamDcatUs(template, templateTransforms, version); const docStream = readableFromArray([dataset]); // no datasets since we're just checking the catalog const feedString = await streamToString(docStream.pipe(dcatStream)); return { feed: JSON.parse(feedString) }; } +describe('generating DCAT-US 1.1 feed', () => { + const version = '1.1'; + it('formats catalog correctly', async function () { + const { feed } = await generateDcatFeed([], {}, {}, version); + + expect(feed['@context']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld'); + expect(feed['@type']).toBe('dcat:Catalog'); + expect(feed['conformsTo']).toBe('https://project-open-data.cio.gov/v1.1/schema'); + expect(feed['describedBy']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.json'); + expect(Array.isArray(feed['dataset'])).toBeTruthy(); + }); + + it('should interprolate dataset stream to feed based upon template', async function () { + const { feed } = await generateDcatFeed(datasetFromApi, { + title: '{{name}}', + description: '{{description}}', + keyword: '{{tags}}', + issued: '{{created:toISO}}', + modified: '{{modified:toISO}}', + publisher: { + name: '{{source}}' + }, + contactPoint: { + '@type': 'vcard:Contact', + fn: '{{owner}}', + hasEmail: '{{orgContactEmail:optional}}' + } + }, + { + toISO: (_key, val) => { + return new Date(val).toISOString(); + } + }, + version); + + expect(feed['@context']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld'); + expect(feed['@type']).toBe('dcat:Catalog'); + expect(feed['conformsTo']).toBe('https://project-open-data.cio.gov/v1.1/schema'); + expect(feed['describedBy']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.json'); + expect(Array.isArray(feed['dataset'])).toBeTruthy(); + expect(feed['dataset'].length).toBe(1); + const feedResponse = feed['dataset'][0]; + expect(feedResponse.title).toBe('Tahoe places of interest'); + expect(feedResponse.description).toBe('Description. Here be Tahoe things. You can do a lot here. Here are some more words. And a few more.

with more words

adding a few more to test how long it takes for our jobs to execute.

Tom was here!
'); + expect(feedResponse.issued).toBe('2021-01-29T15:34:38.000Z'); + expect(feedResponse.modified).toBe('2021-07-27T20:25:19.723Z'); + expect(feedResponse.contactPoint).toStrictEqual({ '@type': 'vcard:Contact', fn: 'thervey_qa_pre_a_hub' }); + expect(feedResponse.publisher).toStrictEqual({ name: 'QA Premium Alpha Hub' }); + expect(feedResponse.keyword).toStrictEqual(['Data collection', 'just modified']); + }); +}); + describe('generating DCAT-US 3.0 feed', () => { + const version = '3.0'; it('formats catalog correctly', async function () { - const { feed } = await generateDcatFeed([], {}, {}); + const { feed } = await generateDcatFeed([], {}, {}, version); - expect(feed['@context']).toStrictEqual(HEADER_V3['@context']); + expect(feed['@context']).toStrictEqual(HEADER_V_3['@context']); expect(feed['conformsTo']).toBe('https://resource.data.gov/profile/dcat-us#'); expect(feed['@type']).toBe('dcat:Catalog'); expect(Array.isArray(feed['dcat:dataset'])).toBeTruthy(); @@ -39,13 +92,15 @@ describe('generating DCAT-US 3.0 feed', () => { header: { '@id': 'hub.arcgis.com' } - }, { - toISO: (_key, val) => { - return new Date(val).toISOString(); - } - }); + }, + { + toISO: (_key, val) => { + return new Date(val).toISOString(); + } + }, + version); - expect(feed['@context']).toStrictEqual(HEADER_V3['@context']); + expect(feed['@context']).toStrictEqual(HEADER_V_3['@context']); expect(feed['@type']).toBe('dcat:Catalog'); expect(feed['@id']).toBe('hub.arcgis.com'); expect(feed['conformsTo']).toBe('https://resource.data.gov/profile/dcat-us#'); diff --git a/src/dcat-us/index.ts b/src/dcat-us/index.ts index 7ec978f..d8e861f 100644 --- a/src/dcat-us/index.ts +++ b/src/dcat-us/index.ts @@ -1,17 +1,31 @@ import { compileDcatFeedEntry } from './compile-dcat-feed'; import { FeedFormatterStream } from './feed-formatter-stream'; import { TransformsList } from 'adlib'; -import { HEADER_V3 } from './constants/contexts'; +import { HEADER_V_3, HEADER_V_1_1 } from './constants/contexts'; -export function getDataStreamDcatUs11(feedTemplate: any, feedTemplateTransforms: TransformsList) { +export function getDataStreamDcatUs(feedTemplate: any, feedTemplateTransforms: TransformsList, version: string) { const footer = '\n\t]\n}'; - const { header: templateHeader, ...restFeedTemplate } = feedTemplate; + let header: string; + let template: Record; + + if (version === '3.0') { + const { header: templateHeader, ...restFeedTemplate } = feedTemplate; + template = restFeedTemplate; + const catalogStr = JSON.stringify({ ...HEADER_V_3, ...templateHeader }, null, '\t'); + header = `${catalogStr.substring(0, catalogStr.length - 2)},\n\t"dcat:dataset": [\n`; + } - const catalogStr = JSON.stringify({ ...HEADER_V3, ...templateHeader }, null, '\t'); - const header = `${catalogStr.substring(0, catalogStr.length - 2)},\n\t"dcat:dataset": [\n`; + if (version === '1.1') { + const catalogStr = JSON.stringify(HEADER_V_1_1, null, '\t'); + header = `${catalogStr.substring( + 0, + catalogStr.length - 2, + )},\n\t"dataset": [\n`; + template = feedTemplate; + } const formatFn = (chunk) => { - return compileDcatFeedEntry(chunk, restFeedTemplate, feedTemplateTransforms); + return compileDcatFeedEntry(chunk, template, feedTemplateTransforms, version); }; return { diff --git a/src/index.test.ts b/src/index.test.ts index 3737f20..f3fdd9e 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -8,6 +8,7 @@ import { createMockKoopApp } from './test-helpers/create-mock-koop-app'; import { readableFromArray } from './test-helpers/stream-utils'; import { DcatUsError } from './dcat-us/dcat-us-error'; import { PassThrough } from 'stream'; +import { HEADER_V_3 } from './dcat-us/constants/contexts'; function buildPluginAndApp(feedTemplate, feedTemplateTransforms) { let Output; @@ -23,7 +24,7 @@ function buildPluginAndApp(feedTemplate, feedTemplateTransforms) { const app = createMockKoopApp(); - app.get('/dcat', function (req, res, next) { + app.get('/dcat/:version', function (req, res, next) { req.app.locals.feedTemplateTransforms = feedTemplateTransforms; res.locals.feedTemplate = feedTemplate; app.use((err, _req, res, _next) => { @@ -86,6 +87,11 @@ describe('Output Plugin', () => { methods: ['get'], handler: 'serve', }, + { + path: '/dcat-us/1.1', + methods: ['get'], + handler: 'serve', + }, { path: '/data.json', methods: ['get'], @@ -99,7 +105,7 @@ describe('Output Plugin', () => { const [plugin, localApp] = buildPluginAndApp(undefined, undefined); try { await request(localApp) - .get('/dcat') + .get('/dcat/1.1') .set('host', siteHostName) .expect('Content-Type', /application\/json/); } catch (error) { @@ -109,10 +115,30 @@ describe('Output Plugin', () => { } }); - it('handles a DCAT US request', async () => { + it('handles a DCAT US 1.1 request', async () => { + // rebuild plugin to trigger initialization code + await request(app) + .get('/dcat/1.1') + .set('host', siteHostName) + .expect('Content-Type', /application\/json/) + .expect(200) + .expect(res => { + expect(res.body).toBeDefined(); + + // perform some basic checks to make sure we have + // something that looks like a DCAT feed + const dcatStream = res.body; + expect(dcatStream['@context']).toBeDefined(); + expect(dcatStream['@type']).toBe('dcat:Catalog'); + expect(dcatStream['dataset']).toBeInstanceOf(Array); + expect(dcatStream['dataset'].length).toBe(1); + }); + }); + + it('handles a DCAT US data.json request', async () => { // rebuild plugin to trigger initialization code await request(app) - .get('/dcat') + .get('/dcat/data.json') .set('host', siteHostName) .expect('Content-Type', /application\/json/) .expect(200) @@ -124,6 +150,26 @@ describe('Output Plugin', () => { const dcatStream = res.body; expect(dcatStream['@context']).toBeDefined(); expect(dcatStream['@type']).toBe('dcat:Catalog'); + expect(dcatStream['dataset']).toBeInstanceOf(Array); + expect(dcatStream['dataset'].length).toBe(1); + }); + }); + + it('handles a DCAT US 3.0 request', async () => { + // rebuild plugin to trigger initialization code + await request(app) + .get('/dcat/3.0') + .set('host', siteHostName) + .expect('Content-Type', /application\/json/) + .expect(200) + .expect(res => { + expect(res.body).toBeDefined(); + const dcatStream = res.body; + expect(dcatStream['@context']).toBeDefined(); + expect(dcatStream['@context']).toStrictEqual(HEADER_V_3['@context']); + expect(dcatStream['@type']).toBe('dcat:Catalog'); + expect(dcatStream['conformsTo']).toBe('https://resource.data.gov/profile/dcat-us#'); + expect(dcatStream['@type']).toBe('dcat:Catalog'); expect(dcatStream['dcat:dataset']).toBeInstanceOf(Array); expect(dcatStream['dcat:dataset'].length).toBe(1); }); @@ -133,7 +179,7 @@ describe('Output Plugin', () => { plugin.model.pullStream.mockRejectedValue(Error('Couldnt get stream')); await request(app) - .get('/dcat') + .get('/dcat/1.1') .set('host', siteHostName) .expect('Content-Type', /application\/json/) .expect(500) @@ -154,7 +200,7 @@ describe('Output Plugin', () => { mockReadable.emit('error', mockError) }, 200) await request(app) - .get('/dcat') + .get('/dcat/1.1') .set('host', siteHostName) .expect('Content-Type', /application\/json/) .expect(500) @@ -171,7 +217,7 @@ describe('Output Plugin', () => { } await request(app) - .get('/dcat') + .get('/dcat/1.1') .set('host', siteHostName) .expect('Content-Type', /application\/json/) .expect(400) diff --git a/src/index.ts b/src/index.ts index 043da8d..5972c33 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,16 +1,21 @@ import { Request, Response } from 'express'; import * as _ from 'lodash'; import { version } from '../package.json'; -import { getDataStreamDcatUs11 } from './dcat-us'; +import { getDataStreamDcatUs } from './dcat-us'; import { TransformsList } from 'adlib'; import { DcatUsError } from './dcat-us/dcat-us-error'; -export = class OutputDcatUs11 { +export = class OutputDcatUs { static type = 'output'; static version = version; static routes = [ { - path: '/dcat-us/3.0', + path: '/dcat-us/3.0', // DCAT US 3.0 is beta + methods: ['get'], + handler: 'serve', + }, + { + path: '/dcat-us/1.1', methods: ['get'], handler: 'serve', }, @@ -44,7 +49,9 @@ export = class OutputDcatUs11 { throw new DcatUsError('DCAT-US 1.1 feed template is not provided.', 400); } - const { stream: dcatStream } = getDataStreamDcatUs11(feedTemplate, feedTemplateTransforms); + const version = this.getVersion(_.get(req, 'path', '')); + + const { stream: dcatStream } = getDataStreamDcatUs(feedTemplate, feedTemplateTransforms, version); const datasetStream = await this.getDatasetStream(req); datasetStream.on('error', (err) => { @@ -78,4 +85,9 @@ export = class OutputDcatUs11 { ), }; } + + private getVersion(reqPath: string) { + const version = reqPath.substring(reqPath.lastIndexOf('/') + 1); + return ['1.1', '3.0'].includes(version) ? version : '1.1'; + } }; \ No newline at end of file