Merge pull request #582 from onursumer/add-chop-nbl-data

Add chop NBL data
ncihtan · Feb 9, 2024 · f211211 · f211211
2 parents 22ebb55 + 688792a
commit f211211
Show file tree

Hide file tree

Showing 12 changed files with 132 additions and 22 deletions.
diff --git a/components/HomePage.tsx b/components/HomePage.tsx
@@ -54,7 +54,7 @@ const HomePage: React.FunctionComponent<IHomePropsProps> = ({
                     }}
                 >
                     <a style={{ color: 'white' }} href="/data-updates">
-                        Data Release V4.1 (Last updated 2023-11-20)
+                        Data Release V4.2 (Last updated 2024-02-09)
                     </a>
                 </div>
                 <Row className="justify-content-md-center">

diff --git a/components/PreReleaseBanner.tsx b/components/PreReleaseBanner.tsx
@@ -3,8 +3,8 @@ import Link from 'next/link';
 
 const PreReleaseBanner = () => (
     <Alert style={{ marginBottom: 0 }} variant={'success'}>
-        New data from SRRS is available as part of{' '}
-        <Link href="/data-updates">Release 4.1</Link>!
+        New data from CHOP is available as part of{' '}
+        <Link href="/data-updates">Release 4.2</Link>!
     </Alert>
 );
 

diff --git a/data/get_syn_data.py b/data/get_syn_data.py
@@ -278,7 +278,14 @@ def generate_json(include_at_risk_populations, include_released_only, do_not_dow
                 if component in MANIFESTS_WITHOUT_ENTITY_ID:
                     pass
                 else:
-                    manifest_df = manifest_df[manifest_df["entityId"].isin(include_release_ids)].copy()
+                    if 'Filename' in manifest_df.columns:
+                        manifest_df = manifest_df[
+                            manifest_df["entityId"].isin(include_release_ids) |
+                            manifest_df['Filename'].astype(str).str.contains("_NBL")  # temporary hack to always have NBL files
+                        ].copy()
+                    else:
+                        manifest_df = manifest_df[manifest_df["entityId"].isin(include_release_ids)].copy()
+
 
             if len(manifest_df) == 0:
                 continue

diff --git a/data/metadata_v4.csv b/data/metadata_v4.csv
@@ -350,3 +350,18 @@ syn51185203,12
 syn39147409,15
 syn52261890,16
 syn35465046,16
+syn53277789,2
+syn53462044,2
+syn53462156,4
+syn53462167,6
+syn53284493,2
+syn53486559,2
+syn53284698,2
+syn51789623,2
+syn52368570,6
+syn51791727,4
+syn53398038,2
+syn52346908,2
+syn52346918,2
+syn52346923,2
+syn52346928,4
diff --git a/data/processSynapseJSON.log b/data/processSynapseJSON.log
@@ -1,10 +1,10 @@
 ncc: Version 0.28.6
 ncc: Compiling file index.js
-ncc: Using typescript@4.3.2 (local user-provided)
+ncc: Using typescript@5.3.3 (local user-provided)
    40kB  sourcemap-register.js
-38149kB  index.js
- 1175kB  index.js.map
-38189kB  [86012ms] - ncc 0.28.6
+40151kB  index.js
+ 1381kB  index.js.map
+40191kB  [62202ms] - ncc 0.28.6
 Missing ParentBiospecimenID:  {
   Component: '10xVisiumSpatialTranscriptomics-RNA-seqLevel1',
   Filename: 'visium_level_1/7794-AS-1_S1_L005_R1_001.fastq.gz',
@@ -3102,5 +3102,5 @@ Missing ParentBiospecimenID:  {
   ParentDataFileID: 'HTA10_05_92131716174070243030005087376361,HTA10_05_63122224365351110135262081126766',
   primaryParents: []
 }
-41226
-2125
+43787
+2148
diff --git a/data/processSynapseJSON.ts b/data/processSynapseJSON.ts
@@ -344,7 +344,7 @@ function processSynapseJSON(
     const filesByAtlas = _.groupBy(returnFiles, (f) => f.atlasid);
     const caseCountByAtlas = _.mapValues(filesByAtlas, (files) => {
         return _.chain(files)
-            .flatMapDeep((f) => f.diagnosisIds)
+            .flatMapDeep((f) => [...f.diagnosisIds, ...f.demographicsIds])
             .uniq()
             .value().length;
     });

diff --git a/data/syn_metadata.json b/data/syn_metadata.json
@@ -1273,6 +1273,81 @@
             "component": "ScRNA-seqLevel4",
             "synapseId": "syn51425823",
             "numItems": 10
+        },
+        {
+            "component": "ScATAC-seqLevel1",
+            "synapseId": "syn51789623",
+            "numItems": 647
+        },
+        {
+            "component": "ScATAC-seqLevel3",
+            "synapseId": "syn51791727",
+            "numItems": 42
+        },
+        {
+            "component": "ScRNA-seqLevel1",
+            "synapseId": "syn52346908",
+            "numItems": 858
+        },
+        {
+            "component": "ScRNA-seqLevel2",
+            "synapseId": "syn52346918",
+            "numItems": 91
+        },
+        {
+            "component": "ScRNA-seqLevel3",
+            "synapseId": "syn52346923",
+            "numItems": 273
+        },
+        {
+            "component": "ScRNA-seqLevel4",
+            "synapseId": "syn52346928",
+            "numItems": 191
+        },
+        {
+            "component": "ScATAC-seqLevel2",
+            "synapseId": "syn52368570",
+            "numItems": 84
+        },
+        {
+            "component": "Biospecimen",
+            "synapseId": "syn53277789",
+            "numItems": 91
+        },
+        {
+            "component": "Demographics",
+            "synapseId": "syn53284493",
+            "numItems": 23
+        },
+        {
+            "component": "NeuroblastomaandGliomaTier3",
+            "synapseId": "syn53284698",
+            "numItems": 47
+        },
+        {
+            "component": "ScATAC-seqLevel4",
+            "synapseId": "syn53398038",
+            "numItems": 3
+        },
+        {
+            "component": "BulkDNALevel1",
+            "synapseId": "syn53462044",
+            "numItems": 312
+        },
+        {
+            "component": "BulkDNALevel2",
+            "synapseId": "syn53462156",
+            "numItems": 98
+        },
+        {
+            "component": "BulkDNALevel3",
+            "synapseId": "syn53462167",
+            "numItems": 15
+        },
+        {
+            "component": "Diagnosis",
+            "synapseId": "syn53486559",
+            "numItems": 23
         }
     ],
     "HTA8": [
@@ -1502,11 +1577,6 @@
         }
     ],
     "HTA10": [
-        {
-            "component": "BulkDNALevel1",
-            "synapseId": "syn39281852",
-            "numItems": 53
-        },
         {
             "component": "BulkRNA-seqLevel1",
             "synapseId": "syn39282161",

diff --git a/lib/helpers.ts b/lib/helpers.ts
@@ -38,7 +38,7 @@ export async function fetchData(): Promise<LoadDataResult> {
     const processedSynURL =
         process.env.NODE_ENV === 'development'
             ? '/processed_syn_data.json'
-            : 'https://d13ch66cwesneh.cloudfront.net/processed_syn_data_20240126_1328.json';
+            : 'https://d13ch66cwesneh.cloudfront.net/processed_syn_data_20240209_1522.json';
     return fetchSynData(processedSynURL);
 }
 

diff --git a/packages/data-portal-commons/src/lib/fillInEntities.ts b/packages/data-portal-commons/src/lib/fillInEntities.ts
@@ -34,7 +34,13 @@ export function fillInEntities(data: LoadDataResult): Entity[] {
         );
         (file as Entity).cases = _.uniqBy(
             mergeCaseData(
-                (file as Entity).diagnosis,
+                _.uniq(
+                    [
+                        ...(file as Entity).diagnosis,
+                        ...(file as Entity).demographics,
+                    ].map((e) => e.ParticipantID)
+                ),
+                diagnosisMap as { [id: string]: Entity },
                 demoMap as { [id: string]: Entity }
             ),
             (c) => c.ParticipantID
@@ -45,11 +51,12 @@ export function fillInEntities(data: LoadDataResult): Entity[] {
 }
 
 function mergeCaseData(
-    diagnosis: Entity[],
+    participantIds: string[],
+    diagnosisByParticipantID: { [participantID: string]: Entity },
     demographicsByParticipantID: { [participantID: string]: Entity }
 ) {
-    return diagnosis.map((d) => ({
-        ...d,
-        ...demographicsByParticipantID[d.ParticipantID],
+    return participantIds.map((id) => ({
+        ...diagnosisByParticipantID[id],
+        ...demographicsByParticipantID[id],
     }));
 }
diff --git a/pages/static/data-updates.html b/pages/static/data-updates.html
@@ -4,6 +4,17 @@
 ---
 
 <h1>Data Updates</h1>
+<h2 id="2024-02-09">February 9th, 2024</h2>
+<p>New data from CHOP is available as part of Release 4.2:</p>
+
+<ul>
+    <li>Level 1-2 single nucleus ATAC-seq data for 28 new participants</li>
+    <li>Level 1-2 single nucleus RNA-seq data for 30 new participants</li>
+    <li>Level 3-4 single nucleus ATAC-seq data for 21 new participants</li>
+    <li>Level 3-4 single nucleus RNA-seq data for 23 new participants</li>
+    <li>Level 1-3 bulk DNA data for 20 new participants</li>
+</ul>
+
 <h2 id="2023-12-04">December 4th, 2023</h2>
 <ul>
     <li>

diff --git a/public/processed_syn_data.json.gz b/public/processed_syn_data.json.gz
diff --git a/public/syn_data.json.gz b/public/syn_data.json.gz