diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a750693..7f7745af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [0.4.8] 2023-10-20 + +### Added + +- Option for redirecting the search to an external url ("public.redirectSearch" key in config) + - An additional key "public.redirectSearchAttribute", defaulting to 'query', will be used as the get parameter attribute. (ie: url + "?redirectSearchAttribute=query") +- Options for using a remote search ending, and merging the results with GNB internal search. + - The 'public.externalSearch' option need to be set to true, and an 'externalSearchOptions' dict need to be set. + - 'url' key is the remote endpoint where the query will be sent + - 'gene_field' is the remote field to get the gene IDs (default to geneId) + - 'query_param' : optional get parameter to use for the query + - 'field_param': optional get parameter to use to restrict the results to the gene_field value + - 'count_param': optional get parameter to restrict the number of results +- Multiple annotations for the same genome + - When adding an annotation, you must now set the '--annot' to set the annotation name. + - When integrating data afterward, you can use the --annot tag to specify the annotation you are aiming for. + - If you have multiple genes with the same ID, and do not specify '--annot', the results may be variables + - You can specify --annotations multiple time when integrating orthogroups +- Will now decode proteins and genes IDs when integrating data. (It was already done when integrating gffs, so there was some mismatch with IDs) + +## Changed + +- Various UI fixes to better fit multiple annotation versions + - Including an 'annotation' selector in the gene list + ## [0.4.7] 2023-09-26 ### Fixed @@ -399,4 +424,3 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. [0.1.3]: https://github.com/genenotebook/genenotebook/compare/v0.1.2...v0.1.3 [0.1.2]: https://github.com/genenotebook/genenotebook/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/genenotebook/genenotebook/compare/v0.1.0...v0.1.1 - diff --git a/cli/genoboo.js b/cli/genoboo.js index bbf85cc3..8e649981 100755 --- a/cli/genoboo.js +++ b/cli/genoboo.js @@ -318,10 +318,14 @@ addAnnotation .arguments('') .option('-u, --username ', 'GeneNoteBook admin username') .option('-p, --password ', 'GeneNoteBook admin password') - .option( + .requiredOption( '-n, --name ', 'Reference genome name to which the annotation should be added.', ) + .requiredOption( + '--annot ', + 'Annotation name', + ) .option( '-r, --re_protein ', 'Replacement string for the protein name using capturing groups defined by --re_protein_capture. Make sure to use JS-style groups ($1 for group 1)', @@ -352,6 +356,7 @@ addAnnotation password, port = 3000, name, + annot, re_protein, re_protein_capture, attr_protein, @@ -389,6 +394,7 @@ addAnnotation { fileName, genomeName: name, + annotationName: annot, re_protein, re_protein_capture: correctProteinCapture, attr_protein, @@ -433,6 +439,10 @@ running GeneNoteBook server.` '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '-fmt, --format [parser]', `Choose a parser for the diamond output format. Parses .xml, .txt @@ -455,7 +465,7 @@ running GeneNoteBook server.` .action( ( file, - { username, password, port = 3000, format, algorithm, matrix, database } + { username, password, port = 3000, annot, format, algorithm, matrix, database } ) => { if (typeof file !== 'string') addDiamond.help(); @@ -490,6 +500,7 @@ file extension is not "xml", "txt"`); fileName, parser: parserType, program: 'diamond', + annot: annot, algorithm: algorithm, matrix: matrix, database: database, @@ -502,7 +513,7 @@ file extension is not "xml", "txt"`); Example: genenotebook add diamond mmucedo.xml -u admin -p admin or - genenotebook add diamond mmucedo.txt --format txt --program blastp --matrix BLOSUM90 -db "Non-reundant protein sequences (nr)" -u admin -p admin + genenotebook add diamond mmucedo.txt --format txt --matrix BLOSUM90 -db "Non-reundant protein sequences (nr)" -u admin -p admin `); }) .exitOverride(customExitOverride(addDiamond)); @@ -525,6 +536,10 @@ running GeneNoteBook server.` '-p, --password ', 'GeneNoteBook admin password' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' @@ -548,7 +563,7 @@ running GeneNoteBook server.` .action( ( file, - { username, password, port = 3000, format, algorithm, matrix, database } + { username, password, port = 3000, annot, format, algorithm, matrix, database } ) => { if (typeof file !== 'string') addBlast.help(); @@ -584,6 +599,7 @@ file extension is not "xml", "txt"`); parser: parserType, program: 'blast', algorithm: algorithm, + annot: annot, matrix: matrix, database: database, } @@ -619,6 +635,10 @@ addExpression '-d, --sample-description ', 'Description of the experiment' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '-r, --replicas ', 'Comma-separated column positions, which are part of the same replica group. Can be set multiple times for multiple groups. The replica group name will be the first column, unless replica-names is set' @@ -639,7 +659,7 @@ addExpression const replicas = opts.replicas || []; const replicaNames = opts.replicaNames || []; const isPublic = opts.public; - + const annot = opts.annot if (!(fileName && username && password)) { program.help(); } @@ -648,6 +668,7 @@ addExpression { fileName, description, + annot, replicas, replicaNames, isPublic @@ -680,6 +701,10 @@ addExpression '-d, --sample-description ', 'Description of the experiment' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '--public', 'Set the generated replica groups as public. Default: false', @@ -692,6 +717,7 @@ addExpression const replicaGroup = opts.replicaGroup || fileName; const description = opts.sampleDescription || 'description'; const isPublic = opts.public + const anot = opts.annot if (!(fileName && username && password)) { program.help(); @@ -701,6 +727,7 @@ addExpression { fileName, sampleName, + annot: annot, replicaGroup, description, isPublic @@ -728,12 +755,16 @@ addInterproscan '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '--format [parser]', `Choose a parser for the interproscan output files. Parses .gff3 and .tsv extensions.` ) - .action((file, { username, password, port = 3000, format }) => { + .action((file, { username, password, port = 3000, format, annot }) => { if (typeof file !== 'string') addInterproscan.help(); const fileName = path.resolve(file); @@ -766,6 +797,7 @@ file extension is not "tsv", "gff3" or "xml".`); { fileName, parser: parserType, + annot: annot } ); }) @@ -794,11 +826,15 @@ addEggnog '-p, --password ', 'GeneNoteBook admin password' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' ) - .action((file, { username, password, port = 3000 }) => { + .action((file, { username, password, port = 3000, annot }) => { if (typeof file !== 'string') addEggnog.help(); const fileName = path.resolve(file); @@ -808,6 +844,7 @@ addEggnog new GeneNoteBookConnection({ username, password, port }).call('addEggnog', { fileName, + annot: annot }); }) .on('--help', () => { @@ -833,11 +870,15 @@ addHectar '-p, --password ', 'GeneNoteBook admin password' ) + .option( + '--annot ', + 'Annotation name', + ) .option( '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' ) - .action((file, { username, password, port = 3000 }) => { + .action((file, { username, password, port = 3000, annot }) => { if (typeof file !== 'string') addHectar.help(); const fileName = path.resolve(file); @@ -847,6 +888,7 @@ addHectar new GeneNoteBookConnection({ username, password, port }).call('addHectar', { fileName, + annot: annot }); }) .on('--help', () => { @@ -878,17 +920,23 @@ addOrthogroups '-f, --force [force]', 'Ignore the use of prefixes.', ) + .option( + '-a, --annotations ', + 'Name of the annotation to use for gene matching. Can be set multiple times' + ) .requiredOption('-u, --username ', 'GeneNoteBook admin username') .requiredOption('-p, --password ', 'GeneNoteBook admin password') .option( '--port [port]', 'Port on which GeneNoteBook is running. Default: 3000' ) - .action((file, { prefixe, list, force, username, password, port = 3000 }) => { + .action((file, { prefixe, list, force, annotations, username, password, port = 3000 }) => { if (typeof file !== 'string') addOrthogroups.help(); const folderName = path.resolve(file); const prefixes = (typeof list !== 'undefined' ? list : path.resolve(prefixe)); + annotations = annotations || [] + if (!(folderName && username && password)) { addOrthogroups.help(); } @@ -899,6 +947,7 @@ addOrthogroups folderName, force, prefixes, + annotations }, ); }) diff --git a/config.json.template b/config.json.template index 85c3399e..3e380878 100644 --- a/config.json.template +++ b/config.json.template @@ -4,6 +4,16 @@ "disable_user_login": false, "disable_user_registration": false, "blast_link": "", - "expression_unit": "My unit" + "expression_unit": "My unit", + "externalSearch": false, + "redirectSearch": "", + "redirectSearchAttribute": "" + }, + "externalSearchOptions": { + "url": "http://0.0.0.0:80", + "gene_field": "gene_id", + "query_param": "q", + "field_param": "display_fields", + "count_param": "max_results" } } diff --git a/imports/api/api.js b/imports/api/api.js index 1c889783..ddcecfb2 100644 --- a/imports/api/api.js +++ b/imports/api/api.js @@ -17,7 +17,6 @@ import './genomes/removeGenome.js'; import './genomes/annotation/removeAnnotationTrack.js'; import './genomes/annotation/addAnnotation.js'; -import './genes/interproscan.js'; import './genes/addInterproscan.js'; import './genes/eggnog/addEggnog.js'; import './genes/hectar/addHectar.js'; diff --git a/imports/api/genes/addInterproscan.js b/imports/api/genes/addInterproscan.js index ad762466..777ad49f 100644 --- a/imports/api/genes/addInterproscan.js +++ b/imports/api/genes/addInterproscan.js @@ -12,14 +12,16 @@ import { Meteor } from 'meteor/meteor'; * @method finalize */ class InterproscanProcessor { - constructor() { + constructor(annot) { this.bulkOp = interproscanCollection.rawCollection().initializeUnorderedBulkOp(); this.geneBulkOp = Genes.rawCollection().initializeUnorderedBulkOp(); this.currentProt = "" this.currentGene = "" this.currentContent = [] this.currentDB = [] - this.currentOnto = [] + this.currentOnto = [], + this.currentAnnotationName = "", + this.annot = annot } finalize = () => { @@ -31,7 +33,7 @@ class InterproscanProcessor { if (this.bulkOp.length > 0){ return this.bulkOp.execute(); } - return { nMatched: 0 } + return { nUpserted: 0 } } updateGenes = () => { @@ -47,11 +49,13 @@ class InterproscanProcessor { this.bulkOp.find({ gene_id: this.currentGene, protein_id: this.currentProt, + annotationName: this.currentAnnotationName }).upsert().update( { $set: { gene_id: this.currentGene, protein_id: this.currentProt, + annotationName: this.currentAnnotationName, protein_domains: this.currentContent }, }, @@ -63,7 +67,7 @@ class InterproscanProcessor { } if (this.currentDB != [] || this.currentOnto != []){ - this.geneBulkOp.find({ID: this.currentGene}).update({ + this.geneBulkOp.find({ID: this.currentGene, annotationName: this.currentAnnotationName}).update({ $addToSet: { 'attributes.Ontology_term': { $each: this.currentOnto }, 'attributes.Dbxref': { $each: this.currentDB } @@ -80,6 +84,10 @@ const addInterproscan = new ValidatedMethod({ name: 'addInterproscan', validate: new SimpleSchema({ fileName: { type: String }, + annot: { + type: String, + optional: true, + }, parser: { type: String, allowedValues: ['tsv', 'gff3'], @@ -88,7 +96,7 @@ const addInterproscan = new ValidatedMethod({ applyOptions: { noRetry: true, }, - run({ fileName, parser }) { + run({ fileName, annot, parser }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); } @@ -96,7 +104,7 @@ const addInterproscan = new ValidatedMethod({ throw new Meteor.Error('not-authorized'); } - const job = new Job(jobQueue, 'addInterproscan', { fileName, parser }); + const job = new Job(jobQueue, 'addInterproscan', { fileName, annot, parser }); const jobId = job.priority('high').save(); // Continue with synchronous processing diff --git a/imports/api/genes/alignment/addSimilarSequence.js b/imports/api/genes/alignment/addSimilarSequence.js index abceca55..aee77c88 100644 --- a/imports/api/genes/alignment/addSimilarSequence.js +++ b/imports/api/genes/alignment/addSimilarSequence.js @@ -19,6 +19,10 @@ const addSimilarSequence = new ValidatedMethod({ optional: true, allowedValues: ['blast', 'diamond'], }, + annot: { + type: String, + optional: true, + }, algorithm: { type: String, optional: true, @@ -44,7 +48,7 @@ const addSimilarSequence = new ValidatedMethod({ applyOptions: { noRetry: true, }, - run({ fileName, parser, program, algorithm, matrix, database }) { + run({ fileName, annot, parser, program, algorithm, matrix, database }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); } @@ -57,6 +61,7 @@ const addSimilarSequence = new ValidatedMethod({ 'addDiamond', { fileName, + annot, parser, program, algorithm, diff --git a/imports/api/genes/alignment/alignment.test.js b/imports/api/genes/alignment/alignment.test.js index 70c0ff96..58f1751a 100644 --- a/imports/api/genes/alignment/alignment.test.js +++ b/imports/api/genes/alignment/alignment.test.js @@ -1,4 +1,4 @@ -7/* eslint-env mocha */ +/* eslint-env mocha */ import chai from 'chai'; import { Meteor } from 'meteor/meteor'; import logger from '/imports/api/util/logger.js'; @@ -36,7 +36,7 @@ describe('alignment', function testAlignment() { // Increase timeout this.timeout(20000); - addTestGenome(annot=true) + addTestGenome(annot=true, multiple=true) const diamondParams = { fileName: 'assets/app/data/Diamond_blastp_bnigra.xml', @@ -45,6 +45,7 @@ describe('alignment', function testAlignment() { algorithm: 'blastx', matrix: 'blosum62', database: 'nr', + annot: "Annotation name" }; // Should fail for non-logged in @@ -59,13 +60,13 @@ describe('alignment', function testAlignment() { let result = addSimilarSequence._execute(adminContext, diamondParams); - const simSeq = similarSequencesCollection.find({iteration_query: "BniB01g000010.2N"}).fetch(); + const simSeq = similarSequencesCollection.find({iteration_query: "Bni|B01g000010.2N", annotationName: "Annotation name"}).fetch(); chai.assert.lengthOf(simSeq, 1, "No similar sequence found") const seq = simSeq[0] chai.assert.equal(seq.algorithm_ref, 'blastx') - chai.assert.equal(seq.protein_id, 'BniB01g000010.2N.1-P') + chai.assert.equal(seq.protein_id, 'Bni|B01g000010.2N.1-P') chai.assert.equal(seq.database_ref, 'nr') chai.assert.equal(seq.program_ref, 'diamond') chai.assert.equal(seq.query_len, 420) @@ -78,6 +79,8 @@ describe('alignment', function testAlignment() { // Increase timeout this.timeout(20000); + addTestGenome(annot=true, multiple=true) + const diamondParams = { fileName: 'assets/app/data/Diamond_blastx_bnigra.txt', parser: 'txt', @@ -85,6 +88,7 @@ describe('alignment', function testAlignment() { algorithm: 'blastx', matrix: 'BLOSUM90', database: 'nr', + annot: "Annotation name" }; // Should fail for non-logged in @@ -101,7 +105,7 @@ describe('alignment', function testAlignment() { //Meteor._sleepForMs(10000); - const simSeq = similarSequencesCollection.find({iteration_query: "BniB01g000010.2N.1-P"}).fetch(); + const simSeq = similarSequencesCollection.find({iteration_query: "Bni|B01g000010.2N", annotationName: "Annotation name"}).fetch(); chai.assert.lengthOf(simSeq, 1, "No similar sequence found") @@ -119,6 +123,8 @@ describe('alignment', function testAlignment() { // Increase timeout this.timeout(20000); + addTestGenome(annot=true, multiple=true) + const diamondParams = { fileName: 'assets/app/data/BLAST_blastx_bnigra.txt', parser: 'txt', @@ -126,6 +132,7 @@ describe('alignment', function testAlignment() { algorithm: 'blastx', matrix: 'BLOSUM90', database: 'nr', + annot: "Annotation name" }; // Should fail for non-logged in @@ -140,7 +147,7 @@ describe('alignment', function testAlignment() { let result = addSimilarSequence._execute(adminContext, diamondParams); - const simSeq = similarSequencesCollection.find({iteration_query: "BniB01g000010.2N.1-P"}).fetch(); + const simSeq = similarSequencesCollection.find({iteration_query: "Bni|B01g000010.2N", annotationName: "Annotation name"}).fetch(); chai.assert.lengthOf(simSeq, 1, "No similar sequence found") const seq = simSeq[0] diff --git a/imports/api/genes/alignment/parser/pairwiseParser.js b/imports/api/genes/alignment/parser/pairwiseParser.js index 66037ba0..663f05db 100644 --- a/imports/api/genes/alignment/parser/pairwiseParser.js +++ b/imports/api/genes/alignment/parser/pairwiseParser.js @@ -7,7 +7,7 @@ class Pairwise { constructor({ iteration_query, query_length, - position_query, + position_query }){ this.iteration_query = iteration_query; this.query_length = query_length; @@ -28,14 +28,16 @@ class Pairwise { * @param {string} database - The reference database (Non-redundant protein sequences (nr)). */ class PairwiseProcessor { - constructor(program, algorithm, matrix, database) { + constructor(program, algorithm, matrix, database, annot) { this.genesDb = Genes.rawCollection(); this.pairWise = new Pairwise({}); + this.currentGene = "" this.program = program; this.algorithm = algorithm; this.matrix = matrix; this.database = database; this.similarSeqBulkOp = similarSequencesCollection.rawCollection().initializeUnorderedBulkOp(); + this.annot = annot; } /** @@ -59,7 +61,23 @@ class PairwiseProcessor { /** In the event that it is the first element of the collection to be created. */ if (typeof this.pairWise.iteration_query === 'undefined') { - this.pairWise.iteration_query = queryClean; + this.pairWise.iteration_query = decodeURIComponent(queryClean); + } + + let geneQuery = { + $or: [ + { 'subfeatures.ID': this.pairWise.iteration_query }, + { 'subfeatures.protein_id': this.pairWise.iteration_query }, + ], + } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + + this.currentGene = Genes.findOne(geneQuery); + if (typeof this.currentGene === 'undefined'){ + logger.warn(`Warning ! No sub-feature was found for ${this.pairWise.iteration_query}.`); + return } /** In the case where a new pairwise must be created. */ @@ -67,8 +85,9 @@ class PairwiseProcessor { && this.pairWise.iteration_query !== queryClean) { /** Update or insert pairwise. */ this.similarSeqBulkOp.find({ - iteration_query: this.pairWise.iteration_query, - protein_id: this.pairWise.iteration_query + iteration_query: this.currentGene.ID, + protein_id: this.pairWise.iteration_query, + annotationName: this.currentGene.annotationName, }).upsert().update( { $set: { @@ -76,7 +95,8 @@ class PairwiseProcessor { algorithm_ref: this.algorithm, matrix_ref: this.matrix, database_ref: this.database, - iteration_query: this.pairWise.iteration_query, + iteration_query: this.currentGene.ID, + annotationName: this.currentGene.annotationName, protein_id: this.pairWise.iteration_query, query_len: this.pairWise.query_length, iteration_hits: this.pairWise.iteration_hits, @@ -91,9 +111,14 @@ class PairwiseProcessor { /** Initializes a new pairwise. */ this.pairWise = new Pairwise({}); - this.pairWise.iteration_query = queryClean; + this.pairWise.iteration_query = decodeURIComponent(queryClean); this.pairWise.iteration_hits = []; } + + if (typeof this.currentGene === 'undefined'){ + return + } + if (/Length/.test(line)) { /** * Get and clean the length of the query sequence according to the program used. @@ -359,9 +384,14 @@ class PairwiseProcessor { */ lastPairwise = () => { + if (typeof this.currentGene === "undefined") { + return { ok:"", writeErrors:"", nInserted:0, nUpserted: 0 } + } + this.similarSeqBulkOp.find({ - iteration_query: this.pairWise.iteration_query, - protein_id: this.pairWise.iteration_query + iteration_query: this.currentGene.ID, + protein_id: this.pairWise.iteration_query, + annotationName: this.currentGene.annotationName, }).upsert().update( { $set: { @@ -369,7 +399,8 @@ class PairwiseProcessor { algorithm_ref: this.algorithm, matrix_ref: this.matrix, database_ref: this.database, - iteration_query: this.pairWise.iteration_query, + iteration_query: this.currentGene.ID, + annotationName: this.currentGene.annotationName, protein_id: this.pairWise.iteration_query, query_len: this.pairWise.query_length, iteration_hits: this.pairWise.iteration_hits, diff --git a/imports/api/genes/alignment/parser/xmlParser.js b/imports/api/genes/alignment/parser/xmlParser.js index cebe7c3c..0195bfdf 100644 --- a/imports/api/genes/alignment/parser/xmlParser.js +++ b/imports/api/genes/alignment/parser/xmlParser.js @@ -16,12 +16,13 @@ import logger from '/imports/api/util/logger.js'; */ class XmlProcessor { - constructor(program, algorithm, matrix, database) { + constructor(program, algorithm, matrix, database, annot) { this.genesDb = Genes.rawCollection(); this.program = program; this.algorithm = algorithm; this.matrix = matrix; this.database = database; + this.annot = annot; this.similarSeqBulkOp = similarSequencesCollection.rawCollection().initializeUnorderedBulkOp(); } @@ -99,11 +100,20 @@ class XmlProcessor { const splitIterationQuery = iterationQuery.split(' '); - await Promise.all(splitIterationQuery.map(async (iter) => { + await Promise.all(splitIterationQuery.map(async (iter_encode) => { /** Chek if the queries exist in the genes collection. */ - const subfeatureIsFound = await this.genesDb.findOne({ $or: [{'subfeatures.ID': iter}, {'subfeatures.protein_id': iter}] }); + + const iter = decodeURIComponent(iter_encode) + + let geneQuery = { $or: [{'subfeatures.ID': iter}, {'subfeatures.protein_id': iter}] } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + + const subfeatureIsFound = await this.genesDb.findOne(geneQuery); if (typeof subfeatureIsFound !== 'undefined' && subfeatureIsFound !== null) { /** Get the total query sequence length. */ + const annotationName = subfeatureIsFound.annotationName const queryLen = blastIteration[i]['iteration_query-len'] /** Get the root tag of hit sequences. */ @@ -185,7 +195,8 @@ class XmlProcessor { /** Mongo bulk-operation. */ this.similarSeqBulkOp.find({ iteration_query: geneIdentifier, - protein_id: iter + protein_id: iter, + annotationName: annotationName }).upsert().update( { $set: { @@ -197,6 +208,7 @@ class XmlProcessor { protein_id: iter, query_len: queryLen, iteration_hits: iterations, + annotationName: annotationName }, }, { diff --git a/imports/api/genes/alignment/similarSequenceCollection.js b/imports/api/genes/alignment/similarSequenceCollection.js index d0e0c918..a4e13d3f 100644 --- a/imports/api/genes/alignment/similarSequenceCollection.js +++ b/imports/api/genes/alignment/similarSequenceCollection.js @@ -20,12 +20,19 @@ const similarSequencesSchema = new SimpleSchema({ }, iteration_query: { type: String, + index: true, label: 'Query sequence name.', }, protein_id: { type: String, + index: true, label: 'Protein_id', }, + annotationName: { + type: String, + index: true, + label: 'Annotation name', + }, iteration_hits: { type: Array, label: 'List of iteration hits.', diff --git a/imports/api/genes/download/download.test.js b/imports/api/genes/download/download.test.js index 8652f59e..5fd30906 100644 --- a/imports/api/genes/download/download.test.js +++ b/imports/api/genes/download/download.test.js @@ -64,7 +64,7 @@ describe('download', function testDownload() { addTestGenome(annot=true) const dlParams = { - query: {ID: "BniB01g000010.2N"}, + query: {ID: "Bni|B01g000010.2N"}, dataType: 'Annotation', options: {}, async: false @@ -74,13 +74,13 @@ describe('download', function testDownload() { dataFile = result.value const stat = fs.statSync(dataFile) - chai.assert.equal(stat.size, 140) + chai.assert.equal(stat.size, 141) const expected = [ '##gff-version 3', 'B1\tAAFC_GIFS\tgene\t13640\t15401\t.\t-\t-\tmyNewAttribute=1', - 'B1\tAAFC_GIFS\tmRNA\t13641\t15400\t.\t-\t.\tParent=BniB01g000010.2N', - 'B1\tAAFC_GIFS\tCDS\t13641\t13653\t.\t-\t.\tParent=BniB01g000010.2N.1' + 'B1\tAAFC_GIFS\tmRNA\t13641\t15400\t.\t-\t.\tParent=Bni|B01g000010.2N', + 'B1\tAAFC_GIFS\tCDS\t13641\t13653\t.\t-\t.\tParent=Bni|B01g000010.2N.1' ] const data = await readFile(dataFile) @@ -95,7 +95,7 @@ describe('download', function testDownload() { addTestGenome(annot=true) const dlParams = { - query: {ID: "BniB01g000010.2N"}, + query: {ID: "Bni|B01g000010.2N"}, dataType: 'Sequence', options: {seqType: "nucl", primaryTranscriptOnly: false}, async: false @@ -105,10 +105,10 @@ describe('download', function testDownload() { dataFile = result.value const stat = fs.statSync(dataFile) - chai.assert.equal(stat.size, 51) + chai.assert.equal(stat.size, 52) const expected = [ - ">BniB01g000010.2N.1", + ">Bni|B01g000010.2N.1", "AGTTTAGAATAC" ] const data = await readFile(dataFile) @@ -124,7 +124,7 @@ describe('download', function testDownload() { const {expId, transcriptomeId} = addTestTranscriptome(genomeId, geneId) const dlParams = { - query: {ID: "BniB01g000010.2N"}, + query: {ID: "Bni|B01g000010.2N"}, dataType: 'Expression', options: {selectedSamples: ["replicaGroup"]}, async: false @@ -134,11 +134,11 @@ describe('download', function testDownload() { dataFile = result.value const stat = fs.statSync(dataFile) - chai.assert.equal(stat.size, 59) + chai.assert.equal(stat.size, 60) const expected = [ "gene_id\treplicaGroup", - "BniB01g000010.2N\t60" + "Bni|B01g000010.2N\t60" ] const data = await readFile(dataFile) diff --git a/imports/api/genes/eggnog/addEggnog.js b/imports/api/genes/eggnog/addEggnog.js index 573eb5ff..1bfe1550 100644 --- a/imports/api/genes/eggnog/addEggnog.js +++ b/imports/api/genes/eggnog/addEggnog.js @@ -8,10 +8,11 @@ import SimpleSchema from 'simpl-schema'; import { Meteor } from 'meteor/meteor'; class EggnogProcessor { - constructor() { + constructor(annot) { // Not a bulk mongo suite. this.genesDb = Genes.rawCollection(); this.nEggnog = 0; + this.annot = annot; } /** @@ -28,7 +29,7 @@ class EggnogProcessor { parse = (line) => { if (!(line[0] === '#' || line.split('\t').length <= 1)) { // Get all eggnog informations line by line and separated by tabs. - const [ + let [ queryName, seedEggnogOrtholog, seedOrthologEvalue, @@ -52,6 +53,8 @@ class EggnogProcessor { pfams, ] = line.split('\t'); + queryName = decodeURIComponent(queryName) + // Organize data in a dictionary. const annotations = { query_name: queryName, @@ -90,38 +93,42 @@ class EggnogProcessor { // If subfeatures is found in genes database (e.g: ID = // MMUCEDO_000002-T1). - const subfeatureIsFound = Genes.findOne({ + + let geneQuery = { $or: [ { 'subfeatures.ID': queryName }, { 'subfeatures.protein_id': queryName }, ], - }); + } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + + const subfeatureIsFound = Genes.findOne(geneQuery); if (typeof subfeatureIsFound !== 'undefined') { // Increment eggnog. this.nEggnog += 1; + let annotationName = subfeatureIsFound.annotationName + annotations['annotationName'] = annotationName // Update or insert if no matching documents were found. const documentEggnog = eggnogCollection.upsert( - { query_name: queryName }, // selector. + { query_name: queryName, annotationName }, // selector. annotations, // modifier. ); // Update eggnogId in genes database. if (typeof documentEggnog.insertedId !== 'undefined') { // Eggnog _id is created. - return this.genesDb.update({ - $or: [ - { 'subfeatures.ID': queryName }, - { 'subfeatures.protein_id': queryName }, - ]}, + return this.genesDb.update(geneQuery, { $set: { eggnogId: documentEggnog.insertedId } }, ); } else { // Eggnog already exists. - const eggnogIdentifiant = eggnogCollection.findOne({ query_name: queryName })._id; + const eggnogIdentifiant = eggnogCollection.findOne({ query_name: queryName, annotationName })._id; return this.genesDb.update( - { $or: [{'subfeatures.ID': queryName}, {'subfeatures.protein_id': queryName}] }, + geneQuery, { $set: { eggnogId: eggnogIdentifiant } }, ); } @@ -139,11 +146,15 @@ const addEggnog = new ValidatedMethod({ name: 'addEggnog', validate: new SimpleSchema({ fileName: { type: String }, + annot: { + type: String, + optional: true, + }, }).validator(), applyOptions: { noRetry: true, }, - run({ fileName }) { + run({ fileName, annot }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); } @@ -152,7 +163,7 @@ const addEggnog = new ValidatedMethod({ } logger.log('file :', { fileName }); - const job = new Job(jobQueue, 'addEggnog', { fileName }); + const job = new Job(jobQueue, 'addEggnog', { fileName, annot }); const jobId = job.priority('high').save(); let { status } = job.doc; diff --git a/imports/api/genes/eggnog/eggnog.test.js b/imports/api/genes/eggnog/eggnog.test.js index cc0cd260..fe8bd98a 100644 --- a/imports/api/genes/eggnog/eggnog.test.js +++ b/imports/api/genes/eggnog/eggnog.test.js @@ -6,6 +6,7 @@ import { eggnogCollection } from './eggnogCollection'; import addEggnog from './addEggnog'; import { addTestUsers, addTestGenome } from '../../../startup/server/fixtures/addTestData'; import '../../jobqueue/process-eggnog'; +import { Genes } from '/imports/api/genes/geneCollection.js'; describe('eggnog', function testEggnog() { let adminId; @@ -29,10 +30,11 @@ describe('eggnog', function testEggnog() { // Increase timeout this.timeout(20000); - addTestGenome(annot = true); + addTestGenome(annot = true, multiple = true); const eggNogParams = { fileName: 'assets/app/data/Bnigra_eggnog.tsv', + annot: "Annotation name" }; // Should fail for non-logged in @@ -49,7 +51,7 @@ describe('eggnog', function testEggnog() { chai.assert.equal(result.nInserted, 1) - const eggs = eggnogCollection.find({ query_name: 'BniB01g000010.2N.1-P' }).fetch(); + const eggs = eggnogCollection.find({ query_name: 'Bni|B01g000010.2N.1-P', annotationName: "Annotation name" }).fetch(); chai.assert.lengthOf(eggs, 1, 'No eggnog data found'); @@ -61,5 +63,13 @@ describe('eggnog', function testEggnog() { chai.assert.lengthOf(egg.eggNOG_OGs, 5); chai.assert.lengthOf(egg.GOs, 18); chai.assert.equal(egg.Description, 'UDP-glucuronic acid decarboxylase'); + + const gene1 = Genes.findOne({ID: 'Bni|B01g000010.2N', annotationName: 'Annotation name'}) + const gene2 = Genes.findOne({ID: 'Bni|B01g000010.2N', annotationName: 'Annotation name 2'}) + + chai.assert.isDefined(gene1.eggnogId, "eggNodeId is not defined for the correct annotation") + chai.assert.isUndefined(gene2.eggnogId, "eggNodeId is defined for the wrong annotation") + + }); }); diff --git a/imports/api/genes/eggnog/eggnogCollection.js b/imports/api/genes/eggnog/eggnogCollection.js index 2c6ef1a5..5c766caa 100644 --- a/imports/api/genes/eggnog/eggnogCollection.js +++ b/imports/api/genes/eggnog/eggnogCollection.js @@ -6,6 +6,10 @@ const eggnogSchema = new SimpleSchema({ type: String, label: 'Query sequence name.', }, + annotationName: { + type: String, + label: 'Annotation name', + }, seed_eggNOG_ortholog: { type: String, label: 'Best protein match in eggNOG.', diff --git a/imports/api/genes/geneCollection.js b/imports/api/genes/geneCollection.js index cc093e3e..03595b8b 100644 --- a/imports/api/genes/geneCollection.js +++ b/imports/api/genes/geneCollection.js @@ -1,5 +1,6 @@ import SimpleSchema from 'simpl-schema'; import { Mongo } from 'meteor/mongo'; +import logger from '/imports/api/util/logger.js'; const VALID_SUBFEATURE_TYPES = [ 'transcript', @@ -73,9 +74,7 @@ const SubfeatureSchema = new SimpleSchema( ID: { type: String, index: true, - unique: true, - // denyUpdate: true, - label: 'Unique subfeature ID', + label: 'Subfeature ID', }, protein_id: { type: String, @@ -129,9 +128,8 @@ const GeneSchema = new SimpleSchema( { ID: { type: String, - unique: true, index: true, - label: 'Unique gene ID', + label: 'Gene ID', }, editing: { type: String, @@ -161,6 +159,11 @@ const GeneSchema = new SimpleSchema( index: true, label: 'Reference genome DB identifier (_id in genome collection)', }, + annotationName: { + type: String, + index: true, + label: 'Annotation name', + }, orthogroup: { type: OrthogroupSchema, index: true, @@ -209,6 +212,12 @@ GeneSchema.extend(IntervalBaseSchema); Genes.attachSchema(GeneSchema); +if (Meteor.isServer) { + Genes.createIndex({ID: 1, annotationName: 1}, {name: 'Id and annotation index', unique: true}) + Genes.createIndex({'subfeatures.ID': 1, annotationName: 1}, {name: 'SubId and annotation index', unique: true}) +} + + export { Genes, GeneSchema, diff --git a/imports/api/genes/hectar/addHectar.js b/imports/api/genes/hectar/addHectar.js index 7c8ebf6d..eaa7bd4e 100644 --- a/imports/api/genes/hectar/addHectar.js +++ b/imports/api/genes/hectar/addHectar.js @@ -8,10 +8,11 @@ import SimpleSchema from 'simpl-schema'; import { Meteor } from 'meteor/meteor'; class HectarProcessor { - constructor() { + constructor(annot) { // Not a bulk mongo suite. this.genesDb = Genes.rawCollection(); this.nHectar = 0; + this.annot = annot; } /** @@ -28,7 +29,7 @@ class HectarProcessor { parse = (line) => { if (!(line.slice(0,10) === 'protein id' || line.split('\t').length <= 1)) { // Get all hectar informations line by line and separated by tabs. - const [ + let [ proteinId, predictedTargetingCategory, signalPeptideScore, @@ -39,6 +40,8 @@ class HectarProcessor { otherScore, ] = line.split('\t'); + proteinId = decodeURIComponent(proteinId) + // Organize data in a dictionary. const annotations = { protein_id: proteinId, @@ -63,39 +66,44 @@ class HectarProcessor { } // If subfeatures is found in genes database (e.g: ID = // MMUCEDO_000002-T1). - const subfeatureIsFound = Genes.findOne({ + + let geneQuery = { $or: [ { 'subfeatures.ID': proteinId }, { 'subfeatures.protein_id': proteinId }, ], - }); + } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + + const subfeatureIsFound = Genes.findOne(geneQuery); if (typeof subfeatureIsFound !== 'undefined') { console.log("if loop" + typeof subfeatureIsFound); + let annotationName = subfeatureIsFound.annotationName // Increment hectar. this.nHectar += 1; + annotations['annotationName'] = annotationName + // Update or insert if no matching documents were found. const documentHectar = hectarCollection.upsert( - { protein_id: proteinId }, // selector. + { protein_id: proteinId, annotationName }, // selector. annotations, // modifier. ); // Update hectarId in genes database. if (typeof documentHectar.insertedId !== 'undefined') { // Hectar _id is created. - return this.genesDb.update({ - $or: [ - { 'subfeatures.ID': proteinId }, - { 'subfeatures.protein_id': proteinId }, - ]}, + return this.genesDb.update(geneQuery, { $set: { hectarId: documentHectar.insertedId } }, ); } else { // Hectar already exists. - const hectarIdentifiant = hectarCollection.findOne({ protein_id: proteinId })._id; + const hectarIdentifiant = hectarCollection.findOne({ protein_id: proteinId, annotationName })._id; return this.genesDb.update( - { $or: [{'subfeatures.ID': proteinId}, {'subfeatures.protein_id': proteinId}] }, + geneQuery, { $set: { hectarId: hectarIdentifiant } }, ); } @@ -113,11 +121,15 @@ const addHectar = new ValidatedMethod({ name: 'addHectar', validate: new SimpleSchema({ fileName: { type: String }, + annot: { + type: String, + optional: true, + }, }).validator(), applyOptions: { noRetry: true, }, - run({ fileName }) { + run({ fileName, annot }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); } @@ -126,7 +138,7 @@ const addHectar = new ValidatedMethod({ } logger.log('file :', { fileName }); - const job = new Job(jobQueue, 'addHectar', { fileName }); + const job = new Job(jobQueue, 'addHectar', { fileName, annot }); const jobId = job.priority('high').save(); let { status } = job.doc; diff --git a/imports/api/genes/hectar/hectar.test.js b/imports/api/genes/hectar/hectar.test.js index dbc210f8..b1f6d702 100644 --- a/imports/api/genes/hectar/hectar.test.js +++ b/imports/api/genes/hectar/hectar.test.js @@ -6,6 +6,7 @@ import { hectarCollection } from './hectarCollection'; import addHectar from './addHectar'; import { addTestUsers, addTestGenome } from '../../../startup/server/fixtures/addTestData'; import '../../jobqueue/process-hectar'; +import { Genes } from '/imports/api/genes/geneCollection.js'; describe('hectar', function testHectar() { let adminId; @@ -29,10 +30,11 @@ describe('hectar', function testHectar() { // Increase timeout this.timeout(20000); - addTestGenome(annot = true); + addTestGenome(annot = true, multiple = true); const hectarParams = { fileName: 'assets/app/data/Bnigra_hectar.tab', + annot: "Annotation name" }; // Should fail for non-logged in @@ -49,7 +51,7 @@ describe('hectar', function testHectar() { chai.assert.equal(result.nInserted, 1) - const hecs = hectarCollection.find({ protein_id: 'BniB01g000010.2N.1' }).fetch(); + const hecs = hectarCollection.find({ protein_id: 'Bni|B01g000010.2N.1', annotationName: "Annotation name" }).fetch(); chai.assert.lengthOf(hecs, 1, 'No hectar data found'); @@ -60,5 +62,11 @@ describe('hectar', function testHectar() { chai.assert.equal(hec.typeII_signal_anchor_score, '0.0228'); chai.assert.equal(hec.mitochondrion_score, '0.1032'); chai.assert.equal(hec.other_score, '0.8968'); + + const gene1 = Genes.findOne({ID: 'Bni|B01g000010.2N', annotationName: 'Annotation name'}) + const gene2 = Genes.findOne({ID: 'Bni|B01g000010.2N', annotationName: 'Annotation name 2'}) + + chai.assert.isDefined(gene1.hectarId, "eggNodeId is not defined for the correct annotation") + chai.assert.isUndefined(gene2.hectarId, "eggNodeId is defined for the wrong annotation") }); }); diff --git a/imports/api/genes/hectar/hectarCollection.js b/imports/api/genes/hectar/hectarCollection.js index bf7dad3f..5033f203 100644 --- a/imports/api/genes/hectar/hectarCollection.js +++ b/imports/api/genes/hectar/hectarCollection.js @@ -6,6 +6,10 @@ const hectarSchema = new SimpleSchema({ type: String, label: 'Query sequence name and type.', }, + annotationName: { + type: String, + label: 'Annotation name', + }, predicted_targeting_category: { type: String, label: 'Predicted sub-cellular localization.', diff --git a/imports/api/genes/interproscan.js b/imports/api/genes/interproscan.js deleted file mode 100644 index 0df060b9..00000000 --- a/imports/api/genes/interproscan.js +++ /dev/null @@ -1,219 +0,0 @@ -/* import { Meteor } from 'meteor/meteor'; -import { Roles } from 'meteor/alanning:roles'; - -import request from 'request'; -import Future from 'fibers/future'; - -import { getGeneSequences } from '/imports/api/util/util.js'; - -import { Genes } from '/imports/api/genes/gene_collection.js'; - -const revcomp = (seq) => { - const comp = { - 'A':'T','a':'t', - 'T':'A','t':'a', - 'C':'G','c':'g', - 'G':'C','g':'c', - 'N':'N','n':'n' - } - const revSeqArray = seq.split('').reverse() - const revCompSeqArray = revSeqArray.map( (nuc) => { - return comp[nuc] - }) - const revCompSeq = revCompSeqArray.join('') - return revCompSeq -} - -const translate = (seq) => { - const trans = { - 'ACC': 'T', 'ACA': 'T', 'ACG': 'T', - 'AGG': 'R', 'AGC': 'S', 'GTA': 'V', - 'AGA': 'R', 'ACT': 'T', 'GTG': 'V', - 'AGT': 'S', 'CCA': 'P', 'CCC': 'P', - 'GGT': 'G', 'CGA': 'R', 'CGC': 'R', - 'TAT': 'Y', 'CGG': 'R', 'CCT': 'P', - 'GGG': 'G', 'GGA': 'G', 'GGC': 'G', - 'TAA': '*', 'TAC': 'Y', 'CGT': 'R', - 'TAG': '*', 'ATA': 'I', 'CTT': 'L', - 'ATG': 'M', 'CTG': 'L', 'ATT': 'I', - 'CTA': 'L', 'TTT': 'F', 'GAA': 'E', - 'TTG': 'L', 'TTA': 'L', 'TTC': 'F', - 'GTC': 'V', 'AAG': 'K', 'AAA': 'K', - 'AAC': 'N', 'ATC': 'I', 'CAT': 'H', - 'AAT': 'N', 'GTT': 'V', 'CAC': 'H', - 'CAA': 'Q', 'CAG': 'Q', 'CCG': 'P', - 'TCT': 'S', 'TGC': 'C', 'TGA': '*', - 'TGG': 'W', 'TCG': 'S', 'TCC': 'S', - 'TCA': 'S', 'GAG': 'E', 'GAC': 'D', - 'TGT': 'C', 'GCA': 'A', 'GCC': 'A', - 'GCG': 'A', 'GCT': 'A', 'CTC': 'L', - 'GAT': 'D'} - const codonArray = seq.match(/.{1,3}/g) - const pepArray = codonArray.map( (codon) => { - let aminoAcid = 'X' - if (codon.indexOf('N') < 0){ - aminoAcid = trans[codon] - } - return aminoAcid - }) - const pep = pepArray.join('') - return pep -} - -const makeFasta = (gene) => { - let transcripts = gene.subfeatures.filter( (subfeature) => { return subfeature.type === 'mRNA' }) - let sequences = transcripts.map( (transcript) => { - let transcriptSeq = `>${transcript.ID}\n`; - let transcriptPep = `>${transcript.ID}\n`; - let cdsArray = gene.subfeatures.filter( (sub) => { - return sub.parents.indexOf(transcript.ID) >= 0 && sub.type === 'CDS' - }).sort( (a,b) => { - return a.start - b.start - }) - - let refStart = 10e99; - //let referenceSubscription = Meteor.subscribe('references',gene.seqid) - - //find all reference fragments overlapping the mRNA feature - let referenceArray = References.find({ - header: gene.seqid, - $and: [ - { start: {$lte: gene.end} }, - { end: {$gte: gene.start} } - ] - }).fetch() - - if (referenceArray.length){ - let reference = referenceArray.sort( (a,b) => { - //sort on start coordinate - return a.start - b.start - }).map( (ref) => { - //find starting position of first reference fragment - refStart = Math.min(refStart,ref.start) - return ref.seq - }).join('') - - seq = cdsArray.map( (cds, index) => { - let start = cds.start - refStart - 1; - let end = cds.end - refStart; - return reference.slice(start,end) - }).join('') - - let phase; - if (this.strand === '-'){ - seq = revcomp(seq) - phase = cdsArray[cdsArray.length -1].phase - } else { - phase = cdsArray[0].phase - } - - if ([1,2].indexOf(phase) >= 0){ - seq = seq.slice(phase) - } - - let pep = translate(seq.toUpperCase()); - - transcriptSeq += seq; - - transcriptPep += pep; - transcriptPep = transcriptPep.split('*').join('X') - } - return {ID:transcript.ID, seq: transcriptSeq, pep: transcriptPep} - }) - return sequences -} - -function submitInterpro(sequenceId,peptide){ - const submitJob = new Future(); - - request.post({ - url: 'http://www.ebi.ac.uk/Tools/services/rest/iprscan5/run/', - form: { - email: 'rens.holmer@gmail.com', - title: `genebook protein ${sequenceId}`, - sequence: peptide - } - }, (error, response, jobId) => { - console.log('error:', error); // Print the error if one occurred - console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received - console.log('requestId:', jobId); - submitJob.return(jobId) - }) - - const jobId = submitJob.wait() - - return jobId -} - -function pollInterpro(jobId,cb){ - const statusRequest = new Future(); - const url = `http://www.ebi.ac.uk/Tools/services/rest/iprscan5/status/${jobId}`; - console.log(`Trying ${url}`) - request.get(url, (error,response,body) => { - console.log(error) - console.log(body) - statusRequest.return(body) - }) - const status = statusRequest.wait() - if (status === 'RUNNING'){ - //figure out a way to call the function with a parameter - Meteor.setTimeout(function(){return pollInterpro(jobId,cb)}, 100000) - } else { - cb(status) - } -} - -function getInterproResults(jobId){ - const future = new Future(); - const url = `http://www.ebi.ac.uk/Tools/services/rest/iprscan5/result/${jobId}/json` - console.log(`Trying ${url}`) - request.get(url, (error,response,body) => { - let interproAnnotation = JSON.parse(body) - future.return(interproAnnotation) - }) - const results = future.wait() - return results -} - -Meteor.methods({ - interproscan(geneId){ - if (! this.userId) { - throw new Meteor.Error('not-authorized'); - } - if (! Roles.userIsInRole(this.userId,'admin')){ - throw new Meteor.Error('not-authorized'); - } - - //this.unblock(); - const gene = Genes.findOne({ID: geneId}) - const sequences = getGeneSequences(gene) - const results = sequences.map((sequence) => { - - // interproscan does not like stop codons, just replace all with X - let pep = sequence.pep.split('*').join('X') - - const jobId = submitInterpro(sequence.ID, pep); - - const fut = new Future(); - pollInterpro(jobId, (status) => { - console.log(`pollInterpro: ${status}`) - fut.return(status) - }) - - const finished = fut.wait(); - - let results; - - if (finished === 'FINISHED'){ - results = getInterproResults(jobId) - console.log(results) - Genes.update({'subfeatures.ID':sequence.ID},{$set:{interproscan:results[0].matches}}) - } - - return results - }) - - return results - } -}) -*/ diff --git a/imports/api/genes/interproscan/interproscan.test.js b/imports/api/genes/interproscan/interproscan.test.js index 1bfbc627..a4e91233 100644 --- a/imports/api/genes/interproscan/interproscan.test.js +++ b/imports/api/genes/interproscan/interproscan.test.js @@ -36,11 +36,12 @@ describe('interproscan', function testInterproscan() { // Increase timeout this.timeout(20000); - addTestGenome(annot=true) + addTestGenome(annot=true, multiple = true) const interproParams = { fileName: 'assets/app/data/Bnigra_interproscan.tsv', parser: "tsv", + annot: "Annotation name" }; // Should fail for non-logged in @@ -57,12 +58,12 @@ describe('interproscan', function testInterproscan() { let result = addInterproscan._execute(adminContext, interproParams); - const gene = Genes.findOne({ID: "BniB01g000010.2N"}) + const gene = Genes.findOne({ID: "Bni|B01g000010.2N"}) chai.assert.deepEqual(gene.attributes.Dbxref, [ 'InterPro:1236' ]) chai.assert.deepEqual(gene.attributes.Ontology_term, [ 'GO:1238' ]) - const interpros = interproscanCollection.find({gene_id: "BniB01g000010.2N"}).fetch(); + const interpros = interproscanCollection.find({gene_id: "Bni|B01g000010.2N", annotationName: "Annotation name"}).fetch(); chai.assert.lengthOf(interpros, 1, "No Interpro document found") const protein_domains = interpros[0].protein_domains @@ -84,11 +85,12 @@ describe('interproscan', function testInterproscan() { // Increase timeout this.timeout(20000); - addTestGenome(annot=true) + addTestGenome(annot=true, multiple = true) const interproParams = { fileName: 'assets/app/data/Bnigra_interproscan.gff', parser: "gff3", + annot: "Annotation name" }; // Should fail for non-logged in @@ -105,7 +107,7 @@ describe('interproscan', function testInterproscan() { let result = addInterproscan._execute(adminContext, interproParams); - const interpros = interproscanCollection.find({gene_id: "BniB01g000010.2N"}).fetch(); + const interpros = interproscanCollection.find({gene_id: "Bni|B01g000010.2N", annotationName: "Annotation name"}).fetch(); chai.assert.lengthOf(interpros, 1, "No Interpro document found") const protein_domains = interpros[0].protein_domains diff --git a/imports/api/genes/interproscan/interproscanCollection.js b/imports/api/genes/interproscan/interproscanCollection.js index 65a2ede8..1a175aed 100644 --- a/imports/api/genes/interproscan/interproscanCollection.js +++ b/imports/api/genes/interproscan/interproscanCollection.js @@ -9,8 +9,14 @@ const interproscanSchema = new SimpleSchema({ }, protein_id: { type: String, + index: true, label: 'Linked protein ID', }, + annotationName: { + type: String, + index: true, + label: 'Annotation name', + }, protein_domains: { type: Array, label: 'Interproscan protein domains', @@ -20,7 +26,7 @@ const interproscanSchema = new SimpleSchema({ type: Object, label: 'Interproscan protein domain', blackbox: true, - }, + } }); const interproscanCollection = new Mongo.Collection('interproscan'); diff --git a/imports/api/genes/orthogroup/addOrthogroupTrees.js b/imports/api/genes/orthogroup/addOrthogroupTrees.js index c00a99e5..f3ed2c1b 100644 --- a/imports/api/genes/orthogroup/addOrthogroupTrees.js +++ b/imports/api/genes/orthogroup/addOrthogroupTrees.js @@ -33,11 +33,19 @@ const addOrthogroupTrees = new ValidatedMethod({ return true; }, }, + annotations: { + type: Array, + optional: true, + defaultValue: [] + }, + 'annotations.$': { + type: String, + }, }).validator(), applyOptions: { noRetry: true, }, - run({ folderName, prefixes }) { + run({ folderName, prefixes, annotations }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); } @@ -51,6 +59,7 @@ const addOrthogroupTrees = new ValidatedMethod({ { folderName, prefixes, + annotations, }, ); const jobId = job.priority('high').save(); diff --git a/imports/api/genes/orthogroup/orthogroupCollection.js b/imports/api/genes/orthogroup/orthogroupCollection.js index 80e88aff..7db7270d 100644 --- a/imports/api/genes/orthogroup/orthogroupCollection.js +++ b/imports/api/genes/orthogroup/orthogroupCollection.js @@ -14,6 +14,14 @@ const orthogroupSchema = new SimpleSchema({ type: String, label: 'Gene ID string', }, + annotations: { + type: Array, + label: 'Array of all annotations names in the orthogroup', + }, + 'annotations.$': { + type: String, + label: 'Annotation name', + }, tree: { type: Object, blackbox: true, diff --git a/imports/api/genes/orthogroup/orthogroups.test.js b/imports/api/genes/orthogroup/orthogroups.test.js index 26d5ea8e..9c627360 100644 --- a/imports/api/genes/orthogroup/orthogroups.test.js +++ b/imports/api/genes/orthogroup/orthogroups.test.js @@ -37,12 +37,13 @@ describe('orthogroups', function testOrthogroups() { // Increase timeout this.timeout(20000); - const {genomeId} = addTestGenome(annot=true) + const {genomeId} = addTestGenome(annot=true, multiple=true) const orthoGroupsParams = { folderName: 'assets/app/data/orthogroups/', force: false, prefixes: "Brassica_nigra", + annotations: ["Annotation name"] }; // Should fail for non-logged in @@ -58,7 +59,7 @@ describe('orthogroups', function testOrthogroups() { let result = addOrthogroupTrees._execute(adminContext, orthoGroupsParams); - const gene = Genes.findOne({ID: "BniB01g000010.2N"}) + let gene = Genes.findOne({ID: "Bni|B01g000010.2N", annotationName: "Annotation name"}) chai.assert.isDefined(gene.orthogroup, 'orthogroup key is undefined') @@ -73,9 +74,13 @@ describe('orthogroups', function testOrthogroups() { chai.assert.deepEqual(ortho.genomes["unknown"], { name: 'unknown', count: 56 }) chai.assert.deepEqual(ortho.genomes[genomeId], { name: 'Test Genome', count: 1 }) - chai.assert.sameMembers(ortho.geneIds, ['BniB01g000010.2N']) + chai.assert.sameMembers(ortho.geneIds, ['Bni|B01g000010.2N']) chai.assert.equal(ortho.size, 84.5) chai.assert.equal(ortho.name, 'OG0000001') + chai.assert.sameMembers(ortho.annotations, ['Annotation name']) + + gene = Genes.findOne({ID: "Bni|B01g000010.2N", annotationName: "Annotation name 2"}) + chai.assert.isUndefined(gene.orthogroup, 'orthogroup key is undefined') }); }) diff --git a/imports/api/genes/orthogroup/parser/treeNewickParser.js b/imports/api/genes/orthogroup/parser/treeNewickParser.js index df0edc65..c87a093e 100644 --- a/imports/api/genes/orthogroup/parser/treeNewickParser.js +++ b/imports/api/genes/orthogroup/parser/treeNewickParser.js @@ -13,9 +13,10 @@ import path from 'path' * @public */ class NewickProcessor { - constructor() { + constructor(annotations) { this.genesDb = Genes.rawCollection(); this.nOrthogroups = 0; + this.annotations = annotations; } /** @@ -115,7 +116,8 @@ class NewickProcessor { * @return {Array} The list of genes without their prefixes. */ - getGeneId = async (prefixes, geneid) => { + getGeneId = async (prefixes, geneIdEncoded) => { + let geneid = decodeURIComponent(geneIdEncoded) return new Promise((resolve, reject) => { let geneName = geneid try { @@ -127,16 +129,19 @@ class NewickProcessor { } } } - const gene = Genes - .findOne( - { - $or: [ - { ID: geneName }, - { 'subfeatures.ID': geneName }, - { 'subfeatures.protein_id': geneName }, - ], - }, - ) + let geneQuery = { + $or: [ + { ID: geneName }, + { 'subfeatures.ID': geneName }, + { 'subfeatures.protein_id': geneName }, + ], + } + + if (this.annotations.length > 0){ + geneQuery['annotationName'] = { $in: this.annotations } + } + + const gene = Genes.findOne(geneQuery) resolve({gene, geneName}) } catch (err) { reject(err); @@ -177,15 +182,22 @@ class NewickProcessor { }); } + let geneQuery = { geneIds: { $in: rmDuplicateGeneIDs } } + + if (this.annotations.length > 0){ + geneQuery['annotations'] = { $in: this.annotations } + } + // Add the orthogroups and link them to their genes. if (rmDuplicateGeneIDs.length !== 0) { const documentOrthogroup = await orthogroupCollection.update( - { geneIds: { $in: rmDuplicateGeneIDs } }, // Selector. + geneQuery, // Selector. { $set: // Modifier. { name: name, geneIds: rmDuplicateGeneIDs, + annotations: this.annotations, tree: tree, size: treeSize, genomes: genomeDict @@ -200,9 +212,15 @@ class NewickProcessor { // Increment orthogroups. this.nOrthogroups += documentOrthogroup; + geneQuery = { ID: { $in: rmDuplicateGeneIDs } } + + if (this.annotations.length > 0){ + geneQuery['annotationName'] = { $in: this.annotations } + } + const orthogroupIdentifiant = orthogroupCollection.findOne({ tree: tree })._id; await this.genesDb.update( - { ID: { $in: rmDuplicateGeneIDs } }, + geneQuery, { $set: // Modifier. { diff --git a/imports/api/genes/parseGff3Interproscan.js b/imports/api/genes/parseGff3Interproscan.js index 5d4a11dd..43c753b1 100644 --- a/imports/api/genes/parseGff3Interproscan.js +++ b/imports/api/genes/parseGff3Interproscan.js @@ -43,7 +43,9 @@ class ParseGff3File extends InterproscanProcessor { parse = (line) => { // Check if the line is different of fasta sequence or others indications. if (!(line[0] === '#' || line.split('\t').length <= 1)) { - const [seqId, source, type, start, end, score, , , attributeString, ] = line.split('\t'); + let [seqId, source, type, start, end, score, , , attributeString, ] = line.split('\t'); + + seqId = decodeURIComponent(seqId) if (this.isProteinMatch(type)) { const attributes = this.parseAllAttributes(attributeString); @@ -57,9 +59,16 @@ class ParseGff3File extends InterproscanProcessor { this.currentProt = seqId this.currentGene = "" - let gene = Genes.findOne({ $or: [{'subfeatures.ID': seqId}, {'subfeatures.protein_id': seqId}] }); + let geneQuery = { $or: [{'subfeatures.ID': seqId}, {'subfeatures.protein_id': seqId}] } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + let gene = Genes.findOne(geneQuery); if (typeof gene !== "undefined"){ this.currentGene = gene.ID + this.currentAnnotationName = gene.annotationName + } else { + logger.warn(`Warning ! No sub-feature was found for ${seqId}.`) } this.currentContent = [] diff --git a/imports/api/genes/parseTsvInterproscan.js b/imports/api/genes/parseTsvInterproscan.js index 8f26694e..fc34390a 100644 --- a/imports/api/genes/parseTsvInterproscan.js +++ b/imports/api/genes/parseTsvInterproscan.js @@ -4,7 +4,7 @@ import logger from '/imports/api/util/logger.js'; class ParseTsvFile extends InterproscanProcessor { parse = async (line) => { - const [ + let [ seqId, md5, length, @@ -22,6 +22,8 @@ class ParseTsvFile extends InterproscanProcessor { pathwaysAnnotations, // Dbxref (gff3) ] = line.split('\t'); + seqId = decodeURIComponent(seqId) + // Add to bulk if protein changes if (seqId !== this.currentProt){ if (seqId !== ""){ @@ -30,9 +32,16 @@ class ParseTsvFile extends InterproscanProcessor { this.currentProt = seqId this.currentGene = "" - let gene = Genes.findOne({ $or: [{'subfeatures.ID': seqId}, {'subfeatures.protein_id': seqId}] }); + let geneQuery = { $or: [{'subfeatures.ID': seqId}, {'subfeatures.protein_id': seqId}] } + if (typeof this.annot !== "undefined"){ + geneQuery['annotationName'] = this.annot + } + let gene = Genes.findOne(geneQuery); if (typeof gene !== "undefined"){ this.currentGene = gene.ID + this.currentAnnotationName = gene.annotationName + } else { + logger.warn(logger.warn(`Warning ! No sub-feature was found for ${seqId}.`)) } this.currentContent = [] diff --git a/imports/api/genomes/annotation/addAnnotation.js b/imports/api/genomes/annotation/addAnnotation.js index b3ef47f9..55de26a3 100644 --- a/imports/api/genomes/annotation/addAnnotation.js +++ b/imports/api/genomes/annotation/addAnnotation.js @@ -15,6 +15,9 @@ const addAnnotation = new ValidatedMethod({ genomeName: { type: String, }, + annotationName: { + type: String, + }, re_protein: { type: String, optional: true, @@ -36,7 +39,7 @@ const addAnnotation = new ValidatedMethod({ noRetry: true, }, run({ - fileName, genomeName, re_protein, re_protein_capture, attr_protein, verbose, + fileName, genomeName, annotationName, re_protein, re_protein_capture, attr_protein, verbose, }) { if (!this.userId || !Roles.userIsInRole(this.userId, 'admin')) { throw new Meteor.Error('not-authorized'); @@ -46,8 +49,11 @@ const addAnnotation = new ValidatedMethod({ if (!existingGenome) { throw new Meteor.Error(`Unknown genome name: ${genomeName}`); } + if (typeof existingGenome.annotationTrack !== 'undefined') { - throw new Meteor.Error(`Genome ${genomeName} already has an annotation track`); + if (existingGenome.annotationTrack.some(annot => annot.name === annotationName)){ + throw new Meteor.Error(`Genome ${genomeName} already has an annotation track with the name ${annotationName}`); + } } const genomeId = existingGenome._id; @@ -58,6 +64,7 @@ const addAnnotation = new ValidatedMethod({ { fileName, genomeName, + annotationName, genomeId, re_protein, re_protein_capture, diff --git a/imports/api/genomes/annotation/addAnnotation.test.js b/imports/api/genomes/annotation/addAnnotation.test.js index 900b4914..5350f8ea 100644 --- a/imports/api/genomes/annotation/addAnnotation.test.js +++ b/imports/api/genomes/annotation/addAnnotation.test.js @@ -28,9 +28,10 @@ describe('AddAnnotation', function testAnnotation() { this.timeout(10000); const { genomeId, genomeSeqId } = addTestGenome(); - const toAnnot = { + let toAnnot = { fileName: 'assets/app/data/Bnigra.gff3', genomeName: 'Test Genome', + annotationName: 'Test annotation', verbose: false, }; @@ -53,7 +54,8 @@ describe('AddAnnotation', function testAnnotation() { const gene = genes[0]; - chai.assert.equal(gene.ID, 'BniB01g000010.2N'); + chai.assert.equal(gene.ID, 'Bni|B01g000010.2N'); + chai.assert.equal(gene.annotationName, 'Test annotation'); chai.assert.equal(gene.seqid, 'B1'); chai.assert.equal(gene.source, 'AAFC_GIFS'); chai.assert.equal(gene.strand, '-'); @@ -64,6 +66,45 @@ describe('AddAnnotation', function testAnnotation() { chai.assert.lengthOf(gene.subfeatures, 13, 'Number of subfeatures is not 13'); }); + it('Should add multiple copies of genes with different annotation names', function addAnnotationGff3() { + // Increase timeout + this.timeout(10000); + + const { genomeId, genomeSeqId } = addTestGenome(); + let toAnnot = { + fileName: 'assets/app/data/Bnigra.gff3', + genomeName: 'Test Genome', + annotationName: 'Test annotation', + verbose: false, + }; + + // Should fail for non-logged in + chai.expect(() => { + addAnnotation._execute({}, toAnnot); + }).to.throw('[not-authorized]'); + + // Should fail for non admin user + chai.expect(() => { + addAnnotation._execute(userContext, toAnnot); + }).to.throw('[not-authorized]'); + + // Add annotation. + addAnnotation._execute(adminContext, toAnnot); + + + toAnnot = { + fileName: 'assets/app/data/Bnigra.gff3', + genomeName: 'Test Genome', + annotationName: 'Test annotation2', + verbose: false, + }; + + addAnnotation._execute(adminContext, toAnnot); + + const genes = Genes.find({ genomeId: genomeId }).fetch(); + + chai.assert.lengthOf(genes, 10, 'Number of created genes is not 10'); + }); it('Should add a default -protein label to the mRNA protein_id', function addAnnotationGff3() { // Increase timeout @@ -73,6 +114,7 @@ describe('AddAnnotation', function testAnnotation() { const toAnnot = { fileName: 'assets/app/data/Bnigra_min.gff3', genomeName: 'Test Genome', + annotationName: 'Test annotation', verbose: false, }; @@ -94,6 +136,7 @@ describe('AddAnnotation', function testAnnotation() { const toAnnot = { fileName: 'assets/app/data/Bnigra_min.gff3', genomeName: 'Test Genome', + annotationName: 'Test annotation', verbose: false, re_protein_capture: '^Bni(.*?)$', re_protein: 'testprot-$1' @@ -104,7 +147,7 @@ describe('AddAnnotation', function testAnnotation() { const genes = Genes.find({ genomeId: genomeId }).fetch(); const mRNA = genes[0].subfeatures[0] - chai.assert.equal("testprot-B01g000010.2N.1", mRNA.protein_id) + chai.assert.equal("testprot-|B01g000010.2N.1", mRNA.protein_id) }); @@ -117,6 +160,7 @@ describe('AddAnnotation', function testAnnotation() { const toAnnot = { fileName: 'assets/app/data/Bnigra_min.gff3', genomeName: 'Test Genome', + annotationName: 'Test annotation', verbose: false, attr_protein: 'protid' }; @@ -126,7 +170,7 @@ describe('AddAnnotation', function testAnnotation() { const genes = Genes.find({ genomeId: genomeId }).fetch(); const mRNA = genes[0].subfeatures[0] - chai.assert.equal("BniB01g000010.2N.1-protattr", mRNA.protein_id) + chai.assert.equal("Bni|B01g000010.2N.1-protattr", mRNA.protein_id) }); @@ -138,6 +182,7 @@ describe('AddAnnotation', function testAnnotation() { const toAnnot = { fileName: 'assets/app/data/Bnigra_min.gff3', genomeName: 'Test Genome', + annotationName: 'Test annotation', verbose: false, attr_protein: 'protid2' }; @@ -147,7 +192,7 @@ describe('AddAnnotation', function testAnnotation() { const genes = Genes.find({ genomeId: genomeId }).fetch(); const mRNA = genes[0].subfeatures[0] - chai.assert.equal("BniB01g000010.2N.1-protattr", mRNA.protein_id) + chai.assert.equal("Bni|B01g000010.2N.1-protattr", mRNA.protein_id) }); }); diff --git a/imports/api/genomes/annotation/parser/annotationParserGff3.js b/imports/api/genomes/annotation/parser/annotationParserGff3.js index 1984140b..8964cb47 100644 --- a/imports/api/genomes/annotation/parser/annotationParserGff3.js +++ b/imports/api/genomes/annotation/parser/annotationParserGff3.js @@ -17,10 +17,11 @@ import { Genes, GeneSchema } from '../../../genes/geneCollection'; * @param {Boolean} verbose - View more details. */ class AnnotationProcessor { - constructor(filename, genomeID, re_protein=undefined, re_protein_capture="^(.*?)$", attr_protein=undefined, verbose = true) { + constructor(filename, annotationName, genomeID, re_protein=undefined, re_protein_capture="^(.*?)$", attr_protein=undefined, verbose = true) { this.filename = filename; this.genomeID = genomeID; this.verbose = verbose; + this.annotationName = annotationName // Protein ID stuff // Regex options . @@ -501,6 +502,7 @@ class AnnotationProcessor { const features = { ID: identifier, genomeId: this.genomeID, + annotationName: this.annotationName, seqid: seqidGff, source: sourceGff, type: typeGff, @@ -594,9 +596,9 @@ class AnnotationProcessor { genomeCollection.update({ _id: this.genomeID, }, { - $set: { + $push: { annotationTrack: { - name: this.filename.split('/').pop(), + name: this.annotationName, }, }, }); diff --git a/imports/api/genomes/genomeCollection.js b/imports/api/genomes/genomeCollection.js index 7ead8563..c8b62309 100644 --- a/imports/api/genomes/genomeCollection.js +++ b/imports/api/genomes/genomeCollection.js @@ -70,24 +70,27 @@ const genomeSchema = new SimpleSchema({ label: 'Organism name', }, annotationTrack: { - type: Object, + type: Array, optional: true, - label: 'Genome annotation', + label: 'Genome annotations', + }, + 'annotationTrack.$': { + type: Object }, - 'annotationTrack.name': { + 'annotationTrack.$.name': { type: String, label: 'Annotation track name', }, - 'annotationTrack.blastDb': { + 'annotationTrack.$.blastDb': { type: Object, optional: true, label: 'Annotation track BLAST database identifiers', }, - 'annotationTrack.blastDb.nucl': { + 'annotationTrack.$.blastDb.nucl': { type: String, label: 'Nucleotide BLAST database', }, - 'annotationTrack.blastDb.prot': { + 'annotationTrack.$.blastDb.prot': { type: String, label: 'Protein BLAST database', }, diff --git a/imports/api/jobqueue/process-annotation.js b/imports/api/jobqueue/process-annotation.js index 028a6591..f7522e1a 100644 --- a/imports/api/jobqueue/process-annotation.js +++ b/imports/api/jobqueue/process-annotation.js @@ -14,17 +14,19 @@ jobQueue.processJobs( const { fileName, genomeName, + annotationName, genomeId, re_protein, re_protein_capture, attr_protein, verbose, } = job.data; - logger.log(`Adding annotation file "${fileName}" to genome "${genomeName}"`); + logger.log(`Adding annotation file "${fileName}" with name "${annotationName}" to genome "${genomeName}"`); if(verbose){ logger.log('file :', fileName); logger.log('name :', genomeName); + logger.log('annotation name : ', annotationName) logger.log('re_protein :', re_protein); logger.log('re_protein_capture', re_protein_capture); logger.log('attr_protein', attr_protein); @@ -33,6 +35,7 @@ jobQueue.processJobs( const lineProcessor = new AnnotationProcessor( fileName, + annotationName, genomeId, re_protein, re_protein_capture, diff --git a/imports/api/jobqueue/process-eggnog.js b/imports/api/jobqueue/process-eggnog.js index b139a9c3..787ca037 100644 --- a/imports/api/jobqueue/process-eggnog.js +++ b/imports/api/jobqueue/process-eggnog.js @@ -11,10 +11,10 @@ jobQueue.processJobs( payload: 1, }, async (job, callback) => { - const { fileName } = job.data; + const { fileName, annot } = job.data; logger.log(`Add ${fileName} eggnog file.`); - const lineProcessor = new EggnogProcessor(); + const lineProcessor = new EggnogProcessor(annot); const rl = readline.createInterface({ input: fs.createReadStream(fileName, 'utf8'), diff --git a/imports/api/jobqueue/process-hectar.js b/imports/api/jobqueue/process-hectar.js index 4338b288..c7ec44ab 100644 --- a/imports/api/jobqueue/process-hectar.js +++ b/imports/api/jobqueue/process-hectar.js @@ -11,10 +11,10 @@ jobQueue.processJobs( payload: 1, }, async (job, callback) => { - const { fileName } = job.data; + const { fileName, annot } = job.data; logger.log(`Add ${fileName} hectar file.`); - const lineProcessor = new HectarProcessor(); + const lineProcessor = new HectarProcessor(annot); const rl = readline.createInterface({ input: fs.createReadStream(fileName, 'utf8'), diff --git a/imports/api/jobqueue/process-interproscan.js b/imports/api/jobqueue/process-interproscan.js index 41ddbd5b..9f07ffdd 100644 --- a/imports/api/jobqueue/process-interproscan.js +++ b/imports/api/jobqueue/process-interproscan.js @@ -12,7 +12,7 @@ jobQueue.processJobs( payload: 1, }, async (job, callback) => { - const { fileName, parser } = job.data; + const { fileName, parser, annot } = job.data; logger.log(`Add ${fileName} interproscan file.`); const rl = readline.createInterface({ @@ -24,11 +24,11 @@ jobQueue.processJobs( switch (parser) { case 'tsv': logger.log('Format : .tsv'); - lineProcessor = new ParseTsvFile(); + lineProcessor = new ParseTsvFile(annot); break; case 'gff3': logger.log('Format : .gff3'); - lineProcessor = new ParseGff3File(); + lineProcessor = new ParseGff3File(annot); break; } diff --git a/imports/api/jobqueue/process-orthogroup.js b/imports/api/jobqueue/process-orthogroup.js index 688922c0..c3a3889a 100644 --- a/imports/api/jobqueue/process-orthogroup.js +++ b/imports/api/jobqueue/process-orthogroup.js @@ -10,10 +10,10 @@ jobQueue.processJobs( payload: 1, }, async (job, callback) => { - const { folderName, prefixes } = job.data; + const { folderName, prefixes, annotations } = job.data; const orthofinder = new OrthoFinderPrefix(prefixes); - const newickProcessor = new NewickProcessor(); + const newickProcessor = new NewickProcessor(annotations); const listprefixes = (typeof prefixes !== 'undefined' ? await orthofinder.getListPrefixes() : null); diff --git a/imports/api/jobqueue/process-similarsequences.js b/imports/api/jobqueue/process-similarsequences.js index 7032911b..90b74ae7 100644 --- a/imports/api/jobqueue/process-similarsequences.js +++ b/imports/api/jobqueue/process-similarsequences.js @@ -13,14 +13,14 @@ jobQueue.processJobs( payload: 1, }, async (job, callback) => { - const { fileName, parser, program, algorithm, matrix, database } = job.data; + const { fileName, parser, program, algorithm, matrix, database, annot } = job.data; logger.log(`Add ${fileName} diamond file.`); // Different parser for the xml file. if (parser === 'xml') { const stream = fs.createReadStream(fileName); const xml = new XmlFlow(stream, { normalize: false }); - const lineProcessor = new XmlProcessor(program, algorithm, matrix, database); + const lineProcessor = new XmlProcessor(program, algorithm, matrix, database, annot); const tag = 'blastoutput'; xml.on(`tag:${tag}`, async (obj) => { @@ -48,7 +48,7 @@ jobQueue.processJobs( input: fs.createReadStream(fileName, 'utf8'), }); - const lineProcessor = new PairwiseProcessor(program, algorithm, matrix, database); + const lineProcessor = new PairwiseProcessor(program, algorithm, matrix, database, annot); for await (const line of lineReader) { try { diff --git a/imports/api/methods/methods.test.js b/imports/api/methods/methods.test.js index d3c1f345..17507021 100644 --- a/imports/api/methods/methods.test.js +++ b/imports/api/methods/methods.test.js @@ -29,7 +29,7 @@ describe('methods', function testMethods() { it('Should get the genes query count', function testGetQueryCount() { addTestGenome(annot=true) - const queryParams = {query: {ID: "BniB01g000010.2N"}} + const queryParams = {query: {ID: "Bni|B01g000010.2N"}} const count = getQueryCount._execute({}, queryParams) chai.assert.equal(count, 1) diff --git a/imports/api/publications.js b/imports/api/publications.js index e82906bb..16a3e58b 100644 --- a/imports/api/publications.js +++ b/imports/api/publications.js @@ -1,6 +1,7 @@ import { Mongo } from 'meteor/mongo'; import { Meteor } from 'meteor/meteor'; import { Roles } from 'meteor/alanning:roles'; +import { fetch } from 'meteor/fetch'; // jobqueue import jobQueue from '/imports/api/jobqueue/jobqueue.js'; // genes @@ -27,6 +28,7 @@ import { fileCollection } from '/imports/api/files/fileCollection.js'; import fetchDbxref from '/imports/api/methods/fetchDbxref.js'; // utilities import { DBXREF_REGEX } from '/imports/api/util/util.js'; +import logger from '/imports/api/util/logger.js' function availableGenomes({ userId }) { const roles = Roles.getRolesForUser(userId); @@ -76,9 +78,37 @@ Meteor.publish({ const queryGenomeIds = hasOwnProperty(query, 'genomeId') ? query.genomeId.$in.filter((genomeId) => genomeIds.includes(genomeId)) : genomeIds; - - const transformedQuery = { ...query, genomeId: { $in: queryGenomeIds } }; - + let transformedQuery = {}; + + let config = Meteor.settings + + if ( query.query !== undefined && config.public.externalSearch && typeof config.externalSearchOptions === "object" && config.externalSearchOptions.url){ + let url = config.externalSearchOptions.url.replace(/,+$/, "") + "/"; + let paramsDict = {} + let geneField = config.externalSearchOptions.gene_field ? config.externalSearchOptions.gene_field : "geneId" + if (config.externalSearchOptions.query_param){ + paramsDict[config.externalSearchOptions.query_param] = query.query + } else { + url += query.query + } + if (config.externalSearchOptions.field_param){ + paramsDict[config.externalSearchOptions.field_param] = geneField + } + + if (config.externalSearchOptions.count_param){ + paramsDict[config.externalSearchOptions.count_param] = limit + } + + let geneResults = [] + url = url + "?" + new URLSearchParams(paramsDict) + const response = HTTP.get(url); + if (response.statusCode === 200){ + geneResults = response.data.data.map(result => result._source[geneField]) + } + transformedQuery = {genomeId: { $in: queryGenomeIds }, ID: { $in: geneResults }} + } else { + transformedQuery = { ...query, genomeId: { $in: queryGenomeIds } }; + } return Genes.find(transformedQuery, { sort, limit }); }, singleGene({ geneId, transcriptId }) { @@ -124,7 +154,7 @@ Meteor.publish({ $or: [{ genomes: { $in: genomeIds } }, { allGenomes: true }], }); }, - geneExpression(geneId) { + geneExpression(geneId, annotationName) { const publication = this; const roles = Roles.getRolesForUser(publication.userId); const permission = { $in: roles }; @@ -138,6 +168,7 @@ Meteor.publish({ return Transcriptomes.find({ geneId, + annotationName, experimentId: { $in: experimentIds, }, @@ -187,6 +218,7 @@ Meteor.publish({ alignment(gene) { const diamond = similarSequencesCollection.find( { + annotationName: gene.annotationName, $or: [ { iteration_query: gene.ID }, { iteration_query: { $in: gene.children } }, @@ -195,8 +227,8 @@ Meteor.publish({ ); return diamond; }, - interpro(query){ - return interproscanCollection.find({gene_id: query}) + interpro(gene){ + return interproscanCollection.find({gene_id: gene.ID, annotationName: gene.annotationName}) }, orthogroups(ID) { return orthogroupCollection.find({ _id: ID }); diff --git a/imports/api/transcriptomes/addExpression.js b/imports/api/transcriptomes/addExpression.js index e4ed1b3d..d345b07f 100644 --- a/imports/api/transcriptomes/addExpression.js +++ b/imports/api/transcriptomes/addExpression.js @@ -12,24 +12,32 @@ import { } from '/imports/api/transcriptomes/transcriptome_collection.js'; import logger from '/imports/api/util/logger.js'; -const getGenomeId = (data) => { - const firstTranscripts = data.slice(0, 10).map((line) => line.gene); +const getGenomeId = (data, firstColumn, annot) => { + const firstTranscripts = data.slice(0, 10).map((line) => decodeURIComponent(line[firstColumn])); logger.debug(firstTranscripts); - const gene = Genes.findOne({ + + let geneQuery = { $or: [ { ID: { $in: firstTranscripts } }, { 'subfeatures.ID': { $in: firstTranscripts } }, ], - }); + } + + if (annot){ + geneQuery['annotationName'] = annot + } + + const gene = Genes.findOne(geneQuery); + if (typeof gene === "undefined"){ - return undefined + return {genomeId: undefined, annotationName: undefined} } logger.debug(gene.genomeId); - return gene.genomeId + return {genomeId: gene.genomeId, annotationName: gene.annotationName} }; const parseExpressionTsv = ({ - fileName, description, replicas = [], replicaNames = [], permission = 'admin', isPublic = false, + fileName, description, annot, replicas = [], replicaNames = [], permission = 'admin', isPublic = false, }) => new Promise((resolve, reject) => { const fileHandle = fs.readFileSync(fileName, { encoding: 'binary' }); const bulkOp = Transcriptomes.rawCollection().initializeUnorderedBulkOp(); @@ -80,7 +88,7 @@ const parseExpressionTsv = ({ } let firstColumn = replicaGroups.shift(); - const genomeId = getGenomeId(data); + const {genomeId, annotationName} = getGenomeId(data, firstColumn, annot); if (typeof genomeId === 'undefined') { reject(new Meteor.Error('Could not find genomeId for first transcript')); @@ -91,6 +99,7 @@ const parseExpressionTsv = ({ const replicaGroup = replicaIndex + 1 in replicaNamesDict ? replicaNamesDict[replicaIndex + 1] : sampleName experiments[sampleName] = ExperimentInfo.insert({ genomeId, + annotationName, sampleName, replicaGroup, description, @@ -100,12 +109,18 @@ const parseExpressionTsv = ({ }); data.forEach((row) => { - const gene = Genes.findOne({ + let geneQuery = { $or: [ - { ID: row[firstColumn] }, - { 'subfeatures.ID': row[firstColumn] }, + { ID: decodeURIComponent(row[firstColumn]) }, + { 'subfeatures.ID': decodeURIComponent(row[firstColumn]) }, ], - }); + } + + if (annot){ + geneQuery['annotationName'] = annot + } + + const gene = Genes.findOne(geneQuery); if (typeof gene === 'undefined') { logger.warn(`${target_id} not found`); @@ -114,6 +129,7 @@ const parseExpressionTsv = ({ replicaGroups.forEach((replicaGroup) => { bulkOp.insert({ geneId: gene.ID, + annotationName, tpm: row[replicaGroup], experimentId: experiments[replicaGroup] }); @@ -136,6 +152,10 @@ const addExpression = new ValidatedMethod({ validate: new SimpleSchema({ fileName: String, description: String, + annot: { + type: String, + optional: true, + }, replicas: { type: Array, optional: true, @@ -158,7 +178,7 @@ const addExpression = new ValidatedMethod({ noRetry: true, }, run({ - fileName, description, replicas, replicaNames, isPublic + fileName, description, annot, replicas, replicaNames, isPublic }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); @@ -167,7 +187,7 @@ const addExpression = new ValidatedMethod({ throw new Meteor.Error('not-authorized'); } return parseExpressionTsv({ - fileName, description, replicas, replicaNames, isPublic + fileName, description, annot, replicas, replicaNames, isPublic }) .catch((error) => { logger.warn(error); diff --git a/imports/api/transcriptomes/addKallistoTranscriptome.js b/imports/api/transcriptomes/addKallistoTranscriptome.js index d6a4ff62..202414ba 100644 --- a/imports/api/transcriptomes/addKallistoTranscriptome.js +++ b/imports/api/transcriptomes/addKallistoTranscriptome.js @@ -12,21 +12,32 @@ import { } from '/imports/api/transcriptomes/transcriptome_collection.js'; import logger from '/imports/api/util/logger.js'; -const getGenomeId = (data) => { - const firstTranscipts = data.slice(0, 10).map((line) => line.target_id); - logger.debug(firstTranscipts); - const { genomeId } = Genes.findOne({ +const getGenomeId = (data, annot) => { + const firstTranscripts = data.slice(0, 10).map((line) => decodeURIComponent(line.target_id)); + logger.debug(firstTranscripts); + + let geneQuery = { $or: [ - { ID: { $in: firstTranscipts } }, - { 'subfeatures.ID': { $in: firstTranscipts } }, + { ID: { $in: firstTranscripts } }, + { 'subfeatures.ID': { $in: firstTranscripts } }, ], - }); - logger.debug(genomeId); - return genomeId; + } + + if (annot){ + geneQuery['annotationName'] = annot + } + + const gene = Genes.findOne(geneQuery); + + if (typeof gene === "undefined"){ + return {genomeId: undefined, annotationName: undefined} + } + logger.debug(gene.genomeId); + return {genomeId: gene.genomeId, annotationName: gene.annotationName} }; const parseKallistoTsv = ({ - fileName, sampleName, replicaGroup, + fileName, annot, sampleName, replicaGroup, description, permission = 'admin', isPublic = false, }) => new Promise((resolve, reject) => { const fileHandle = fs.readFileSync(fileName, { encoding: 'binary' }); @@ -43,7 +54,7 @@ const parseKallistoTsv = ({ complete({ data }, _file) { let nInserted = 0; - const genomeId = getGenomeId(data); + const {genomeId, annotationName} = getGenomeId(data, annot); if (typeof genomeId === 'undefined') { reject(new Meteor.Error('Could not find genomeId for first transcript')); @@ -53,6 +64,7 @@ const parseKallistoTsv = ({ const experimentId = ExperimentInfo.insert({ genomeId, + annotationName, sampleName, replicaGroup, description, @@ -62,12 +74,18 @@ const parseKallistoTsv = ({ }); data.forEach(({ target_id, tpm, est_counts }) => { - const gene = Genes.findOne({ + let geneQuery = { $or: [ - { ID: target_id }, - { 'subfeatures.ID': target_id }, + { ID: decodeURIComponent(target_id) }, + { 'subfeatures.ID': decodeURIComponent(target_id) }, ], - }); + } + + if (annot){ + geneQuery['annotationName'] = annot + } + + const gene = Genes.findOne(geneQuery); if (typeof gene === 'undefined') { logger.warn(`${target_id} not found`); @@ -75,6 +93,7 @@ const parseKallistoTsv = ({ nInserted += 1; bulkOp.insert({ geneId: gene.ID, + annotationName, tpm, est_counts, experimentId, @@ -96,6 +115,10 @@ const addKallistoTranscriptome = new ValidatedMethod({ name: 'addKallistoTranscriptome', validate: new SimpleSchema({ fileName: String, + annot: { + type: String, + optional: true, + }, sampleName: String, replicaGroup: String, description: String, @@ -105,7 +128,7 @@ const addKallistoTranscriptome = new ValidatedMethod({ noRetry: true, }, run({ - fileName, sampleName, replicaGroup, description, isPublic + fileName, annot, sampleName, replicaGroup, description, isPublic }) { if (!this.userId) { throw new Meteor.Error('not-authorized'); @@ -114,7 +137,7 @@ const addKallistoTranscriptome = new ValidatedMethod({ throw new Meteor.Error('not-authorized'); } return parseKallistoTsv({ - fileName, sampleName, replicaGroup, description, isPublic + fileName, annot, sampleName, replicaGroup, description, isPublic }) .catch((error) => { logger.warn(error); diff --git a/imports/api/transcriptomes/transcriptome_collection.js b/imports/api/transcriptomes/transcriptome_collection.js index 574256cd..3ce2fc15 100644 --- a/imports/api/transcriptomes/transcriptome_collection.js +++ b/imports/api/transcriptomes/transcriptome_collection.js @@ -8,6 +8,10 @@ const ExperimentInfoSchema = new SimpleSchema({ type: String, label: 'Genome ID', }, + annotationName: { + type: String, + label: 'Annotation name', + }, sampleName: { type: String, label: 'Short name for the sample', @@ -40,6 +44,10 @@ const TranscriptomeSchema = new SimpleSchema({ label: 'Gene ID', index: true, }, + annotationName: { + type: String, + label: 'Annotation name', + }, experimentId: { type: String, label: 'Experiment ID', diff --git a/imports/api/transcriptomes/transcriptomes.test.js b/imports/api/transcriptomes/transcriptomes.test.js index a52a60d9..43228ef6 100644 --- a/imports/api/transcriptomes/transcriptomes.test.js +++ b/imports/api/transcriptomes/transcriptomes.test.js @@ -37,10 +37,11 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId} = addTestGenome(annot=true, multiple=true) const transcriParams = { fileName: 'assets/app/data/Bnigra_kallisto_abundance.tsv', + annot: "Annotation name", sampleName: "mySample", replicaGroup: "replicaGroup", description: "A new description", @@ -69,6 +70,7 @@ describe('transcriptomes', function testTranscriptomes() { chai.assert.equal(exp.sampleName, 'mySample') chai.assert.equal(exp.replicaGroup, 'replicaGroup') chai.assert.equal(exp.description, 'A new description') + chai.assert.equal(exp.annotationName, "Annotation name") const transcriptomes = Transcriptomes.find({experimentId: exp._id}).fetch() @@ -76,8 +78,9 @@ describe('transcriptomes', function testTranscriptomes() { const transcriptome = transcriptomes[0] - chai.assert.equal(transcriptome.geneId, 'BniB01g000010.2N') + chai.assert.equal(transcriptome.geneId, 'Bni|B01g000010.2N') chai.assert.equal(transcriptome.tpm, '1.80368') + chai.assert.equal(transcriptome.annotationName, "Annotation name") chai.assert.equal(transcriptome.est_counts, '21') }) @@ -86,10 +89,11 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId} = addTestGenome(annot=true, multiple=true) const transcriParams = { fileName: 'assets/app/data/Bnigra_abundance.tsv', + annot: "Annotation name", description: "A new description", isPublic: false }; @@ -116,10 +120,12 @@ describe('transcriptomes', function testTranscriptomes() { chai.assert.equal(exp.sampleName, 'sample1') chai.assert.equal(exp.replicaGroup, 'sample1') chai.assert.equal(exp.description, 'A new description') + chai.assert.equal(exp.annotationName, "Annotation name") chai.assert.equal(exps[1].sampleName, 'sample2') chai.assert.equal(exps[1].replicaGroup, 'sample2') chai.assert.equal(exps[1].description, 'A new description') + chai.assert.equal(exps[1].annotationName, "Annotation name") const transcriptomes = Transcriptomes.find({experimentId: exp._id}).fetch() @@ -127,8 +133,9 @@ describe('transcriptomes', function testTranscriptomes() { const transcriptome = transcriptomes[0] - chai.assert.equal(transcriptome.geneId, 'BniB01g000010.2N') + chai.assert.equal(transcriptome.geneId, 'Bni|B01g000010.2N') chai.assert.equal(transcriptome.tpm, '40') + chai.assert.equal(transcriptome.annotationName, "Annotation name") chai.assert.isUndefined(transcriptome.est_counts) }) @@ -137,10 +144,11 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId} = addTestGenome(annot=true, multiple=true) const transcriParams = { fileName: 'assets/app/data/Bnigra_abundance.tsv', + annot: "Annotation name", description: "A new description", replicas: ["1,2"], isPublic: false @@ -168,14 +176,17 @@ describe('transcriptomes', function testTranscriptomes() { chai.assert.equal(exp.sampleName, 'sample1') chai.assert.equal(exp.replicaGroup, 'sample1') chai.assert.equal(exp.description, 'A new description') + chai.assert.equal(exp.annotationName, "Annotation name") chai.assert.equal(exps[1].sampleName, 'sample2') chai.assert.equal(exps[1].replicaGroup, 'sample1') chai.assert.equal(exps[1].description, 'A new description') + chai.assert.equal(exps[1].annotationName, "Annotation name") chai.assert.equal(exps[2].sampleName, 'sample3') chai.assert.equal(exps[2].replicaGroup, 'sample3') chai.assert.equal(exps[2].description, 'A new description') + chai.assert.equal(exps[2].annotationName, "Annotation name") const transcriptomes = Transcriptomes.find({experimentId: exp._id}).fetch() @@ -183,8 +194,9 @@ describe('transcriptomes', function testTranscriptomes() { const transcriptome = transcriptomes[0] - chai.assert.equal(transcriptome.geneId, 'BniB01g000010.2N') + chai.assert.equal(transcriptome.geneId, 'Bni|B01g000010.2N') chai.assert.equal(transcriptome.tpm, '40') + chai.assert.equal(transcriptome.annotationName, "Annotation name") chai.assert.isUndefined(transcriptome.est_counts) }) @@ -193,10 +205,11 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId} = addTestGenome(annot=true, multiple=true) const transcriParams = { fileName: 'assets/app/data/Bnigra_abundance.tsv', + annot: "Annotation name", description: "A new description", replicas: ["1,2"], replicaNames: ["My replica group name", "Another group name"], @@ -225,18 +238,22 @@ describe('transcriptomes', function testTranscriptomes() { chai.assert.equal(exp.sampleName, 'sample1') chai.assert.equal(exp.replicaGroup, 'My replica group name') chai.assert.equal(exp.description, 'A new description') + chai.assert.equal(exp.annotationName, "Annotation name") chai.assert.equal(exps[1].sampleName, 'sample2') chai.assert.equal(exps[1].replicaGroup, 'My replica group name') chai.assert.equal(exps[1].description, 'A new description') + chai.assert.equal(exps[1].annotationName, "Annotation name") chai.assert.equal(exps[2].sampleName, 'sample3') chai.assert.equal(exps[2].replicaGroup, 'Another group name') chai.assert.equal(exps[2].description, 'A new description') + chai.assert.equal(exps[2].annotationName, "Annotation name") chai.assert.equal(exps[3].sampleName, 'sample4') chai.assert.equal(exps[3].replicaGroup, 'sample4') chai.assert.equal(exps[3].description, 'A new description') + chai.assert.equal(exps[3].annotationName, "Annotation name") const transcriptomes = Transcriptomes.find({experimentId: exp._id}).fetch() @@ -244,9 +261,10 @@ describe('transcriptomes', function testTranscriptomes() { const transcriptome = transcriptomes[0] - chai.assert.equal(transcriptome.geneId, 'BniB01g000010.2N') + chai.assert.equal(transcriptome.geneId, 'Bni|B01g000010.2N') chai.assert.equal(transcriptome.tpm, '40') chai.assert.isUndefined(transcriptome.est_counts) + chai.assert.equal(transcriptome.annotationName, "Annotation name") }) @@ -254,10 +272,11 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId} = addTestGenome(annot=true, multiple=true) const transcriParams = { fileName: 'assets/app/data/Bnigra_abundance.tsv', + annot: "Annotation name", description: "A new description", replicaNames: ["TestReplica1", "TestReplica2"], isPublic: false @@ -285,14 +304,17 @@ describe('transcriptomes', function testTranscriptomes() { chai.assert.equal(exp.sampleName, 'sample1') chai.assert.equal(exp.replicaGroup, 'TestReplica1') chai.assert.equal(exp.description, 'A new description') + chai.assert.equal(exp.annotationName, "Annotation name") chai.assert.equal(exps[1].sampleName, 'sample2') chai.assert.equal(exps[1].replicaGroup, 'TestReplica2') chai.assert.equal(exps[1].description, 'A new description') + chai.assert.equal(exps[1].annotationName, "Annotation name") chai.assert.equal(exps[2].sampleName, 'sample3') chai.assert.equal(exps[2].replicaGroup, 'sample3') chai.assert.equal(exps[2].description, 'A new description') + chai.assert.equal(exps[2].annotationName, "Annotation name") const transcriptomes = Transcriptomes.find({experimentId: exp._id}).fetch() @@ -300,8 +322,9 @@ describe('transcriptomes', function testTranscriptomes() { const transcriptome = transcriptomes[0] - chai.assert.equal(transcriptome.geneId, 'BniB01g000010.2N') + chai.assert.equal(transcriptome.geneId, 'Bni|B01g000010.2N') chai.assert.equal(transcriptome.tpm, '40') + chai.assert.equal(transcriptome.annotationName, "Annotation name") chai.assert.isUndefined(transcriptome.est_counts) }) @@ -311,7 +334,7 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId, geneId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId, geneId} = addTestGenome(annot=true, multiple=true) const {expId, transcriptomeId} = addTestTranscriptome(genomeId, geneId) const updateParams = { @@ -347,7 +370,7 @@ describe('transcriptomes', function testTranscriptomes() { // Increase timeout this.timeout(20000); - const {genomeId, genomeSeqId, geneId} = addTestGenome(annot=true) + const {genomeId, genomeSeqId, geneId} = addTestGenome(annot=true, multiple=true) const {expId, transcriptomeId} = addTestTranscriptome(genomeId, geneId) const updateParams = { diff --git a/imports/api/util/util.test.js b/imports/api/util/util.test.js index b7af9cad..973c2f5e 100644 --- a/imports/api/util/util.test.js +++ b/imports/api/util/util.test.js @@ -42,8 +42,8 @@ describe('util', function testUtils() { const gene = Genes.findOne({ID: geneId}) const seq = getGeneSequences(gene) const expected = [{ - ID: "BniB01g000010.2N.1", - protein_id: "BniB01g000010.2N.1-P", + ID: "Bni|B01g000010.2N.1", + protein_id: "Bni|B01g000010.2N.1-P", nucl: "AGTTTAGAATAC", prot: "SLEY" }] diff --git a/imports/startup/server/fixtures/addDefaultAttributes.js b/imports/startup/server/fixtures/addDefaultAttributes.js index 07f0c459..bf2e3f72 100644 --- a/imports/startup/server/fixtures/addDefaultAttributes.js +++ b/imports/startup/server/fixtures/addDefaultAttributes.js @@ -6,36 +6,48 @@ const PERMANENT_ATTRIBUTES = [ { name: 'Note', query: 'attributes.Note', + display: false }, { name: 'Dbxref', query: 'attributes.Dbxref', + display: false }, { name: 'Ontology Term', query: 'attributes.Ontology_term', + display: false }, { name: 'Orthogroup', query: 'orthogroup.name', + display: false }, { name: 'Gene ID', query: 'ID', + display: true }, { name: 'Has changes', query: 'changed', + display: false }, { name: 'Genome', query: 'genomeId', + display: true + }, + { + name: 'Annotation', + query: 'annotationName', + display: true }, ]; export default function addDefaultAttributes() { // add some default attributes to filter on - PERMANENT_ATTRIBUTES.forEach(({ name, query }) => { + PERMANENT_ATTRIBUTES.forEach(({ name, query, display }) => { const existingAttribute = attributeCollection.findOne({ name }); if (typeof existingAttribute === 'undefined') { logger.log(`Adding default filter option: ${name}`); @@ -47,8 +59,8 @@ export default function addDefaultAttributes() { $setOnInsert: { name, query, - defaultShow: false, - defaultSearch: false, + defaultShow: display, + defaultSearch: display, allGenomes: true, }, }, diff --git a/imports/startup/server/fixtures/addTestData.js b/imports/startup/server/fixtures/addTestData.js index 150cc595..e55b810a 100644 --- a/imports/startup/server/fixtures/addTestData.js +++ b/imports/startup/server/fixtures/addTestData.js @@ -49,9 +49,9 @@ export function addTestUsers() { return { adminId, newUserId, curatorId } } -export function addTestGenome(annot=false) { +export function addTestGenome(annot=false, multiple=false) { - const annotObj = annot ? { name: 'myfilename.gff'} : undefined + const annotObj = annot ? [{ name: 'myfilename.gff'}] : undefined const genomeId = genomeCollection.insert({ name: "Test Genome", @@ -75,11 +75,11 @@ export function addTestGenome(annot=false) { let geneId if (annot) { - const subfeature = {ID: "BniB01g000010.2N.1", phase: '.', protein_id: "BniB01g000010.2N.1-P", type: 'mRNA', parents: ['BniB01g000010.2N'], seq: 'GTATTCTAAACT', start:13641, end:15400, score: '.', attributes: {}} - const cds = {ID: "BniB01g000010.2N.1.cds1", phase: '.', type: 'CDS', parents: ['BniB01g000010.2N.1'], seq: 'GTATTCTAAACT', start:13641, end:13653, score: '.', attributes: {}} + const subfeature = {ID: "Bni|B01g000010.2N.1", phase: '.', protein_id: "Bni|B01g000010.2N.1-P", type: 'mRNA', parents: ['Bni|B01g000010.2N'], seq: 'GTATTCTAAACT', start:13641, end:15400, score: '.', attributes: {}} + const cds = {ID: "Bni|B01g000010.2N.1.cds1", phase: '.', type: 'CDS', parents: ['Bni|B01g000010.2N.1'], seq: 'GTATTCTAAACT', start:13641, end:13653, score: '.', attributes: {}} Genes.insert({ - ID: 'BniB01g000010.2N', + ID: 'Bni|B01g000010.2N', seqid: 'B1', source: 'AAFC_GIFS', strand: '-', @@ -87,14 +87,33 @@ export function addTestGenome(annot=false) { start: 13640, end: 15401, genomeId: genomeId, + annotationName: "Annotation name", score: '.', subfeatures: [subfeature, cds], seq: 'AAAA', attributes: {"myNewAttribute": 1} }) + + if (multiple){ + Genes.insert({ + ID: 'Bni|B01g000010.2N', + seqid: 'B1', + source: 'AAFC_GIFS', + strand: '-', + type: 'gene', + start: 13640, + end: 15401, + genomeId: genomeId, + annotationName: "Annotation name 2", + score: '.', + subfeatures: [subfeature, cds], + seq: 'AAAA', + attributes: {"myNewAttribute": 1} + }) + } } - return { genomeId, genomeSeqId, geneId: "BniB01g000010.2N" } + return { genomeId, genomeSeqId, geneId: "Bni|B01g000010.2N" } } export function addTestTranscriptome(genomeId, geneId) { @@ -102,6 +121,7 @@ export function addTestTranscriptome(genomeId, geneId) { const expId = ExperimentInfo.insert({ genomeId: genomeId, sampleName: "sampleName", + annotationName: "Annotation name", replicaGroup: "replicaGroup", description: 'description', permission: 'admin', @@ -110,6 +130,7 @@ export function addTestTranscriptome(genomeId, geneId) { const transcriptomeId = Transcriptomes.insert({ geneId: geneId, + annotationName: "Annotation name", tpm: "60", est_counts: "1000", experimentId: expId diff --git a/imports/ui/genetable/GeneTable.jsx b/imports/ui/genetable/GeneTable.jsx index ef60c562..9761d2f0 100644 --- a/imports/ui/genetable/GeneTable.jsx +++ b/imports/ui/genetable/GeneTable.jsx @@ -62,20 +62,25 @@ function searchTracker({ const attributeString = queryString.get('attributes') || ''; const searchAttributes = attributeString.split(','); - const searchValue = queryString.get('search') || ''; - const searchQuery = { $or: [] }; - attributes - .filter(({ name }) => new RegExp(name).test(searchAttributes)) - .forEach((attribute) => { - searchQuery.$or.push({ - [attribute.query]: { - $regex: searchValue, - $options: 'i', - }, + let searchQuery + + if (searchValue !== "" && Meteor.settings.public.externalSearch === true){ + searchQuery = {query: searchValue, $or: []} + } else { + searchQuery = { $or: [] }; + attributes + .filter(({ name }) => new RegExp(name).test(searchAttributes)) + .forEach((attribute) => { + searchQuery.$or.push({ + [attribute.query]: { + $regex: searchValue, + $options: 'i', + }, + }); }); - }); + } const selectedAttributes = attributes .filter( diff --git a/imports/ui/genetable/GeneTableBody.jsx b/imports/ui/genetable/GeneTableBody.jsx index 66fc60aa..29258bc7 100644 --- a/imports/ui/genetable/GeneTableBody.jsx +++ b/imports/ui/genetable/GeneTableBody.jsx @@ -45,7 +45,7 @@ function dataTracker({ }) { const geneSub = Meteor.subscribe('genes', { query, sort, limit }); const loading = !geneSub.ready(); - const genes = Genes.find(query, { limit, sort }).fetch(); + const genes = Genes.find({}, { limit, sort }).fetch(); return { genes, @@ -122,11 +122,11 @@ function Loading({ selectedColumns, ...props }) { } function AttributeColumn({ - attributeName, attributeValue, geneId, genomeDataCache, + attributeName, attributeValue, gene, genomeDataCache, }) { switch (attributeName) { case 'Gene ID': - return ; + return ; case 'Genome': return ( diff --git a/imports/ui/genetable/columns/GeneLink.jsx b/imports/ui/genetable/columns/GeneLink.jsx index e338df1e..05b199a8 100644 --- a/imports/ui/genetable/columns/GeneLink.jsx +++ b/imports/ui/genetable/columns/GeneLink.jsx @@ -3,9 +3,15 @@ import { Meteor } from 'meteor/meteor'; import React from 'react'; import { Link } from 'react-router-dom'; -export default function GeneLink({ geneId }) { +export default function GeneLink({ gene }) { + const geneId = gene.ID + + const query = new URLSearchParams(); + query.set("annotation", gene.annotationName); + const url = `/gene/${geneId}?${query.toString()}` + return ( - + { geneId } ); diff --git a/imports/ui/genetable/filteroptions/GenomeSelect.jsx b/imports/ui/genetable/filteroptions/GenomeSelect.jsx index 638598b3..a5be7727 100644 --- a/imports/ui/genetable/filteroptions/GenomeSelect.jsx +++ b/imports/ui/genetable/filteroptions/GenomeSelect.jsx @@ -24,6 +24,7 @@ function genomeDataTracker({ ...props }) { annotationTrack: { $exists: true }, }) .fetch(); + return { loading, genomes, @@ -38,6 +39,16 @@ function GenomeSelect({ new Set(genomes.map((genome) => genome._id)), ); + const [selectedAnnotations, setSelectedAnnotations] = useState( + new Set(genomes.flatMap((genome) => { + return genome.annotationTrack.map((annotation) => annotation.name); + })), + ); + + let annotations = genomes.flatMap((genome) => { + return genome.annotationTrack.map((annotation) => annotation.name) + }) + function toggleGenomeSelect(genomeId) { const newSelection = cloneDeep(selectedGenomes); const newQuery = cloneDeep(query); @@ -57,7 +68,26 @@ function GenomeSelect({ updateQuery(newQuery); } - function selectAll() { + function toggleAnnotationSelect(annotationName) { + const newSelection = cloneDeep(selectedAnnotations); + const newQuery = cloneDeep(query); + + if (newSelection.has(annotationName)) { + newSelection.delete(annotationName); + } else { + newSelection.add(annotationName); + } + setSelectedAnnotations(newSelection); + + if (newSelection.size < annotations.length) { + newQuery.annotationName = { $in: [...newSelection] }; + } else if (hasOwnProperty(query, 'annotationName')) { + delete newQuery.annotationName; + } + updateQuery(newQuery); + } + + function selectAllGenomes() { const newSelection = new Set(genomes.map((genome) => genome._id)); setSelectedGenomes(newSelection); @@ -66,7 +96,7 @@ function GenomeSelect({ updateQuery(newQuery); } - function unselectAll() { + function unselectAllGenomes() { const newSelection = new Set(); setSelectedGenomes(newSelection); @@ -75,7 +105,28 @@ function GenomeSelect({ updateQuery(newQuery); } + function selectAllAnnotations() { + const newSelection = new Set(genomes.flatMap((genome) => { + genome.annotationTrack.map((annotation) => annotation.name) + })); + setSelectedAnnotations(newSelection); + + const newQuery = cloneDeep(query); + newQuery.annotationName = { $in: [...newSelection] }; + updateQuery(newQuery); + } + + function unselectAllAnnotations() { + const newSelection = new Set(); + setSelectedAnnotations(newSelection); + + const newQuery = cloneDeep(query); + newQuery.annotationName = { $in: [...newSelection] }; + updateQuery(newQuery); + } + return ( + <>
+ +
+
+ + +
+
+ +
+
+
+
+ Select annotations +
+ {annotations.map((name) => { + const checked = selectedAnnotations.has(name); + return ( +
+ +
+ ); + })} +
+ -
+ ); } diff --git a/imports/ui/main/SearchBar.jsx b/imports/ui/main/SearchBar.jsx index cfad6db7..7aeba315 100644 --- a/imports/ui/main/SearchBar.jsx +++ b/imports/ui/main/SearchBar.jsx @@ -5,7 +5,7 @@ import { Meteor } from 'meteor/meteor'; import { withTracker } from 'meteor/react-meteor-data'; import React, { useState, useEffect, useRef } from 'react'; -import { Redirect, withRouter } from 'react-router-dom'; +import { Redirect, withRouter, useHistory } from 'react-router-dom'; import { cloneDeep } from 'lodash'; import { attributeCollection } from '/imports/api/genes/attributeCollection.js'; @@ -57,7 +57,7 @@ function SearchBar({ const [selectedAttributes, setSelectedAttributes] = useState( new Set(['Gene ID', ...initialSelectedAttributes]), ); - + let history = useHistory() const inputRef = useRef(); useEffect(() => { if (highLightSearch) { @@ -90,6 +90,13 @@ function SearchBar({ function submit(event) { event.preventDefault(); + if (Meteor.settings.public.redirectSearch){ + const query = new URLSearchParams(); + const searchUrl = Meteor.settings.public.redirectSearch + const searchAttr = Meteor.settings.public.redirectSearchAttribute ? Meteor.settings.public.redirectSearchAttribute : 'query' + query.set(searchAttr, searchString.trim()); + location.href = searchUrl + `?${query.toString()}` + } setRedirect(true); } @@ -99,6 +106,7 @@ function SearchBar({ if (redirect) { const query = new URLSearchParams(); + let searchUrl = "/genes" query.set('attributes', [...selectedAttributes]); query.set('search', searchString.trim()); const queryString = `?${query.toString()}`; @@ -107,7 +115,7 @@ function SearchBar({
+ {display_attr &&
@@ -135,7 +147,7 @@ function SearchBar({
+ }
+ + Annotation name + + {`${gene.annotationName} `} + + Genome coordinates diff --git a/imports/ui/singleGenePage/ProteinDomains.jsx b/imports/ui/singleGenePage/ProteinDomains.jsx index e8f8144a..27dec0c2 100644 --- a/imports/ui/singleGenePage/ProteinDomains.jsx +++ b/imports/ui/singleGenePage/ProteinDomains.jsx @@ -295,10 +295,10 @@ function NoProteinDomains({ showHeader }) { } function InterproDataTracker({ gene }) { - const interproSub = Meteor.subscribe('interpro', gene.ID); + const interproSub = Meteor.subscribe('interpro', gene); const loading = !interproSub.ready(); - const proteinDomains = interproscanCollection.find({}).fetch() + const proteinDomains = interproscanCollection.find({gene_id: gene.ID, annotationName: gene.annotationName}).fetch() return { loading, @@ -372,7 +372,7 @@ function ProteinDomains({ ); }) - let axisTransform = `translate(0,${15 + currentTranslate})` + let axisTransform = `translate(0,${15 + currentTranslate})` let gTransform = `translate(0,${40 + currentTranslate})` let data = ( diff --git a/imports/ui/singleGenePage/SingleGenePage.jsx b/imports/ui/singleGenePage/SingleGenePage.jsx index 983cce10..39e91f62 100644 --- a/imports/ui/singleGenePage/SingleGenePage.jsx +++ b/imports/ui/singleGenePage/SingleGenePage.jsx @@ -2,6 +2,7 @@ import { Meteor } from 'meteor/meteor'; import { withTracker } from 'meteor/react-meteor-data'; import React from 'react'; +import { Link } from 'react-router-dom'; import hash from 'object-hash'; import { Genes } from '/imports/api/genes/geneCollection.js'; @@ -36,24 +37,69 @@ function isLoading({ loading }) { return loading; } -function isNotFound({ gene }) { - return typeof gene === 'undefined'; +function isNotFound({ genes }) { + return genes.length === 0; } -function geneDataTracker({ match, genomeDataCache }) { +function isMultiple({ genes }) { + return genes.length > 1; +} + +function Multiple( {genes} ){ + let gene = genes[0] + + let content = genes.map(gene => { + const query = new URLSearchParams(); + query.set("annotation", gene.annotationName); + const url = `/gene/${gene.ID}?${query.toString()}` + return ( +

+ { gene.annotationName } +

+ ); + }) + + return ( +
+
+
+

+ {`${gene.ID} `} +

+
+

This gene is defined in several annotations. Please select one:

+
+ {content} +
+
+
+
+
+ ); +} + +function geneDataTracker({ match, genomeDataCache, location }) { const { geneId } = match.params; + const annotation = new URLSearchParams(location.search).get("annotation"); const geneSub = Meteor.subscribe('singleGene', { geneId }); - const gene = Genes.findOne({ ID: geneId }); + let genes + if (annotation) { + genes = Genes.find({ ID: geneId, annotationName: annotation }).fetch(); + } else { + genes = Genes.find({ ID: geneId }).fetch(); + } + const loading = !geneSub.ready(); return { loading, - gene, + genes, genomeDataCache, }; } -function genomeDataTracker({ gene, genomeDataCache }) { +function genomeDataTracker({ genes, genomeDataCache }) { // const genomeSub = Meteor.subscribe('genomes'); + let gene = genes[0] const { genomeId } = gene; let genome; let genomeSub; @@ -82,7 +128,8 @@ function SingleGenePage({ gene, genome = {} }) {

{`${gene.ID} `} - {genome.name} + {genome.name}  + {gene.annotationName}

    @@ -176,6 +223,7 @@ export default compose( withTracker(geneDataTracker), branch(isLoading, Loading), branch(isNotFound, NotFound), + branch(isMultiple, Multiple), withTracker(genomeDataTracker), branch(isLoading, Loading), )(SingleGenePage); diff --git a/imports/ui/singleGenePage/geneExpression/ExpressionPlot.jsx b/imports/ui/singleGenePage/geneExpression/ExpressionPlot.jsx index 16db79ef..6bddb97c 100644 --- a/imports/ui/singleGenePage/geneExpression/ExpressionPlot.jsx +++ b/imports/ui/singleGenePage/geneExpression/ExpressionPlot.jsx @@ -29,13 +29,14 @@ import './expressionPlot.scss'; function expressionDataTracker({ gene, samples, loading, }) { - const transcriptomeSub = Meteor.subscribe('geneExpression', gene.ID); + const transcriptomeSub = Meteor.subscribe('geneExpression', gene.ID, gene.annotationName); const sampleInfo = groupBy(samples, '_id'); const sampleIds = samples.map((sample) => sample._id); const values = Transcriptomes.find({ geneId: gene.ID, + annotationName: gene.annotationName, experimentId: { $in: sampleIds, }, @@ -82,6 +83,23 @@ function Loading() { ); } +function hasNoExpression({ values }) { + return values.length == 0; +} + +function NoExpression() { + return( +
    +
    +
    +

    No expression data for the selected samples

    +
    +
    +
    + ) +} + + function YAxis({ scale, numTicks }) { const range = scale.range(); const [start, end] = scale.domain(); @@ -406,4 +424,5 @@ export default compose( branch(hasNoSamples, NoSamples), withTracker(expressionDataTracker), branch(isLoading, Loading), + branch(hasNoExpression, NoExpression) )(ExpressionPlot); diff --git a/imports/ui/singleGenePage/geneExpression/SampleSelection.jsx b/imports/ui/singleGenePage/geneExpression/SampleSelection.jsx index e6cbe4bf..bf09162d 100644 --- a/imports/ui/singleGenePage/geneExpression/SampleSelection.jsx +++ b/imports/ui/singleGenePage/geneExpression/SampleSelection.jsx @@ -10,13 +10,17 @@ import { ExperimentInfo } from '/imports/api/transcriptomes/transcriptome_collec import { Dropdown, DropdownMenu, DropdownButton } from '/imports/ui/util/Dropdown.jsx'; +import { + branch, compose, round, /* ErrorBoundary, */ +} from '/imports/ui/util/uiUtil.jsx'; + import './sampleSelection.scss'; function dataTracker({ gene, showHeader, children }) { - const { genomeId } = gene; + const { genomeId, annotationName } = gene; const experimentSub = Meteor.subscribe('experimentInfo'); const loading = !experimentSub.ready(); - const experiments = ExperimentInfo.find({ genomeId }).fetch(); + const experiments = ExperimentInfo.find({ genomeId, annotationName }).fetch(); const replicaGroups = groupBy(experiments, 'replicaGroup'); return { showHeader, @@ -27,6 +31,31 @@ function dataTracker({ gene, showHeader, children }) { }; } + +function hasNoExpression({ experiments }) { + return experiments.length === 0; +} + +function NoExpression({ showHeader }) { + return ( + <> + { showHeader && + <> +
    +

    Gene Expression

    + + } +
    +
    +
    +

    No samples found

    +
    +
    +
    + + ); +} + const customStyles = { control: (provided) => ({ ...provided, @@ -70,10 +99,13 @@ function SampleSelection({ })); } + let className = showHeader ? "is-pulled-right" : "" + let style = showHeader ? {} : {"text-align": "right"} + return ( <> { showHeader &&
    } -
    +