From e5f9f3538fe8f33966befba196fd1c4fab710a6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Haian=20Huang=28=E6=B7=B1=E5=BA=A6=E7=9C=B8=29?=
<1286304229@qq.com>
Date: Tue, 26 Dec 2023 18:20:24 +0800
Subject: [PATCH] Update README and refine of MM-GDINO (#11298)
---
configs/glip/README.md | 9 +-
...retrain_obj365-goldg_zeroshot_flickr30k.py | 8 +-
...dino_swin-t-pretrain_zeroshot_flickr30k.py | 6 +-
configs/mm_grounding_dino/README.md | 432 +++++--
...no_swin-t_finetune_8xb4_50e_brain_tumor.py | 10 +-
...ino_swin-t_finetune_8xb4_50e_cityscapes.py | 2 +-
...ding_dino_swin-t_finetune_16xb4_1x_coco.py | 6 +-
...ino_swin-t_finetune_16xb4_1x_coco_48_17.py | 33 +-
..._dino_swin-t_finetune_16xb4_1x_sft_coco.py | 93 ++
.../dataset_prepare_zh-CN.md | 1123 +++++++++++++++++
...ounding_dino_swin-t-pretrain_flickr30k.py} | 6 +-
.../grounding_dino_swin-l_pretrain_all.py | 135 +-
...nding_dino_swin-t_finetune_8xb4_20e_cat.py | 102 ++
.../grounding_dino_swin-t_pretrain_obj365.py | 4 +-
...nding_dino_swin-t_pretrain_obj365_goldg.py | 2 +-
...ino_swin-t_pretrain_obj365_goldg_grit9m.py | 2 +-
...in-t_pretrain_obj365_goldg_grit9m_v3det.py | 2 +-
...dino_swin-t_pretrain_obj365_goldg_v3det.py | 101 ++
...ino_swin-t_pretrain_pseudo-labeling_cat.py | 43 +
...n-t_pretrain_pseudo-labeling_flickr30k.py} | 12 +-
...ding_dino_swin-t_finetune_16xb4_1x_lvis.py | 2 +-
...o_swin-t_finetune_16xb4_1x_lvis_866_337.py | 6 +-
configs/mm_grounding_dino/metafile.yml | 54 +
...-t_finetune_8xb4_50e_people_in_painting.py | 2 +-
..._dino_swin-t_finetune_8xb4_5e_grefcoco.py} | 19 +-
...g_dino_swin-t_finetune_8xb4_5e_refcoco.py} | 4 +-
...o_swin-t_finetune_8xb4_5e_refcoco_plus.py} | 4 +-
..._dino_swin-t_finetune_8xb4_5e_refcocog.py} | 38 +-
...nding_dino_swin-t_finetune_8xb4_1x_rtts.py | 2 +-
...nding_dino_swin-t_finetune_8xb4_1x_ruod.py | 2 +-
configs/mm_grounding_dino/usage_zh-CN.md | 491 +++++++
mmdet/datasets/flickr30k.py | 40 +-
mmdet/datasets/odvg.py | 5 +-
.../datasets/transforms/text_transformers.py | 3 +
mmdet/engine/hooks/__init__.py | 7 +-
mmdet/engine/hooks/visualization_hook.py | 205 ++-
mmdet/evaluation/metrics/dump_odvg_results.py | 103 +-
mmdet/evaluation/metrics/flickr30k_metric.py | 9 +-
mmdet/models/detectors/glip.py | 4 +-
model-index.yml | 1 +
projects/XDecoder/README.md | 2 +-
tests/test_models/test_detectors/test_glip.py | 22 +-
tools/analysis_tools/browse_grounding_raw.py | 14 +-
tools/dataset_converters/coco2odvg.py | 5 +-
tools/dataset_converters/coco2ovd.py | 70 +
tools/dataset_converters/fix_o365_names.py | 7 +-
tools/dataset_converters/goldg2odvg.py | 2 +-
tools/dataset_converters/lvis2ovd.py | 41 +
tools/dataset_converters/openimages2odvg.py | 2 +-
...y => remove_cocotrain2017_from_refcoco.py} | 0
....csv => zhiyuan_objv2_train_names_fix.csv} | 0
51 files changed, 2956 insertions(+), 341 deletions(-)
create mode 100644 configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_sft_coco.py
create mode 100644 configs/mm_grounding_dino/dataset_prepare_zh-CN.md
rename configs/mm_grounding_dino/flickr30k/{grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py => grounding_dino_swin-t-pretrain_flickr30k.py} (90%)
create mode 100644 configs/mm_grounding_dino/grounding_dino_swin-t_finetune_8xb4_20e_cat.py
create mode 100644 configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py
create mode 100644 configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_cat.py
rename configs/mm_grounding_dino/{grounding_dino_swin-b_pretrain_pl.py => grounding_dino_swin-t_pretrain_pseudo-labeling_flickr30k.py} (79%)
create mode 100644 configs/mm_grounding_dino/metafile.yml
rename configs/mm_grounding_dino/refcoco/{grounding_dino_swin-t_finetune_grefcoco.py => grounding_dino_swin-t_finetune_8xb4_5e_grefcoco.py} (92%)
rename configs/mm_grounding_dino/refcoco/{grounding_dino_swin-t_finetune_refcoco.py => grounding_dino_swin-t_finetune_8xb4_5e_refcoco.py} (95%)
rename configs/mm_grounding_dino/refcoco/{grounding_dino_swin-t_finetune_refcoco_plus.py => grounding_dino_swin-t_finetune_8xb4_5e_refcoco_plus.py} (95%)
rename configs/mm_grounding_dino/refcoco/{grounding_dino_swin-t_finetune_refcocog.py => grounding_dino_swin-t_finetune_8xb4_5e_refcocog.py} (82%)
create mode 100644 configs/mm_grounding_dino/usage_zh-CN.md
create mode 100644 tools/dataset_converters/coco2ovd.py
create mode 100644 tools/dataset_converters/lvis2ovd.py
rename tools/dataset_converters/{exclude_cocotrain2017_from_refcoco.py => remove_cocotrain2017_from_refcoco.py} (100%)
rename tools/dataset_converters/{objects365_v2_names_fix.csv => zhiyuan_objv2_train_names_fix.csv} (100%)
diff --git a/configs/glip/README.md b/configs/glip/README.md
index 5c3015ec011..e74e98d1b57 100644
--- a/configs/glip/README.md
+++ b/configs/glip/README.md
@@ -166,7 +166,8 @@ Learning visual representations from natural language supervision has recently s
### Results on Flickr30k
-| Model | Official | Pre-Train Data | Val R@1 | Val R@5 | Val R@10 | Test R@1 | Test R@5 | Test R@10 |
-| ------------- | -------- | -------------- | ------- | ------- | -------- | -------- | -------- | --------- |
-| **GLIP-T(C)** | ✔ | O365, GoldG | 84.8 | 94.9 | 96.3 | 85.5 | 95.4 | 96.6 |
-| **GLIP-T(C)** | | O365, GoldG | 84.9 | 94.9 | 96.3 | 85.6 | 95.4 | 96.7 |
+| Model | Official | Pre-Train Data | Val R@1 | Val R@5 | Val R@10 | Test R@1 | Test R@5 | Test R@10 |
+| ------------- | -------- | ------------------- | ------- | ------- | -------- | -------- | -------- | --------- |
+| **GLIP-T(C)** | ✔ | O365, GoldG | 84.8 | 94.9 | 96.3 | 85.5 | 95.4 | 96.6 |
+| **GLIP-T(C)** | | O365, GoldG | 84.9 | 94.9 | 96.3 | 85.6 | 95.4 | 96.7 |
+| **GLIP-T** | | O365,GoldG,CC3M,SBU | 85.3 | 95.5 | 96.9 | 86.0 | 95.9 | 97.2 |
diff --git a/configs/glip/flickr30k/glip_atss_swin-t_c_fpn_dyhead_pretrain_obj365-goldg_zeroshot_flickr30k.py b/configs/glip/flickr30k/glip_atss_swin-t_c_fpn_dyhead_pretrain_obj365-goldg_zeroshot_flickr30k.py
index c494bfcdec5..14d6e8aaa63 100644
--- a/configs/glip/flickr30k/glip_atss_swin-t_c_fpn_dyhead_pretrain_obj365-goldg_zeroshot_flickr30k.py
+++ b/configs/glip/flickr30k/glip_atss_swin-t_c_fpn_dyhead_pretrain_obj365-goldg_zeroshot_flickr30k.py
@@ -2,10 +2,10 @@
lang_model_name = 'bert-base-uncased'
-model = dict(bbox_head=dict(early_fuse=True), )
+model = dict(bbox_head=dict(early_fuse=True))
dataset_type = 'Flickr30kDataset'
-data_root = 'data/flickr30k/'
+data_root = 'data/flickr30k_entities/'
test_pipeline = [
dict(
@@ -27,7 +27,7 @@
dataset_Flickr30k_val = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_val.json',
+ ann_file='final_flickr_separateGT_val.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
@@ -35,7 +35,7 @@
dataset_Flickr30k_test = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_test.json',
+ ann_file='final_flickr_separateGT_test.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
diff --git a/configs/grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py b/configs/grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py
index e2df152fef4..c1996567588 100644
--- a/configs/grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py
+++ b/configs/grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py
@@ -1,7 +1,7 @@
_base_ = '../grounding_dino_swin-t_pretrain_obj365_goldg_cap4m.py'
dataset_type = 'Flickr30kDataset'
-data_root = 'data/flickr30k/'
+data_root = 'data/flickr30k_entities/'
test_pipeline = [
dict(
@@ -23,7 +23,7 @@
dataset_Flickr30k_val = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_val.json',
+ ann_file='final_flickr_separateGT_val.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
@@ -31,7 +31,7 @@
dataset_Flickr30k_test = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_test.json',
+ ann_file='final_flickr_separateGT_test.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
diff --git a/configs/mm_grounding_dino/README.md b/configs/mm_grounding_dino/README.md
index 346dd97cd51..eda2c1da5f1 100644
--- a/configs/mm_grounding_dino/README.md
+++ b/configs/mm_grounding_dino/README.md
@@ -1,147 +1,353 @@
-# Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection
-
-[Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://arxiv.org/abs/2303.05499)
+# MM Grounding DINO
## Abstract
-In this paper, we present an open-set object detector, called Grounding DINO, by marrying Transformer-based detector DINO with grounded pre-training, which can detect arbitrary objects with human inputs such as category names or referring expressions. The key solution of open-set object detection is introducing language to a closed-set detector for open-set concept generalization. To effectively fuse language and vision modalities, we conceptually divide a closed-set detector into three phases and propose a tight fusion solution, which includes a feature enhancer, a language-guided query selection, and a cross-modality decoder for cross-modality fusion. While previous works mainly evaluate open-set object detection on novel categories, we propose to also perform evaluations on referring expression comprehension for objects specified with attributes. Grounding DINO performs remarkably well on all three settings, including benchmarks on COCO, LVIS, ODinW, and RefCOCO/+/g. Grounding DINO achieves a 52.5 AP on the COCO detection zero-shot transfer benchmark, i.e., without any training data from COCO. It sets a new record on the ODinW zero-shot benchmark with a mean 26.1 AP.
+TODO
+
+## Dataset Preparation
+
+Please refer to [dataset_prepare.md](dataset_prepare.md) or [中文版数据准备](dataset_prepare_zh-CN.md)
+
+## Usage
-
-
-
(O365,GoldG,Cap4M) | GroundingDINO-T-V2
(O365,GoldG) | GroundingDINO-T-V2
(O365,GoldG,GRIT) | GroundingDINO-T-V2
(O365,GoldG,GRIT,V3Det) |
-| --------------------- | ---------------------------------------- | ------------------------------------- | ------------------------------------------ | ------------------------------------------------ |
-| AerialMaritimeDrone | 0.173 | 0.133 | 0.155 | 0.151 |
-| Aquarium | 0.195 | 0.252 | 0.261 | 0.283 |
-| CottontailRabbits | 0.799 | 0.771 | 0.810 | 0.786 |
-| EgoHands | 0.608 | 0.499 | 0.537 | 0.519 |
-| NorthAmericaMushrooms | 0.507 | 0.331 | 0.462 | 0.767 |
-| Packages | 0.687 | 0.707 | 0.687 | 0.706 |
-| PascalVOC | 0.563 | 0.565 | 0.580 | 0.566 |
-| pistols | 0.726 | 0.585 | 0.709 | 0.729 |
-| pothole | 0.215 | 0.136 | 0.285 | 0.243 |
-| Raccoon | 0.549 | 0.469 | 0.511 | 0.535 |
-| ShellfishOpenImages | 0.393 | 0.321 | 0.437 | 0.488 |
-| thermalDogsAndPeople | 0.657 | 0.556 | 0.603 | 0.542 |
-| VehiclesOpenImages | 0.613 | 0.566 | 0.603 | 0.615 |
-| Average | **0.514** | **0.453** | **0.511** | **0.533** |
+| Method | GDINO-T
(O365,GoldG,Cap4M) | MM-GDINO-T
(O365,GoldG) | MM-GDINO-T
(O365,GoldG,GRIT) | MM-GDINO-T
(O365,GoldG,V3Det) | MM-GDINO-T
(O365,GoldG,GRIT,V3Det) |
+| --------------------- | -------------------------------- | ----------------------------- | ---------------------------------- | ----------------------------------- | ---------------------------------------- |
+| AerialMaritimeDrone | 0.173 | 0.133 | 0.155 | 0.177 | 0.151 |
+| Aquarium | 0.195 | 0.252 | 0.261 | 0.266 | 0.283 |
+| CottontailRabbits | 0.799 | 0.771 | 0.810 | 0.778 | 0.786 |
+| EgoHands | 0.608 | 0.499 | 0.537 | 0.506 | 0.519 |
+| NorthAmericaMushrooms | 0.507 | 0.331 | 0.462 | 0.669 | 0.767 |
+| Packages | 0.687 | 0.707 | 0.687 | 0.710 | 0.706 |
+| PascalVOC | 0.563 | 0.565 | 0.580 | 0.556 | 0.566 |
+| pistols | 0.726 | 0.585 | 0.709 | 0.671 | 0.729 |
+| pothole | 0.215 | 0.136 | 0.285 | 0.199 | 0.243 |
+| Raccoon | 0.549 | 0.469 | 0.511 | 0.553 | 0.535 |
+| ShellfishOpenImages | 0.393 | 0.321 | 0.437 | 0.519 | 0.488 |
+| thermalDogsAndPeople | 0.657 | 0.556 | 0.603 | 0.493 | 0.542 |
+| VehiclesOpenImages | 0.613 | 0.566 | 0.603 | 0.614 | 0.615 |
+| Average | **0.514** | **0.453** | **0.511** | **0.516** | **0.533** |
+
+- The MM-GDINO-T config file is [odinw13](odinw/grounding_dino_swin-t_pretrain_odinw13.py)
### Results and models of ODinW35
-| Method | GroundingDINO-T
(O365,GoldG,Cap4M) | GroundingDINO-T-V2
(O365,GoldG) | GroundingDINO-T-V2
(O365,GoldG,GRIT) | GroundingDINO-T-V2
(O365,GoldG,GRIT,V3Det) |
-| --------------------------- | ---------------------------------------- | ------------------------------------- | ------------------------------------------ | ------------------------------------------------ |
-| AerialMaritimeDrone_large | 0.173 | 0.133 | 0.155 | 0.151 |
-| AerialMaritimeDrone_tiled | 0.206 | 0.170 | 0.225 | 0.206 |
-| AmericanSignLanguageLetters | 0.002 | 0.016 | 0.020 | 0.007 |
-| Aquarium | 0.195 | 0.252 | 0.261 | 0.283 |
-| BCCD | 0.161 | 0.069 | 0.118 | 0.077 |
-| boggleBoards | 0.000 | 0.002 | 0.001 | 0.002 |
-| brackishUnderwater | 0.021 | 0.033 | 0.021 | 0.025 |
-| ChessPieces | 0.000 | 0.000 | 0.000 | 0.000 |
-| CottontailRabbits | 0.806 | 0.771 | 0.810 | 0.786 |
-| dice | 0.004 | 0.002 | 0.005 | 0.001 |
-| DroneControl | 0.042 | 0.047 | 0.097 | 0.074 |
-| EgoHands_generic | 0.608 | 0.527 | 0.537 | 0.519 |
-| EgoHands_specific | 0.002 | 0.001 | 0.005 | 0.003 |
-| HardHatWorkers | 0.046 | 0.048 | 0.070 | 0.108 |
-| MaskWearing | 0.004 | 0.009 | 0.004 | 0.009 |
-| MountainDewCommercial | 0.430 | 0.453 | 0.465 | 0.430 |
-| NorthAmericaMushrooms | 0.471 | 0.331 | 0.462 | 0.767 |
-| openPoetryVision | 0.000 | 0.001 | 0.000 | 0.000 |
-| OxfordPets_by_breed | 0.003 | 0.002 | 0.004 | 0.004 |
-| OxfordPets_by_species | 0.011 | 0.019 | 0.016 | 0.015 |
-| PKLot | 0.001 | 0.004 | 0.002 | 0.007 |
-| Packages | 0.695 | 0.707 | 0.687 | 0.706 |
-| PascalVOC | 0.563 | 0.565 | 0.580 | 0.566 |
-| pistols | 0.726 | 0.585 | 0.709 | 0.729 |
-| plantdoc | 0.005 | 0.005 | 0.007 | 0.011 |
-| pothole | 0.215 | 0.136 | 0.219 | 0.168 |
-| Raccoons | 0.549 | 0.469 | 0.511 | 0.535 |
-| selfdrivingCar | 0.089 | 0.091 | 0.076 | 0.083 |
-| ShellfishOpenImages | 0.393 | 0.321 | 0.437 | 0.488 |
-| ThermalCheetah | 0.087 | 0.063 | 0.081 | 0.045 |
-| thermalDogsAndPeople | 0.657 | 0.556 | 0.603 | 0.543 |
-| UnoCards | 0.006 | 0.012 | 0.010 | 0.005 |
-| VehiclesOpenImages | 0.613 | 0.566 | 0.603 | 0.615 |
-| WildfireSmoke | 0.134 | 0.106 | 0.154 | 0.127 |
-| websiteScreenshots | 0.012 | 0.02 | 0.016 | 0.016 |
-| Average | **0.227** | **0.202** | **0.228** | **0.284** |
-
-## Referring Expression Comprehension Results
-
-| Method | GroundingDINO-T
(O365,GoldG,Cap4M) | GroundingDINO-T-V2
(O365,GoldG) | GroundingDINO-T-V2
(O365,GoldG,GRIT) | GroundingDINO-T-V2
(O365,GoldG,GRIT,V3Det) |
-| --------------------------------------- | ---------------------------------------- | ------------------------------------- | ------------------------------------------ | ------------------------------------------------ |
-| RefCOCO val @1,5,10 | 50.77/89.45/94.86 | 53.06/89.91/94.69 | 53.4/90.3/95.5 | 53.1/89.7/95.1 |
-| RefCOCO testA @1,5,10 | 57.45/91.29/95.62 | 59.70/91.50/95.88 | 58.8/91.70/96.2 | 59.1/91.0/95.5 |
-| RefCOCO testB @1,5,10 | 44.97/86.54/92.88 | 46.38/86.87/92.21 | 46.8/87.7/93.3 | 46.8/87.8/93.6 |
-| RefCOCO+ val @1,5,10 | 51.64/86.35/92.57 | 53.11/87.00/92.79 | 53.5/88.00/93.7 | 52.7/87.7/93.5 |
-| RefCOCO+ testA @1,5,10 | 57.25/86.74/92.65 | 58.94/87.34/92.91 | 59.0/88.1/93.7 | 58.7/87.2/93.1 |
-| RefCOCO+ testB @1,5,10 | 46.35/84.05/90.67 | 47.92/84.31/91.04 | 47.9/85.5/92.7 | 48.4/85.8/92.1 |
-| RefCOCOg val @1,5,10 | 60.42/92.10/96.18 | 61.23/92.61/96.14 | 62.7/93.3/97.0 | 62.9/93.3/97.2 |
-| RefCOCOg test @1,5,10 | 59.74/92.08/96.28 | 61.13/93.26/96.72 | 62.6/94.9/97.1 | 62.9/93.9/97.43 |
-| gRefCOCO val Pr@(F1=1, IoU≥0.5),N-acc | 41.32/91.82 | 39.76/84.65 | 40.7/89.7 | 41.0/91.3 |
-| gRefCOCO testA Pr@(F1=1, IoU≥0.5),N-acc | 27.23/90.24 | 26.25/89.04 | 26.0/91.9 | 26.1/93.0 |
-| gRefCOCO testB Pr@(F1=1, IoU≥0.5),N-acc | 29.70/93.49 | 31.31/84.79 | 30.6/90.2 | 30.4/92.3 |
-
-## Description Detection Dataset
+| Method | GDINO-T
(O365,GoldG,Cap4M) | MM-GDINO-T
(O365,GoldG) | MM-GDINO-T
(O365,GoldG,GRIT) | MM-GDINO-T
(O365,GoldG,V3Det) | MM-GDINO-T
(O365,GoldG,GRIT,V3Det) |
+| --------------------------- | -------------------------------- | ----------------------------- | ---------------------------------- | ----------------------------------- | ---------------------------------------- |
+| AerialMaritimeDrone_large | 0.173 | 0.133 | 0.155 | 0.177 | 0.151 |
+| AerialMaritimeDrone_tiled | 0.206 | 0.170 | 0.225 | 0.184 | 0.206 |
+| AmericanSignLanguageLetters | 0.002 | 0.016 | 0.020 | 0.011 | 0.007 |
+| Aquarium | 0.195 | 0.252 | 0.261 | 0.266 | 0.283 |
+| BCCD | 0.161 | 0.069 | 0.118 | 0.083 | 0.077 |
+| boggleBoards | 0.000 | 0.002 | 0.001 | 0.001 | 0.002 |
+| brackishUnderwater | 0.021 | 0.033 | 0.021 | 0.025 | 0.025 |
+| ChessPieces | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 |
+| CottontailRabbits | 0.806 | 0.771 | 0.810 | 0.778 | 0.786 |
+| dice | 0.004 | 0.002 | 0.005 | 0.001 | 0.001 |
+| DroneControl | 0.042 | 0.047 | 0.097 | 0.088 | 0.074 |
+| EgoHands_generic | 0.608 | 0.527 | 0.537 | 0.506 | 0.519 |
+| EgoHands_specific | 0.002 | 0.001 | 0.005 | 0.007 | 0.003 |
+| HardHatWorkers | 0.046 | 0.048 | 0.070 | 0.070 | 0.108 |
+| MaskWearing | 0.004 | 0.009 | 0.004 | 0.011 | 0.009 |
+| MountainDewCommercial | 0.430 | 0.453 | 0.465 | 0.194 | 0.430 |
+| NorthAmericaMushrooms | 0.471 | 0.331 | 0.462 | 0.669 | 0.767 |
+| openPoetryVision | 0.000 | 0.001 | 0.000 | 0.000 | 0.000 |
+| OxfordPets_by_breed | 0.003 | 0.002 | 0.004 | 0.006 | 0.004 |
+| OxfordPets_by_species | 0.011 | 0.019 | 0.016 | 0.020 | 0.015 |
+| PKLot | 0.001 | 0.004 | 0.002 | 0.008 | 0.007 |
+| Packages | 0.695 | 0.707 | 0.687 | 0.710 | 0.706 |
+| PascalVOC | 0.563 | 0.565 | 0.580 | 0.566 | 0.566 |
+| pistols | 0.726 | 0.585 | 0.709 | 0.671 | 0.729 |
+| plantdoc | 0.005 | 0.005 | 0.007 | 0.008 | 0.011 |
+| pothole | 0.215 | 0.136 | 0.219 | 0.077 | 0.168 |
+| Raccoons | 0.549 | 0.469 | 0.511 | 0.553 | 0.535 |
+| selfdrivingCar | 0.089 | 0.091 | 0.076 | 0.094 | 0.083 |
+| ShellfishOpenImages | 0.393 | 0.321 | 0.437 | 0.519 | 0.488 |
+| ThermalCheetah | 0.087 | 0.063 | 0.081 | 0.030 | 0.045 |
+| thermalDogsAndPeople | 0.657 | 0.556 | 0.603 | 0.493 | 0.543 |
+| UnoCards | 0.006 | 0.012 | 0.010 | 0.009 | 0.005 |
+| VehiclesOpenImages | 0.613 | 0.566 | 0.603 | 0.614 | 0.615 |
+| WildfireSmoke | 0.134 | 0.106 | 0.154 | 0.042 | 0.127 |
+| websiteScreenshots | 0.012 | 0.02 | 0.016 | 0.016 | 0.016 |
+| Average | **0.227** | **0.202** | **0.228** | **0.214** | **0.284** |
+
+- The MM-GDINO-T config file is [odinw35](odinw/grounding_dino_swin-t_pretrain_odinw35.py)
+
+## Zero-Shot Referring Expression Comprehension Results
+
+| Method | GDINO-T
(O365,GoldG,Cap4M) | MM-GDINO-T
(O365,GoldG) | MM-GDINO-T
(O365,GoldG,GRIT) | MM-GDINO-T
(O365,GoldG,V3Det) | MM-GDINO-T
(O365,GoldG,GRIT,V3Det) |
+| ---------------------- | -------------------------------- | ----------------------------- | ---------------------------------- | ----------------------------------- | ---------------------------------------- |
+| RefCOCO val @1,5,10 | 50.8/89.5/94.9 | 53.1/89.9/94.7 | 53.4/90.3/95.5 | 52.1/89.8/95.0 | 53.1/89.7/95.1 |
+| RefCOCO testA @1,5,10 | 57.4/91.3/95.6 | 59.7/91.5/95.9 | 58.8/91.70/96.2 | 58.4/86.8/95.6 | 59.1/91.0/95.5 |
+| RefCOCO testB @1,5,10 | 45.0/86.5/92.9 | 46.4/86.9/92.2 | 46.8/87.7/93.3 | 45.4/86.2/92.6 | 46.8/87.8/93.6 |
+| RefCOCO+ val @1,5,10 | 51.6/86.4/92.6 | 53.1/87.0/92.8 | 53.5/88.0/93.7 | 52.5/86.8/93.2 | 52.7/87.7/93.5 |
+| RefCOCO+ testA @1,5,10 | 57.3/86.7/92.7 | 58.9/87.3/92.9 | 59.0/88.1/93.7 | 58.1/86.7/93.5 | 58.7/87.2/93.1 |
+| RefCOCO+ testB @1,5,10 | 46.4/84.1/90.7 | 47.9/84.3/91.0 | 47.9/85.5/92.7 | 46.9/83.7/91.5 | 48.4/85.8/92.1 |
+| RefCOCOg val @1,5,10 | 60.4/92.1/96.2 | 61.2/92.6/96.1 | 62.7/93.3/97.0 | 61.7/92.9/96.6 | 62.9/93.3/97.2 |
+| RefCOCOg test @1,5,10 | 59.7/92.1/96.3 | 61.1/93.3/96.7 | 62.6/94.9/97.1 | 61.0/93.1/96.8 | 62.9/93.9/97.4 |
+
+| Method | thresh_score | GDINO-T
(O365,GoldG,Cap4M) | MM-GDINO-T
(O365,GoldG) | MM-GDINO-T
(O365,GoldG,GRIT) | MM-GDINO-T
(O365,GoldG,V3Det) | MM-GDINO-T
(O365,GoldG,GRIT,V3Det) |
+| --------------------------------------- | ------------ | -------------------------------- | ----------------------------- | ---------------------------------- | ----------------------------------- | ---------------------------------------- |
+| gRefCOCO val Pr@(F1=1, IoU≥0.5),N-acc | 0.5 | 39.3/70.4 | | | | 39.4/67.5 |
+| gRefCOCO val Pr@(F1=1, IoU≥0.5),N-acc | 0.6 | 40.5/83.8 | | | | 40.6/83.1 |
+| gRefCOCO val Pr@(F1=1, IoU≥0.5),N-acc | 0.7 | 41.3/91.8 | 39.8/84.7 | 40.7/89.7 | 40.3/88.8 | 41.0/91.3 |
+| gRefCOCO val Pr@(F1=1, IoU≥0.5),N-acc | 0.8 | 41.5/96.8 | | | | 41.1/96.4 |
+| gRefCOCO testA Pr@(F1=1, IoU≥0.5),N-acc | 0.5 | 31.9/70.4 | | | | 33.1/69.5 |
+| gRefCOCO testA Pr@(F1=1, IoU≥0.5),N-acc | 0.6 | 29.3/82.9 | | | | 29.2/84.3 |
+| gRefCOCO testA Pr@(F1=1, IoU≥0.5),N-acc | 0.7 | 27.2/90.2 | 26.3/89.0 | 26.0/91.9 | 25.4/91.8 | 26.1/93.0 |
+| gRefCOCO testA Pr@(F1=1, IoU≥0.5),N-acc | 0.8 | 25.1/96.3 | | | | 23.8/97.2 |
+| gRefCOCO testB Pr@(F1=1, IoU≥0.5),N-acc | 0.5 | 30.9/72.5 | | | | 33.0/69.6 |
+| gRefCOCO testB Pr@(F1=1, IoU≥0.5),N-acc | 0.6 | 30.0/86.1 | | | | 31.6/96.7 |
+| gRefCOCO testB Pr@(F1=1, IoU≥0.5),N-acc | 0.7 | 29.7/93.5 | 31.3/84.8 | 30.6/90.2 | 30.7/89.9 | 30.4/92.3 |
+| gRefCOCO testB Pr@(F1=1, IoU≥0.5),N-acc | 0.8 | 29.1/97.4 | | | | 29.5/84.2 |
+
+- The MM-GDINO-T config file is [here](refcoco/grounding_dino_swin-t_pretrain_zeroshot_refexp.py)
+
+## Zero-Shot Description Detection Dataset(DOD)
```shell
pip install ddd-dataset
```
-| Method | mode | Grounding DINO-T
(O365,GoldG,Cap4M) | GroundingDINO-T-V2
(O365,GoldG) | GroundingDINO-T-V2
(O365,GoldG,GRIT) | GroundingDINO-T-V2
(O365,GoldG,GRIT,V3Det) |
-| -------------------------------- | -------- | ----------------------------------------- | ------------------------------------- | ------------------------------------------ | ------------------------------------------------ |
-| FULL/short/middle/long/very long | concat | 17.2/18.0/18.7/14.8/16.3 | 15.6/17.3/16.7/14.3/13.1 | 17.0/17.7/18.0/15.7/15.7 | 17.5/23.4/18.3/14.7/13.8 |
-| FULL/short/middle/long/very long | parallel | 22.3/28.2/24.8/19.1/13.9 | | 22.5/25.6/25.1/20.5/14.9 | 22.9/28.1/25.4/20.4/14.4 |
-| PRES/short/middle/long/very long | concat | 17.8/18.3/19.2/15.2/17.3 | 16.4/18.4/17.3/14.5/14.2 | 17.9/19.0/18.3/16.5/17.5 | 18.0/23.7/18.6/15.4/13.3 |
-| PRES/short/middle/long/very long | parallel | 21.0/27.0/22.8/17.5/12.5 | | 21.5/25.2/23.0/19.0/15.0 | 21.9/27.4/23.2/19.1/14.2 |
-| ABS/short/middle/long/very long | concat | 15.4/17.1/16.4/13.6/14.9 | 13.4/13.4/14.5/13.5/11.9 | 14.5/13.1/16.7/13.6/13.3 | 15.9/22.2/17.1/12.5/14.4 |
-| ABS/short/middle/long/very long | parallel | 26.0/32.0/33.0/23.6/15.5 | | 25.6/26.8/33.9/24.5/14.7 | 26.0/30.3/34.1/23.9/14.6 |
+| Method | mode | GDINO-T
(O365,GoldG,Cap4M) | MM-GDINO-T
(O365,GoldG) | MM-GDINO-T
(O365,GoldG,GRIT) | MM-GDINO-T
(O365,GoldG,V3Det) | MM-GDINO-T
(O365,GoldG,GRIT,V3Det) |
+| -------------------------------- | -------- | -------------------------------- | ----------------------------- | ---------------------------------- | ----------------------------------- | ---------------------------------------- |
+| FULL/short/middle/long/very long | concat | 17.2/18.0/18.7/14.8/16.3 | 15.6/17.3/16.7/14.3/13.1 | 17.0/17.7/18.0/15.7/15.7 | 16.2/17.4/16.8/14.9/15.4 | 17.5/23.4/18.3/14.7/13.8 |
+| FULL/short/middle/long/very long | parallel | 22.3/28.2/24.8/19.1/13.9 | 21.7/24.7/24.0/20.2/13.7 | 22.5/25.6/25.1/20.5/14.9 | 22.3/25.6/24.5/20.6/14.7 | 22.9/28.1/25.4/20.4/14.4 |
+| PRES/short/middle/long/very long | concat | 17.8/18.3/19.2/15.2/17.3 | 16.4/18.4/17.3/14.5/14.2 | 17.9/19.0/18.3/16.5/17.5 | 16.6/18.8/17.1/15.1/15.0 | 18.0/23.7/18.6/15.4/13.3 |
+| PRES/short/middle/long/very long | parallel | 21.0/27.0/22.8/17.5/12.5 | 21.3/25.5/22.8/19.2/12.9 | 21.5/25.2/23.0/19.0/15.0 | 21.6/25.7/23.0/19.5/14.8 | 21.9/27.4/23.2/19.1/14.2 |
+| ABS/short/middle/long/very long | concat | 15.4/17.1/16.4/13.6/14.9 | 13.4/13.4/14.5/13.5/11.9 | 14.5/13.1/16.7/13.6/13.3 | 14.8/12.5/15.6/14.3/15.8 | 15.9/22.2/17.1/12.5/14.4 |
+| ABS/short/middle/long/very long | parallel | 26.0/32.0/33.0/23.6/15.5 | 22.8/22.2/28.7/22.9/14.7 | 25.6/26.8/33.9/24.5/14.7 | 24.1/24.9/30.7/23.8/14.7 | 26.0/30.3/34.1/23.9/14.6 |
Note:
1. Considering that the evaluation time for Inter-scenario is very long and the performance is low, it is temporarily not supported. The mentioned metrics are for Intra-scenario.
2. `concat` is the default inference mode for Grounding DINO, where it concatenates multiple sub-sentences with "." to form a single sentence for inference. On the other hand, "parallel" performs inference on each sub-sentence in a for-loop.
+3. The MM-GDINO-T config file is [concat_dod](dod/grounding_dino_swin-t_pretrain_zeroshot_concat_dod.py) and [parallel_dod](dod/grounding_dino_swin-t_pretrain_zeroshot_parallel_dod.py)
-## Flickr30k Results
+## Pretrain Flickr30k Results
-| Model | Pre-Train Data | Val R@1 | Val R@5 | Val R@10 | Tesst R@1 | Test R@5 | Test R@10 | Config | Download |
-| :-----------------: | :-------------------: | ------- | ------- | -------- | --------- | -------- | --------- | :-------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
-| Grounding DINO-T | O365,GoldG,Cap4M | 87.8 | 96.6 | 98.0 | 88.1 | 96.9 | 98.2 | [config](grounding_dino_swin-t_finetune_16xb2_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v3.0/grounding_dino/grounding_dino_swin-t_finetune_16xb2_1x_coco/grounding_dino_swin-t_finetune_16xb2_1x_coco_20230921_152544-5f234b20.pth) \| [log](https://download.openmmlab.com/mmdetection/v3.0/grounding_dino/grounding_dino_swin-t_finetune_16xb2_1x_coco/grounding_dino_swin-t_finetune_16xb2_1x_coco_20230921_152544.log.json) |
-| Grounding DINO-T-V2 | O365,GoldG | | | | | | | | |
-| Grounding DINO-T-V2 | O365,GoldG,GRIT | | | | | | | | |
-| Grounding DINO-T-V2 | O365,GoldG,GRIT,V3Det | | | | | | | | |
+| Model | Pre-Train Data | Val R@1 | Val R@5 | Val R@10 | Test R@1 | Test R@5 | Test R@10 |
+| :--------: | :-------------------: | ------- | ------- | -------- | -------- | -------- | --------- |
+| GLIP-T | O365,GoldG | 84.9 | 94.9 | 96.3 | 85.6 | 95.4 | 96.7 |
+| GLIP-T | O365,GoldG,CC3M,SBU | 85.3 | 95.5 | 96.9 | 86.0 | 95.9 | 97.2 |
+| GDINO-T | O365,GoldG,Cap4M | 87.8 | 96.6 | 98.0 | 88.1 | 96.9 | 98.2 |
+| MM-GDINO-T | O365,GoldG | 85.5 | 95.6 | 97.2 | 86.2 | 95.7 | 97.4 |
+| MM-GDINO-T | O365,GoldG,GRIT | 86.7 | 95.8 | 97.6 | 87.0 | 96.2 | 97.7 |
+| MM-GDINO-T | O365,GoldG,V3Det | 85.9 | 95.7 | 97.4 | 86.3 | 95.7 | 97.4 |
+| MM-GDINO-T | O365,GoldG,GRIT,V3Det | 86.7 | 96.0 | 97.6 | 87.2 | 96.2 | 97.7 |
Note:
1. `@1,5,10` refers to precision at the top 1, 5, and 10 positions in a predicted ranked list.
+2. The MM-GDINO-T config file is [here](flickr30k/grounding_dino_swin-t-pretrain_flickr30k.py)
+
+## Validating the generalization of a pre-trained model through fine-tuning
+
+### RTTS
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | ------- | -------- |
+| Faster R-CNN | R-50 | 1x | 48.1 |
+| Cascade R-CNN | R-50 | 1x | 50.8 |
+| ATSS | R-50 | 1x | 48.2 |
+| TOOD | R-50 | 1X | 50.8 |
+| MM-GDINO(zero-shot) | Swin-T | | 49.8 |
+| MM-GDINO | Swin-T | 1x | **69.1** |
+
+- The reference metrics come from https://github.com/BIGWangYuDong/lqit/tree/main/configs/detection/rtts_dataset
+- The MM-GDINO-T config file is [here](rtts/grounding_dino_swin-t_finetune_8xb4_1x_rtts.py)
+
+### RUOD
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | ------- | -------- |
+| Faster R-CNN | R-50 | 1x | 52.4 |
+| Cascade R-CNN | R-50 | 1x | 55.3 |
+| ATSS | R-50 | 1x | 55.7 |
+| TOOD | R-50 | 1X | 57.4 |
+| MM-GDINO(zero-shot) | Swin-T | | 29.8 |
+| MM-GDINO | Swin-T | 1x | **65.5** |
+
+- The reference metrics come from https://github.com/BIGWangYuDong/lqit/tree/main/configs/detection/ruod_dataset
+- The MM-GDINO-T config file is [here](ruod/grounding_dino_swin-t_finetune_8xb4_1x_ruod.py)
+
+### Brain Tumor
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------: | :------: | ------- | ------ |
+| Faster R-CNN | R-50 | 50e | 43.5 |
+| Cascade R-CNN | R-50 | 50e | 46.2 |
+| DINO | R-50 | 50e | 46.4 |
+| Cascade-DINO | R-50 | 50e | 48.6 |
+| MM-GDINO | Swin-T | 50e | 47.5 |
+
+- The reference metrics come from https://arxiv.org/abs/2307.11035
+- The MM-GDINO-T config file is [here](brain_tumor/grounding_dino_swin-t_finetune_8xb4_50e_brain_tumor.py)
+
+### Cityscapes
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | ------- | -------- |
+| Faster R-CNN | R-50 | 50e | 30.1 |
+| Cascade R-CNN | R-50 | 50e | 31.8 |
+| DINO | R-50 | 50e | 34.5 |
+| Cascade-DINO | R-50 | 50e | 34.8 |
+| MM-GDINO(zero-shot) | Swin-T | | 34.2 |
+| MM-GDINO | Swin-T | 50e | **51.5** |
+
+- The reference metrics come from https://arxiv.org/abs/2307.11035
+- The MM-GDINO-T config file is [here](cityscapes/grounding_dino_swin-t_finetune_8xb4_50e_cityscapes.py)
+
+### People in Painting
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | ------- | -------- |
+| Faster R-CNN | R-50 | 50e | 17.0 |
+| Cascade R-CNN | R-50 | 50e | 18.0 |
+| DINO | R-50 | 50e | 12.0 |
+| Cascade-DINO | R-50 | 50e | 13.4 |
+| MM-GDINO(zero-shot) | Swin-T | | 23.1 |
+| MM-GDINO | Swin-T | 50e | **38.9** |
+
+- The reference metrics come from https://arxiv.org/abs/2307.11035
+- The MM-GDINO-T config file is [here](people_in_painting/grounding_dino_swin-t_finetune_8xb4_50e_people_in_painting.py)
+
+### COCO
+
+**(1) Closed-set performance**
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | ------- | ------ |
+| Faster R-CNN | R-50 | 1x | 37.4 |
+| Cascade R-CNN | R-50 | 1x | 40.3 |
+| ATSS | R-50 | 1x | 39.4 |
+| TOOD | R-50 | 1X | 42.4 |
+| DINO | R-50 | 1X | 50.1 |
+| GLIP(zero-shot) | Swin-T | | 46.6 |
+| GDINO(zero-shot) | Swin-T | | 48.5 |
+| MM-GDINO(zero-shot) | Swin-T | | 50.4 |
+| GLIP | Swin-T | 1x | 55.4 |
+| GDINO | Swin-T | 1x | 58.1 |
+| MM-GDINO | Swin-T | 1x | 58.2 |
+
+- The MM-GDINO-T config file is [here](coco/grounding_dino_swin-t_finetune_16xb4_1x_coco.py)
+
+**(2) Open-set continuing pretraining performance**
+
+| Architecture | Backbone | Lr schd | box AP |
+| :-----------------: | :------: | :-----: | :----: |
+| GLIP(zero-shot) | Swin-T | | 46.7 |
+| GDINO(zero-shot) | Swin-T | | 48.5 |
+| MM-GDINO(zero-shot) | Swin-T | | 50.4 |
+| MM-GDINO | Swin-T | 1x | 54.7 |
+
+- The MM-GDINO-T config file is [here](coco/grounding_dino_swin-t_finetune_16xb4_1x_sft_coco.py)
+- Due to the small size of the COCO dataset, continuing pretraining solely on COCO can easily lead to overfitting. The results shown above are from the third epoch. I do not recommend you train using this approach.
+
+**(3) Open vocabulary performance**
+
+| Architecture | Backbone | Lr schd | box AP | Base box AP | Novel box AP | box AP@50 | Base box AP@50 | Novel box AP@50 |
+| :-----------------: | :------: | :-----: | :----: | :---------: | :----------: | :-------: | :------------: | :-------------: |
+| MM-GDINO(zero-shot) | Swin-T | | 51.1 | 48.4 | 58.9 | 66.7 | 64.0 | 74.2 |
+| MM-GDINO | Swin-T | 1x | 57.2 | 56.1 | 60.4 | 73.6 | 73.0 | 75.3 |
+
+- The MM-GDINO-T config file is [here](coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py)
+
+### LVIS 1.0
+
+**(1) Open-set continuing pretraining performance**
+
+| Architecture | Backbone | Lr schd | MiniVal APr | MiniVal APc | MiniVal APf | MiniVal AP | Val1.0 APr | Val1.0 APc | Val1.0 APf | Val1.0 AP |
+| :-----------------: | :------: | :-----: | :---------: | :---------: | :---------: | :--------: | :--------: | :--------: | :--------: | :-------: |
+| GLIP(zero-shot) | Swin-T | | 18.1 | 21.2 | 33.1 | 26.7 | 10.8 | 14.7 | 29.0 | 19.6 |
+| GDINO(zero-shot) | Swin-T | | 18.8 | 24.2 | 34.7 | 28.8 | 10.1 | 15.3 | 29.9 | 20.1 |
+| MM-GDINO(zero-shot) | Swin-T | | 34.2 | 37.4 | 46.2 | 41.4 | 23.6 | 27.6 | 40.5 | 31.9 |
+| MM-GDINO | Swin-T | 1x | 50.7 | 58.8 | 60.1 | 58.7 | 45.2 | 50.2 | 56.1 | 51.7 |
+
+- The MM-GDINO-T config file is [here](lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py)
+
+**(2) Open vocabulary performance**
+
+| Architecture | Backbone | Lr schd | MiniVal APr | MiniVal APc | MiniVal APf | MiniVal AP |
+| :-----------------: | :------: | :-----: | :---------: | :---------: | :---------: | :--------: |
+| MM-GDINO(zero-shot) | Swin-T | | 34.2 | 37.4 | 46.2 | 41.4 |
+| MM-GDINO | Swin-T | 1x | 43.2 | 57.4 | 59.3 | 57.1 |
+
+- The MM-GDINO-T config file is [here](lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py)
+
+### RefEXP
+
+#### RefCOCO
+
+| Architecture | Backbone | Lr schd | val @1 | val @5 | val @10 | testA @1 | testA @5 | testA @10 | testB @1 | testB @5 | testB @10 |
+| :-----------------: | :------: | :-----: | :----: | :----: | :-----: | :------: | :------: | :-------: | :------: | :------: | :-------: |
+| GDINO(zero-shot) | Swin-T | | 50.8 | 89.5 | 94.9 | 57.5 | 91.3 | 95.6 | 45.0 | 86.5 | 92.9 |
+| MM-GDINO(zero-shot) | Swin-T | | 53.1 | 89.7 | 95.1 | 59.1 | 91.0 | 95.5 | 46.8 | 87.8 | 93.6 |
+| GDINO | Swin-T | UNK | 89.2 | | | 91.9 | | | 86.0 | | |
+| MM-GDINO | Swin-T | 5e | 89.5 | 98.6 | 99.4 | 91.4 | 99.2 | 99.8 | 86.6 | 97.9 | 99.1 |
+
+- The MM-GDINO-T config file is [here](refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco.py)
+
+#### RefCOCO+
+
+| Architecture | Backbone | Lr schd | val @1 | val @5 | val @10 | testA @1 | testA @5 | testA @10 | testB @1 | testB @5 | testB @10 |
+| :-----------------: | :------: | :-----: | :----: | :----: | :-----: | :------: | :------: | :-------: | :------: | :------: | :-------: |
+| GDINO(zero-shot) | Swin-T | | 51.6 | 86.4 | 92.6 | 57.3 | 86.7 | 92.7 | 46.4 | 84.1 | 90.7 |
+| MM-GDINO(zero-shot) | Swin-T | | 52.7 | 87.7 | 93.5 | 58.7 | 87.2 | 93.1 | 48.4 | 85.8 | 92.1 |
+| GDINO | Swin-T | UNK | 81.1 | | | 87.4 | | | 74.7 | | |
+| MM-GDINO | Swin-T | 5e | 82.1 | 97.8 | 99.2 | 87.5 | 99.2 | 99.7 | 74.0 | 96.3 | 96.4 |
+
+- The MM-GDINO-T config file is [here](refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco_plus.py)
+
+#### RefCOCOg
+
+| Architecture | Backbone | Lr schd | val @1 | val @5 | val @10 | test @1 | test @5 | test @10 |
+| :-----------------: | :------: | :-----: | :----: | :----: | :-----: | :-----: | :-----: | :------: |
+| GDINO(zero-shot) | Swin-T | | 60.4 | 92.1 | 96.2 | 59.7 | 92.1 | 96.3 |
+| MM-GDINO(zero-shot) | Swin-T | | 62.9 | 93.3 | 97.2 | 62.9 | 93.9 | 97.4 |
+| GDINO | Swin-T | UNK | 84.2 | | | 84.9 | | |
+| MM-GDINO | Swin-T | 5e | 85.5 | 98.4 | 99.4 | 85.8 | 98.6 | 99.4 |
+
+- The MM-GDINO-T config file is [here](refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcocog.py)
+
+#### gRefCOCO
+
+| Architecture | Backbone | Lr schd | val Pr@(F1=1, IoU≥0.5) | val N-acc | testA Pr@(F1=1, IoU≥0.5) | testA N-acc | testB Pr@(F1=1, IoU≥0.5) | testB N-acc |
+| :-----------------: | :------: | :-----: | :--------------------: | :-------: | :----------------------: | :---------: | :----------------------: | :---------: |
+| GDINO(zero-shot) | Swin-T | | 41.3 | 91.8 | 27.2 | 90.2 | 29.7 | 93.5 |
+| MM-GDINO(zero-shot) | Swin-T | | 41.0 | 91.3 | 26.1 | 93.0 | 30.4 | 92.3 |
+| MM-GDINO | Swin-T | 5e | 45.1 | 64.7 | 42.5 | 65.5 | 40.3 | 63.2 |
+
+- The MM-GDINO-T config file is [here](refcoco/grounding_dino_swin-t_finetune_8xb4_5e_grefcoco.py)
diff --git a/configs/mm_grounding_dino/brain_tumor/grounding_dino_swin-t_finetune_8xb4_50e_brain_tumor.py b/configs/mm_grounding_dino/brain_tumor/grounding_dino_swin-t_finetune_8xb4_50e_brain_tumor.py
index b0c09f0a9e4..1172da5b641 100644
--- a/configs/mm_grounding_dino/brain_tumor/grounding_dino_swin-t_finetune_8xb4_50e_brain_tumor.py
+++ b/configs/mm_grounding_dino/brain_tumor/grounding_dino_swin-t_finetune_8xb4_50e_brain_tumor.py
@@ -3,6 +3,8 @@
# https://universe.roboflow.com/roboflow-100/brain-tumor-m2pbp/dataset/2
data_root = 'data/brain_tumor_v2/'
class_name = ('label0', 'label1', 'label2')
+label_name = '_annotations.coco.json'
+
palette = [(220, 20, 60), (255, 0, 0), (0, 0, 142)]
metainfo = dict(classes=class_name, palette=palette)
@@ -64,20 +66,20 @@
pipeline=train_pipeline,
return_classes=True,
data_prefix=dict(img='train/'),
- ann_file='train/_annotations.coco.json')))
+ ann_file='train/' + label_name)))
val_dataloader = dict(
dataset=dict(
metainfo=metainfo,
data_root=data_root,
return_classes=True,
- ann_file='valid/_annotations.coco.json',
+ ann_file='valid/' + label_name,
data_prefix=dict(img='valid/')))
test_dataloader = val_dataloader
val_evaluator = dict(
type='CocoMetric',
- ann_file=data_root + 'valid/_annotations.coco.json',
+ ann_file=data_root + 'valid/' + label_name,
metric='bbox',
format_only=False)
test_evaluator = val_evaluator
@@ -107,4 +109,4 @@
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/cityscapes/grounding_dino_swin-t_finetune_8xb4_50e_cityscapes.py b/configs/mm_grounding_dino/cityscapes/grounding_dino_swin-t_finetune_8xb4_50e_cityscapes.py
index 46b2dbd68fe..c4283413c4b 100644
--- a/configs/mm_grounding_dino/cityscapes/grounding_dino_swin-t_finetune_8xb4_50e_cityscapes.py
+++ b/configs/mm_grounding_dino/cityscapes/grounding_dino_swin-t_finetune_8xb4_50e_cityscapes.py
@@ -107,4 +107,4 @@
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco.py b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco.py
index 1253f43470e..792297accd3 100644
--- a/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco.py
+++ b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco.py
@@ -64,7 +64,7 @@
custom_keys={
'absolute_pos_embed': dict(decay_mult=0.),
'backbone': dict(lr_mult=0.1),
- # 'language_model': dict(lr_mult=0),
+ 'language_model': dict(lr_mult=0.1),
}))
# learning policy
@@ -75,11 +75,11 @@
begin=0,
end=max_epochs,
by_epoch=True,
- milestones=[11],
+ milestones=[8, 11],
gamma=0.1)
]
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py
index 43503fb8bea..e68afbb4328 100644
--- a/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py
+++ b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py
@@ -8,21 +8,20 @@
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'pizza', 'donut', 'chair', 'bed', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'microwave', 'oven', 'toaster',
- 'refrigerator', 'book', 'clock', 'vase', 'toothbrush')
+ 'refrigerator', 'book', 'clock', 'vase', 'toothbrush') # 48
novel_classes = ('airplane', 'bus', 'cat', 'dog', 'cow', 'elephant',
'umbrella', 'tie', 'snowboard', 'skateboard', 'cup', 'knife',
- 'cake', 'couch', 'keyboard', 'sink', 'scissors')
-all_classes = ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
- 'train', 'truck', 'boat', 'bench', 'bird', 'cat', 'dog',
- 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
- 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
- 'skis', 'snowboard', 'kite', 'skateboard', 'surfboard',
- 'bottle', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana',
- 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'pizza',
- 'donut', 'cake', 'chair', 'couch', 'bed', 'toilet', 'tv',
- 'laptop', 'mouse', 'remote', 'keyboard', 'microwave', 'oven',
- 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
- 'scissors', 'toothbrush')
+ 'cake', 'couch', 'keyboard', 'sink', 'scissors') # 17
+all_classes = (
+ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
+ 'truck', 'boat', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
+ 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag',
+ 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'kite', 'skateboard',
+ 'surfboard', 'bottle', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana',
+ 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'pizza', 'donut',
+ 'cake', 'chair', 'couch', 'bed', 'toilet', 'tv', 'laptop', 'mouse',
+ 'remote', 'keyboard', 'microwave', 'oven', 'toaster', 'sink',
+ 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'toothbrush') # 65
train_metainfo = dict(classes=base_classes)
test_metainfo = dict(
@@ -95,7 +94,7 @@
type='CocoDataset',
metainfo=train_metainfo,
data_root=data_root,
- ann_file='zero-shot/instances_train2017_seen_2.json',
+ ann_file='annotations/instances_train2017_seen_2.json',
data_prefix=dict(img='train2017/'),
return_classes=True,
filter_cfg=dict(filter_empty_gt=False, min_size=32),
@@ -111,7 +110,7 @@
type='CocoDataset',
metainfo=test_metainfo,
data_root=data_root,
- ann_file='zero-shot/instances_val2017_all_2.json',
+ ann_file='annotations/instances_val2017_all_2.json',
data_prefix=dict(img='val2017/'),
test_mode=True,
pipeline=test_pipeline,
@@ -121,7 +120,7 @@
val_evaluator = dict(
type='OVCocoMetric',
- ann_file=data_root + 'zero-shot/instances_val2017_all_2.json',
+ ann_file=data_root + 'annotations/instances_val2017_all_2.json',
metric='bbox',
format_only=False)
test_evaluator = val_evaluator
@@ -155,4 +154,4 @@
checkpoint=dict(
max_keep_ckpts=1, save_best='coco/novel_ap50', rule='greater'))
-load_from = 'epoch_30.pth'
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_sft_coco.py b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_sft_coco.py
new file mode 100644
index 00000000000..5505df58b8b
--- /dev/null
+++ b/configs/mm_grounding_dino/coco/grounding_dino_swin-t_finetune_16xb4_1x_sft_coco.py
@@ -0,0 +1,93 @@
+_base_ = '../grounding_dino_swin-t_pretrain_obj365.py'
+
+data_root = 'data/coco/'
+
+train_pipeline = [
+ dict(type='LoadImageFromFile'),
+ dict(type='LoadAnnotations', with_bbox=True),
+ dict(type='RandomFlip', prob=0.5),
+ dict(
+ type='RandomChoice',
+ transforms=[
+ [
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ],
+ [
+ dict(
+ type='RandomChoiceResize',
+ # The radio of all image in train dataset < 7
+ # follow the original implement
+ scales=[(400, 4200), (500, 4200), (600, 4200)],
+ keep_ratio=True),
+ dict(
+ type='RandomCrop',
+ crop_type='absolute_range',
+ crop_size=(384, 600),
+ allow_negative_crop=True),
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ]
+ ]),
+ dict(
+ type='RandomSamplingNegPos',
+ tokenizer_name=_base_.lang_model_name,
+ num_sample_negative=20, # ======= important =====
+ label_map_file='data/coco/annotations/coco2017_label_map.json',
+ max_tokens=256),
+ dict(
+ type='PackDetInputs',
+ meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+ 'scale_factor', 'flip', 'flip_direction', 'text',
+ 'custom_entities', 'tokens_positive', 'dataset_mode'))
+]
+
+train_dataloader = dict(
+ dataset=dict(
+ _delete_=True,
+ type='ODVGDataset',
+ need_text=False,
+ data_root=data_root,
+ ann_file='annotations/instances_train2017_od.json',
+ label_map_file='annotations/coco2017_label_map.json',
+ data_prefix=dict(img='train2017/'),
+ return_classes=True,
+ filter_cfg=dict(filter_empty_gt=False, min_size=32),
+ pipeline=train_pipeline))
+
+optim_wrapper = dict(
+ _delete_=True,
+ type='OptimWrapper',
+ optimizer=dict(type='AdamW', lr=0.00005, weight_decay=0.0001),
+ clip_grad=dict(max_norm=0.1, norm_type=2),
+ paramwise_cfg=dict(
+ custom_keys={
+ 'absolute_pos_embed': dict(decay_mult=0.),
+ 'backbone': dict(lr_mult=0.1),
+ 'language_model': dict(lr_mult=0.0),
+ }))
+
+# learning policy
+max_epochs = 12
+param_scheduler = [
+ dict(
+ type='MultiStepLR',
+ begin=0,
+ end=max_epochs,
+ by_epoch=True,
+ milestones=[8, 11],
+ gamma=0.1)
+]
+train_cfg = dict(max_epochs=max_epochs, val_interval=1)
+
+default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
+
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/dataset_prepare_zh-CN.md b/configs/mm_grounding_dino/dataset_prepare_zh-CN.md
new file mode 100644
index 00000000000..0af692e4ceb
--- /dev/null
+++ b/configs/mm_grounding_dino/dataset_prepare_zh-CN.md
@@ -0,0 +1,1123 @@
+# 数据准备和处理
+
+## MM-GDINO-T 预训练数据准备和处理
+
+MM-GDINO-T 模型中我们一共提供了 5 种不同数据组合的预训练配置，数据采用逐步累加的方式进行训练，因此用户可以根据自己的实际需求准备数据。
+
+### 1 Object365 v1
+
+对应的训练配置为 [grounding_dino_swin-t_pretrain_obj365](./grounding_dino_swin-t_pretrain_obj365.py)
+
+Objects365_v1 可以从 [opendatalab](https://opendatalab.com/OpenDataLab/Objects365_v1) 下载，其提供了 CLI 和 SDK 两者下载方式。
+
+下载并解压后，将其放置或者软链接到 `data/objects365v1` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── objects365v1
+│ │ ├── objects365_train.json
+│ │ ├── objects365_val.json
+│ │ ├── train
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── test
+```
+
+然后使用 [coco2odvg.py](../../tools/dataset_converters/coco2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/coco2odvg.py data/objects365v1/objects365_train.json -d o365v1
+```
+
+程序运行完成后会在 `data/objects365v1` 目录下创建 `o365v1_train_od.json` 和 `o365v1_label_map.json` 两个新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── objects365v1
+│ │ ├── objects365_train.json
+│ │ ├── objects365_val.json
+│ │ ├── o365v1_train_od.json
+│ │ ├── o365v1_label_map.json
+│ │ ├── train
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── test
+```
+
+### 2 COCO 2017
+
+上述配置在训练过程中会评估 COCO 2017 数据集的性能，因此需要准备 COCO 2017 数据集。你可以从 [COCO](https://cocodataset.org/) 官网下载或者从 [opendatalab](https://opendatalab.com/OpenDataLab/COCO_2017) 下载
+
+下载并解压后，将其放置或者软链接到 `data/coco` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+### 3 GoldG
+
+下载该数据集后就可以训练 [grounding_dino_swin-t_pretrain_obj365_goldg](./grounding_dino_swin-t_pretrain_obj365_goldg.py) 配置了。
+
+GoldG 数据集包括 `GQA` 和 `Flickr30k` 两个数据集，来自 GLIP 论文中提到的 MixedGrounding 数据集，其排除了 COCO 数据集。下载链接为 [mdetr_annotations](https://huggingface.co/GLIPModel/GLIP/tree/main/mdetr_annotations)，我们目前需要的是 `mdetr_annotations/final_mixed_train_no_coco.json` 和 `mdetr_annotations/final_flickr_separateGT_train.json` 文件。
+
+然后下载 [GQA images](https://nlp.stanford.edu/data/gqa/images.zip) 图片。下载并解压后，将其放置或者软链接到 `data/gqa` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── gqa
+| | ├── final_mixed_train_no_coco.json
+│ │ ├── images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+然后下载 [Flickr30k images](http://shannon.cs.illinois.edu/DenotationGraph/) 图片。这个数据下载需要先申请，再获得下载链接后才可以下载。下载并解压后，将其放置或者软链接到 `data/flickr30k_entities` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── flickr30k_entities
+│ │ ├── final_flickr_separateGT_train.json
+│ │ ├── flickr30k_images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+对于 GQA 数据集，你需要使用 [goldg2odvg.py](../../tools/dataset_converters/goldg2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/goldg2odvg.py data/gqa/final_mixed_train_no_coco.json
+```
+
+程序运行完成后会在 `data/gqa` 目录下创建 `final_mixed_train_no_coco_vg.json` 新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── gqa
+| | ├── final_mixed_train_no_coco.json
+| | ├── final_mixed_train_no_coco_vg.json
+│ │ ├── images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+对于 Flickr30k 数据集，你需要使用 [goldg2odvg.py](../../tools/dataset_converters/goldg2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/goldg2odvg.py data/flickr30k_entities/final_flickr_separateGT_train.json
+```
+
+程序运行完成后会在 `data/flickr30k_entities` 目录下创建 `final_flickr_separateGT_train_vg.json` 新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── flickr30k_entities
+│ │ ├── final_flickr_separateGT_train.json
+│ │ ├── final_flickr_separateGT_train_vg.json
+│ │ ├── flickr30k_images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 4 GRIT-20M
+
+对应的训练配置为 [grounding_dino_swin-t_pretrain_obj365_goldg_grit9m](./grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py)
+
+### 5 V3Det
+
+对应的训练配置为
+
+- [grounding_dino_swin-t_pretrain_obj365_goldg_v3det](./grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py)
+- [grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det](./grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py)
+
+V3Det 数据集下载可以从 [opendatalab](https://opendatalab.com/V3Det/V3Det) 下载，下载并解压后，将其放置或者软链接到 `data/v3det` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── v3det
+│ │ ├── annotations
+│ │ | ├── v3det_2023_v1_train.json
+│ │ ├── images
+│ │ │ ├── a00000066
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+然后使用 [coco2odvg.py](../../tools/dataset_converters/coco2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/coco2odvg.py data/v3det/annotations/v3det_2023_v1_train.json -d v3det
+```
+
+程序运行完成后会在 `data/v3det/annotations` 目录下创建目录下创建 `v3det_2023_v1_train_od.json` 和 `v3det_2023_v1_label_map.json` 两个新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── v3det
+│ │ ├── annotations
+│ │ | ├── v3det_2023_v1_train.json
+│ │ | ├── v3det_2023_v1_train_od.json
+│ │ | ├── v3det_2023_v1_label_map.json
+│ │ ├── images
+│ │ │ ├── a00000066
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 6 数据切分和可视化
+
+考虑到用户需要准备的数据集过多，不方便对图片和标注进行训练前确认，因此我们提供了一个数据切分和可视化的工具，可以将数据集切分为 tiny 版本，然后使用可视化脚本查看图片和标签正确性。
+
+1. 切分数据集
+
+脚本位于 [这里](../../tools/misc/split_odvg.py), 以 `Object365 v1` 为例，切分数据集的命令如下：
+
+```shell
+python tools/misc/split_odvg.py data/object365_v1/ o365v1_train_od.json train your_output_dir --label-map-file o365v1_label_map.json -n 200
+```
+
+上述脚本运行后会在 `your_output_dir` 目录下创建和 `data/object365_v1/` 一样的文件夹结构，但是只会保存 200 张训练图片和对应的 json，方便用户查看。
+
+2. 可视化原始数据集
+
+脚本位于 [这里](../../tools/analysis_tools/browse_grounding_raw.py), 以 `Object365 v1` 为例，可视化数据集的命令如下：
+
+```shell
+python tools/analysis_tools/browse_grounding_raw.py data/object365_v1/ o365v1_train_od.json train --label-map-file o365v1_label_map.json -o your_output_dir --not-show
+```
+
+上述脚本运行后会在 `your_output_dir` 目录下生成同时包括图片和标签的图片，方便用户查看。
+
+3. 可视化 dataset 输出的数据集
+
+脚本位于 [这里](../../tools/analysis_tools/browse_grounding_dataset.py), 用户可以通过该脚本查看 dataset 输出的结果即包括了数据增强的结果。 以 `Object365 v1` 为例，可视化数据集的命令如下：
+
+```shell
+python tools/analysis_tools/browse_grounding_dataset.py configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py -o your_output_dir --not-show
+```
+
+上述脚本运行后会在 `your_output_dir` 目录下生成同时包括图片和标签的图片，方便用户查看。
+
+## MM-GDINO-L 预训练数据准备和处理
+
+### 1 Object365 v2
+
+Objects365_v2 可以从 [opendatalab](https://opendatalab.com/OpenDataLab/Objects365) 下载，其提供了 CLI 和 SDK 两者下载方式。
+
+下载并解压后，将其放置或者软链接到 `data/objects365v2` 目录下，目录结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── objects365v2
+│ │ ├── annotations
+│ │ │ ├── zhiyuan_objv2_train.json
+│ │ ├── train
+│ │ │ ├── patch0
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+由于 objects365v2 类别中有部分类名是错误的，因此需要先进行修正。
+
+```shell
+python tools/dataset_converters/fix_o365_names.py
+```
+
+会在 `data/objects365v2/annotations` 下生成新的标注文件 `zhiyuan_objv2_train_fixname.json`。
+
+然后使用 [coco2odvg.py](../../tools/dataset_converters/coco2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/coco2odvg.py data/objects365v2/annotations/zhiyuan_objv2_train_fixname.json -d o365v2
+```
+
+程序运行完成后会在 `data/objects365v2` 目录下创建 `zhiyuan_objv2_train_fixname_od.json` 和 `o365v2_label_map.json` 两个新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── objects365v2
+│ │ ├── annotations
+│ │ │ ├── zhiyuan_objv2_train.json
+│ │ │ ├── zhiyuan_objv2_train_fixname.json
+│ │ │ ├── zhiyuan_objv2_train_fixname_od.json
+│ │ │ ├── o365v2_label_map.json
+│ │ ├── train
+│ │ │ ├── patch0
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 2 OpenImages v6
+
+OpenImages v6 可以从 [官网](https://storage.googleapis.com/openimages/web/download_v6.html) 下载，由于数据集比较大，需要花费一定的时间，下载完成后文件结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── OpenImages
+│ │ ├── annotations
+| │ │ ├── oidv6-train-annotations-bbox.csv
+| │ │ ├── class-descriptions-boxable.csv
+│ │ ├── OpenImages
+│ │ │ ├── train
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+然后使用 [openimages2odvg.py](../../tools/dataset_converters/openimages2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/openimages2odvg.py data/OpenImages/annotations
+```
+
+程序运行完成后会在 `data/OpenImages/annotations` 目录下创建 `oidv6-train-annotation_od.json` 和 `openimages_label_map.json` 两个新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── OpenImages
+│ │ ├── annotations
+| │ │ ├── oidv6-train-annotations-bbox.csv
+| │ │ ├── class-descriptions-boxable.csv
+| │ │ ├── oidv6-train-annotations_od.json
+| │ │ ├── openimages_label_map.json
+│ │ ├── OpenImages
+│ │ │ ├── train
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 3 V3Det
+
+参见前面的 MM-GDINO-T 预训练数据准备和处理 数据准备部分，完整数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── v3det
+│ │ ├── annotations
+│ │ | ├── v3det_2023_v1_train.json
+│ │ | ├── v3det_2023_v1_train_od.json
+│ │ | ├── v3det_2023_v1_label_map.json
+│ │ ├── images
+│ │ │ ├── a00000066
+│ │ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 4 LVIS 1.0
+
+参加后面的 `微调数据集准备` 的 `2 LVIS 1.0` 部分。完整数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── lvis_v1_train.json
+│ │ │ ├── lvis_v1_val.json
+│ │ │ ├── lvis_v1_train_od.json
+│ │ │ ├── lvis_v1_label_map.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── lvis_v1_minival_inserted_image_name.json
+│ │ │ ├── lvis_od_val.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+### 5 COCO2017 OD
+
+数据准备可以参考前面的 `MM-GDINO-T 预训练数据准备和处理` 部分。为了方便后续处理，请将下载的 [mdetr_annotations](https://huggingface.co/GLIPModel/GLIP/tree/main/mdetr_annotations) 文件夹软链接或者移动到 `data/coco` 路径下
+完整数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── ...
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+由于 COCO2017 train 和 RefCOCO/RefCOCO+/RefCOCOg/gRefCOCO val 中存在部分重叠，如果不提前移除，在评测 RefExp 时候会存在数据泄露。
+
+```shell
+python tools/dataset_converters/remove_cocotrain2017_from_refcoco.py data/coco/mdetr_annotations data/coco/annotations/instances_train2017.json
+```
+
+会在 `data/coco/annotations` 目录下创建 `instances_train2017_norefval.json` 新文件。最后使用 [coco2odvg.py](../../tools/dataset_converters/coco2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/coco2odvg.py data/coco/annotations/instances_train2017_norefval.json -d coco
+```
+
+会在 `data/coco/annotations` 目录下创建 `instances_train2017_norefval_od.json` 和 `coco_label_map.json` 两个新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2017_norefval_od.json
+│ │ │ ├── coco_label_map.json
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── ...
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+注意： COCO2017 train 和 LVIS 1.0 val 数据集有 15000 张图片重复，因此一旦在训练中使用了 COCO2017 train，那么 LVIS 1.0 val 的评测结果就存在数据泄露问题，LVIS 1.0 minival 没有这个问题。
+
+### 6 GoldG
+
+参见 MM-GDINO-T 预训练数据准备和处理 部分
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── flickr30k_entities
+│ │ ├── final_flickr_separateGT_train.json
+│ │ ├── final_flickr_separateGT_train_vg.json
+│ │ ├── flickr30k_images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ ├── gqa
+| | ├── final_mixed_train_no_coco.json
+| | ├── final_mixed_train_no_coco_vg.json
+│ │ ├── images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 7 COCO2014 VG
+
+MDetr 中提供了 COCO2014 train 的 Phrase Grounding 版本标注， 最原始标注文件为 `final_mixed_train.json`，和之前类似，文件结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_mixed_train.json
+│ │ │ ├── ...
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+我们可以从 `final_mixed_train.json` 中提取出 COCO 部分数据
+
+```shell
+python tools/dataset_converters/extract_coco_from_mixed.py data/coco/mdetr_annotations/final_mixed_train.json
+```
+
+会在 `data/coco/mdetr_annotations` 目录下创建 `final_mixed_train_only_coco.json` 新文件，最后使用 [goldg2odvg.py](../../tools/dataset_converters/goldg2odvg.py) 转换为训练所需的 ODVG 格式：
+
+```shell
+python tools/dataset_converters/goldg2odvg.py data/coco/mdetr_annotations/final_mixed_train_only_coco.json
+```
+
+会在 `data/coco/mdetr_annotations` 目录下创建 `final_mixed_train_only_coco_vg.json` 新文件，完整结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_mixed_train.json
+│ │ │ ├── final_mixed_train_only_coco.json
+│ │ │ ├── final_mixed_train_only_coco_vg.json
+│ │ │ ├── ...
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+注意： COCO2014 train 和 COCO2017 val 没有重复图片，因此不用担心 COCO 评测的数据泄露问题。
+
+### 8 Referring Expression Comprehension
+
+其一共包括 4 个数据集。数据准备部分请参见 微调数据集准备 部分。
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── finetune_refcoco_testB.json
+│ │ │ ├── finetune_refcoco+_testA.json
+│ │ │ ├── finetune_refcoco+_testB.json
+│ │ │ ├── finetune_refcocog_test.json
+│ │ │ ├── finetune_refcoco_train_vg.json
+│ │ │ ├── finetune_refcoco+_train_vg.json
+│ │ │ ├── finetune_refcocog_train_vg.json
+│ │ │ ├── finetune_grefcoco_train_vg.json
+```
+
+### 9 GRIT-20M
+
+参见 MM-GDINO-T 预训练数据准备和处理 部分
+
+## 评测数据集准备
+
+### 1 COCO 2017
+
+数据准备流程和前面描述一致，最终结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+### 2 LVIS 1.0
+
+LVIS 1.0 val 数据集包括 mini 和全量两个版本，mini 版本存在的意义是：
+
+1. LVIS val 全量评测数据集比较大，评测一次需要比较久的时间
+2. LVIS val 全量数据集中包括了 15000 张 COCO2017 train, 如果用户使用了 COCO2017 数据进行训练，那么将存在数据泄露问题
+
+LVIS 1.0 图片和 COCO2017 数据集图片完全一样，只是提供了新的标注而已，minival 标注文件可以从 [这里](https://huggingface.co/GLIPModel/GLIP/blob/main/lvis_v1_minival_inserted_image_name.json)下载， val 1.0 标注文件可以从 [这里](https://huggingface.co/GLIPModel/GLIP/blob/main/lvis_od_val.json) 下载。 最终结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── lvis_v1_minival_inserted_image_name.json
+│ │ │ ├── lvis_od_val.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+### 3 ODinW
+
+ODinw 全称为 Object Detection in the Wild，是用于验证 grounding 预训练模型在不同实际场景中的泛化能力的数据集，其包括两个子集，分别是 ODinW13 和 ODinW35，代表是由 13 和 35 个数据集组成的。你可以从 [这里](https://huggingface.co/GLIPModel/GLIP/tree/main/odinw_35)下载，然后对每个文件进行解压，最终结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── odinw
+│ │ ├── AerialMaritimeDrone
+│ │ | |── large
+│ │ | | ├── test
+│ │ | | ├── train
+│ │ | | ├── valid
+│ │ | |── tiled
+│ │ ├── AmericanSignLanguageLetters
+│ │ ├── Aquarium
+│ │ ├── BCCD
+│ │ ├── ...
+```
+
+在评测 ODinW3535 时候由于需要自定义 prompt，因此需要提前对标注的 json 文件进行处理，你可以使用 [override_category.py](./odinw/override_category.py) 脚本进行处理，处理后会生成新的标注文件，不会覆盖原先的标注文件。
+
+```shell
+python configs/mm_grounding_dino/odinw/override_category.py data/odinw/
+```
+
+### 4 DOD
+
+DOD 来自 [Described Object Detection: Liberating Object Detection with Flexible Expressions](https://arxiv.org/abs/2307.12813)。其数据集可以从 [这里](https://github.com/shikras/d-cube?tab=readme-ov-file#download)下载，最终的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── d3
+│ │ ├── d3_images
+│ │ ├── d3_json
+│ │ ├── d3_pkl
+```
+
+### 5 Flickr30k Entities
+
+在前面 GoldG 数据准备章节中我们已经下载了 Flickr30k 训练所需文件，评估所需的文件是 2 个 json 文件，你可以从 [这里](https://huggingface.co/GLIPModel/GLIP/blob/main/mdetr_annotations/final_flickr_separateGT_val.json) 和 [这里](https://huggingface.co/GLIPModel/GLIP/blob/main/mdetr_annotations/final_flickr_separateGT_test.json)下载，最终的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── flickr30k_entities
+│ │ ├── final_flickr_separateGT_train.json
+│ │ ├── final_flickr_separateGT_val.json
+│ │ ├── final_flickr_separateGT_test.json
+│ │ ├── final_flickr_separateGT_train_vg.json
+│ │ ├── flickr30k_images
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+```
+
+### 6 Referring Expression Comprehension
+
+指代性表达式理解包括 4 个数据集： RefCOCO, RefCOCO+, RefCOCOg, gRefCOCO。这 4 个数据集所采用的图片都来自于 COCO2014 train，和 COCO2017 类似，你可以从 COCO 官方或者 opendatalab 中下载，而标注可以直接从 [这里](https://huggingface.co/GLIPModel/GLIP/tree/main/mdetr_annotations) 下载，mdetr_annotations 文件夹里面包括了其他大量的标注，你如果觉得数量过多，可以只下载所需要的几个 json 文件即可。最终的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── finetune_refcoco_testB.json
+│ │ │ ├── finetune_refcoco+_testA.json
+│ │ │ ├── finetune_refcoco+_testB.json
+│ │ │ ├── finetune_refcocog_test.json
+│ │ │ ├── finetune_refcocog_test.json
+```
+
+注意 gRefCOCO 是在 [GREC: Generalized Referring Expression Comprehension](https://arxiv.org/abs/2308.16182) 被提出，并不在 `mdetr_annotations` 文件夹中，需要自行处理。具体步骤为：
+
+1. 下载 [gRefCOCO](https://github.com/henghuiding/gRefCOCO?tab=readme-ov-file#grefcoco-dataset-download)，并解压到 data/coco/ 文件夹中
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ ├── grefs
+│ │ │ ├── grefs(unc).json
+│ │ │ ├── instances.json
+```
+
+2. 转换为 coco 格式
+
+你可以使用 gRefCOCO 官方提供的[转换脚本](https://github.com/henghuiding/gRefCOCO/blob/b4b1e55b4d3a41df26d6b7d843ea011d581127d4/mdetr/scripts/fine-tuning/grefexp_coco_format.py)。注意需要将被注释的 161 行打开，并注释 160 行才可以得到全量的 json 文件。
+
+```shell
+# 需要克隆官方 repo
+git clone https://github.com/henghuiding/gRefCOCO.git
+cd gRefCOCO/mdetr
+python scripts/fine-tuning/grefexp_coco_format.py --data_path ../../data/coco/grefs --out_path ../../data/coco/mdetr_annotations/ --coco_path ../../data/coco
+```
+
+会在 `data/coco/mdetr_annotations/` 文件夹中生成 4 个 json 文件，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── finetune_refcoco_testB.json
+│ │ │ ├── finetune_grefcoco_train.json
+│ │ │ ├── finetune_grefcoco_val.json
+│ │ │ ├── finetune_grefcoco_testA.json
+│ │ │ ├── finetune_grefcoco_testB.json
+```
+
+## 微调数据集准备
+
+### 1 COCO 2017
+
+COCO 是检测领域最常用的数据集，我们希望能够更充分探索其微调模式。从目前发展来看，一共有 3 种微调方式：
+
+1. 闭集微调，即微调后文本端将无法修改描述，转变为闭集算法，在 COCO 上性能能够最大化，但是失去了通用性。
+2. 开集继续预训练微调，即对 COCO 数据集采用和预训练一致的预训练手段。此时有两种做法，第一种是降低学习率并固定某些模块，仅仅在 COCO 数据上预训练，第二种是将 COCO 数据和部分预训练数据混合一起训练，两种方式的目的都是在尽可能不降低泛化性时提高 COCO 数据集性能
+3. 开放词汇微调，即采用 OVD 领域常用做法，将 COCO 类别分成 base 类和 novel 类，训练时候仅仅在 base 类上进行，评测在 base 和 novel 类上进行。这种方式可以验证 COCO OVD 能力，目的也是在尽可能不降低泛化性时提高 COCO 数据集性能
+
+**(1) 闭集微调**
+
+这个部分无需准备数据，直接用之前的数据即可。
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+**(2) 开集继续预训练微调**
+这种方式需要将 COCO 训练数据转换为 ODVG 格式，你可以使用如下命令转换：
+
+```shell
+python tools/dataset_converters/coco2odvg.py data/coco/annotations/instances_train2017.json -d coco
+```
+
+会在 `data/coco/annotations/` 下生成新的 `instances_train2017_od.json` 和 `coco2017_label_map.json`，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_train2017_od.json
+│ │ │ ├── coco2017_label_map.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+在得到数据后，你可以自行选择单独预习还是混合预训练方式。
+
+**(3) 开放词汇微调**
+这种方式需要将 COCO 训练数据转换为 OVD 格式，你可以使用如下命令转换：
+
+```shell
+python tools/dataset_converters/coco2ovd.py data/coco/
+```
+
+会在 `data/coco/annotations/` 下生成新的 `instances_val2017_all_2.json` 和 `instances_val2017_seen_2.json`，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_train2017_od.json
+│ │ │ ├── instances_val2017_all_2.json
+│ │ │ ├── instances_val2017_seen_2.json
+│ │ │ ├── coco2017_label_map.json
+│ │ │ ├── instances_val2017.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+然后可以直接使用 [配置](coco/grounding_dino_swin-t_finetune_16xb4_1x_coco_48_17.py) 进行训练和测试。
+
+### 2 LVIS 1.0
+
+LVIS 是一个包括 1203 类的数据集，同时也是一个长尾联邦数据集，对其进行微调很有意义。 由于其类别过多，我们无法对其进行闭集微调，因此只能采用开集继续预训练微调和开放词汇微调。
+
+你需要先准备好 LVIS 训练 JSON 文件，你可以从 [这里](https://www.lvisdataset.org/dataset) 下载，我们只需要 `lvis_v1_train.json` 和 `lvis_v1_val.json`，然后将其放到 `data/coco/annotations/` 下，然后运行如下命令：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── lvis_v1_train.json
+│ │ │ ├── lvis_v1_val.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── lvis_v1_minival_inserted_image_name.json
+│ │ │ ├── lvis_od_val.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+(1) 开集继续预训练微调
+
+使用如下命令转换为 ODVG 格式：
+
+```shell
+python tools/dataset_converters/lvis2odvg.py data/coco/annotations/lvis_v1_train.json
+```
+
+会在 `data/coco/annotations/` 下生成新的 `lvis_v1_train_od.json` 和 `lvis_v1_label_map.json`，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── lvis_v1_train.json
+│ │ │ ├── lvis_v1_val.json
+│ │ │ ├── lvis_v1_train_od.json
+│ │ │ ├── lvis_v1_label_map.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── lvis_v1_minival_inserted_image_name.json
+│ │ │ ├── lvis_od_val.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+然后可以直接使用 [配置](lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py) 进行训练测试，或者你修改配置将其和部分预训练数据集混合使用。
+
+**(2) 开放词汇微调**
+
+使用如下命令转换为 OVD 格式：
+
+```shell
+python tools/dataset_converters/lvis2ovd.py data/coco/
+```
+
+会在 `data/coco/annotations/` 下生成新的 `lvis_v1_train_od_norare.json` 和 `lvis_v1_label_map_norare.json`，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── lvis_v1_train.json
+│ │ │ ├── lvis_v1_val.json
+│ │ │ ├── lvis_v1_train_od.json
+│ │ │ ├── lvis_v1_label_map.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── lvis_v1_minival_inserted_image_name.json
+│ │ │ ├── lvis_od_val.json
+│ │ │ ├── lvis_v1_train_od_norare.json
+│ │ │ ├── lvis_v1_label_map_norare.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+```
+
+然后可以直接使用 [配置](lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py) 进行训练测试
+
+### 3 RTTS
+
+RTTS 是一个浓雾天气数据集，该数据集包含 4,322 张雾天图像，包含五个类：自行车 (bicycle)、公共汽车 (bus)、汽车 (car)、摩托车 (motorbike) 和人 (person)。可以从 [这里](https://drive.google.com/file/d/15Ei1cHGVqR1mXFep43BO7nkHq1IEGh1e/view)下载, 然后解压到 `data/RTTS/` 文件夹中。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── RTTS
+│ │ ├── annotations_json
+│ │ ├── annotations_xml
+│ │ ├── ImageSets
+│ │ ├── JPEGImages
+```
+
+### 4 RUOD
+
+RUOD 是一个水下目标检测数据集，你可以从 [这里](https://drive.google.com/file/d/1hxtbdgfVveUm_DJk5QXkNLokSCTa_E5o/view)下载, 然后解压到 `data/RUOD/` 文件夹中。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── RUOD
+│ │ ├── Environment_pic
+│ │ ├── Environmet_ANN
+│ │ ├── RUOD_ANN
+│ │ ├── RUOD_pic
+```
+
+### 5 Brain Tumor
+
+Brain Tumor 是一个医学领域的 2d 检测数据集，你可以从 [这里](https://universe.roboflow.com/roboflow-100/brain-tumor-m2pbp/dataset/2)下载, 请注意选择 `COCO JSON` 格式。然后解压到 `data/brain_tumor_v2/` 文件夹中。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── brain_tumor_v2
+│ │ ├── test
+│ │ ├── train
+│ │ ├── valid
+```
+
+### 6 Cityscapes
+
+Cityscapes 是一个城市街景数据集，你可以从 [这里](https://www.cityscapes-dataset.com/) 或者 opendatalab 中下载, 然后解压到 `data/cityscapes/` 文件夹中。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── cityscapes
+│ │ ├── annotations
+│ │ ├── leftImg8bit
+│ │ │ ├── train
+│ │ │ ├── val
+│ │ ├── gtFine
+│ │ │ ├── train
+│ │ │ ├── val
+```
+
+在下载后，然后使用 [cityscapes.py](../../tools/dataset_converters/cityscapes.py) 脚本生成我们所需要的 json 格式
+
+```shell
+python tools/dataset_converters/cityscapes.py data/cityscapes/
+```
+
+会在 annotations 中生成 3 个新的 json 文件。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── cityscapes
+│ │ ├── annotations
+│ │ │ ├── instancesonly_filtered_gtFine_train.json
+│ │ │ ├── instancesonly_filtered_gtFine_val.json
+│ │ │ ├── instancesonly_filtered_gtFine_test.json
+│ │ ├── leftImg8bit
+│ │ │ ├── train
+│ │ │ ├── val
+│ │ ├── gtFine
+│ │ │ ├── train
+│ │ │ ├── val
+```
+
+### 7 People in Painting
+
+People in Painting 是一个油画数据集，你可以从 [这里](https://universe.roboflow.com/roboflow-100/people-in-paintings/dataset/2), 请注意选择 `COCO JSON` 格式。然后解压到 `data/people_in_painting_v2/` 文件夹中。完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── people_in_painting_v2
+│ │ ├── test
+│ │ ├── train
+│ │ ├── valid
+```
+
+### 8 Referring Expression Comprehension
+
+指代性表达式理解的微调和前面一样，也是包括 4 个数据集，在评测数据准备阶段已经全部整理好了，完整的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── finetune_refcoco_testB.json
+│ │ │ ├── finetune_refcoco+_testA.json
+│ │ │ ├── finetune_refcoco+_testB.json
+│ │ │ ├── finetune_refcocog_test.json
+│ │ │ ├── finetune_refcocog_test.json
+```
+
+然后我们需要将其转换为所需的 ODVG 格式，请使用 [refcoco2odvg.py](../../tools/dataset_converters/refcoco2odvg.py) 脚本转换，
+
+```shell
+python tools/dataset_converters/refcoco2odvg.py data/coco/mdetr_annotations
+```
+
+会在 `data/coco/mdetr_annotations` 中生成新的 4 个 json 文件。 转换后的数据集结构如下：
+
+```text
+mmdetection
+├── configs
+├── data
+│ ├── coco
+│ │ ├── annotations
+│ │ │ ├── instances_train2017.json
+│ │ │ ├── instances_val2017.json
+│ │ │ ├── instances_train2014.json
+│ │ ├── train2017
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── val2017
+│ │ │ ├── xxxx.jpg
+│ │ │ ├── ...
+│ │ ├── train2014
+│ │ │ ├── xxx.jpg
+│ │ │ ├── ...
+│ │ ├── mdetr_annotations
+│ │ │ ├── final_refexp_val.json
+│ │ │ ├── finetune_refcoco_testA.json
+│ │ │ ├── finetune_refcoco_testB.json
+│ │ │ ├── finetune_refcoco+_testA.json
+│ │ │ ├── finetune_refcoco+_testB.json
+│ │ │ ├── finetune_refcocog_test.json
+│ │ │ ├── finetune_refcoco_train_vg.json
+│ │ │ ├── finetune_refcoco+_train_vg.json
+│ │ │ ├── finetune_refcocog_train_vg.json
+│ │ │ ├── finetune_grefcoco_train_vg.json
+```
diff --git a/configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py b/configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_flickr30k.py
similarity index 90%
rename from configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py
rename to configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_flickr30k.py
index b0c94e31f2b..e9eb783da97 100644
--- a/configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_zeroshot_flickr30k.py
+++ b/configs/mm_grounding_dino/flickr30k/grounding_dino_swin-t-pretrain_flickr30k.py
@@ -1,7 +1,7 @@
_base_ = '../grounding_dino_swin-t_pretrain_obj365.py'
dataset_type = 'Flickr30kDataset'
-data_root = 'data/flickr30k/'
+data_root = 'data/flickr30k_entities/'
test_pipeline = [
dict(
@@ -23,7 +23,7 @@
dataset_Flickr30k_val = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_val.json',
+ ann_file='final_flickr_separateGT_val.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
@@ -31,7 +31,7 @@
dataset_Flickr30k_test = dict(
type=dataset_type,
data_root=data_root,
- ann_file='mdetr_annotations/final_flickr_separateGT_test.json',
+ ann_file='final_flickr_separateGT_test.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
)
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-l_pretrain_all.py b/configs/mm_grounding_dino/grounding_dino_swin-l_pretrain_all.py
index 8523eb6b01e..46241e2e03b 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-l_pretrain_all.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-l_pretrain_all.py
@@ -31,12 +31,13 @@
decoder=dict(layer_cfg=dict(cross_attn_cfg=dict(num_levels=num_levels))))
# --------------------------- object365v2 od dataset---------------------------
-objv2_backend_args = dict(
- backend='petrel',
- path_mapping=dict({
- './data/objects365v2/': 'yudong:s3://wangyudong/obj365_v2/',
- 'data/objects365v2/': 'yudong:s3://wangyudong/obj365_v2/'
- }))
+# objv2_backend_args = dict(
+# backend='petrel',
+# path_mapping=dict({
+# './data/objects365v2/': 'yudong:s3://wangyudong/obj365_v2/',
+# 'data/objects365v2/': 'yudong:s3://wangyudong/obj365_v2/'
+# }))
+objv2_backend_args = None
objv2_train_pipeline = [
dict(type='LoadImageFromFile', backend_args=objv2_backend_args),
@@ -97,17 +98,18 @@
filter_cfg=dict(filter_empty_gt=False),
pipeline=objv2_train_pipeline,
return_classes=True,
- need_text=False, # change this
+ need_text=False,
backend_args=None,
)
# --------------------------- openimagev6 od dataset---------------------------
-oi_backend_args = dict(
- backend='petrel',
- path_mapping=dict({
- './data/': 's3://openmmlab/datasets/detection/',
- 'data/': 's3://openmmlab/datasets/detection/'
- }))
+# oi_backend_args = dict(
+# backend='petrel',
+# path_mapping=dict({
+# './data/': 's3://openmmlab/datasets/detection/',
+# 'data/': 's3://openmmlab/datasets/detection/'
+# }))
+oi_backend_args = None
oi_train_pipeline = [
dict(type='LoadImageFromFile', backend_args=oi_backend_args),
@@ -162,11 +164,11 @@
oiv6_dataset = dict(
type='ODVGDataset',
data_root='data/OpenImages/',
- ann_file='annotations/oidv6-train-annotations-vg.jsonl',
+ ann_file='annotations/oidv6-train-annotations_od.json',
label_map_file='annotations/openimages_label_map.json',
data_prefix=dict(img='OpenImages/train/'),
filter_cfg=dict(filter_empty_gt=False),
- need_text=False, # change this
+ need_text=False,
pipeline=oi_train_pipeline,
return_classes=True,
backend_args=None)
@@ -231,11 +233,76 @@
label_map_file='annotations/v3det_2023_v1_label_map.json',
data_prefix=dict(img=''),
filter_cfg=dict(filter_empty_gt=False),
- need_text=False, # change this
+ need_text=False,
pipeline=v3d_train_pipeline,
return_classes=True,
backend_args=None))
+# --------------------------- lvis od dataset---------------------------
+lvis_train_pipeline = [
+ dict(type='LoadImageFromFile', backend_args=_base_.backend_args),
+ dict(type='LoadAnnotations', with_bbox=True),
+ dict(type='RandomFlip', prob=0.5),
+ dict(
+ type='RandomChoice',
+ transforms=[
+ [
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ],
+ [
+ dict(
+ type='RandomChoiceResize',
+ # The radio of all image in train dataset < 7
+ # follow the original implement
+ scales=[(400, 4200), (500, 4200), (600, 4200)],
+ keep_ratio=True),
+ dict(
+ type='RandomCrop',
+ crop_type='absolute_range',
+ crop_size=(384, 600),
+ allow_negative_crop=True),
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ]
+ ]),
+ dict(type='FilterAnnotations', min_gt_bbox_wh=(1e-2, 1e-2)),
+ dict(
+ type='RandomSamplingNegPos',
+ tokenizer_name=_base_.lang_model_name,
+ num_sample_negative=85,
+ # change this
+ label_map_file='data/coco/annotations/lvis_v1_label_map.json',
+ max_tokens=256),
+ dict(
+ type='PackDetInputs',
+ meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+ 'scale_factor', 'flip', 'flip_direction', 'text',
+ 'custom_entities', 'tokens_positive', 'dataset_mode'))
+]
+lvis_dataset = dict(
+ type='ClassBalancedDataset',
+ oversample_thr=1e-3,
+ dataset=dict(
+ type='ODVGDataset',
+ data_root='data/coco/',
+ ann_file='annotations/lvis_v1_train_od.json',
+ label_map_file='annotations/lvis_v1_label_map.json',
+ data_prefix=dict(img=''),
+ filter_cfg=dict(filter_empty_gt=False),
+ need_text=False, # change this
+ pipeline=lvis_train_pipeline,
+ return_classes=True,
+ backend_args=None))
+
# --------------------------- coco2017 od dataset---------------------------
coco2017_train_dataset = dict(
type='RepeatDataset',
@@ -243,8 +310,8 @@
dataset=dict(
type='ODVGDataset',
data_root='data/coco/',
- ann_file='instance_train2017_norefval_od.json',
- label_map_file='coco2017_label_map.json',
+ ann_file='annotations/instance_train2017_norefval_od.json',
+ label_map_file='annotations/coco2017_label_map.json',
data_prefix=dict(img='train2017'),
filter_cfg=dict(filter_empty_gt=False),
pipeline=_base_.train_pipeline,
@@ -351,12 +418,13 @@
backend_args=None))
# --------------------------- grit vg dataset---------------------------
-grit_backend_args = dict(
- backend='petrel',
- path_mapping=dict({
- './data/grit/': 'yichen:s3://chenyicheng/grit/',
- 'data/grit/': 'yichen:s3://chenyicheng/grit/'
- }))
+# grit_backend_args = dict(
+# backend='petrel',
+# path_mapping=dict({
+# './data/grit/': 'yichen:s3://chenyicheng/grit/',
+# 'data/grit/': 'yichen:s3://chenyicheng/grit/'
+# }))
+grit_backend_args = None
grit_train_pipeline = [
dict(type='LoadImageFromFile', backend_args=grit_backend_args),
@@ -425,14 +493,15 @@
_delete_=True,
type='CustomSampleSizeSampler',
ratio_mode=True,
- # OD ~ 1.74+1.67*0.5+0.18*2+0.12*2=3.175
- # vg ~ 0.15*2+0.62*1+0.49*1+0.12*2+0.12*2+0.08*3+0.19*2+9*0.09=3.32
- dataset_size=[-1, 0.5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0.09]),
+ # OD ~ 1.74+1.67*0.5+0.18*2+0.12*2+0.1=3.2
+ # vg ~ 0.15*2+0.62*1+0.49*1+0.12*2+0.12*2+0.08*3+0.19*2+9*0.09=3.3
+ dataset_size=[-1, 0.5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0.09]),
dataset=dict(datasets=[
o365v2_dataset, # 1.74M
oiv6_dataset, # 1.67M
v3det_dataset, # 0.18M
coco2017_train_dataset, # 0.12M
+ lvis_dataset, # 0.1M
flickr30k_dataset, # 0.15M
gqa_dataset, # 0.62M
coco2014_vg_dataset, # 0.49M
@@ -446,11 +515,11 @@
# bs=256
optim_wrapper = dict(optimizer=dict(lr=0.0008))
-# one epoch = (3.175+3.32)M/256 = 25371 iter
-# 24e=608904 iter
-# 16e=405936 iter
-# 20e=507420 iter
-max_iter = 608904
+# one epoch = (3.2+3.3)M/256 = 25390 iter
+# 24e=609360 iter
+# 16e=406240 iter
+# 20e=507800 iter
+max_iter = 609360
train_cfg = dict(
_delete_=True,
type='IterBasedTrainLoop',
@@ -464,7 +533,7 @@
begin=0,
end=max_iter,
by_epoch=False,
- milestones=[405936, 507420],
+ milestones=[406240, 507800],
gamma=0.1)
]
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_finetune_8xb4_20e_cat.py b/configs/mm_grounding_dino/grounding_dino_swin-t_finetune_8xb4_20e_cat.py
new file mode 100644
index 00000000000..bf3b35894eb
--- /dev/null
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_finetune_8xb4_20e_cat.py
@@ -0,0 +1,102 @@
+_base_ = 'grounding_dino_swin-t_pretrain_obj365.py'
+
+data_root = 'data/cat/'
+class_name = ('cat', )
+num_classes = len(class_name)
+metainfo = dict(classes=class_name, palette=[(220, 20, 60)])
+
+model = dict(bbox_head=dict(num_classes=num_classes))
+
+train_pipeline = [
+ dict(type='LoadImageFromFile'),
+ dict(type='LoadAnnotations', with_bbox=True),
+ dict(type='RandomFlip', prob=0.5),
+ dict(
+ type='RandomChoice',
+ transforms=[
+ [
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ],
+ [
+ dict(
+ type='RandomChoiceResize',
+ # The radio of all image in train dataset < 7
+ # follow the original implement
+ scales=[(400, 4200), (500, 4200), (600, 4200)],
+ keep_ratio=True),
+ dict(
+ type='RandomCrop',
+ crop_type='absolute_range',
+ crop_size=(384, 600),
+ allow_negative_crop=True),
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ]
+ ]),
+ dict(
+ type='PackDetInputs',
+ meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+ 'scale_factor', 'flip', 'flip_direction', 'text',
+ 'custom_entities'))
+]
+
+train_dataloader = dict(
+ dataset=dict(
+ _delete_=True,
+ type='CocoDataset',
+ data_root=data_root,
+ metainfo=metainfo,
+ return_classes=True,
+ pipeline=train_pipeline,
+ filter_cfg=dict(filter_empty_gt=False, min_size=32),
+ ann_file='annotations/trainval.json',
+ data_prefix=dict(img='images/')))
+
+val_dataloader = dict(
+ dataset=dict(
+ metainfo=metainfo,
+ data_root=data_root,
+ ann_file='annotations/test.json',
+ data_prefix=dict(img='images/')))
+
+test_dataloader = val_dataloader
+
+val_evaluator = dict(ann_file=data_root + 'annotations/test.json')
+test_evaluator = val_evaluator
+
+max_epoch = 20
+
+default_hooks = dict(
+ checkpoint=dict(interval=1, max_keep_ckpts=1, save_best='auto'),
+ logger=dict(type='LoggerHook', interval=5))
+train_cfg = dict(max_epochs=max_epoch, val_interval=1)
+
+param_scheduler = [
+ dict(
+ type='MultiStepLR',
+ begin=0,
+ end=max_epoch,
+ by_epoch=True,
+ milestones=[15],
+ gamma=0.1)
+]
+
+optim_wrapper = dict(
+ optimizer=dict(lr=0.0001),
+ paramwise_cfg=dict(
+ custom_keys={
+ 'absolute_pos_embed': dict(decay_mult=0.),
+ 'backbone': dict(lr_mult=0.0),
+ 'language_model': dict(lr_mult=0.0)
+ }))
+
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py
index 782487434fe..66060f45ea7 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py
@@ -189,7 +189,7 @@
coco_od_dataset = dict(
type=dataset_type,
data_root=data_root,
- ann_file='o365v1_train_odvg.jsonl',
+ ann_file='o365v1_train_odvg.json',
label_map_file='o365v1_label_map.json',
data_prefix=dict(img='train/'),
filter_cfg=dict(filter_empty_gt=False),
@@ -243,3 +243,5 @@
# USER SHOULD NOT CHANGE ITS VALUES.
# base_batch_size = (16 GPUs) x (2 samples per GPU)
auto_scale_lr = dict(base_batch_size=64)
+
+default_hooks = dict(visualization=dict(type='GroundingVisualizationHook'))
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg.py
index a86abd7997e..b7f388bdd4e 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg.py
@@ -3,7 +3,7 @@
o365v1_od_dataset = dict(
type='ODVGDataset',
data_root='data/objects365v1/',
- ann_file='o365v1_train_odvg.jsonl',
+ ann_file='o365v1_train_odvg.json',
label_map_file='o365v1_label_map.json',
data_prefix=dict(img='train/'),
filter_cfg=dict(filter_empty_gt=False),
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py
index 1cd659f063e..8e9f5ca4aab 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py
@@ -3,7 +3,7 @@
o365v1_od_dataset = dict(
type='ODVGDataset',
data_root='data/objects365v1/',
- ann_file='o365v1_train_odvg.jsonl',
+ ann_file='o365v1_train_odvg.json',
label_map_file='o365v1_label_map.json',
data_prefix=dict(img='train/'),
filter_cfg=dict(filter_empty_gt=False),
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py
index 5a7d3b58947..56e500c8693 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py
@@ -3,7 +3,7 @@
o365v1_od_dataset = dict(
type='ODVGDataset',
data_root='data/objects365v1/',
- ann_file='o365v1_train_odvg.jsonl',
+ ann_file='o365v1_train_odvg.json',
label_map_file='o365v1_label_map.json',
data_prefix=dict(img='train/'),
filter_cfg=dict(filter_empty_gt=False),
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py
new file mode 100644
index 00000000000..c89014fbbe4
--- /dev/null
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py
@@ -0,0 +1,101 @@
+_base_ = 'grounding_dino_swin-t_pretrain_obj365.py'
+
+o365v1_od_dataset = dict(
+ type='ODVGDataset',
+ data_root='data/objects365v1/',
+ ann_file='o365v1_train_odvg.json',
+ label_map_file='o365v1_label_map.json',
+ data_prefix=dict(img='train/'),
+ filter_cfg=dict(filter_empty_gt=False),
+ pipeline=_base_.train_pipeline,
+ return_classes=True,
+ backend_args=None,
+)
+
+flickr30k_dataset = dict(
+ type='ODVGDataset',
+ data_root='data/flickr30k_entities/',
+ ann_file='final_flickr_separateGT_train_vg.json',
+ label_map_file=None,
+ data_prefix=dict(img='flickr30k_images/'),
+ filter_cfg=dict(filter_empty_gt=False),
+ pipeline=_base_.train_pipeline,
+ return_classes=True,
+ backend_args=None)
+
+gqa_dataset = dict(
+ type='ODVGDataset',
+ data_root='data/gqa/',
+ ann_file='final_mixed_train_no_coco_vg.json',
+ label_map_file=None,
+ data_prefix=dict(img='images/'),
+ filter_cfg=dict(filter_empty_gt=False),
+ pipeline=_base_.train_pipeline,
+ return_classes=True,
+ backend_args=None)
+
+v3d_train_pipeline = [
+ dict(type='LoadImageFromFile', backend_args=_base_.backend_args),
+ dict(type='LoadAnnotations', with_bbox=True),
+ dict(type='RandomFlip', prob=0.5),
+ dict(
+ type='RandomChoice',
+ transforms=[
+ [
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ],
+ [
+ dict(
+ type='RandomChoiceResize',
+ # The radio of all image in train dataset < 7
+ # follow the original implement
+ scales=[(400, 4200), (500, 4200), (600, 4200)],
+ keep_ratio=True),
+ dict(
+ type='RandomCrop',
+ crop_type='absolute_range',
+ crop_size=(384, 600),
+ allow_negative_crop=True),
+ dict(
+ type='RandomChoiceResize',
+ scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333),
+ (608, 1333), (640, 1333), (672, 1333), (704, 1333),
+ (736, 1333), (768, 1333), (800, 1333)],
+ keep_ratio=True)
+ ]
+ ]),
+ dict(type='FilterAnnotations', min_gt_bbox_wh=(1e-2, 1e-2)),
+ dict(
+ type='RandomSamplingNegPos',
+ tokenizer_name=_base_.lang_model_name,
+ num_sample_negative=85,
+ # change this
+ label_map_file='data/V3Det/annotations/v3det_2023_v1_label_map.json',
+ max_tokens=256),
+ dict(
+ type='PackDetInputs',
+ meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+ 'scale_factor', 'flip', 'flip_direction', 'text',
+ 'custom_entities', 'tokens_positive', 'dataset_mode'))
+]
+v3det_dataset = dict(
+ type='ODVGDataset',
+ data_root='data/V3Det/',
+ ann_file='annotations/v3det_2023_v1_train_od.json',
+ label_map_file='annotations/v3det_2023_v1_label_map.json',
+ data_prefix=dict(img=''),
+ filter_cfg=dict(filter_empty_gt=False),
+ need_text=False, # change this
+ pipeline=v3d_train_pipeline,
+ return_classes=True,
+ backend_args=None)
+
+train_dataloader = dict(
+ dataset=dict(datasets=[
+ o365v1_od_dataset, flickr30k_dataset, gqa_dataset, v3det_dataset
+ ]))
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_cat.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_cat.py
new file mode 100644
index 00000000000..6dc8dcd8df4
--- /dev/null
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_cat.py
@@ -0,0 +1,43 @@
+_base_ = 'grounding_dino_swin-t_pretrain_obj365.py'
+
+test_pipeline = [
+ dict(
+ type='LoadImageFromFile', backend_args=None,
+ imdecode_backend='pillow'),
+ dict(
+ type='FixScaleResize',
+ scale=(800, 1333),
+ keep_ratio=True,
+ backend='pillow'),
+ dict(type='LoadTextAnnotations'),
+ dict(
+ type='PackDetInputs',
+ meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+ 'scale_factor', 'text', 'custom_entities',
+ 'tokens_positive'))
+]
+
+data_root = 'data/cat/'
+
+val_dataloader = dict(
+ batch_size=1,
+ num_workers=2,
+ persistent_workers=False,
+ dataset=dict(
+ type='ODVGDataset',
+ data_root=data_root,
+ label_map_file='cat_label_map.json',
+ ann_file='cat_train_od.json',
+ data_prefix=dict(img='images/'),
+ pipeline=test_pipeline,
+ return_classes=True))
+test_dataloader = val_dataloader
+
+val_evaluator = dict(
+ _delete_=True,
+ outfile_path=data_root + 'cat_train_od_v1.json',
+ img_prefix=data_root + 'images/',
+ score_thr=0.7,
+ nms_thr=0.5,
+ type='DumpODVGResults')
+test_evaluator = val_evaluator
diff --git a/configs/mm_grounding_dino/grounding_dino_swin-b_pretrain_pl.py b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_flickr30k.py
similarity index 79%
rename from configs/mm_grounding_dino/grounding_dino_swin-b_pretrain_pl.py
rename to configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_flickr30k.py
index 31591e55643..78bf1c344bf 100644
--- a/configs/mm_grounding_dino/grounding_dino_swin-b_pretrain_pl.py
+++ b/configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_pseudo-labeling_flickr30k.py
@@ -1,6 +1,4 @@
-_base_ = '../grounding_dino/grounding_dino_swin-b_pretrain_mixeddata.py'
-
-model = dict(test_cfg=dict(max_per_img=10))
+_base_ = 'grounding_dino_swin-t_pretrain_obj365.py'
test_pipeline = [
dict(
@@ -19,7 +17,7 @@
'tokens_positive'))
]
-data_root = 'data/'
+data_root = 'data/flickr30k_entities/'
val_dataloader = dict(
batch_size=1,
@@ -28,7 +26,7 @@
dataset=dict(
type='ODVGDataset',
data_root=data_root,
- ann_file='final_flickr_separateGT_train_vg.json',
+ ann_file='flickr_simple_train_vg.json',
data_prefix=dict(img='flickr30k_images/'),
pipeline=test_pipeline,
return_classes=True))
@@ -36,7 +34,9 @@
val_evaluator = dict(
_delete_=True,
- outfile_path='aa.json',
+ outfile_path=data_root + 'flickr_simple_train_vg_v1.json',
img_prefix=data_root + 'flickr30k_images/',
+ score_thr=0.4,
+ nms_thr=0.5,
type='DumpODVGResults')
test_evaluator = val_evaluator
diff --git a/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py b/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py
index 3ca34c88509..3ba12c90675 100644
--- a/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py
+++ b/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis.py
@@ -117,4 +117,4 @@
checkpoint=dict(
max_keep_ckpts=1, save_best='lvis_fixed_ap/AP', rule='greater'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py b/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py
index 07d129c39b8..28d0141d3e2 100644
--- a/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py
+++ b/configs/mm_grounding_dino/lvis/grounding_dino_swin-t_finetune_16xb4_1x_lvis_866_337.py
@@ -1,6 +1,6 @@
_base_ = '../grounding_dino_swin-t_pretrain_obj365.py'
-data_root = 'data/lvis/'
+data_root = 'data/coco/'
model = dict(test_cfg=dict(
max_per_img=300,
@@ -48,7 +48,7 @@
tokenizer_name=_base_.lang_model_name,
num_sample_negative=85,
# change this
- label_map_file='data/lvis/annotations/lvis_v1_label_map_norare.json',
+ label_map_file='data/coco/annotations/lvis_v1_label_map_norare.json',
max_tokens=256),
dict(
type='PackDetInputs',
@@ -117,4 +117,4 @@
checkpoint=dict(
max_keep_ckpts=3, save_best='lvis_fixed_ap/AP', rule='greater'))
-load_from = 'epoch_30.pth'
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/metafile.yml b/configs/mm_grounding_dino/metafile.yml
new file mode 100644
index 00000000000..3071686e7ac
--- /dev/null
+++ b/configs/mm_grounding_dino/metafile.yml
@@ -0,0 +1,54 @@
+Collections:
+ - Name: MM Grounding DINO
+ Metadata:
+ Training Data: Objects365, GoldG, GRIT and V3Det
+ Training Techniques:
+ - AdamW
+ - Multi Scale Train
+ - Gradient Clip
+ Training Resources: 3090 GPUs
+ Architecture:
+ - Swin Transformer
+ - BERT
+ README: configs/mm_grounding_dino/README.md
+ Code:
+ URL:
+ Version: v3.0.0
+
+Models:
+ - Name: grounding_dino_swin-t_pretrain_obj365_goldg
+ In Collection: MM Grounding DINO
+ Config: configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg.py
+ Results:
+ - Task: Object Detection
+ Dataset: COCO
+ Metrics:
+ box AP: 50.4
+ Weights: https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg/grounding_dino_swin-t_pretrain_obj365_goldg_20231122_132602-4ea751ce.pth
+ - Name: grounding_dino_swin-t_pretrain_obj365_goldg_grit9m
+ In Collection: MM Grounding DINO
+ Config: configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m.py
+ Results:
+ - Task: Object Detection
+ Dataset: COCO
+ Metrics:
+ box AP: 50.5
+ Weights: https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_20231128_200818-169cc352.pth
+ - Name: grounding_dino_swin-t_pretrain_obj365_goldg_v3det
+ In Collection: MM Grounding DINO
+ Config: configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det.py
+ Results:
+ - Task: Object Detection
+ Dataset: COCO
+ Metrics:
+ box AP: 50.6
+ Weights: https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_v3det_20231218_095741-e316e297.pth
+ - Name: grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det
+ In Collection: MM Grounding DINO
+ Config: configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det.py
+ Results:
+ - Task: Object Detection
+ Dataset: COCO
+ Metrics:
+ box AP: 50.4
+ Weights: https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth
diff --git a/configs/mm_grounding_dino/people_in_painting/grounding_dino_swin-t_finetune_8xb4_50e_people_in_painting.py b/configs/mm_grounding_dino/people_in_painting/grounding_dino_swin-t_finetune_8xb4_50e_people_in_painting.py
index ae9617ef30f..449d8682f89 100644
--- a/configs/mm_grounding_dino/people_in_painting/grounding_dino_swin-t_finetune_8xb4_50e_people_in_painting.py
+++ b/configs/mm_grounding_dino/people_in_painting/grounding_dino_swin-t_finetune_8xb4_50e_people_in_painting.py
@@ -106,4 +106,4 @@
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_grefcoco.py b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_grefcoco.py
similarity index 92%
rename from configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_grefcoco.py
rename to configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_grefcoco.py
index a6ce25e904d..983ffe5c6f3 100644
--- a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_grefcoco.py
+++ b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_grefcoco.py
@@ -74,11 +74,12 @@
pipeline=_base_.test_pipeline,
backend_args=None)
val_evaluator_all_val = dict(
- type='RefExpMetric',
+ type='gRefCOCOMetric',
ann_file=data_root + ann_file,
metric='bbox',
iou_thrs=0.5,
- topk=(1, 5, 10))
+ thresh_score=0.7,
+ thresh_f1=1.0)
# -------------------------------------------------#
ann_file = 'mdetr_annotations/finetune_grefcoco_testA.json'
@@ -93,11 +94,12 @@
backend_args=None)
val_evaluator_refcoco_testA = dict(
- type='RefExpMetric',
+ type='gRefCOCOMetric',
ann_file=data_root + ann_file,
metric='bbox',
iou_thrs=0.5,
- topk=(1, 5, 10))
+ thresh_score=0.7,
+ thresh_f1=1.0)
# -------------------------------------------------#
ann_file = 'mdetr_annotations/finetune_grefcoco_testB.json'
@@ -112,11 +114,12 @@
backend_args=None)
val_evaluator_refcoco_testB = dict(
- type='RefExpMetric',
+ type='gRefCOCOMetric',
ann_file=data_root + ann_file,
metric='bbox',
iou_thrs=0.5,
- topk=(1, 5, 10))
+ thresh_score=0.7,
+ thresh_f1=1.0)
# -------------------------------------------------#
datasets = [
@@ -164,6 +167,4 @@
]
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
-default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco.py b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco.py
similarity index 95%
rename from configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco.py
rename to configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco.py
index d26bf98c0f7..d91af473a23 100644
--- a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco.py
+++ b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco.py
@@ -164,6 +164,4 @@
]
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
-default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco_plus.py b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco_plus.py
similarity index 95%
rename from configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco_plus.py
rename to configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco_plus.py
index ff084b8c514..871adc8efb4 100644
--- a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcoco_plus.py
+++ b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcoco_plus.py
@@ -164,6 +164,4 @@
]
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
-default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcocog.py b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcocog.py
similarity index 82%
rename from configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcocog.py
rename to configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcocog.py
index 79ec375c756..a351d6f9d12 100644
--- a/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_refcocog.py
+++ b/configs/mm_grounding_dino/refcoco/grounding_dino_swin-t_finetune_8xb4_5e_refcocog.py
@@ -81,8 +81,8 @@
topk=(1, 5, 10))
# -------------------------------------------------#
-ann_file = 'mdetr_annotations/finetune_refcocog_testA.json'
-val_dataset_refcoco_testA = dict(
+ann_file = 'mdetr_annotations/finetune_refcocog_test.json'
+val_dataset_refcoco_test = dict(
type='MDETRStyleRefCocoDataset',
data_root=data_root,
ann_file=ann_file,
@@ -92,7 +92,7 @@
pipeline=_base_.test_pipeline,
backend_args=None)
-val_evaluator_refcoco_testA = dict(
+val_evaluator_refcoco_test = dict(
type='RefExpMetric',
ann_file=data_root + ann_file,
metric='bbox',
@@ -100,33 +100,9 @@
topk=(1, 5, 10))
# -------------------------------------------------#
-ann_file = 'mdetr_annotations/finetune_refcocog_testB.json'
-val_dataset_refcoco_testB = dict(
- type='MDETRStyleRefCocoDataset',
- data_root=data_root,
- ann_file=ann_file,
- data_prefix=dict(img='train2014/'),
- test_mode=True,
- return_classes=True,
- pipeline=_base_.test_pipeline,
- backend_args=None)
-
-val_evaluator_refcoco_testB = dict(
- type='RefExpMetric',
- ann_file=data_root + ann_file,
- metric='bbox',
- iou_thrs=0.5,
- topk=(1, 5, 10))
-
-# -------------------------------------------------#
-datasets = [
- val_dataset_all_val, val_dataset_refcoco_testA, val_dataset_refcoco_testB
-]
-dataset_prefixes = ['refcocog_val', 'refcocog_testA', 'refcocog_testB']
-metrics = [
- val_evaluator_all_val, val_evaluator_refcoco_testA,
- val_evaluator_refcoco_testB
-]
+datasets = [val_dataset_all_val, val_dataset_refcoco_test]
+dataset_prefixes = ['refcocog_val', 'refcocog_test']
+metrics = [val_evaluator_all_val, val_evaluator_refcoco_test]
val_dataloader = dict(
dataset=dict(_delete_=True, type='ConcatDataset', datasets=datasets))
@@ -166,4 +142,4 @@
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/rtts/grounding_dino_swin-t_finetune_8xb4_1x_rtts.py b/configs/mm_grounding_dino/rtts/grounding_dino_swin-t_finetune_8xb4_1x_rtts.py
index db167f671c1..95c2be058e2 100644
--- a/configs/mm_grounding_dino/rtts/grounding_dino_swin-t_finetune_8xb4_1x_rtts.py
+++ b/configs/mm_grounding_dino/rtts/grounding_dino_swin-t_finetune_8xb4_1x_rtts.py
@@ -103,4 +103,4 @@
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/ruod/grounding_dino_swin-t_finetune_8xb4_1x_ruod.py b/configs/mm_grounding_dino/ruod/grounding_dino_swin-t_finetune_8xb4_1x_ruod.py
index 16a6a6cbb7a..f57682b29d9 100644
--- a/configs/mm_grounding_dino/ruod/grounding_dino_swin-t_finetune_8xb4_1x_ruod.py
+++ b/configs/mm_grounding_dino/ruod/grounding_dino_swin-t_finetune_8xb4_1x_ruod.py
@@ -105,4 +105,4 @@
train_cfg = dict(max_epochs=max_epochs, val_interval=1)
default_hooks = dict(checkpoint=dict(max_keep_ckpts=1, save_best='auto'))
-load_from = ''
+load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
diff --git a/configs/mm_grounding_dino/usage_zh-CN.md b/configs/mm_grounding_dino/usage_zh-CN.md
new file mode 100644
index 00000000000..0e5e1a766df
--- /dev/null
+++ b/configs/mm_grounding_dino/usage_zh-CN.md
@@ -0,0 +1,491 @@
+# 用法说明
+
+## 安装
+
+在按照 [get_started](../../docs/zh_cn/get_started.md) 一节的说明安装好 MMDet 之后，需要安装额外的依赖包：
+
+```shell
+cd $MMDETROOT
+
+pip install -r requirements/multimodal.txt
+pip install emoji ddd-dataset
+pip install git+https://github.com/lvis-dataset/lvis-api.git"
+```
+
+请注意由于 LVIS 第三方库暂时不支持 numpy 1.24，因此请确保您的 numpy 版本符合要求。建议安装 numpy 1.23 版本。
+
+## 说明
+
+### BERT 权重下载
+
+MM Grounding DINO 采用了 BERT 作为语言模型，需要访问 https://huggingface.co/, 如果您因为网络访问问题遇到连接错误，可以在有网络访问权限的电脑上下载所需文件并保存在本地。最后，修改配置文件中的 `lang_model_name` 字段为本地路径即可。具体请参考以下代码：
+
+```python
+from transformers import BertConfig, BertModel
+from transformers import AutoTokenizer
+
+config = BertConfig.from_pretrained("bert-base-uncased")
+model = BertModel.from_pretrained("bert-base-uncased", add_pooling_layer=False, config=config)
+tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+
+config.save_pretrained("your path/bert-base-uncased")
+model.save_pretrained("your path/bert-base-uncased")
+tokenizer.save_pretrained("your path/bert-base-uncased")
+```
+
+### NLTK 权重下载
+
+MM Grounding DINO 在进行 Phrase Grounding 推理时候可能会进行名词短语提取，虽然会在运行时候下载特定的模型，但是考虑到有些用户运行环境无法联网，因此可以提前下载到 `~/nltk_data` 路径下
+
+```python
+import nltk
+nltk.download('punkt', download_dir='~/nltk_data')
+nltk.download('averaged_perceptron_tagger', download_dir='~/nltk_data')
+```
+
+### MM Grounding DINO-T 模型权重下载
+
+为了方便演示，您可以提前下载 MM Grounding DINO-T 模型权重到当前路径下
+
+```shell
+wget load_from = 'https://download.openmmlab.com/mmdetection/v3.0/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det/grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth' # noqa
+```
+
+## 推理
+
+在推理前，为了更好的体验不同图片的推理效果，建议您先下载 [这些图片](https://github.com/microsoft/X-Decoder/tree/main/inference_demo/images) 到当前路径下
+
+MM Grounding DINO 支持了闭集目标检测，开放词汇目标检测，Phrase Grounding 和指代性表达式理解 4 种推理方式，下面详细说明。
+
+**(1) 闭集目标检测**
+
+由于 MM Grounding DINO 是预训练模型，理论上可以应用于任何闭集检测数据集，目前我们支持了常用的 coco/voc/cityscapes/objects365v1/lvis 等，下面以 coco 为例
+
+```shell
+python demo/image_demo.py images/animals.png \
+ configs/mm_grounding_dino/grounding_dino_swin-t_pretrain_obj365.py \
+ --weights grounding_dino_swin-t_pretrain_obj365_goldg_grit9m_v3det_20231204_095047-b448804b.pth \
+ --texts '$: coco'
+```
+
+会在当前路径下生成 `outputs/vis/animals.png` 的预测结果，如下图所示
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+