Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor ic #20

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
+283 −224
Open

refactor ic #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d998076
refactor ic
MikeLippincott Dec 28, 2024
4d3cfba
add args
MikeLippincott Dec 30, 2024
183bf09
fix slurm scripts
MikeLippincott Dec 30, 2024
d483461
run on HPC
MikeLippincott Dec 30, 2024
821e09e
run on HPC
MikeLippincott Jan 3, 2025
0bc902f
fix multiprocessing
MikeLippincott Jan 3, 2025
016f187
run on HPC
MikeLippincott Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ data/2023*
data/objects_db
environments/SAM2_install/segment-anything-2
data/models
data/preprocessed_data/*
data/raw_data/*
data/test_data/*



# logs
Expand All @@ -25,6 +29,7 @@ data/models
2.cellprofiler_ic_processing/illum_directory_test_small
2.cellprofiler_ic_processing/illum_directory_test


# cellprofiler logs
4.cellprofiler_analysis/scripts/logs
4.cellprofiler_analysis/scripts/*.log
Expand Down Expand Up @@ -58,7 +63,12 @@ data/models
3b.run_sam/masks/*.csv
3b.run_sam/notebooks/videos
3b.run_sam/sam2_processing_dir/*
3b.run_sam/stardist_processing_dir/*

3c.segment_cells/results/*
3c.segment_cells/figures/*
3c.segment_cells/tmp_output/*
!3c.segment_cells/figures/C-02_F0001_tracks.gif

# cytomining results
5.process_CP_features/data/
Expand Down
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
repos:
# Update the pre-commit hooks
- repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update
rev: v0.5.1post1 # Insert the latest tag here
rev: v0.6.1 # Insert the latest tag here
hooks:
- id: pre-commit-update
args: [--exclude, black, --keep, isort]
Expand All @@ -22,7 +22,7 @@ repos:
language_version: python3.10

- repo: https://github.com/nbQA-dev/nbQA
rev: 1.9.0
rev: 1.9.1
hooks:
- id: nbqa-isort
additional_dependencies: [isort==5.6.4]
Expand Down
157 changes: 86 additions & 71 deletions 2.cellprofiler_ic_processing/notebooks/0.perform_ic.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,25 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import argparse\n",
"import pathlib\n",
"import sys\n",
"\n",
"sys.path.append(\"../../utils\")\n",
"import cp_parallel\n",
"import cp_utils as cp_utils\n",
"import tqdm"
"import tqdm\n",
"\n",
"# check if in a jupyter notebook\n",
"try:\n",
" cfg = get_ipython().config\n",
" in_notebook = True\n",
"except NameError:\n",
" in_notebook = False"
]
},
{
Expand All @@ -43,14 +51,37 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"run_name = \"illumination_correction\"\n",
"# path to folder for IC images\n",
"illum_directory = pathlib.Path(\"../illum_directory\").resolve()\n",
"# make sure the directory exists\n",
"if not in_notebook:\n",
" print(\"Running as script\")\n",
" # set up arg parser\n",
" parser = argparse.ArgumentParser(description=\"Segment the nuclei of a tiff image\")\n",
"\n",
" parser.add_argument(\n",
" \"--test_data\",\n",
" action=\"store_true\",\n",
" help=\"Use the test data instead of the full dataset\",\n",
" )\n",
"\n",
" args = parser.parse_args()\n",
" run_test_data = args.test_data\n",
"else:\n",
" print(\"Running in a notebook\")\n",
" run_test_data = True\n",
"\n",
"\n",
"if not run_test_data:\n",
" preprocessed_data_path = pathlib.Path(\"../../data/preprocessed_data/\").resolve()\n",
" illum_directory = pathlib.Path(\"../illum_directory\").resolve()\n",
"\n",
"else:\n",
" preprocessed_data_path = pathlib.Path(\"../../data/test_data/\").resolve()\n",
" illum_directory = pathlib.Path(\"../illum_directory_test\").resolve()\n",
"\n",
"\n",
"illum_directory.mkdir(exist_ok=True, parents=True)"
]
},
Expand All @@ -67,55 +98,41 @@
"metadata": {},
"outputs": [],
"source": [
"dict_of_inputs = {\n",
" # \"run_20230920ChromaLiveTL_24hr4ch_MaxIP\": {\n",
" # \"path_to_images\": pathlib.Path(\n",
" # \"../../data/20230920ChromaLiveTL_24hr4ch_MaxIP\"\n",
" # ).resolve(),\n",
" # \"path_to_output\": pathlib.Path(\n",
" # f\"{illum_directory}/20230920ChromaLiveTL_24hr4ch_MaxIP/\"\n",
" # ).resolve(),\n",
" # \"path_to_pipeline\": pathlib.Path(\"../pipelines/illum_4ch.cppipe\").resolve(),\n",
" # },\n",
" # \"run_20231017ChromaLive_6hr_4ch_MaxIP\": {\n",
" # \"path_to_images\": pathlib.Path(\n",
" # \"../../data/20231017ChromaLive_6hr_4ch_MaxIP\"\n",
" # ).resolve(),\n",
" # \"path_to_output\": pathlib.Path(\n",
" # f\"{illum_directory}/20231017ChromaLive_6hr_4ch_MaxIP/\"\n",
" # ).resolve(),\n",
" # \"path_to_pipeline\": pathlib.Path(\"../pipelines/illum_4ch.cppipe\").resolve(),\n",
" # },\n",
" # \"run_20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP\": {\n",
" # \"path_to_images\": pathlib.Path(\n",
" # \"../../data/20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP\"\n",
" # ).resolve(),\n",
" # \"path_to_output\": pathlib.Path(\n",
" # f\"{illum_directory}/20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP/\"\n",
" # ).resolve(),\n",
" # \"path_to_pipeline\": pathlib.Path(\"../pipelines/illum_2ch.cppipe\").resolve(),\n",
" # },\n",
" # testing small datasets to make sure the pipeline works\n",
" # these have both Well C02 FOV 1 and Well E11 FOV 4\n",
" \"run_20231017ChromaLive_6hr_4ch_MaxIP_test_small\": {\n",
" \"path_to_images\": pathlib.Path(\n",
" \"../../data/20231017ChromaLive_6hr_4ch_MaxIP_test_small\"\n",
" ).resolve(),\n",
" \"path_to_output\": pathlib.Path(\n",
" f\"{illum_directory}/20231017ChromaLive_6hr_4ch_MaxIP_test_small/\"\n",
" ).resolve(),\n",
" \"path_to_pipeline\": pathlib.Path(\"../pipelines/illum_4ch.cppipe\").resolve(),\n",
" },\n",
" \"run_20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_test_small\": {\n",
" \"path_to_images\": pathlib.Path(\n",
" \"../../data/20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_test_small\"\n",
" ).resolve(),\n",
" \"path_to_output\": pathlib.Path(\n",
" f\"{illum_directory}/20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_test_small/\"\n",
" ).resolve(),\n",
" \"path_to_pipeline\": pathlib.Path(\"../pipelines/illum_2ch.cppipe\").resolve(),\n",
" },\n",
"}"
"dict_of_inputs = {}"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# get the list of dirs in the raw_data_path\n",
"dirs = [x for x in preprocessed_data_path.iterdir() if x.is_dir()]\n",
"# get the list of all dirs in the dir\n",
"for dir in dirs:\n",
" # get the list of all dirs in the dir\n",
" subdirs = [x for x in dir.iterdir() if x.is_dir()]\n",
" for subdir in subdirs:\n",
" run_name = f\"{dir.name}_{subdir.name}\"\n",
" if \"4ch\" in dir.name:\n",
" dict_of_inputs[run_name] = {\n",
" \"path_to_images\": pathlib.Path(subdir).resolve(strict=True),\n",
" \"path_to_output\": pathlib.Path(illum_directory / run_name).resolve(),\n",
" \"path_to_pipeline\": pathlib.Path(\n",
" \"../pipelines/illum_4ch.cppipe\"\n",
" ).resolve(),\n",
" }\n",
" elif \"2ch\" in dir.name:\n",
" dict_of_inputs[run_name] = {\n",
" \"path_to_images\": pathlib.Path(subdir).resolve(strict=True),\n",
" \"path_to_output\": pathlib.Path(illum_directory / run_name).resolve(),\n",
" \"path_to_pipeline\": pathlib.Path(\n",
" \"../pipelines/illum_2ch.cppipe\"\n",
" ).resolve(),\n",
" }\n",
" else:\n",
" ValueError(\"The directory name does not contain 2ch or 4ch\")"
]
},
{
Expand All @@ -129,20 +146,23 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of processes: 5\n",
"Number of processes: 8\n",
"All processes have been completed!\n",
"20230920ChromaLiveTL_24hr4ch_MaxIP 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP_test_small 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_test_small 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_C-04_F0001 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_C-03_F0001 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_C-02_F0001 0\n",
"20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP_C-05_F0001 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP_C-04_F0001 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP_C-03_F0001 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP_C-02_F0001 0\n",
"20231017ChromaLive_6hr_4ch_MaxIP_C-05_F0001 0\n",
"All results have been converted to log files!\n"
]
}
Expand All @@ -156,7 +176,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "pe2loaddata",
"display_name": "cellprofiler_timelapse_env",
"language": "python",
"name": "python3"
},
Expand All @@ -170,14 +190,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.19"
"version": "3.8.20"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "8e682b7bd16cdbc4be2393bc1b1eed6b87cf8a0c86d477c0593cdffdecdf8222"
}
}
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
Expand Down
Loading