-
Notifications
You must be signed in to change notification settings - Fork 2
/
Submit_preprocess.sh
executable file
·122 lines (108 loc) · 4.74 KB
/
Submit_preprocess.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/bin/sh
# ICOADS3.0 is 28GB. ICOADS_Step_00_download.csh is the script we use to
# download from RDA dataset 548.0. Please download files to
# $home_ICOADS3/ICOADS_00_raw_zip/. After downloading,
# run ICOADS_Step_00_unzip.sh to unzip files,
# which are placed in $home_ICOADS3/ICOADS_00_raw/.
export partition_preprocess="huce_intel" # TODO
export group_account="huybers_lab" # TODO
mkdir logs
# ############################################################
# Convert ICOADS3.0 data from ASCII format to .mat files
# ############################################################
export JOB_ascii2mat=$(sbatch << EOF | egrep -o -e "\b[0-9]+$"
#!/bin/sh
#SBATCH --account=${group_account}
#SBATCH -p ${partition_preprocess}
#SBATCH -J ASCII2mat
#SBATCH --array=1-1200
#SBATCH -n 1
#SBATCH -t 1500
#SBATCH --mem-per-cpu=20000
#SBATCH -e logs/err_step_01_ascii2mat.%A.%a
#SBATCH -o logs/log_step_01_ascii2mat.%A.%a
matlab -nosplash -nodesktop -nojvm -nodisplay -r "HM_load_package; num=\$SLURM_ARRAY_TASK_ID; ICOADS_Step_01_ascii2mat_sub; quit;"
EOF
)
echo submitted job ${JOB_ascii2mat} for converting ICOADS3.0 data from ASCII format to .mat files
# This step outputs 3,234 'IMMA1_R3.0.0_YYYY-MM.mat' files to $home_ICOADS3/ICOADS_01_mat_files/
# ############################################################
# Assign missing country information and measurement method
# ############################################################
export JOB_assign_missing=$(sbatch << EOF | egrep -o -e "\b[0-9]+$"
#!/bin/sh
#SBATCH --account=${group_account}
#SBATCH -p ${partition_preprocess}
#SBATCH -J Assign_missing
#SBATCH --array=1-1200
#SBATCH -n 1
#SBATCH -t 500
#SBATCH --mem-per-cpu=10000
#SBATCH -e logs/err_step_02_preQC.%A.%a
#SBATCH -o logs/log_step_02_preQC.%A.%a
#SBATCH --dependency=afterok:${JOB_ascii2mat}
matlab -nosplash -nodesktop -nojvm -nodisplay -r "HM_load_package; num=\$SLURM_ARRAY_TASK_ID; ICOADS_Step_02_pre_QC_sub; quit;"
EOF
)
echo submitted job ${JOB_assign_missing} for assigning missing country information and measurement method
# This step outputs 3,234 'IMMA1_R3.0.0_YYYY-MM_preQC.mat files to $home_ICOADS3/ICOADS_02_pre_QC/
# ############################################################
# Compute winsorized mean of 5-day SST at 1 degree resolution
# ############################################################
export JOB_winsorize=$(sbatch << EOF | egrep -o -e "\b[0-9]+$"
#!/bin/sh
#SBATCH --account=${group_account}
#SBATCH -p ${partition_preprocess}
#SBATCH -J Winsorize
#SBATCH --array=1-1200
#SBATCH -n 1
#SBATCH -t 500
#SBATCH --mem-per-cpu=5000
#SBATCH -e logs/err_step_03_winsorize.%A.%a
#SBATCH -o logs/log_step_03_winsorize.%A.%a
#SBATCH --dependency=afterok:${JOB_assign_missing}
matlab -nosplash -nodesktop -nojvm -nodisplay -r "HM_load_package; num=\$SLURM_ARRAY_TASK_ID; ICOADS_Step_03_WM_sub; quit;"
EOF
)
echo submitted job ${JOB_winsorize} for computing winsorized mean of 5-day SST at 1 degree resolution
# This step outputs 3234 'IMMA1_R3.0.0_YYYY-MM_WM_SST.mat' files and
# 3234 'IMMA1_R3.0.0_YYYY-MM_WM_NMAT.mat' files to $home_ICOADS3/ICOADS_03_WM/
# ############################################################
# Computes between-neighbor standard deviation
# ############################################################
export JOB_neighbor_sigma=$(sbatch << EOF | egrep -o -e "\b[0-9]+$"
#!/bin/sh
#SBATCH --account=${group_account}
#SBATCH -p ${partition_preprocess}
#SBATCH -J neighbor_sigma
#SBATCH -n 1
#SBATCH -t 500
#SBATCH --mem-per-cpu=10000
#SBATCH -e logs/err_step_04_neighbor_sigma.%A.%a
#SBATCH -o logs/log_step_04_neighbor_sigma.%A.%a
#SBATCH --dependency=afterok:${JOB_winsorize}
matlab -nosplash -nodesktop -nojvm -nodisplay -r "HM_load_package; ICOADS_Step_04_Neighbor_std_sub; quit;"
EOF
)
echo submitted job ${JOB_neighbor_sigma} for computing between-neighbor standard deviation
# This step outputs 'Buddy_std_SST.mat' and 'Buddy_std_NMAT.mat' to $home_ICOADS3/ICOADS_Mis/
# ############################################################
# Perform buddy check and other quality controls
# ############################################################
export JOB_buddy=$(sbatch << EOF | egrep -o -e "\b[0-9]+$"
#!/bin/sh
#SBATCH --account=${group_account}
#SBATCH -p ${partition_preprocess}
#SBATCH -J Buddy
#SBATCH --array=1-1200
#SBATCH -n 1
#SBATCH -t 500
#SBATCH --mem-per-cpu=5000
#SBATCH -e logs/err_step_05_buddy.%A.%a
#SBATCH -o logs/log_step_05_buddy.%A.%a
#SBATCH --dependency=afterok:${JOB_assign_missing}
matlab -nosplash -nodesktop -nojvm -nodisplay -r "HM_load_package; num=\$SLURM_ARRAY_TASK_ID; ICOADS_Step_05_Buddy_check_sub; quit;"
EOF
)
echo submitted job ${JOB_neighbor_sigma} for performing buddy check and other quality controls
# This step outputs 3234 'IMMA1_R3.0.0_YYYY-MM_QCed.mat' files to $home_ICOADS3/ICOADS_QCed/