From 781947fe2d5e5d3bc85ba0c46b06a523e4dc255c Mon Sep 17 00:00:00 2001
From: doh16101 <dong.han@uconn.edu>
Date: Thu, 25 Jan 2024 14:15:28 -0500
Subject: [PATCH] Tried to solve the error. Running it on my Linux computer now

---
 .gitignore                                    |   1 +
 .../ss_active_learning.cpython-311.pyc        | Bin 6909 -> 6919 bytes
 .../active_learning/ss_active_learning.py     |   4 +-
 BML_project/ss_main.py                        |  61 +++++++++++++++++-
 .../__pycache__/data_loader.cpython-311.pyc   | Bin 18912 -> 18814 bytes
 BML_project/utils_gp/data_loader.py           |   8 +--
 6 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index 11d1435..233cecd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ VAE.py
 model_checkpoint.pt
 GP_original_data.py
 Attention_network.py
+*.pt
diff --git a/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc b/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc
index 9504984498b04e14db8bf2c7a9373efccf469c21..bfc5c60abc83f888ab6d48e2330808b7d3362830 100644
GIT binary patch
delta 87
zcmexs+HS_ZoR^o20SH3YZQ98Fm5Y<RAhD<<GchNAG9z~$<IKqg+)5lwtlA$KfW+ik
m+`%$8I3%ufs9fSuSrBrOL+c8M)&(&7z{bEMvzeXe8y^6G_8L(D

delta 81
zcmZoS`)kU*oR^o20SG*nuG`4{m5Y&iGADN)<MhcD+)5nGtjZr4fW+ip0n5p^xuv8Y
ga7bL|kh#PmGb7|8htd@er3+y6fLnUAAkQ~G0Ij7M4FCWD

diff --git a/BML_project/active_learning/ss_active_learning.py b/BML_project/active_learning/ss_active_learning.py
index a488281..0de3533 100644
--- a/BML_project/active_learning/ss_active_learning.py
+++ b/BML_project/active_learning/ss_active_learning.py
@@ -55,8 +55,8 @@ def run_minibatch_kmeans(data_loader, n_clusters, device, batch_size=100):
     # Iterate through data_loader and fit MiniBatchKMeans
     for batch in data_loader:
         data = batch['data'].view(batch['data'].size(0), -1).to(device).cpu().numpy()
-        # minibatch_kmeans.partial_fit(data)
-        minibatch_kmeans.fit(data) # Dong, 01/22/2024: Debug
+        minibatch_kmeans.partial_fit(data)
+        # minibatch_kmeans.fit(data) # Dong, 01/22/2024: Debug
 
     return minibatch_kmeans
 
diff --git a/BML_project/ss_main.py b/BML_project/ss_main.py
index 2716179..73d6fb6 100644
--- a/BML_project/ss_main.py
+++ b/BML_project/ss_main.py
@@ -11,9 +11,36 @@
 from utils_gp.ss_evaluation import stochastic_evaluation, evaluate_model_on_all_data
 from active_learning.ss_active_learning import stochastic_uncertainty_sampling, run_minibatch_kmeans, stochastic_compare_kmeans_gp_predictions
 from utils_gp.visualization import plot_comparative_results, plot_training_performance, plot_results
+import os
+import pickle
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
+class CheckpointManager:
+    def __init__(self, checkpoint_dir):
+        self.checkpoint_dir = checkpoint_dir  # Store the directory path for checkpoints
+        if not os.path.exists(checkpoint_dir):  # Check if the directory exists
+            os.makedirs(checkpoint_dir)  # Create the directory if it does not exist
+
+    def save_checkpoint(self, loader_name, iteration, additional_state):
+        # Construct the checkpoint file path using the loader name
+        checkpoint_path = os.path.join(self.checkpoint_dir, f"{loader_name}_checkpoint.pkl")
+        checkpoint = {
+            'iteration': iteration,  # Store the current iteration
+            'additional_state': additional_state  # Store any additional state information
+        }
+        with open(checkpoint_path, 'wb') as f:  # Open the file in write-binary mode
+            pickle.dump(checkpoint, f)  # Serialize the checkpoint dictionary to the file
+
+    def load_checkpoint(self, loader_name):
+        # Construct the checkpoint file path using the loader name
+        checkpoint_path = os.path.join(self.checkpoint_dir, f"{loader_name}_checkpoint.pkl")
+        try:
+            with open(checkpoint_path, 'rb') as f:  # Open the file in read-binary mode
+                return pickle.load(f)  # Deserialize the checkpoint file and return it
+        except FileNotFoundError:  # Handle the case where the checkpoint file does not exist
+            return None  # Return None if the file is not found
+        
 def main():
     # Set parameters like n_classes, batch_size, etc.
     n_classes = 4
@@ -21,7 +48,7 @@ def main():
     clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled = split_uids()
     data_format = 'pt'
     # Preprocess data
-    train_loader, val_loader, test_loader = preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size)
+    train_loader, val_loader, test_loader, saving_path = preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size)
     print('Debug: len(train_loader)',len(train_loader))
 
     kmeans_model = run_minibatch_kmeans(train_loader, n_clusters=n_classes, device=device)
@@ -43,9 +70,27 @@ def main():
     results['validation_metrics']['f1'].extend(training_metrics['f1_score'])
     # results['validation_metrics']['auc_roc'].extend(training_metrics['auc_roc'])
 
+    # --- Dong: copied from GP_Original_Checkpoint.py ---
+    # Initialize the CheckpointManager
+    checkpoint_manager = CheckpointManager(saving_path)
+
+    # Attempt to load a training checkpoint
+    train_checkpoint = checkpoint_manager.load_checkpoint('train')
+    start_iteration = train_checkpoint['iteration'] if train_checkpoint else 0
+    # Dong, 01/25/2024: save it first before entering the active learning.
+    additional_state = {
+        'model_state': model.state_dict(),
+        'likelihood':likelihood,
+        'val_loader':val_loader,
+        'train_loader':train_loader
+        # Include other states like optimizer, scheduler, etc.
+    }
+    checkpoint_manager.save_checkpoint('train', start_iteration, additional_state)
+    # ---------------------------------------------------
+    
     active_learning_iterations = 10
     # Active Learning Iterations
-    for iteration in tqdm(range(active_learning_iterations), desc='Active Learning', unit='iteration', leave=True):
+    for iteration in tqdm(range(start_iteration,active_learning_iterations), desc='Active Learning', unit='iteration', leave=True):
         # Perform uncertainty sampling to select new samples from the validation set
         uncertain_sample_indices = stochastic_uncertainty_sampling(model, likelihood, val_loader, n_samples=batch_size, n_batches=5)
 
@@ -58,9 +103,19 @@ def main():
         # Store the validation metrics after each active learning iteration
         results['validation_metrics']['precision'].append(val_metrics['precision'])
         results['validation_metrics']['recall'].append(val_metrics['recall'])
-        results['validation_metrics']['f1'].append(val_metrics['f1'])
+        results['validation_metrics']['f1'].append(val_metrics['f1_score'])
         # results['validation_metrics']['auc_roc'].append(val_metrics['auc_roc'])
 
+        # Save checkpoint at the end of each iteration
+        additional_state = {
+            'model_state': model.state_dict(),
+            'likelihood':likelihood,
+            'val_loader':val_loader,
+            'train_loader':train_loader
+            # Include other states like optimizer, scheduler, etc.
+        }
+        checkpoint_manager.save_checkpoint('train', iteration, additional_state)
+
     # Compare K-Means with GP model predictions after retraining
     gp_vs_kmeans_data, original_labels = stochastic_compare_kmeans_gp_predictions(kmeans_model, model, train_loader, n_batches=5, device=device)
     
diff --git a/BML_project/utils_gp/__pycache__/data_loader.cpython-311.pyc b/BML_project/utils_gp/__pycache__/data_loader.cpython-311.pyc
index e131c154219f7dc3144ea923de24d050f059f45a..f58fbb2dc35e02ef41ed0b0ae0340b3857ee87d0 100644
GIT binary patch
delta 1178
zcmY+ET}TvB6vsWz?)bHu3MIRiZB$Exu3<`qrD5o%eU}z(RNLTAXl=W4XDi#Z%#X-E
zp(7IIX7x}}^kKITp&m-adMM~E7%J?8C<^i+=*_xk#s^*4-<*5yIp?1FpSwGXFn%7=
zZ=|QECWz1bdk?g41L+#TGp@svq|_EyOYUx|HbE)mp_GqyzJ3}4Y~x=t3nBHeNXQaj
zJ&)&RXWH#!js%CB`*ViDc1*Z&rqhX~&cDfRVxdTe8;^S2=y$oYx+w_++^kxVo#36W
z#gN@$#<42#Fr@vBLDr3TD`w#5Ty;8a8M4+K)xtq`!iww0tuudRx6vYztrZV;hH#E=
znH7do(>^O4MpA>StI;U4VlI*^B@|JEfuI^y6+5GD45=~L%;J>a$H((VVL8Pq@}#;u
zXp;)E%LH@ZN@kbH(I;TYVeMc*bROrk=IoQ8kDr=18@y)si4onfHL1}C!?q{V7-sEe
zyw=>PM|I|>3blkfd^bNY=Qx>kqLOO%XT924ba)oaS7<Me)t<R<nuk1}?HNaeCuS?G
z*AFPbZsYcXeV*qOL{MCell_qJh(NPp_Xu71yTB<=9Gl^-gifsWE^wO7Y!Ly5J*KgU
zuB2E!#IeWgg{wH^RiK~$@|H;O0Ou6Bmb@}!^#-*`V<S}frGO$?StO1j>-B0#W4a_3
zZui94-=JP}5{}?_QT~dPltzPPy@U&dYlQ2B(}aG)8$u;v6~Vkdtz@<0-bI5BGJTBn
z_+i6^#W5JbTgAS7Gr}&sN}BchBtE0Duup_eLKopU&n;<|MN>Vc%d%bwkC9U6s|_@T
zB4SEI_^Grke<Q_hBA7zGCTpt1O0rd~9&45s%ckD#ynX3EfI)08d$g`4dACjcq@;Cq
zLZnG&i%Q}u|Dl54$#z&}4v^yzAx1DeHZO=(89T~%!BhNN{y6!5{@b*&x3)5VW%)S3
z8Qx!!A?19eqyfUHfH?6R+1GKna<jOguB!6@y?Do`OrP2bO9T<tgFk&YSCODTAEQbN
z_!P;KxK42zNr6BxQXdFd3hboH!vcofq=tj4Zr*7_+EBBG(dgL-epu;GGZmzN57x>n
Ss#im1`B;JslYo6SmHz?pe>uVc

delta 1307
zcmZ9MZERCj7{~kc_TKJ=;<^Cadh0rZQ*TDg-b7Q`2v|DaG~#BAVyrj27gsada&C2V
zD?`B1Yz8o#7$C68OEUJs805HFqhC}AkSM8fAISZH6%&GqCPaxJ_yC;eT#Un;^mm^=
z_kW)A^nae)yN94e!FR^zt8npaYUVTZ=g)m6l->5aT=`)BQJQOif}Mmh?^;8xLy5TF
zf}J(F!2dFVoNA~>WYqniZl{{#x}3UA>UL=>SbO<3S^nL_OR(xkD8N=WRTY7X98dW9
z$LeC&{8e(K6l^~0p9{7X)%K#N_Np40-Z7`P6a%eAPtDcP(*-CjEl6{rRYgy*R2gD_
zFF6nNo7!48$<v=|@4#;QL){wxI2Hts2C|-$jIsK_N~r#T$C^ae@tZBZlQhUU-KGvK
z#<Z=NgIP0WkTFLyPXAK>sNRiV+~j$FBL@&4v0K3uL`8g%Cv0z@Zt2-9aWuy;Ro<W0
zjfA0RbxkJNi7h3KK{AQ!C)knt60Am@rp&AR20p}s<P(H=?kpz9F)_hmD=Bjz!MmPh
zUp8!)V1hM17J{(o-kr8=Pe#x7*`Bx4$rRZu^yXmF%39<#tk8paneJ$;ugPPGJ8Bi8
zKj)_%q*IM6m6P~vl3s6I2K$-v_;q>VE}j|kM0%}vH6WkTf10+h`xZqAEkM4&cm^?x
z*n>EW*vDa)aZ9a%B=5_e%Gk0_i2mNctxPYE!6^N6`IB{`!B<c(51AxwY2{8|lQa?z
z!zucDSc4O6U$|WYM!#+jJabuStv)?tl8d<6B@TYF=(aG-R;*sV-z1jAGYt(7#^4I|
zwT#0qdZeXk?T0AE!zPCjM-itHXAt`lClFT<F~mlM7^e}8MyT9cSb$-iphsj2z1%tk
z=V+`g-Xs(<&uW#9C3qkNkBa<?7)78%B+q8r1{K~^B)ZD~9naY1J@M{Dra#S3?t3~M
z?Pz)lb=?SI)Q=cFY`a$G6f@{=(Kbb#!Cn@8_8!124R*|I7O$Z6LJzlS-p;%ZAMV>&
z?k2_yoFW657(xsqM89HWoVqmB`6hft-|w7Nij~bpPc!RWeHY*W>sYf;TDAq3oI{j2
z_<lDq7M=<_y!Ilx^tnZJPwXi^z=hZ$fWvfCTwDBbU&IZ3*~fG&KD`mmjmHUA)*!A)
zj@><4o$XB|jC5}z;kc2&oi1?LN=8o^x+Mn9mR{Vjncx_Zi*&f_)k@(e{_Btt_Dk0$
PsA{?8QeYlvbVKYu`ovlj

diff --git a/BML_project/utils_gp/data_loader.py b/BML_project/utils_gp/data_loader.py
index 921467d..5b84b8d 100644
--- a/BML_project/utils_gp/data_loader.py
+++ b/BML_project/utils_gp/data_loader.py
@@ -98,9 +98,9 @@ def split_uids():
     print(f'Clinical trial: selected {len(clinical_trial_test)} UIDs for testing {clinical_trial_test}')
     print(f'Clinical trial: selected {len(clinical_trial_unlabeled)} UIDs for unlabeled {clinical_trial_unlabeled}')
     
-    clinical_trial_train = [clinical_trial_train[0]]
-    clinical_trial_test = [clinical_trial_test[0]]
-    clinical_trial_unlabeled = clinical_trial_unlabeled[0:4]
+    # clinical_trial_train = [clinical_trial_train[0]]
+    # clinical_trial_test = [clinical_trial_test[0]]
+    # clinical_trial_unlabeled = clinical_trial_unlabeled[0:4]
     
     return clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled
 
@@ -260,7 +260,7 @@ def preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clin
     train_loader = load_data_split_batched(data_path, labels_path, clinical_trial_train, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels)
     val_loader = load_data_split_batched(data_path, labels_path, clinical_trial_test, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels)
     test_loader = load_data_split_batched(data_path, labels_path, clinical_trial_unlabeled, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels)
-    return train_loader, val_loader, test_loader
+    return train_loader, val_loader, test_loader, saving_path
 
 def map_samples_to_uids(uncertain_sample_indices, dataset):
     """