From ddbd109b5bef61f52486a6c11ec82455a7abd050 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 23 Jan 2024 16:05:12 -0500 Subject: [PATCH 1/6] Tried to run Luis' code. Encountered error again. Going to add checkpoint to save progress. --- .../ss_active_learning.cpython-311.pyc | Bin 6942 -> 6909 bytes .../active_learning/ss_active_learning.py | 3 +- BML_project/cassey_CS330_torch.yml | 272 ++++++++++++++++++ .../__pycache__/ss_gp_model.cpython-311.pyc | Bin 12179 -> 12155 bytes BML_project/models/ss_gp_model.py | 3 +- BML_project/ss_main.py | 7 +- .../__pycache__/data_loader.cpython-311.pyc | Bin 17950 -> 18912 bytes .../__pycache__/ss_evaluation.cpython-311.pyc | Bin 9772 -> 9750 bytes .../__pycache__/visualization.cpython-311.pyc | Bin 5734 -> 5677 bytes BML_project/utils_gp/data_loader.py | 28 +- transfer_data/tar_PT_files.sh | 19 ++ 11 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 BML_project/cassey_CS330_torch.yml create mode 100644 transfer_data/tar_PT_files.sh diff --git a/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc b/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc index 45c9875deaf1bad595378c59b1e08a1296ac1ec3..9504984498b04e14db8bf2c7a9373efccf469c21 100644 GIT binary patch delta 204 zcmbPd_SckmIWI340}yyDU6<;yk(YxhGFLx0uSCB{-__4ODBd|EKTqE!KQCQBpt1zS zc8d4)i4Q2s&q_@$(e)|KEJhL0cLK^5KxFk3lS?woQsZ+{6N~aPfhKHjXPU&p$UIqz zdm;xjtMUg1ATfD2cd+;a4vFg=GM6}HW`tbiP`bjQbODSutMa(AFnVv!7P!sE7`@qD Uyn|VUi&5qS11e!K`LUEW0EaU`U;qFB delta 260 zcmexsI?s%EIWI340}!Zs%t__g$jiZ0-{x!;6Iz^FR2-92lxt*UU|<^KpO@-Vlv$Rl zpsNs?2c(ONGfOHJ^3xQY^YijjlS?v_OG{#0@{>z*Q}arSW85=KGD?$ToZ@|b;sc8E zvw$jMoPg{CC_53TqbxN(CpEDsFEcMarnopBA+))gsg;A1yCAWsBr`E5eljEXL=Gla z?GFq;I^mffF`Yx7z5 zGA2gT$u3+wwLO6L{%Bx;!4FJwtXv-$un-Q6lMT7WH`{UB@-upFu8}I@Vw|@5y$TOI m&zkJ1?*#x_p%j$T1!ks{DFRg%7wtJp0WIW})$ zFJodfnXJvVQ_CGF@uPtO20t*#vT}W3z(Uw<=I0LKXY|;dB2~o2ICb-N6&`lRRhyq^ h%Q5R4GD?47z$7MAOsT&jZt@W%^#viHG1*Yx3jhUmUONB) diff --git a/BML_project/models/ss_gp_model.py b/BML_project/models/ss_gp_model.py index c18f06f..e364cec 100644 --- a/BML_project/models/ss_gp_model.py +++ b/BML_project/models/ss_gp_model.py @@ -20,7 +20,8 @@ class MultitaskGPModel(gpytorch.models.ApproximateGP): def __init__(self): # Let's use a different set of inducing points for each latent function - inducing_points = torch.rand(num_latents, num_inducing_points, 127 * 128) # Assuming flattened 128x128 images + inducing_points = torch.rand(num_latents, num_inducing_points, 128 * 128) # Assuming flattened 128x128 images + # Dong, 01/22/2024: I will use 128 * 128. # We have to mark the CholeskyVariationalDistribution as batch # so that we learn a variational distribution for each task diff --git a/BML_project/ss_main.py b/BML_project/ss_main.py index a610684..2716179 100644 --- a/BML_project/ss_main.py +++ b/BML_project/ss_main.py @@ -4,13 +4,13 @@ @author: lrm22005 """ -import tqdm +from tqdm import tqdm import torch -from utils.data_loader import preprocess_data, split_uids, update_train_loader_with_uncertain_samples +from utils_gp.data_loader import preprocess_data, split_uids, update_train_loader_with_uncertain_samples from models.ss_gp_model import MultitaskGPModel, train_gp_model from utils_gp.ss_evaluation import stochastic_evaluation, evaluate_model_on_all_data from active_learning.ss_active_learning import stochastic_uncertainty_sampling, run_minibatch_kmeans, stochastic_compare_kmeans_gp_predictions -from utils.visualization import plot_comparative_results, plot_training_performance, plot_results +from utils_gp.visualization import plot_comparative_results, plot_training_performance, plot_results device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -22,6 +22,7 @@ def main(): data_format = 'pt' # Preprocess data train_loader, val_loader, test_loader = preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size) + print('Debug: len(train_loader)',len(train_loader)) kmeans_model = run_minibatch_kmeans(train_loader, n_clusters=n_classes, device=device) diff --git a/BML_project/utils_gp/__pycache__/data_loader.cpython-311.pyc b/BML_project/utils_gp/__pycache__/data_loader.cpython-311.pyc index 0b1a7ebdb4b25cf36cb492dfb1b7d20074e1bb08..e131c154219f7dc3144ea923de24d050f059f45a 100644 GIT binary patch delta 5497 zcmcH+ZE#f8_1*p6&2Bay`@NfF^9f5Jn*<^xfiMY6h%qD~gb}g0ta+P|xY>>Oz3>s= zHb7B*U@=-4_7WrnzwDYk0W{=rd3rBmB;-peN0U^{lE zy~&>U?)kdsoO|xM=bb!Ct}>E)+F~(rFiySrZ1fLnkLS9`3&$7TWg)yOmbWdB*B|7z zQ(fJ6Po92s6-bBb`>8Tct9 zN#Y)m(#}ZPvh`-Tc-=xNiST-NhRjWAIF5Zl3N#`Fh5f0@VH%mWk4;uNCk@&1D7_k_ z;c1-H0yERxY(k;u+=MaDNLuD7AelRZHD_a$Sv^_`O!J7Sej-Q4T4zM(JEUiu%50Bn zazjZKpU3A9sgmj;jxTtcBsK6Xgr^pE)(&``q)zH3P0}D~B^_!_8pV8EPZ_*JEMyP2 zd2~;3yi?M%F@r7dCi*02dGdQHZd@VJnYL>cckc0(IDX~4;l{pLfk1}JTuFjv>);TRmE3*F;&G$1yZ48lkEJW zvx`r`!8rpzWh5P9wfweRhLVHl{=J>qCh-BbL$|EaIip}k{=bm^tu9#Zl3bEWPGL9X zkwByE(xw zdFM3Ot;$NaC0V=U%=Q}6uv&87e5OE%^%{!EQWiHjvJX_YKJQKXmPuaFXW#CepRhwD z>6APRFdoSZSygktYA&%MBETnk2h~u@gWWgmvBW?mwlg7!HS!yu7>x|aCq`JLp_v3( zaHX5AG&$^Afh1cy&#Dk$kd??tvSH$dlDqnXd)X!TvXsU#t#_x2R$SDtNaGFzAGkZI8j-|~ybhV^1P=lbppAgtkq7Nyh0aP6V~x(Qz{Y&jIY{=ie>hw7 zk02wq1Ab)>53?Ru8QI0CYYFh3aJ^L>0%Cy33k1{@b0MvU_)u9=jgtP%{*Vlv2x5X)Dvu!1|n$~}un2fNKvNha96o-MG@c~3{tek7~tMtIPN5Ih1P zV;fuQEhh(9uh$2@k9x~Ulg#oku%z`nBSJ(JsUrFWqKvUbgb(u(F%rB#Y7pDxobCxv80x3Jwq`)sHQ@|J?;am&^5aqW4-Uto{^%a46~Ya&lwz3DdLcSA z8jXvv_wKR-W~^niiFLZHhdjeRDJ$ubb570h64EN@mOXV`kqeuhV@O8BVeb<;6O= zh#q6t%a`k>04wydnhJ+5-?-*OE}wN)G}x?`HCOWNDM#=*(}ld|OL@(yyk>Tw;_brE zu3CNH8cbO$QickOZVaf{^NqH1$0{2&g_AI!&|sk87@vg+?P6~P20GD2`f>&YJoJb7 z{vv{x5F9}890DvUX#*T#VfQe&t%{GOwQyxc_GUbNU`Ys$(rZf=yJi1xM}F-@lqQ6L zQE~euGgmJoPqU`#igMY{Dd3rKt{WG`#Apc4sSw3Wa^}$f}5Qq6oS>UlMM8)=eiUYpd!Yk{JhrRafT0E!VP?!=R_ShDP{~zqx1@_p#tUZqZr9F`^jE_~s?qg=~3;K50P54-r^m3o< z>Yi}V)bE=cJ8A?KthC{$h3Wi`-gOJQzBe0=YN-i1cHZ)%%8AnTeSJNBrT#m5w)FW! zw}$-PJ^lU->-+uv9qawQt^FM}aPmu6uFn4zh@_4E?VH16u>=(DU$gki=2|`N13cY~ zKrUmy$G7aF7%(XP6%QnKd1dXSTqN;urgMbIuHpDl7;gpv^9iqf9CF1-=R)@@HzW!J zh4zqp6Mn+?A?^2ZSF(G{D;i_URGF$;bwSg3Nz<6pG|ulfr)_uewD-=G z!3F-dkI2$JV5&K=!YF`JC z)7CK>h3}4ms36FZp`QVk5LVIrGBkPbHeV+JEW2=PVXv2IKFKhB7p4N%uz9Vgre!Iy zvAUKeI=QZfSfs^okr`FUxC5H6hg#ga&tYOD><29YSmJVvN9EiE82SQ&s{mjX6r!h@ zp|zg8#hP2o$O(2=Yn6PRq!+WUb#8ksa(6Tq<-irFXv~CPOvm%0ggQwNKw{}qs zC_T?ALwEGb2SLKQd=cRmE1QiwL}hQ#XNra_xEgO5R*xq5@mRElT7e4x*vPyZI_jV1 mEc@iYYnrWu^uUFg^$=NXeMepy30~u7K__9atnbsHm;VBY?yEY zaPHP7kYb3d1Pv{OG%d*wM+$Be6{)S1Rt;&SqE?mMDpk=EkXxlcqCVm%P4j4)^qhNb z<6Y>Fj=bN@J#)^PGjqLTPY<+Ce&<{VLoDEJAl4p-@9BjLwP6~bX1-!4>*i@IqL zxU?jMob8|idcA*16MPwNeT`}J#@i^S1xfR*8IZIrP)&tY^(&$-0Zhv#QT|Hr3mWZW;LQWVYqU|M`(?M^CUN1VtGN2n2A!k%{Ig_GS45%z;mTl@f zDu_N;!th za!z2h=E~*T#bpSQ9k<3xd7-3OJ!T-Jkd|}Fb+;r=1q)|SkB25hho~4nVLlb?BGbTm zX^A`Mo(#gQAg;LctUwf}Qm(iZw^%b%JFN#<_-QTYksB7r5TYKD+<6lqpp^4gV-5G3 zf)Q__(49q=BotF+ZG}>ySk&P4L0tJ29|TYkLdCAoQNlcmUvohGHrGsrfIpy6;Qxxs z?}Zpx3vzzwn~QwD7z-!7Q3BdJ+iqWHLqS~D75L9v=naV2dc}(uY0dJBR4C)W zO{PKd-*)*xhQDU6)HNyrxOwhg#k)MX2Wif$_=_lhB>;gJd20Ec3n;*+1P=O80ds?c40gsCsR^% zFgqZ|$Xnu>k+dA`>)*q=;9J6& z42LHsSU0lt0Pr)mFzw+N>>Ft>|AYMr>gIGC)9wDJF5p>cSBnLvQ4DuotGPz;_#5u;2lHGq{E_ZV$DDPzkW3xp`yah@41^?6^id!5V15hS<(Cdds(`LpZZcGdmPQUA81 ze%4&idN03JSwlCflBa+tZyJh8F;9v!4ki&cGEso6)`R|bU(`Dn(R3QbzalYzJr8w z0C{6N@pu%@4HuIjo5Q(51BW%O*hDA!(2CPR_2ic-5nwUH50Ci6Qf{jW3Xgomn`)lv zo>`uF4x{(grszdM)QNi0FeuCzi?X&sF)XiVE}HR;@`_N66=t2BQFcLb5V0*QcMewR$RA;?n46w29#poi|B|K};s#o2OSkS&B(uDIWZFyXg0M zvaWik8ftn~g6#m7_fVE$?fKr9&>}W13Wbc6*Joq$P+osHlT54E2}2X^y2N&&$Zr0} zx~dRfDTW@6Sk(~GVo~+3g*)rlLRAC|=lS;fDx;csPV!X!T{Y-qsRNuhL9Y1%leTM4 z*Q9aIRdM#7c~WM%7l2=>U$3*Y)9#`Jgs zaNqF7W;=MSDZUyv#a>^8NPzti$FCy1hM;DLS8z1J-*5687dB&W?xN$ozq!%B59js( zsqrM0#6DnGaO5i2^VFQVJaj`A>{FXnqu73bmiQ|zhHa!gR`@^>&KF<+_GLs0kyC^{2uFDzH4n+Wp9w-K&7=dCG zP4H!gFALQ4M6_xJDvR-8W3x*qmP~`DOYqZL&ZyG$VrdcGg!B~#(N;t^r*D8(dy!U0 z5!e5(briIgE!WB`?{@Q0W0UKD*tYdfwmD?e;?4;Cqf(fro}`LVF)3!XesT5j{Tp8A z4fjm(pWgEm0mAI;#&+k=Kss;P+tVM-rZRBFf6o83aZ`)F`11S;*x)`)Ah#JF^;9xF zawKnr(NH$d@88r~zx4R#OQhInGCdf@EFxJz0>WKukj7u!f%8Lblf6c|1wJlPq6WqD^nsOSR?gX-b^?w7~aZNGzpdQA`)&@zmApDpa)K**QlE3DZPg{m$I z=b)Wf;W@Map!72cUjh_68#S#O$wDEKh{`OMOh>t=x5}Uco5FlUZ~udM`y&P|!SwVn z+*(BF>50&uh)$#Kjrj7WXjII^qfsp;4*(PU1*$e?p)!qWuToyOd&m6@69>D(Z}jfJ zZxn}Wp28r|!U28A)D7dYFtnp6bAD!SAI^v)sl;Yxg<1F`(gXaP+ovk#^}8tD1!=UP ZJILF1YUb&Imn>{E|KR2&LzewNJ&pjyKIU_$$-z7gUT|c0*1jKfV s_w|VnD9X=DO)k;(Da|ZK5z%)7$`(Lm^-D`KbBg2B3-mYdW6zTV0DkBtHUIzs delta 153 zcmbQ{v&M&eIWI340}#YZPT$Dw&0b&bY!wq)oLW>IlT(yyWMp7q8sndr>Qa;fpev?Md9IHtHbKD8_{r!=u7Ge1wSpz@YzaY=r1#^(L(;c@_*lsBsY diff --git a/BML_project/utils_gp/__pycache__/visualization.cpython-311.pyc b/BML_project/utils_gp/__pycache__/visualization.cpython-311.pyc index a8c3afd30ad9aefea43425eec7756a9d74ec4254..fe8ddd586cc0009554b2a6bedcddf59cc949fd24 100644 GIT binary patch delta 108 zcmaE+vsQUb&Hb0&7sBer{fgev!VbpLX(*e<`l=L7wB(RV{;Gy0GlZ!wEzGB delta 166 zcmZ3h^Gt_(IWI340}yyzY1qg;fwg{?vsFxJacWUI& Lj@kT{)j Date: Thu, 25 Jan 2024 14:15:28 -0500 Subject: [PATCH 2/6] Tried to solve the error. Running it on my Linux computer now --- .gitignore | 1 + .../ss_active_learning.cpython-311.pyc | Bin 6909 -> 6919 bytes .../active_learning/ss_active_learning.py | 4 +- BML_project/ss_main.py | 61 +++++++++++++++++- .../__pycache__/data_loader.cpython-311.pyc | Bin 18912 -> 18814 bytes BML_project/utils_gp/data_loader.py | 8 +-- 6 files changed, 65 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 11d1435..233cecd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ VAE.py model_checkpoint.pt GP_original_data.py Attention_network.py +*.pt diff --git a/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc b/BML_project/active_learning/__pycache__/ss_active_learning.cpython-311.pyc index 9504984498b04e14db8bf2c7a9373efccf469c21..bfc5c60abc83f888ab6d48e2330808b7d3362830 100644 GIT binary patch delta 87 zcmexs+HS_ZoR^o20SH3YZQ98Fm5Y^O4MpA>StI;U4VlI*^B@|JEfuI^y6+5GD45=~L%;J>a$H((VVL8Pq@}#;u zXp;)E%LH@ZN@kbH(I;TYVeMc*bROrk=IoQ8kDr=18@y)si4onfHL1}C!?q{V7-sEe zyw=>PM|I|>3blkfd^bNY=Qx>kqLOO%XT924ba)oaS7-B0#W4a_3 zZui94-=JP}5{}?_QT~dPltzPPy@U&dYlQ2B(}aG)8$u;v6~Vkdtz@<0-bI5BGJTBn z_+i6^#W5JbTgAS7Gr}&sN}BchBtE0Duup_eLKopU&n;<|MN>Vc%d%bwkC9U6s|_@T zB4SEI_^Grken Ss#im1`B;JslYo6SmHz?pe>uVc delta 1307 zcmZ9MZERCj7{~kc_TKJ=;<^Cadh0rZQ*TDg-b7Q`2v|DaG~#BAVyrj27gsada&C2V zD?`B1Yz8o#7$C68OEUJs805HFqhC}AkSM8fAISZH6%&GqCPaxJ_yC;eT#Un;^mm^= z_kW)A^nae)yN94e!FR^zt8npaYUVTZ=g)m6l->5aT=`)BQJQOif}Mmh?^;8xLy5TF zf}J(F!2dFVoNA~>WYqniZl{{#x}3UA>UL=>SbO<3S^nL_OR(xkD8N=WRTY7X98dW9 z$LeC&{8e(K6l^~0p9{7X)%K#N_Np40-Z7`P6a%eAPtDcP(*-CjEl6{rRYgy*R2gD_ zFF6nNo7!48$sNRiV+~j$FBL@&4v0K3uL`8g%Cv0z@Zt2-9aWuy;RoVE}j|kM0%}vH6WkTf10+h`xZqAEkM4&cm^?x z*n>EW*vDa)aZ9a%B=5_e%Gk0_i2mNctxPYE!6^N6`IB{`!B& z?k2_yoFW657(xsqM89HWoVqmB`6hft-|w7Nij~bpPc!RWeHY*W>sYf;TDAq3oI{j2 z_<4o$XB|jC5}z;kc2&oi1?LN=8o^x+Mn9mR{Vjncx_Zi*&f_)k@(e{_Btt_Dk0$ PsA{?8QeYlvbVKYu`ovlj diff --git a/BML_project/utils_gp/data_loader.py b/BML_project/utils_gp/data_loader.py index 921467d..5b84b8d 100644 --- a/BML_project/utils_gp/data_loader.py +++ b/BML_project/utils_gp/data_loader.py @@ -98,9 +98,9 @@ def split_uids(): print(f'Clinical trial: selected {len(clinical_trial_test)} UIDs for testing {clinical_trial_test}') print(f'Clinical trial: selected {len(clinical_trial_unlabeled)} UIDs for unlabeled {clinical_trial_unlabeled}') - clinical_trial_train = [clinical_trial_train[0]] - clinical_trial_test = [clinical_trial_test[0]] - clinical_trial_unlabeled = clinical_trial_unlabeled[0:4] + # clinical_trial_train = [clinical_trial_train[0]] + # clinical_trial_test = [clinical_trial_test[0]] + # clinical_trial_unlabeled = clinical_trial_unlabeled[0:4] return clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled @@ -260,7 +260,7 @@ def preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clin train_loader = load_data_split_batched(data_path, labels_path, clinical_trial_train, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels) val_loader = load_data_split_batched(data_path, labels_path, clinical_trial_test, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels) test_loader = load_data_split_batched(data_path, labels_path, clinical_trial_unlabeled, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels) - return train_loader, val_loader, test_loader + return train_loader, val_loader, test_loader, saving_path def map_samples_to_uids(uncertain_sample_indices, dataset): """ From cb8a5af16b4a911fd8e8883a888352f5fe406f7d Mon Sep 17 00:00:00 2001 From: doh16101 Date: Fri, 16 Feb 2024 11:14:09 -0500 Subject: [PATCH 3/6] Tar single UID's PT files. --- transfer_data/tar_PT_files_single_UID.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 transfer_data/tar_PT_files_single_UID.sh diff --git a/transfer_data/tar_PT_files_single_UID.sh b/transfer_data/tar_PT_files_single_UID.sh new file mode 100644 index 0000000..dc3f3c0 --- /dev/null +++ b/transfer_data/tar_PT_files_single_UID.sh @@ -0,0 +1,5 @@ +source_path="/mnt/r/ENGR_Chon/Dong/MATLAB_generate_results/NIH_PulseWatch/PT_format" +dest_path="/mnt/r/ENGR_Chon/Dong/MATLAB_generate_results/NIH_PulseWatch/tar_PT_format" +sub_d="120" +dest_tar="${dest_path}/${sub_d}.tar" +tar -C $source_path -cvf $dest_tar $sub_d \ No newline at end of file From 861e7420a147d34802c29fb2d53a2eba8f9ab6ff Mon Sep 17 00:00:00 2001 From: doh16101 Date: Thu, 4 Apr 2024 12:18:20 -0400 Subject: [PATCH 4/6] Cassey manually updated the data loader changes. --- .../Colab_example_dataloader_2024_04_04.ipynb | 22 ++ BML_project/models/ss_gp_model.py | 138 ++++++++++- BML_project/ss_main.py | 74 +++++- BML_project/utils_gp/data_loader.py | 219 +++++++++++------- 4 files changed, 348 insertions(+), 105 deletions(-) create mode 100644 BML_project/models/Colab_example_dataloader_2024_04_04.ipynb diff --git a/BML_project/models/Colab_example_dataloader_2024_04_04.ipynb b/BML_project/models/Colab_example_dataloader_2024_04_04.ipynb new file mode 100644 index 0000000..4495514 --- /dev/null +++ b/BML_project/models/Colab_example_dataloader_2024_04_04.ipynb @@ -0,0 +1,22 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# R:\\ENGR_Chon\\Darren\\NIH_Pulsewatch\\Poincare_pt\\128x128\n", + "# Darren created the PT files again (because UID 120 has missing files in the original csv file)\n", + "# I need to prepare for my interview, and I will tar those PT files again and test your code on Colab later." + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/BML_project/models/ss_gp_model.py b/BML_project/models/ss_gp_model.py index e364cec..355a6fd 100644 --- a/BML_project/models/ss_gp_model.py +++ b/BML_project/models/ss_gp_model.py @@ -4,12 +4,15 @@ @author: lrm22005 """ +import os import numpy as np from tqdm import tqdm import torch import gpytorch from sklearn.metrics import precision_recall_fscore_support, roc_auc_score from sklearn.preprocessing import label_binarize +from utils_gp.data_loader import preprocess_data_train_val,preprocess_data_test +import time num_latents = 6 # This should match the complexity of your data or the number of tasks num_tasks = 4 # This should match the number of output classes or tasks @@ -70,11 +73,49 @@ def forward(self, x): return latent_pred -def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, patience=10, checkpoint_path='model_checkpoint_full.pt'): +def train_gp_model(train_loader, val_loader, batch_size,\ + data_format, clinical_trial_train, clinical_trial_test,\ + clinical_trial_unlabeled,\ + num_iterations=50, n_classes=4, patience=10, checkpoint_path='model_checkpoint_full.pt',\ + resume_training=False,\ + datackpt_name = 'dataset_checkpoint.pt',modelckpt_name = 'model_checkpoint_full.pt'): + print(f'Debug: resume_training:{resume_training}, checkpoint_path: {checkpoint_path}') model = MultitaskGPModel().to(device) likelihood = gpytorch.likelihoods.SoftmaxLikelihood(num_features=4, num_classes=4).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=0.1) mll = gpytorch.mlls.VariationalELBO(likelihood, model, num_data=len(train_loader.dataset)) + + # Load checkpoint if resuming training for gp model. + start_epoch = 0 + flag_reload_dataloader = False # We do not need to reset train loader in the new epoch. + ckpt_model_file = os.path.join(checkpoint_path,modelckpt_name) + if resume_training and os.path.exists(ckpt_model_file): + print(f'Debug: loading ckpt: {ckpt_model_file}') + checkpoint = torch.load(ckpt_model_file) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + start_epoch = checkpoint.get('epoch', 0) # Resume from the same epoch because you did not finished it. + + # Update the dataloader if there are segments finished. + finished_seg_names = checkpoint['finished_seg_names'] + + if len(finished_seg_names) > 0: + # There were segments used in training. Only update the train loader. + flag_reload_dataloader = True + print('Debug: renewing train_loader now...') + startTime_for_tictoc = time.time() + # ---- Dong, 02/15/2024: I want to test training on large dataset and resume training. ---- + # train_loader,_,_ = preprocess_data_train_val(data_format, clinical_trial_train, clinical_trial_test, batch_size, finished_seg_names,\ + # read_all_labels=False) + train_loader = preprocess_data_test(data_format = data_format, \ + clinical_trial_unlabeled=clinical_trial_unlabeled, \ + batch_size=batch_size,\ + finished_seg_names=finished_seg_names,\ + read_all_labels=False) + endTime_for_tictoc = time.time() - startTime_for_tictoc + print(f'Debug: took {endTime_for_tictoc} to renew the train_loader') + best_val_loss = float('inf') epochs_no_improve = 0 @@ -86,19 +127,69 @@ def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, pat 'train_loss': [] # Add a list to store training losses } - for epoch in tqdm(range(num_iterations), desc='Training', unit='epoch', leave=False): - for train_batch in train_loader: + for epoch in tqdm(range(start_epoch,num_iterations), desc='Training', unit='epoch', leave=False): + finished_idx = [] + finished_seg_names = [] + for batch_index, train_batch in enumerate(train_loader): + print(f'Debug: now in a new batch of data! {batch_index}/{len(train_loader)}') # train_batch is the image data. model.train() likelihood.train() optimizer.zero_grad() + train_x = train_batch['data'].reshape(train_batch['data'].size(0), -1).to(device) # Use reshape here train_y = train_batch['label'].to(device) + # Get finished segment index in dataloader and segment name. + temp_finished_idx = train_batch['idx'] + temp_finished_seg_names = train_batch['segment_name'] + print('Debug: temp_finished_idx:',temp_finished_idx) + print('Debug: temp_finished_segment_name:',temp_finished_seg_names) + finished_idx.append(temp_finished_idx) + finished_seg_names.append(temp_finished_seg_names) output = model(train_x) loss = -mll(output, train_y) metrics['train_loss'].append(loss.item()) # Store the training loss loss.backward() optimizer.step() + save_ckpt_model_path = os.path.join(checkpoint_path,modelckpt_name) + torch.save({ + 'epoch': epoch, + 'model_state_dict': model.state_dict(), + 'likelihood_state_dict': likelihood.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'best_val_loss': best_val_loss, + 'finished_seg_names':finished_seg_names, + 'finished_idx':finished_idx + # Include other metrics as needed + }, save_ckpt_model_path) + + # Optionally, save the dataset state at intervals or after certain conditions + save_ckpt_dataset_path = os.path.join(checkpoint_path,datackpt_name) + train_loader.dataset.save_checkpoint(save_ckpt_dataset_path) # Here, manage the index as needed + + # import sys + # if epoch == 3 and batch_index == 5: + # sys.exit(f"Debug: Manually stop the program at epoch {epoch} batch {batch_index}.") + + # Reset the finished segments again because we finished one epoch. + finished_idx = [] + finished_seg_names = [] + if flag_reload_dataloader: + print('Debug: reset the train_loader now...') + # Reset the train dataloader now. + startTime_for_tictoc = time.time() + # --- Dong, 02/15/2024: + # train_loader,_,_ = preprocess_data_train_val(data_format, clinical_trial_train, clinical_trial_test, batch_size, finished_seg_names,\ + # read_all_labels=False) + train_loader = preprocess_data_test(data_format = data_format, \ + clinical_trial_unlabeled=clinical_trial_unlabeled, \ + batch_size=batch_size,\ + finished_seg_names=finished_seg_names,\ + read_all_labels=False) + endTime_for_tictoc = time.time() - startTime_for_tictoc + print(f'Debug: took {endTime_for_tictoc} to reset the train_loader') + flag_reload_dataloader = False # Turn off the flag for reseting train dataloader. + # Stochastic validation model.eval() likelihood.eval() @@ -131,19 +222,46 @@ def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, pat if val_loss < best_val_loss: best_val_loss = val_loss epochs_no_improve = 0 - torch.save({'model_state_dict': model.state_dict(), - 'likelihood_state_dict': likelihood.state_dict(), - 'optimizer_state_dict': optimizer.state_dict()}, checkpoint_path) + # torch.save({'model_state_dict': model.state_dict(), + # 'likelihood_state_dict': likelihood.state_dict(), + # 'optimizer_state_dict': optimizer.state_dict(), + # 'train_loader':train_loader, + # 'val_loader':val_loader + # }, checkpoint_path) else: epochs_no_improve += 1 if epochs_no_improve >= patience: print(f"Early stopping triggered at epoch {epoch+1}") break + + # Save checkpoint at the end of each epoch + save_ckpt_model_path = os.path.join(checkpoint_path,modelckpt_name) + torch.save({ + 'epoch': epoch, + 'model_state_dict': model.state_dict(), + 'likelihood_state_dict': likelihood.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'best_val_loss': best_val_loss, + 'finished_seg_names':finished_seg_names, + 'finished_idx':finished_idx + # Include other metrics as needed + }, save_ckpt_model_path) + print('Debug: saved model checkpoint with epoch.',save_ckpt_model_path) + + # Optionally, save the dataset state at intervals or after certain conditions + save_ckpt_dataset_path = os.path.join(checkpoint_path,datackpt_name) + train_loader.dataset.save_checkpoint(save_ckpt_dataset_path) # Finished all batches, so start from zero. + + if epochs_no_improve >= patience: + print(f"Early stopping triggered at epoch {epoch+1}") + break - checkpoint = torch.load(checkpoint_path) - model.load_state_dict(checkpoint['model_state_dict']) - likelihood.load_state_dict(checkpoint['likelihood_state_dict']) - optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + # Optionally, load the best model at the end of training + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) return model, likelihood, metrics diff --git a/BML_project/ss_main.py b/BML_project/ss_main.py index bf34fbf..326f80f 100644 --- a/BML_project/ss_main.py +++ b/BML_project/ss_main.py @@ -6,13 +6,18 @@ """ from tqdm import tqdm import torch -from utils_gp.data_loader import preprocess_data, split_uids, update_train_loader_with_uncertain_samples +from utils_gp.data_loader import preprocess_data_train_val, split_uids, update_train_loader_with_uncertain_samples, preprocess_data_test from models.ss_gp_model import MultitaskGPModel, train_gp_model from utils_gp.ss_evaluation import stochastic_evaluation, evaluate_model_on_all_data from active_learning.ss_active_learning import stochastic_uncertainty_sampling, run_minibatch_kmeans, stochastic_compare_kmeans_gp_predictions from utils_gp.visualization import plot_comparative_results, plot_training_performance, plot_results import os import pickle +from datetime import datetime +now = datetime.now() # Get the time now for model checkpoint saving. + +dt_string = now.strftime("%Y_%m_%d_%H_%M_%S") # YYYY_mm_dd_HH_MM_SS, for model saving. +print("The date and time suffix of the model file is", dt_string) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -48,8 +53,29 @@ def main(): clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled = split_uids() data_format = 'pt' # Preprocess data - train_loader, val_loader, test_loader, saving_path = preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size) + # ---- Dong, 02/15/2024: I want to test loading large amount dataset. ---- + # train_loader, val_loader, saving_path = preprocess_data_train_val(data_format = data_format, \ + _, val_loader, saving_path = preprocess_data_train_val(data_format = data_format, \ + clinical_trial_train=clinical_trial_train, \ + clinical_trial_test=clinical_trial_test, \ + batch_size=batch_size,\ + finished_seg_names = [],\ + read_all_labels=False) + # ---- Dong, 02/15/2024: I want to test loading large amount dataset. ---- + # test_loader = preprocess_data_test(data_format = data_format, \ + train_loader = preprocess_data_test(data_format = data_format, \ + clinical_trial_unlabeled=clinical_trial_unlabeled, \ + batch_size=batch_size,\ + finished_seg_names=[],\ + read_all_labels=False) + + menu_segment_names = train_loader.dataset.segment_names # All the segments to be run in the training dataset. + menu_labels = train_loader.dataset.labels # All the ground truth labels + print('Debug: len(menu_segment_names)',len(menu_segment_names)) + print('Debug: len(menu_labels)',len(menu_labels)) + print('Debug: len(train_loader)',len(train_loader)) + print('Debug: dir(train_loader.dataset)',dir(train_loader.dataset)) kmeans_model = run_minibatch_kmeans(train_loader, n_clusters=n_classes, device=device) @@ -61,7 +87,21 @@ def main(): } # Initial model training - model, likelihood, training_metrics = train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=n_classes) + model, likelihood, training_metrics = train_gp_model( + train_loader = train_loader, + val_loader = val_loader, + num_iterations=50, + n_classes=n_classes, + patience=10, + checkpoint_path=saving_path, + resume_training=True, + datackpt_name = 'dataset_checkpoint.pt', + modelckpt_name = 'model_checkpoint_full.pt', + batch_size=batch_size, + data_format = data_format, + clinical_trial_train = clinical_trial_train, + clinical_trial_test = clinical_trial_test, + clinical_trial_unlabeled=clinical_trial_unlabeled) # Dong: remember to change this function in its code. # Save the training metrics for future visualization results['train_loss'].extend(training_metrics['train_loss']) @@ -77,6 +117,7 @@ def main(): # Attempt to load a training checkpoint train_checkpoint = checkpoint_manager.load_checkpoint('train') start_iteration = train_checkpoint['iteration'] if train_checkpoint else 0 + print('Debug: start_iteration is:',start_iteration) # Dong, 01/25/2024: save it first before entering the active learning. additional_state = { 'model_state': model.state_dict(), @@ -91,15 +132,20 @@ def main(): active_learning_iterations = 10 # Active Learning Iterations for iteration in tqdm(range(start_iteration,active_learning_iterations), desc='Active Learning', unit='iteration', leave=True): + print(f"Active Learning Iteration: {iteration+1}/{active_learning_iterations}") # Perform uncertainty sampling to select new samples from the validation set - uncertain_sample_indices = stochastic_uncertainty_sampling(model, likelihood, val_loader, n_samples=batch_size, n_batches=5) - + uncertain_sample_indices = stochastic_uncertainty_sampling(model, likelihood, val_loader, n_samples=50, n_batches=5, device=device) + labeled_samples = label_samples(uncertain_sample_indices, val_loader.dataset) # Update the training loader with uncertain samples - train_loader = update_train_loader_with_uncertain_samples(train_loader, uncertain_sample_indices, batch_size) - print(f"Updated training data size: {len(train_loader.dataset)}") + train_loader = update_train_loader_with_uncertain_samples(train_loader, labeled_samples, batch_size) + + # Optionally, save the dataset state at intervals or after certain conditions + train_loader.dataset.save_checkpoint(dataset_checkpoint_path) # Here, manage the index as needed # Re-train the model with the updated training data - model, likelihood, val_metrics = train_gp_model(train_loader, val_loader, num_iterations=10, n_classes=n_classes, patience=10, checkpoint_path='model_checkpoint_last.pt') + model, likelihood, val_metrics = train_gp_model( + train_loader, val_loader, num_iterations=10, n_classes=n_classes, patience=10, + checkpoint_path=saving_path, resume_training=True, batch_size=batch_size) # Store the validation metrics after each active learning iteration results['validation_metrics']['precision'].append(val_metrics['precision']) @@ -123,13 +169,21 @@ def main(): plot_comparative_results(gp_vs_kmeans_data, original_labels) # Final evaluation on test set + import subprocess + print('Start to run bash script!') + subprocess.call("./BML_project/untar_unlabeled_PT.sh") + print('End to run bash script!') + + test_loader = preprocess_data_test(data_format = data_format, \ + clinical_trial_unlabeled=clinical_trial_unlabeled, \ + batch_size=batch_size,\ + finished_seg_names=[],\ + read_all_labels=False) test_metrics = evaluate_model_on_all_data(model, likelihood, test_loader, device, n_classes) test_kmeans_model = run_minibatch_kmeans(test_loader, n_clusters=n_classes, device=device) results['test_metrics'] = test_metrics test_gp_vs_kmeans_data, test_original_labels = stochastic_compare_kmeans_gp_predictions(test_kmeans_model, model, test_loader, n_batches=5, device=device) - - print(f"Length of original_labels: {len(original_labels)}, Length of gp_predictions: {len(gp_predictions)}") plot_comparative_results(test_gp_vs_kmeans_data, test_original_labels) # Visualization of results diff --git a/BML_project/utils_gp/data_loader.py b/BML_project/utils_gp/data_loader.py index e0a6823..bd22a79 100644 --- a/BML_project/utils_gp/data_loader.py +++ b/BML_project/utils_gp/data_loader.py @@ -5,6 +5,8 @@ @author: lrm22005 """ import os +# For saving checkpoints +from pathlib import Path import numpy as np import pandas as pd from PIL import Image @@ -13,6 +15,12 @@ from sklearn.preprocessing import StandardScaler from torchvision.transforms import ToTensor import socket +# Downsampling image +import cv2 +# import torchvision.transforms as T +# transform for rectangular resize +img_size = 32 # Dong, 01/30/2024: this is for testing the CIFAR10 models. +# transform = T.Resize((img_size,img_size)) def split_uids(): # ====== Load the per subject arrythmia summary ====== @@ -26,6 +34,8 @@ def split_uids(): df_summary = pd.read_csv(r'R:\ENGR_Chon\NIH_Pulsewatch_Database\Adjudication_UConn\final_attemp_4_1_Dong_Ohm_summary_20231025.csv') elif your_computer_name == 'Luis_computer_name': df_summary = pd.read_csv(r'\\grove.ad.uconn.edu\research\ENGR_Chon\NIH_Pulsewatch_Database\Adjudication_UConn\final_attemp_4_1_Dong_Ohm_summary_20231025.csv') + else: + df_summary = pd.read_csv(r'/content/drive/MyDrive/Adjudication_UConn/final_attemp_4_1_Dong_Ohm_summary_20231025.csv') df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] @@ -103,111 +113,96 @@ def split_uids(): # clinical_trial_unlabeled = clinical_trial_unlabeled[0:4] return clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled +def extract_segment_names_and_labels(UIDs,labels_path,read_all_labels=False): + # Extract all segment names and labels when starting the main function. + # Output: + # segment_names: list of string. + # labels: dictionary, with segment_names as key and label as value. + segment_names = [] + labels = {} + + for UID in UIDs: + label_file = os.path.join(labels_path, UID + "_final_attemp_4_1_Dong.csv") + if os.path.exists(label_file): + print('Debug: this file exists',label_file) + label_data = pd.read_csv(label_file, sep=',', header=0, names=['segment', 'label']) + label_segment_names = label_data['segment'].apply(lambda x: x.split('.')[0]) + for idx, segment_name in enumerate(label_segment_names): + label_val = label_data['label'].values[idx] + if read_all_labels: + # Assign -1 if label is not in [0, 1, 2, 3] + labels[segment_name] = label_val if label_val in [0, 1, 2, 3] else -1 + if segment_name not in segment_names: + segment_names.append(segment_name) + else: + # Only add segments with labels in [0, 1, 2, 3] + if label_val in [0, 1, 2, 3] and segment_name not in segment_names: + segment_names.append(segment_name) + labels[segment_name] = label_val + print('>>> Number of segments in this dataloader:',len(segment_names)) # Dong, 01/29/2024: know the number of segments before running training epochs. + print('>>> Number of labels in this dataloader:',len(labels)) + return segment_names, labels + +def remove_finished_segment_names_and_labels(labels,finished_seg_names): + # From extract_segment_names_and_labels: + # Input: + # labels: dictionary, with segment_names as key and label as value. + # finished_seg_names: list of string. + remain_labels = labels.copy() + print('Debug: type(remain_labels)',type(remain_labels)) + for batch in finished_seg_names: + for key in batch: + remain_labels.pop(key) + print('Debug: len(labels)',len(labels)) + print('Debug: len(remain_labels)',len(remain_labels)) + + return remain_labels + class CustomDataset(Dataset): - def __init__(self, data_path, labels_path, UIDs, standardize=True, data_format='csv', read_all_labels=False, start_idx=0): + def __init__(self, data_path, labels_path, batch_size,segment_names,labels, standardize=True, data_format='csv', read_all_labels=False): self.data_path = data_path self.labels_path = labels_path - self.UIDs = UIDs self.standardize = standardize self.data_format = data_format self.read_all_labels = read_all_labels self.transforms = ToTensor() - self.start_idx = start_idx # Initial batch index to start from, useful for resuming training - self.refresh_dataset() - - # Initialize the current batch index to None, this could be used if you want to track batch progress within the dataset itself - self.current_batch_index = None + self.segment_names = segment_names + self.labels = labels - def refresh_dataset(self): - self.segment_names, self.labels = self.extract_segment_names_and_labels() - - def add_uids(self, new_uids): - unique_new_uids = [uid for uid in new_uids if uid not in self.UIDs] - self.UIDs.extend(unique_new_uids) - self.refresh_dataset() + # Initialize the current batch index to None + self.batch_size = batch_size def __len__(self): return len(self.segment_names) def save_checkpoint(self, checkpoint_path): - # Enhanced to automatically include 'start_idx' in the checkpoint checkpoint = { 'segment_names': self.segment_names, - 'labels': self.labels, - 'UIDs': self.UIDs, - 'start_idx': self.start_idx # Now also saving start_idx + 'labels': self.labels + # Save the current batch index if provided } torch.save(checkpoint, checkpoint_path) def load_checkpoint(self, checkpoint_path): checkpoint = torch.load(checkpoint_path) + print('Debug: loaded dataset checkpoint!',checkpoint_path) self.segment_names = checkpoint['segment_names'] self.labels = checkpoint['labels'] - self.UIDs = checkpoint['UIDs'] - # Now also loading and setting start_idx from checkpoint - self.start_idx = checkpoint.get('start_idx', 0) self.refresh_dataset() + # Load the current batch index if it exists in the checkpoint def __getitem__(self, idx): - actual_idx = (idx + self.start_idx) % len(self.segment_names) # Adjust index based on start_idx and wrap around if needed - segment_name = self.segment_names[actual_idx] + segment_name = self.segment_names[idx] label = self.labels[segment_name] - if hasattr(self, 'all_data') and actual_idx < len(self.all_data): - time_freq_tensor = self.all_data[actual_idx] + if hasattr(self, 'all_data') and idx < len(self.all_data): + time_freq_tensor = self.all_data[idx] else: time_freq_tensor = self.load_data(segment_name) - return {'data': time_freq_tensor, 'label': label, 'segment_name': segment_name} - def set_current_batch_index(self, index): - self.current_batch_index = index - - def get_current_batch_index(self): - return self.current_batch_index - - def set_start_idx(self, index): - self.start_idx = index - - def add_data_label_pair(self, data, label): - # Assign a unique ID or name for the new data - new_id = len(self.segment_names) - segment_name = f"new_data_{new_id}" - - # Append the new data and label - self.segment_names.append(segment_name) - self.labels[segment_name] = label - - # Append the new data tensor to an attribute that holds all the data - if hasattr(self, 'all_data'): - self.all_data.append(data) - else: - self.all_data = [data] - - def extract_segment_names_and_labels(self): - segment_names = [] - labels = {} - - for UID in self.UIDs: - label_file = os.path.join(self.labels_path, UID + "_final_attemp_4_1_Dong.csv") - if os.path.exists(label_file): - print('Debug: this file exists',label_file) - label_data = pd.read_csv(label_file, sep=',', header=0, names=['segment', 'label']) - label_segment_names = label_data['segment'].apply(lambda x: x.split('.')[0]) - for idx, segment_name in enumerate(label_segment_names): - label_val = label_data['label'].values[idx] - if self.read_all_labels: - # Assign -1 if label is not in [0, 1, 2, 3] - labels[segment_name] = label_val if label_val in [0, 1, 2, 3] else -1 - if segment_name not in segment_names: - segment_names.append(segment_name) - else: - # Only add segments with labels in [0, 1, 2, 3] - if label_val in [0, 1, 2, 3] and segment_name not in segment_names: - segment_names.append(segment_name) - labels[segment_name] = label_val - - return segment_names, labels + + return {'data': time_freq_tensor, 'label': label, 'segment_name': segment_name, 'idx': idx} def load_data(self, segment_name): data_path_UID = os.path.join(self.data_path, segment_name.split('_')[0]) @@ -232,15 +227,30 @@ def load_data(self, segment_name): except Exception as e: print(f"Error processing segment: {segment_name}. Exception: {str(e)}") - return torch.zeros((1, 128, 128)) # Return zeros in case of an error + return torch.zeros((1, img_size, img_size)) # Return zeros in case of an error def standard_scaling(self, data): scaler = StandardScaler() data = scaler.fit_transform(data.reshape(-1, data.shape[-1])).reshape(data.shape) return torch.Tensor(data) -def load_data_split_batched(data_path, labels_path, UIDs, batch_size, standardize=False, data_format='csv', read_all_labels=False, drop_last=False, num_workers=4, start_idx=0): - dataset = CustomDataset(data_path, labels_path, UIDs, standardize, data_format, read_all_labels, start_idx=start_idx) +def load_data_split_batched(data_path, labels_path, UIDs, batch_size, standardize=False, data_format='pt', read_all_labels=False, drop_last=False, num_workers=4,\ + finished_seg_names = []): + # Run the main from the beginning. Load all data into the dataloader. + segment_names, labels = extract_segment_names_and_labels(UIDs,labels_path,read_all_labels=read_all_labels) + if len(finished_seg_names) > 0: + # If any segments have been trained. + remain_labels = remove_finished_segment_names_and_labels(labels,finished_seg_names) + segment_names = list(remain_labels.keys()) + labels = remain_labels.copy() + dataset = CustomDataset(data_path=data_path, \ + labels_path=labels_path, \ + standardize=standardize, \ + data_format=data_format, \ + read_all_labels=read_all_labels, \ + batch_size=batch_size, + segment_names = segment_names, + labels = labels) dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, drop_last=drop_last, num_workers=num_workers, prefetch_factor=2) return dataloader @@ -261,8 +271,12 @@ def get_data_paths(data_format): labels_base_path = "R:\ENGR_Chon\\NIH_Pulsewatch_Database\Adjudication_UConn" saving_base_path = r"\\grove.ad.uconn.edu\research\ENGR_Chon\Luis\Research\Casseys_case" else: - print('ERROR! YOUR DID NOT GET THE PATH.') - raise ValueError + print('Debug: You are in Google Colab.') + base_path = '/content' + labels_base_path = '/content/drive/MyDrive/Adjudication_UConn' + saving_base_path = '/content/drive/MyDrive/Checkpoint_Colab' + # print('ERROR! YOUR DID NOT GET THE PATH.') + # raise ValueError if data_format == 'csv': data_path = os.path.join(base_path, "TFS_csv") @@ -278,16 +292,51 @@ def get_data_paths(data_format): saving_path = os.path.join(saving_base_path, "Project_1_analysis") else: raise ValueError("Invalid data format. Choose 'csv' or 'png.") + + # Create the parent path for checkpoints. + Path(saving_path).mkdir(parents=True, exist_ok=True) + return data_path, labels_path, saving_path # Function to extract and preprocess data -def preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size, read_all_labels=False, current_batch_index=0): - start_idx = current_batch_index * batch_size +def preprocess_data_train_val(data_format, clinical_trial_train, clinical_trial_test, batch_size, finished_seg_names,\ + read_all_labels=False): + # Extracts paths and loads data into train, validation, and test loaders + data_path, labels_path, saving_path = get_data_paths(data_format) + + train_loader = load_data_split_batched(data_path=data_path, \ + labels_path=labels_path, \ + UIDs=clinical_trial_train, \ + batch_size = batch_size, \ + standardize=True, \ + data_format=data_format, \ + read_all_labels=read_all_labels,\ + finished_seg_names = finished_seg_names) + # Usually the validation set will not need to resume training. + val_loader = load_data_split_batched(data_path=data_path, \ + labels_path=labels_path, \ + UIDs=clinical_trial_test, \ + batch_size=batch_size, \ + standardize=True, \ + data_format=data_format, \ + read_all_labels=read_all_labels, \ + finished_seg_names = []) + return train_loader, val_loader, saving_path + +# Function to extract and preprocess data +def preprocess_data_test(data_format, clinical_trial_unlabeled, batch_size, finished_seg_names,\ + read_all_labels=False): + # Extracts paths and loads data into train, validation, and test loaders data_path, labels_path, saving_path = get_data_paths(data_format) - train_loader = load_data_split_batched(data_path, labels_path, clinical_trial_train, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels, start_idx=start_idx) - val_loader = load_data_split_batched(data_path, labels_path, clinical_trial_test, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels, start_idx=start_idx) - test_loader = load_data_split_batched(data_path, labels_path, clinical_trial_unlabeled, batch_size, standardize=True, data_format=data_format, read_all_labels=read_all_labels, start_idx=start_idx) - return train_loader, val_loader, test_loader + test_loader = load_data_split_batched(data_path=data_path, \ + labels_path=labels_path, \ + UIDs=clinical_trial_unlabeled, \ + batch_size=batch_size, \ + standardize=True, \ + data_format=data_format, \ + read_all_labels=read_all_labels,\ + finished_seg_names=finished_seg_names) + return test_loader def map_samples_to_uids(uncertain_sample_indices, dataset): """ From bfe5b1c121edb53b593ea9253f24c4ae05a452e0 Mon Sep 17 00:00:00 2001 From: Luis Roberto Mercado Diaz Date: Sat, 20 Apr 2024 14:59:42 -0400 Subject: [PATCH 5/6] Interpretation and addition of changes based on Darren Codes and Casseys codes I did an addition of some changes that Cassey did before to ensure the availability of the code to run in my new environment, the change is saving the data loader stepping process, allowing to understand really well and restart the model without losing the previous downloading time. Darren code is a renovated version of our previous methods that ensures flexibility and ability to run the codes in any environment. The reference for now is a code in debugging, later it would be transpose in a more suitable definition structure. --- main_darren_v1-8GJQ9R3.py | 260 ++ main_darren_v1.py | 265 ++ utils/__pycache__/dataloader.cpython-310.pyc | Bin 0 -> 19552 bytes utils/__pycache__/dataloader.cpython-311.pyc | Bin 0 -> 19661 bytes utils/__pycache__/dataloader.cpython-312.pyc | Bin 0 -> 37393 bytes utils/__pycache__/dataloader.cpython-39.pyc | Bin 0 -> 20774 bytes .../dataloader_batch.cpython-310.pyc | Bin 0 -> 11546 bytes .../dataloader_database.cpython-310.pyc | Bin 0 -> 6203 bytes .../dataloader_smote.cpython-310.pyc | Bin 0 -> 6462 bytes utils/__pycache__/get_paths.cpython-310.pyc | Bin 0 -> 3278 bytes utils/__pycache__/get_paths.cpython-311.pyc | Bin 0 -> 5898 bytes utils/__pycache__/misc_func.cpython-310.pyc | Bin 0 -> 757 bytes utils/__pycache__/model_func.cpython-310.pyc | Bin 0 -> 35849 bytes utils/__pycache__/model_func.cpython-311.pyc | Bin 0 -> 16736 bytes utils/__pycache__/model_func.cpython-312.pyc | Bin 0 -> 92196 bytes .../model_func_batch.cpython-310.pyc | Bin 0 -> 7818 bytes utils/__pycache__/pathmaster.cpython-310.pyc | Bin 0 -> 7890 bytes utils/__pycache__/pathmaster.cpython-312.pyc | Bin 0 -> 11785 bytes utils/__pycache__/pathmaster.cpython-39.pyc | Bin 0 -> 8470 bytes .../plot_save_func.cpython-310.pyc | Bin 0 -> 12901 bytes .../plot_save_func.cpython-311.pyc | Bin 0 -> 6303 bytes .../plot_save_func.cpython-312.pyc | Bin 0 -> 26641 bytes .../__pycache__/plot_save_func.cpython-39.pyc | Bin 0 -> 13081 bytes utils/__pycache__/train_func.cpython-310.pyc | Bin 0 -> 7573 bytes utils/__pycache__/train_func.cpython-311.pyc | Bin 0 -> 13150 bytes utils/dataloader.py | 895 +++++++ utils/dataloader_database.py | 223 ++ utils/dataloader_smote.py | 215 ++ utils/get_paths.py | 154 ++ utils/misc_func.py | 26 + utils/model_func.py | 2145 +++++++++++++++++ utils/pathmaster.py | 321 +++ utils/plot_save_func.py | 542 +++++ utils/smote.py | 158 ++ utils/smote_accelerated.py | 178 ++ utils/smote_accelerated_lab.py | 177 ++ utils/smote_transfer_location.py | 93 + 37 files changed, 5652 insertions(+) create mode 100644 main_darren_v1-8GJQ9R3.py create mode 100644 main_darren_v1.py create mode 100644 utils/__pycache__/dataloader.cpython-310.pyc create mode 100644 utils/__pycache__/dataloader.cpython-311.pyc create mode 100644 utils/__pycache__/dataloader.cpython-312.pyc create mode 100644 utils/__pycache__/dataloader.cpython-39.pyc create mode 100644 utils/__pycache__/dataloader_batch.cpython-310.pyc create mode 100644 utils/__pycache__/dataloader_database.cpython-310.pyc create mode 100644 utils/__pycache__/dataloader_smote.cpython-310.pyc create mode 100644 utils/__pycache__/get_paths.cpython-310.pyc create mode 100644 utils/__pycache__/get_paths.cpython-311.pyc create mode 100644 utils/__pycache__/misc_func.cpython-310.pyc create mode 100644 utils/__pycache__/model_func.cpython-310.pyc create mode 100644 utils/__pycache__/model_func.cpython-311.pyc create mode 100644 utils/__pycache__/model_func.cpython-312.pyc create mode 100644 utils/__pycache__/model_func_batch.cpython-310.pyc create mode 100644 utils/__pycache__/pathmaster.cpython-310.pyc create mode 100644 utils/__pycache__/pathmaster.cpython-312.pyc create mode 100644 utils/__pycache__/pathmaster.cpython-39.pyc create mode 100644 utils/__pycache__/plot_save_func.cpython-310.pyc create mode 100644 utils/__pycache__/plot_save_func.cpython-311.pyc create mode 100644 utils/__pycache__/plot_save_func.cpython-312.pyc create mode 100644 utils/__pycache__/plot_save_func.cpython-39.pyc create mode 100644 utils/__pycache__/train_func.cpython-310.pyc create mode 100644 utils/__pycache__/train_func.cpython-311.pyc create mode 100644 utils/dataloader.py create mode 100644 utils/dataloader_database.py create mode 100644 utils/dataloader_smote.py create mode 100644 utils/get_paths.py create mode 100644 utils/misc_func.py create mode 100644 utils/model_func.py create mode 100644 utils/pathmaster.py create mode 100644 utils/plot_save_func.py create mode 100644 utils/smote.py create mode 100644 utils/smote_accelerated.py create mode 100644 utils/smote_accelerated_lab.py create mode 100644 utils/smote_transfer_location.py diff --git a/main_darren_v1-8GJQ9R3.py b/main_darren_v1-8GJQ9R3.py new file mode 100644 index 0000000..a84b9a3 --- /dev/null +++ b/main_darren_v1-8GJQ9R3.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Apr 18 12:52:53 2024 + +@author: lrmercadod +""" +import torch +import torch.nn as nn +import time +import datetime as dt +import gpytorch +from sklearn.metrics import precision_recall_fscore_support, roc_auc_score +from sklearn.preprocessing import label_binarize + +# Import my own functions and classes +from utils.pathmaster import PathMaster +from utils.dataloader import preprocess_data + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +num_latents = 6 # This should match the complexity of your data or the number of tasks +num_tasks = 4 # This should match the number of output classes or tasks +num_inducing_points = 50 # This is independent and should be sufficient for the input space + +class MultitaskGPModel(gpytorch.models.ApproximateGP): + def __init__(self): + # Let's use a different set of inducing points for each latent function + inducing_points = torch.rand(num_latents, num_inducing_points, 128 * 128) # Assuming flattened 128x128 images + + # We have to mark the CholeskyVariationalDistribution as batch + # so that we learn a variational distribution for each task + variational_distribution = gpytorch.variational.CholeskyVariationalDistribution( + inducing_points.size(-2), batch_shape=torch.Size([num_latents]) + ) + + # We have to wrap the VariationalStrategy in a LMCVariationalStrategy + # so that the output will be a MultitaskMultivariateNormal rather than a batch output + variational_strategy = gpytorch.variational.LMCVariationalStrategy( + gpytorch.variational.VariationalStrategy( + self, inducing_points, variational_distribution, learn_inducing_locations=True + ), + num_tasks=num_tasks, + num_latents=num_latents, + latent_dim=-1 + ) + + super().__init__(variational_strategy) + + # The mean and covariance modules should be marked as batch + # so we learn a different set of hyperparameters + self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_latents])) + self.covar_module = gpytorch.kernels.ScaleKernel( + gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents])), + batch_shape=torch.Size([num_latents]) + ) + + def forward(self, x): + mean_x = self.mean_module(x) + covar_x = self.covar_module(x) + latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x) + return latent_pred + +def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, patience=10, + checkpoint_path='model_checkpoint.pt', resume_training=False): + model = MultitaskGPModel().to(device) + likelihood = gpytorch.likelihoods.SoftmaxLikelihood(num_features=4, num_classes=4).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=0.1) + mll = gpytorch.mlls.VariationalELBO(likelihood, model, num_data=len(train_loader.dataset)) + + start_epoch = 0 + if resume_training and os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + start_epoch = checkpoint.get('epoch', 0) + + best_val_loss = float('inf') + epochs_no_improve = 0 + + metrics = { + 'precision': [], + 'recall': [], + 'f1_score': [], + 'auc_roc': [], + 'train_loss': [] + } + + for epoch in range(start_epoch, num_iterations): + model.train() + likelihood.train() + for train_batch in train_loader: + optimizer.zero_grad() + train_x = train_batch['data'].reshape(train_batch['data'].size(0), -1).to(device) + train_y = train_batch['label'].to(device) + output = model(train_x) + loss = -mll(output, train_y) + metrics['train_loss'].append(loss.item()) + loss.backward() + optimizer.step() + + # Stochastic validation + model.eval() + likelihood.eval() + with torch.no_grad(): + val_indices = torch.randperm(len(val_loader.dataset))[:int(0.1 * len(val_loader.dataset))] + val_loss = 0.0 + val_labels = [] + val_predictions = [] + for idx in val_indices: + val_batch = val_loader.dataset[idx] + val_x = val_batch['data'].reshape(-1).unsqueeze(0).to(device) + val_y = torch.tensor([val_batch['label']], device=device) + val_output = model(val_x) + val_loss_batch = -mll(val_output, val_y).sum() + val_loss += val_loss_batch.item() + val_labels.append(val_y.item()) + val_predictions.append(val_output.mean.argmax(dim=-1).item()) + + precision, recall, f1, _ = precision_recall_fscore_support(val_labels, val_predictions, average='macro') + auc_roc = roc_auc_score(label_binarize(val_labels, classes=range(n_classes)), + label_binarize(val_predictions, classes=range(n_classes)), + multi_class='ovr') + + metrics['precision'].append(precision) + metrics['recall'].append(recall) + metrics['f1_score'].append(f1) + metrics['auc_roc'].append(auc_roc) + val_loss /= len(val_indices) + + if val_loss < best_val_loss: + best_val_loss = val_loss + epochs_no_improve = 0 + torch.save({'model_state_dict': model.state_dict(), + 'likelihood_state_dict': likelihood.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'epoch': epoch}, checkpoint_path) + else: + epochs_no_improve += 1 + if epochs_no_improve >= patience: + print(f"Early stopping triggered at epoch {epoch+1}") + break + + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + + return model, likelihood, metrics + +def evaluate_gp_model(test_loader, model, likelihood, n_classes=4): + model.eval() + likelihood.eval() + test_labels = [] + test_predictions = [] + + with torch.no_grad(): + for test_batch in test_loader: + test_x = test_batch['data'].reshape(test_batch['data'].size(0), -1).to(device) + test_y = test_batch['label'].to(device) + test_output = model(test_x) + test_labels.extend(test_y.tolist()) + test_predictions.extend(test_output.mean.argmax(dim=-1).tolist()) + + precision, recall, f1, _ = precision_recall_fscore_support(test_labels, test_predictions, average='macro') + auc_roc = roc_auc_score(label_binarize(test_labels, classes=range(n_classes)), + label_binarize(test_predictions, classes=range(n_classes)), + multi_class='ovr') + + metrics = { + 'precision': precision, + 'recall': recall, + 'f1_score': f1, + 'auc_roc': auc_roc + } + + return metrics + +def main(): + # Device and drives + is_linux = False + is_hpc = False + is_internal = False + is_external = True + binary = False + + # Input + is_tfs = True + + # Database + database = 'mimic3' + + # Initialize the focus + focus = 'thesis_results_database_multiclass' + + # Initialize the file tag + file_tag = 'MIMIC_III' + + # Image resolution + img_res = '128x128_float16' + + # Data type: the type to convert the data into when it is loaded in + data_type = torch.float32 + + # Model type + model_type = torch.float32 + + # Create a PathMaster object + pathmaster = PathMaster(is_linux, is_hpc, is_tfs, is_internal, is_external, focus, file_tag, img_res) + + # Image dimensions + img_channels = 1 + img_size = 128 + downsample = None + standardize = True + + # Run parameters + n_epochs = 100 + if binary: + n_classes = 2 + else: + n_classes = 3 + patience = round(n_epochs / 10) if n_epochs > 50 else 5 + save = True + + # Resume checkpoint + resume_checkpoint_path = None + + # Data loading details + data_format = 'pt' + batch_size = 256 + + # Preprocess database data + test_loader = preprocess_data(database, batch_size, standardize, img_channels, img_size, + downsample, data_type, pathmaster, binary) + + # Training and validation + start_time = time.time() + model, likelihood, metrics = train_gp_model(train_loader, val_loader, n_epochs, + n_classes, patience, save, pathmaster) + end_time = time.time() + time_passed = end_time - start_time + print('\nTraining and validation took %.2f minutes' % (time_passed / 60)) + + # Evaluation + start_time = time.time() + test_metrics = evaluate_gp_model(test_loader, model, likelihood, n_classes) + end_time = time.time() + time_passed = end_time - start_time + print('\nTesting took %.2f seconds' % time_passed) + + print('Test Metrics:') + print('Precision: %.4f' % test_metrics['precision']) + print('Recall: %.4f' % test_metrics['recall']) + print('F1 Score: %.4f' % test_metrics['f1_score']) + print('AUC-ROC: %.4f' % test_metrics['auc_roc']) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/main_darren_v1.py b/main_darren_v1.py new file mode 100644 index 0000000..29ec642 --- /dev/null +++ b/main_darren_v1.py @@ -0,0 +1,265 @@ +import os +import torch +import gpytorch +from sklearn.metrics import precision_recall_fscore_support, roc_auc_score +from sklearn.preprocessing import label_binarize +from torch.utils.data import Dataset, DataLoader +import numpy as np +import random +import time + +# Seeds +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +num_latents = 6 # This should match the complexity of your data or the number of tasks +num_tasks = 4 # This should match the number of output classes or tasks +num_inducing_points = 50 # This is independent and should be sufficient for the input space + +class MultitaskGPModel(gpytorch.models.ApproximateGP): + def __init__(self): + # Let's use a different set of inducing points for each latent function + inducing_points = torch.rand(num_latents, num_inducing_points, 128 * 128) # Assuming flattened 128x128 images + + # We have to mark the CholeskyVariationalDistribution as batch + # so that we learn a variational distribution for each task + variational_distribution = gpytorch.variational.CholeskyVariationalDistribution( + inducing_points.size(-2), batch_shape=torch.Size([num_latents]) + ) + + # We have to wrap the VariationalStrategy in a LMCVariationalStrategy + # so that the output will be a MultitaskMultivariateNormal rather than a batch output + variational_strategy = gpytorch.variational.LMCVariationalStrategy( + gpytorch.variational.VariationalStrategy( + self, inducing_points, variational_distribution, learn_inducing_locations=True + ), + num_tasks=num_tasks, + num_latents=num_latents, + latent_dim=-1 + ) + + super().__init__(variational_strategy) + + # The mean and covariance modules should be marked as batch + # so we learn a different set of hyperparameters + self.mean_module = gpytorch.means.ConstantMean(batch_shape=torch.Size([num_latents])) + self.covar_module = gpytorch.kernels.ScaleKernel( + gpytorch.kernels.RBFKernel(batch_shape=torch.Size([num_latents])), + batch_shape=torch.Size([num_latents]) + ) + + def forward(self, x): + mean_x = self.mean_module(x) + covar_x = self.covar_module(x) + latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x) + return latent_pred + +class CustomDataset(Dataset): + def __init__(self, data_path, labels_path, binary=False): + self.data_path = data_path + self.labels_path = labels_path + self.binary = binary + self.segment_names, self.labels = self.extract_segment_names_and_labels() + + def __len__(self): + return len(self.segment_names) + + def __getitem__(self, idx): + segment_name = self.segment_names[idx] + label = self.labels[segment_name] + data_tensor = torch.load(os.path.join(self.data_path, segment_name + '.pt')) + return {'data': data_tensor, 'label': label, 'segment_name': segment_name} + + def extract_segment_names_and_labels(self): + segment_names = [] + labels = {} + + with open(self.labels_path, 'r') as file: + lines = file.readlines() + for line in lines[1:]: # Skip the header line + segment_name, label = line.strip().split(',') + label = int(float(label)) # Convert the label to float first, then to int + if self.binary and label == 2: + label = 0 # Convert PAC/PVC to non-AF (0) for binary classification + segment_names.append(segment_name) + labels[segment_name] = label + + return segment_names, labels + +def load_data(data_path, labels_path, batch_size, binary=False): + dataset = CustomDataset(data_path, labels_path, binary) + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True) + return dataloader + +def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, patience=10, + checkpoint_path='model_checkpoint.pt', resume_training=False): + model = MultitaskGPModel().to(device) + likelihood = gpytorch.likelihoods.SoftmaxLikelihood(num_features=4, num_classes=4).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=0.1) + mll = gpytorch.mlls.VariationalELBO(likelihood, model, num_data=len(train_loader.dataset)) + + start_epoch = 0 + if resume_training and os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + start_epoch = checkpoint.get('epoch', 0) + + best_val_loss = float('inf') + epochs_no_improve = 0 + + metrics = { + 'precision': [], + 'recall': [], + 'f1_score': [], + 'auc_roc': [], + 'train_loss': [] + } + + for epoch in range(start_epoch, num_iterations): + model.train() + likelihood.train() + for train_batch in train_loader: + optimizer.zero_grad() + train_x = train_batch['data'].reshape(train_batch['data'].size(0), -1).to(device) + train_y = train_batch['label'].to(device) + output = model(train_x) + loss = -mll(output, train_y) + metrics['train_loss'].append(loss.item()) + loss.backward() + optimizer.step() + + # Stochastic validation + model.eval() + likelihood.eval() + with torch.no_grad(): + val_indices = torch.randperm(len(val_loader.dataset))[:int(0.1 * len(val_loader.dataset))] + val_loss = 0.0 + val_labels = [] + val_predictions = [] + for idx in val_indices: + val_batch = val_loader.dataset[idx] + val_x = val_batch['data'].reshape(-1).unsqueeze(0).to(device) + val_y = torch.tensor([val_batch['label']], device=device) + val_output = model(val_x) + val_loss_batch = -mll(val_output, val_y).sum() + val_loss += val_loss_batch.item() + val_labels.append(val_y.item()) + val_predictions.append(val_output.mean.argmax(dim=-1).item()) + + precision, recall, f1, _ = precision_recall_fscore_support(val_labels, val_predictions, average='macro') + auc_roc = roc_auc_score(label_binarize(val_labels, classes=range(n_classes)), + label_binarize(val_predictions, classes=range(n_classes)), + multi_class='ovr') + + metrics['precision'].append(precision) + metrics['recall'].append(recall) + metrics['f1_score'].append(f1) + metrics['auc_roc'].append(auc_roc) + val_loss /= len(val_indices) + + if val_loss < best_val_loss: + best_val_loss = val_loss + epochs_no_improve = 0 + torch.save({'model_state_dict': model.state_dict(), + 'likelihood_state_dict': likelihood.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'epoch': epoch}, checkpoint_path) + else: + epochs_no_improve += 1 + if epochs_no_improve >= patience: + print(f"Early stopping triggered at epoch {epoch+1}") + break + + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path) + model.load_state_dict(checkpoint['model_state_dict']) + likelihood.load_state_dict(checkpoint['likelihood_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + + return model, likelihood, metrics + +def evaluate_gp_model(test_loader, model, likelihood, n_classes=4): + model.eval() + likelihood.eval() + test_labels = [] + test_predictions = [] + + with torch.no_grad(): + for test_batch in test_loader: + test_x = test_batch['data'].reshape(test_batch['data'].size(0), -1).to(device) + test_y = test_batch['label'].to(device) + test_output = model(test_x) + test_labels.extend(test_y.tolist()) + test_predictions.extend(test_output.mean.argmax(dim=-1).tolist()) + + precision, recall, f1, _ = precision_recall_fscore_support(test_labels, test_predictions, average='macro') + auc_roc = roc_auc_score(label_binarize(test_labels, classes=range(n_classes)), + label_binarize(test_predictions, classes=range(n_classes)), + multi_class='ovr') + + metrics = { + 'precision': precision, + 'recall': recall, + 'f1_score': f1, + 'auc_roc': auc_roc + } + + return metrics + +def main(): + # Paths + base_path = r"\\grove.ad.uconn.edu\\research\\ENGR_Chon\Darren\\NIH_Pulsewatch" + smote_type = 'Cassey5k_SMOTE' + split = 'holdout_60_10_30' + data_path_train = os.path.join(base_path, "TFS_pt", smote_type, split, "train") + data_path_val = os.path.join(base_path, "TFS_pt", smote_type, split, "validate") + data_path_test = os.path.join(base_path, "TFS_pt", smote_type, split, "test") + labels_path_train = os.path.join(base_path, "TFS_pt", smote_type, split, "Cassey5k_SMOTE_train_names_labels.csv") + labels_path_val = os.path.join(base_path, "TFS_pt", smote_type, split, "Cassey5k_SMOTE_validate_names_labels.csv") + labels_path_test = os.path.join(base_path, "TFS_pt", smote_type, split, "Cassey5k_SMOTE_test_names_labels.csv") + + # Parameters + binary = False + n_epochs = 100 + if binary: + n_classes = 2 + else: + n_classes = 3 + patience = round(n_epochs / 10) if n_epochs > 50 else 5 + save = True + resume_checkpoint_path = None + batch_size = 256 + + # Data loading + train_loader = load_data(data_path_train, labels_path_train, batch_size, binary) + val_loader = load_data(data_path_val, labels_path_val, batch_size, binary) + test_loader = load_data(data_path_test, labels_path_test, batch_size, binary) + + # Training and validation + start_time = time.time() + model, likelihood, metrics = train_gp_model(train_loader, val_loader, n_epochs, + n_classes, patience, save) + end_time = time.time() + time_passed = end_time - start_time + print('\nTraining and validation took %.2f minutes' % (time_passed / 60)) + + # Evaluation + start_time = time.time() + test_metrics = evaluate_gp_model(test_loader, model, likelihood, n_classes) + end_time = time.time() + time_passed = end_time - start_time + print('\nTesting took %.2f seconds' % time_passed) + + print('Test Metrics:') + print('Precision: %.4f' % test_metrics['precision']) + print('Recall: %.4f' % test_metrics['recall']) + print('F1 Score: %.4f' % test_metrics['f1_score']) + print('AUC-ROC: %.4f' % test_metrics['auc_roc']) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/__pycache__/dataloader.cpython-310.pyc b/utils/__pycache__/dataloader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae5efbeafd8d424bcf3ebbd72456f4172f9bd4de GIT binary patch literal 19552 zcmeHvX_Oq-bzW8V-qYRFvtV|BO>k`r9L|O{5J?aqC`!bTB0$M%gDN+=YG!)S8(!4_ z#H2>11u7zyAfb~u8>Mq*V8;ovI8H*@au#PjR$?bkPLe;VIEfbfB(R*7$vF<~Ncp~7 zy<g~&Y_uZ|go=zt<{PuokdgYz3YT7sX()~-}2W8@gIMTwEL^WARRtE|Lk{+p~s_8;n;?YW`I#?KN zmd_TlC?BgB)uF%&iu^=nPj##?rfYV5d)$t%Pi#-_XzF<@S<|h7 z9le0G?I|m@qoYJmAiVDx&Dv}2drhoIE%;fE~atd^hO zJeWMG?4oO1xq2;k2G42xLT=$uZvM!zg`>v~9?dPxEi5IIPZl@awYoZ%vr21rEqRMu zxt!Pi$g|brs+~9d*r}phbZpm8GV)x#XxR$+nX_)OW)+onwp6T0O8i{?oLzJ3%#A8! z)~k$_*6QVwjaaDUT=v7I4XcP~W8>B%=&1x8Y5yh=i{m+o$H^F)=4sp7j<%({`nKWe zJNlO48QUSGggju;q_!P)BRl$fWIJjZJ37;jYk;9i!tkUv&M?vpqmmbsytu@pL0ll> z$w|!`xM-+Pxp6Dy#op4a^p`@{ zfIpX>LN0noZrHhqIC^WxF&@U%B`>_4@M6TbOHZ|1$h2Bm)vUoA`c@oa7GVP65W*zF zVT1z+M-ZkEjv`DW+=DQKa17xf!f}LIFYYBU|Kzx~HRKI=Da6x=4|^GJ5b-SHBi@KN zjQ9}ZqZrAAoAgGn>DHv1bO*Lm-sp~gO}`e_HRMepZ>TG8s4H(8dBa_K!(DkZ$Q$X( z8)@ZXd;{n^gTD882^k;8wKe7?IZhehxHrBv?(Ol$yz!L~aC+~uAA0H8Q@29E4?nbg z_C-JX!ikeFeBz`ZI{S%pe#$9U8x`BE)oUkC`v#swGX>G%?o3k^-E!TJoO}MGy}LKJR&;W;dd{&|t9H$GW?R_|-<@w|pE!LWhk+c()m08-c?^Tuy-Ggo zhvw$z36=mB=Lil1Eb;XqUk@H6I0A5(X-5_TjJ5OUyrr z)WwB^`8eWptTn#?u)sDK*wO;~S!C;rC^@%uI6s159;u6Lb#V^G7w3?_$o3cK(cR)a zI$2~Biwi8Wz#?pHaRC`itg*xzORTXpho+Y18Dov5g?Wx`X^wFYc4=WDpFmX>TbyT` zT!=-Ea*3}?^BD9J*JWv*Z7wX{9LE6J{v7MfF@KKZnBzF+7C4IqE))kdx5TtVoaiB0 zJQhBJX6KHu+z~c#gyoJRV}1_h=Go~y8=Vhkh$2gKM{df3veSc7XyIlGaZYt^kqs}h zk%N3a$Sw}?^)UN6%sPiz|Nb^_x|@N1jK35zdozu*vQe!T)m5`mbk|gZ2P%%+!ZJyt z6K);pAnuMIEjsSihV6$OSNW06m2#!x$IGs*RQ-zMM;fYJb5W?#uxpkds@OH(s5Sg( zFw2`0ev<81(TJ`51ZvyTp`Uh(7Z7g_!cST&?RZAn)nd6;uC1CJtRE@YEc+Edgbr02 z<1C~%%9dGkl<6vzD-5(=FBdC*s`cX9j$4qo;!3k%l6iPzIjO`P7RxfDQ7knsmzqU0 zT`4R+*!9jT7(X}EWHLtLr#i326+hc~3tHuZHj5+5QJcI^V`y3;akk0WO-=2P7#&o{ z&Og2U%!}s9wR-LRsiIPL?fggTwYqZ5b8EI!cFu3O<%)CO!i`*!8+o>I)gN&hm9ooG zIp(1`bAHZToZE}`oR0#;jbu2kPZ={tN}n;pp{x-F<1xu~iu$jT$7#5)2Ux6O0gy66_%uBjA;+ z#tFz1s!4#Gd(@+hKSnS|K>ktfBUl0e^<6ru?&0gb!RsMX>BB<)c{v<8qV8wz13~W5 zqv|2PJ}j^EypiVTkEjE9y%|>1%$gxsB-qc?l$v6EmSCP>fq(~0RX5;~Gj;Mo-A0{|T@G*kt2+k1vK7tPuJVEdgf)fN!5s)QTrwE=Uc%I-zg3|=g z0Nf0zry1v2S7(`!P=^>lLaR`TZd~>dykx)Xxz7 z6@tG;@Yf06Cb;`kGPBz$snr8P=1-wb=k5+l*a+)kefQfM2j%C{;s*z%DW-P`*%YvO zKt!;nz)iuL_cjG<3fvT|DR5J;roc_XngaLtBUt}`aZvhB#_!E_QmaEK`d8oEDG7&o zGQK-ci&OHI_c|r&r;rQf!}~)}@&kR0ONVDse-0gWn3RA&`AbYASJELu`Rw=0OLRDh zUgn{TqW!@8wkv_VP^=VdCELp4W>!DP+57^)oe9$iLd5?C?cb1S|3k6O!K&ENZT_`t(|g2?wNf*&SO1T_Ma;1z-k z1SJBSpiHn1fZCTBe~I9e1V2PjARvpYssukmU=jR&f;z!PfSVcRF#a-uORz!kX@biH zR|u{W6bUv7RtQ!J)(AX+d?H}!MSc*`p^vW@fkDnU#Q+pP#HxMr#gj}EAwxt9kt!%Z zrIoD=5+Px}g3fd;B87TLgcG;7b7hU`f>-hYBwX+ljx;WGY zR-`g%Y3@0Y?z`kogS+<*ylG7R5(at4E8u;IfG4JJCbx`6VzCKJKh`?JH z*Y%@J5M=IPl_0C2mS7bH{s5r_ZHZ9%!7g<{2So)tE=+J?`!Q8(|@V5xwr!%Df9vl1n1cK#%hq1p)z;mJgHo-3v@Eir0FK0kb!;U%u=uwV{(iMO;b zg=xiXrY9wR*h}`M4@mlmH_%FN3Puo|C^WqR8|rpaET@mda+=v1^ajOpnnegJ>(-Fi z76+}wq_(50pGI$cyevjA*0g|r0X#tBvabFS!elc{1)((#8|uWx5c4EHDe?4n+RM_O z`nrY~G!3w>8)XxbEVc{W>mYTZ;iM^uOoaz|Gf5jeZ>$9ef)LU zdyzNl4tv9}jgHb%e?7dl#~tzZSX#S--PV_{oX`t@0;nhLge9!I2A^fWrq8-7gLjY_d( zuYP6U=f3svul(W@w^)61@?-_}L)2+4w1b5!(GdnKPMczG1=jlZ(qqLtiOr?o9POla zmOdW}cu3i13IRyC7eSztrx1aJ^Gpy%f!A?#5>Wm`ju2yBlj-h!-mrF{Kv9PMn5; z6S!xdF0PfU_FB{2x##wDtV3buwq%R;&N?iNj@{K*qp0eyTy?5w_r&Q!rd=zr&!#)? zY;?H&+FsYfRH?aP&C#Jq3(I$7tP62>t?UR!4RhXReZ?%nTwR7W)OHHvH8Hz06^81n zW73*kE?q24G*taU@j@9k(yOM0imH48X4HDk4_m5u<+fo2mRj*9DNObaF~~#n4S#gE ztlKMK@_}bcVWPhRv!tn!U2^>X;_Y<{j?u3YD}uAsIW+$mhtVOr^%`7KMmz%hF6_Y6 z8prhs`HP3CDWg3&u4fP$lA6?$y76i%6gIL*9}PvJON)MICOjiLwWN{M!>`VS3Y|7BC{Tc>R!qf9h;_XR1GE7^8d!SY@;1Fk6aNV*Z7M!Qxju%Es zXeDXItpr@&B4lyk;uf`1RvMwOHVDDUSVIUC*042#Flmiidk_v-W7asrlr>>ZB21GZ zK$x*+ti1>at$o%#2(wlUBrNBrPi{DFz1nnmJ13|1bYP796%6&(s{qTT3}EZWIEfCg z2j>ESr+b>Ic?KXH(f|!F3>fkvfMG8R7;&T5KvK;Z7%|O^!x2g|!G-}Q;ryhTaMl7$ zd1=5jT(>kc1NL%oM}u3Vc2>KoFTTBpveVyPN1m7h1b%Ng@ z_zwjCk>D1=Zvr%@%2w64nD|=+J8f`-W#Xm@n(vzC5_ZF(;2$-jdh0LVj!8-=Wo(Z1 zZ1rrbde2R$qnQvM+1yb?0xti2u({Y;Zio8!$bi37&VB`pSfU??c#N5~V%2s`(52=| z`{|w>KN@VnOTDX-DE%&Sdn+@X;USJ9l;RO;~IyeC|0j?Z=nE?kN6Ttod z4V20o>YpI+LvTe={}Zubb~hONX8>o=u1su7+kB*VVnxdm2bSHgecY~W-H&3a;cp~s zgfuRnAFtV0q+($(a6U3y2|?d73J$=Jx4L+aqagYZi=#_FO0u{57A%YQH;qM@;YvzKj(tFG_;U#67A4e0+Y}Pp^Q)Bs&RxSnCh6A}B_y!a2;)Nuj^M7K0n)ceBxcT<)uL+q{W@GViAl zaf!E0S73^jppI|~wjhil7D9~9zofLrc7*rH4i77+E=A<#nnag(o4b&Cu=Kx+PW&*P zs`^U#p;g-r2ow*e`Zhry85(50uK9nuHGkfw{L8?Ls(=1{b$aGc^lZ=n0$zB(tix5= zWoTT}$22%a69=#YaEk^ecvyk0u!~#A(ytlnLC=t~k!$)z=u%Mv7=jX!8g8WRs274; zbp*E)9G)%Xi*&OME#r*ff*|DvVGA?#BSNjP z37MX>6%CGR2v&O$0H88;!Iql1I4*%9qAPj-0uPubc7c2yB3xWv zz@xIX6GjToX&~*B{l}?Yqi1PRyZ0yuS9gb|>%mDe0{Z+DDBeFBP;}7y)^)js7S->d z*yeQa2zpADNSa$e9%;S{dQ*;Uk&`VPXfFh**?=cH2n_t+!z9*ZEU?AlJ)?W#K=#d2jEs>{7Fb)()b5;=e48i_t)x zp_hU0P#Wk^8iWuU2*U_N2!YS*P~g0LV=L-bOr@8Gzs@VK^PY{eukLdSdcWjDGW zBae9WR~Psin1^YraOFD(kI zGtF{uH}l>GuNXqTH;|si#|Ce7%EEXitz0^TGEge1w@@a5GH>0XOax`#Mwua$dAqBO zww_f#Yh~N5wcEWkg2P~}P_Q3f&OdNh_ME%2=KLPpa`N}wm96ZqEM-5qo$SFM#2ER5 zm?GaFL*(X^xdNWQVuIL15H^?0dGl1gwmJ(R@^|$E&@v?~yHc*wUEGf~FAkyeiqaHx zIH^*-vQe!CtTF_Ti{*w%$8uz)p;d5fv)-V9?)WJYTbge@R04ECNa$IFavP9Ix44yged(Ufdns-s8n_b0wsW1lkzmr*o(|Dm7E6 z`B|xnTS20-1fwvv@uB1nR*2>JtQaV_ z*UIuHb<^t^>NwW-Swq*2_V}foj6dTAZFS9RdxqsXMuhjMU%~m#@fOE>?2|a*Va>hw zP5ml95ZwZ%Z7#a|tznNQtr7P=-p(!41IHrc{%y?ae!Q`dV1(d-KM7XjQ_!z$4SZ(c ztkzXRX0>u#iJ;^a{R&jP%f4>Pf!-VdMXb2y*>k7Q&2A2x7I;`H8!0@MiprjqgS;6g zg**rPIEimB>#KH64yxHr3_+rJxcpF~wuaM)H#;f^9camR9I-D1tUL@6>Dk=Vuas<|EXQ)V z@27cWX9)HZd<_k#3C6y~*slZlaY*3Km2&gzA(dn9Ys~$en7$WzgPo2SO! zSt~-3(^2t|&idgxJ|I%p8I9m;mm&z*M$N&8Qud~;Aj?8S^$AijnM0rz3Y9J|;4@Qv z3qd@3M{ zWOSh{B+E>SOzn9g4x%P+0o9XNp?R5(%+jdTU6tZ+L(r8KL}4an20LYf>FIVq1x%va zw@3kLhnfNlOjT`Yb=EJg@4MTwB;l?+& z*U2D;memx-j<0@!eKAFm#!s(67uVJ)szuOH~00DL(~XT1816k(lo1eYXje#nSRPNFG1`Kasu3p ztzdoJ#Rc`>*qMm-|BJDA2_SN53SYp&rH3FxfX~l0xv&)XOorDrTrwVe!wmKJH*bFvVg50m5>yRWX~Fi zP#wbS2v&P-8)`h-d0Q!#0b4t+tnDzqcOSPxc!P*U#va0c1foL%q?pR)G`|V9!q;gf zkHYeeFQ_U0KtKw4Lv9SV_UQE_zZS-B$Ckx}c=}vR`0t$L-^6&-e=}l82TX zYTC=iGME!!A9q4=5Bvd^ZgvPe-q)D%b%JjYJjK2LCSxRr0e5wZX-^aEqQ~xA8ed;k zA3;$+MhXEH0krCf@T}f;1pM*#5jF!OkF2L{-NR!shW?YVy}P>)Ci+5x`|J(BJkL>6k-fKKh0s1c0i3c8YCm>L4nlBK9}r#X=eCDglP zqXY||Xl;vC7-&YN;0~JUI^N3|<71o$Wqp2Qn9&u}O6j0KfkgZvJSXuuKLcRFL*E4PqbGhMWIq+uh+1`i;2U~AMoEmU!Fr~l_e2WuR58?e5jkoYH`z8}QX!Jc|5Oi~b zGIA?9ve$C*fUDBhzhc-CNt#R3jThLuU{wNi13xGe7YK7sGp=; zVKY7;AF%F@x;)KIrbF~Z2f5V;&}i7m;2w*DugB*r62eULzf)nR^v055^}`rI&+XMh zWVEI^!vI}I4k&@Bkk5yT6;w;+@HR8@Pq6PIHM~U&KJ4BjOFAm6iXNJ zop#P$v&$-10&%dNMjbWii4ky0IWg(*Q-2!SW|PwL3AO4Rlh~u3R?X&r8))`;Eca}u zYjn^l*zO)9VMdb{%r+Zrz8=frtKU{$t}d=3xhres(i#R@<1`$pU4`(+@nV9#L*|#~ zU1NUHb~jY5J?!T|TUm!CIJ@4uy-N|S(CrflMtmEsAt4>%7LiDa@hG+>rXvrGMX%wA zl4#3tq?f_AzJSf^q7yV6{Sc9dTR*04K7`pe*I8`*{Oq|~{KlljuSA;28-hv2)(SL( z)|pEY*MM{fHmhavz8>X283Ecxi&tA{`%(iK0E{JS$m!$mp29WpoLmz*^4~dVXDe58 zB3jcwNd5`NZrEAc;CJN0m*MUVPHTW!&7AI*#YLvOSKLr7tZa7GTKr?V6FK@}$Ui#R zz<=Oy4&>}&2RAw9qF^svo2b;6e}qtmvlB)hjJwrxhfO$^dx|VWj_iHluY)#>@cr?I4dC1GJA+MWVAW{z)h#QV_mGKAnLeM1j~#h9{ayg)+t{JUi3{Or!^kc{P!P7c3@~4zL(E9?TKfXp9K8Oc5?DO#%Kw zvWI>`L;-kabyf_fM_D4C6(u~itW#8(NGJ@`s?bX(gQ`Em;=hf)RgtkB#stWGAGzav z`_H%Iwr31+DQRTL5s^aW5BM;{TJ@4?ilpmD0#Q}u&uH^MUlxC)^W|NeqbWPa4LFXI*y)>WNBzWNx!FSD7SB=|DH&l7xw z;1>vfiQuaQzeXTD?vt!W22r>zxnYH>2~x;9buIl4tI7Juj|alk46$QXWsG@$vQ7-+jFIeeb>Ref+cHVmpIp;^==2{i90^^PlJ=^O)0~FAf_R z<~@dEh8fPl8N-HQgZefOo76WuY{uIZwoDcc7a6oP)?q8c*syKVK5U5Ia~$+F=xHOv|=pR5?JNXJ(WSE_YY4ObbNkaeOu(@6Mh zGXA7HtK%vj7>4V)N~CX4L+ZIIglyre5z@%j05)+ATrI*pobeLl+cMK-@8&~+ScvmP zBc5~k>kHlRbnf@;Ie579(BZB_p3aWWeRlhaz;tXZ${+S{!Ld-p{yEh==rbthfyuzl zkk6W2p8bFOTj=`4Y3YIhe8n{O0gCmkv(mX$PS zy?LIPCHJAvKam~fB%4c<^KmA9daCURqVZ;Pwc^X1`EgM3L(mAcU80562LyPpG zy05%ADt*SGw!ogwncx2m<>a~mpB@hr2F}4bZyFQEo4~d|F(gcQm*CBEF0K^e=D0Co ziJKBdaW-zI`V!VyajrnFEau96#Vz^~`VqsG$1Oi%e$48_4b0b+QKV0cGAiOlzh@cN z4P{iut(%pB-d3Tvwwp}Cev|o5U7Se*>oz|6QR`u@e{TfXm2f1Sac4fxC5rK#H$#b% zSVgY133ZjDZ=|y4W>w!>zEU?j)$wACLCuc~u+P9e?2Z@5Ew>H)X}pW@?!()L_e=3& z&YdfrtG%UCg0R0Jye=Ky7PlvXMH`=ZvA$&n=H0sIq%0{+`SAO(ToagyEY5KCj~fcp z#_b7LY)h_w9ZtFQsktpPz-xV2qBPc&%l&ShKK48HbM>PJuJM8O-MXGa{4blUL#&Z& z+B`PaoGUl(ikHUAIM3te53oLbm|Y_hY# zqB>q3w>`J?Z_smn>1y9=(06ydrl9ZcZ{Bz0uZ`Q0|A0DnRiM2+7(+MSy9#J6Lxar) z;TamNir0SQdJpQ>hZY^RRO!bJE63J&)n+T-*OcMjP)2Lq{d<;CyP=G2@!HMGzzVz_ zD{vLwKD^y{x8YqIuZp|lwWB6*E;|Ni-X0mb$wzO8+5=qsbTAr;w1>Fq5k4e@0(@|6 zq<66YqQ84A8W|ZJIORV-9Tq}&0GD9&=fol?rol@W z70daP-RG}#E2c|Vh7_j|n4AiS{E=wnWS?ThpDi5%Z+U*CpQk1$rhOfq0DC%o)rx6v z#~uO**xS)XU_XHa0DC$JQ0yLx-E)w@A%ISb=%k2Fis;-&fbw=ywoWA3*MTJasHlBZ z)V>Z1+q2JSB|lQ`+q0K|>Z3$d;yx;IACBJeXw`5kd<#ZN^{J0sO%59&;0{+x4P^A#QlrNK57Sj1R4sn4dFzHxl?vim|;z zG2B&(fDV34xj>We@yvd8{+l9WPk`q`k&#o;NR$`+Lt`OfTo{>-jfaI14u~06Ip_AN zdtcd)gvW(gFgiJPY^G6|3XjM9)8m}bek2?PFD)E{Clh}jjSy(B{Bo8Nn~rL3F_1JQ zxBb+X4fp~H7+vp)?Nxi<2hR_T>E3Sv1IVENChGqT*pqx#`(2}+h&eSydZZdbn1;`3 zd@<^#a46zq`5t`ny#R_W7@dy9Xw-PBPoS~K(9QDwl;jk^@9O6T%C-kU?dN5&iQLmb zvMjkL`N~fZ{j~lQ)6&3Sm3&xC!(^!Vq5GBdT^VW2REHiOmz&@@Pr5%3#`CwNAqd=Alp-?_jvqR&+3!g7cXtTA=$ z-^)|O`GWvzK-WQjC;W)&+NTB(fp+ZO&+nxa<%)5NQ>+lpI6o~kmVgkuHx*J$LX20; zGo$0-uwt2xjK47*Qp{8QcqGQ3rP8e9u@KKk?+6Mz8HkN3=5Zksh#>paR4Bsn#K-a{ zDWRFiWN;{NdXgvUN?{3`d<}}78_leBiZu`p`>B^o^~pZJO6~r>z}Wa?XbfMVc%^D% zIL1P0*bvN(b65$1!a^vou&Dqay?alX#F~pmu_T*IE4(=OO@)?cKeJ=2)CJR%mTn!A zWY!g>gbz(>tCH3S^LT^{-BnB&9-dfF#TpwM7YHAfa_YQ4I_eKXARPxCh6JT1GCg@C z#8W5+byD!hqW-Dz;4Q^H#Yb-hZj6H>@A)}o1SQuf(>j zxoQ`-$}Z2UYuk!zo8Z2X0Hmg)GQ!w~l)9f}_l#vkUY43J%Lry0o;lF(7P0kcs&eO2Xx(IR9{n`~@EZcN zJ-@}+wz&!>qV*glH7a=vy<{17LUFH zLb+;q$@r_Qul>q@LkuP)|9#niAE76VCy9PuH>j^+qyZh#E1d&h$GRy+bGK1sx1{Q~ z&^WlZV~)(bR4`{+!)$DyJ3e?)Vaj>v@5)y0)w{M)MvO05=^aVs)G7Ah#%phtG+-Rd?m8IDAdMG9tbnT)7ey z;lfdI>J2%}i?{Esgzt&rdnsqjs&mJRbH`%0CMJ2mvpZL?D*mg{6JubH% z#}~PwG0h{fw`BH~$lg*(@FpcZDzzSyTaStMo8&HrG>^oN$?TZOj;RR+O8Ao0+AX(s zi*|wB)i*Sc#KvScCbBW~v0=AZ-%gHor|Nczn!`{vb&8r}8`d06V)H(6|Dfo=x-zJ8 zF+r@h*+*64;OmPKfcKaXP=sTKjjHmulCwh5YvOEB`?8!J(5$MGEu=;sF5-#-ty~G9 zjdKCoxl%v}R|e?h$^na^P-ccpxFYa%RZ4O9v=EC(a{FauBr?xA0@COlr zk_eCr5`)b?ED{U`JZ5QOW3zBsk4M^QW&--T<@Y9AN+hk

Li#|s2O=z*EdPgnEDOkRQ+=tfeXLB(R9NtH7d6C`A9T&cGZF7sr| zHvPy3IPSUfG@-t*6@cW-x-*(f?p+E#t^7r;xc@ZV!d~qSs17w+K;)!YfDa^T(PScw z=2)>tLU+`<6qhEl`7;5k`rvhsh5!(^(rqdvsZ2Q0rEnSm?Mh|Zh1M-@2axK{$aQDt zTq&#bp>5UbUa`6tnk1`Fw)#Y?51|Dl1S*328ggkvMf!8Lc-~2yUJZ~T2%{>7@{f>} z|08N`4!@NmzaJ_AEErY#nMenJpOO;}%rehh<@5XGimj5XRd%(C*4B;KOd9~r@fV{2 zMASJX$!{h6SsH+!a;Ohyc%MxeV|m-a1eK^h*q96-AlHZ4oLQWOK11!=50Fq>K54!5 zSvCt*%&{o+kXUsCw?jWqqk;>^r@>oOQ3&kXzF^Mo!jq5gZwSAodn6v-Z$=FIB+UE~1yd_z8%hugu z#%&yFB29Iq6?pXckUMw9h+(F4r`8E$+{hVnhduerCt~u>GZ2%V6~vfyd8lsni5rvI z`N@RQn`1e%ey&iSC6fY6eU=5tgwgv0{C$`WNYAciaI;xd`5T{=|0$9~M*{6!K`O;` zGZfRdO`>`J9{?20B<1>=nJkF@2^A0o03B4-JsM8Bq^b_Ns$CcyRW>f%mMgKlKD4dXdVkXR!^TC&(gmsZfLwb3fhAv0{aHLmrP=NY z`WD3iy%56*YMp#0$<09?FSDC^EX{oT<{vM~;QFa;b05{FwvG^g^BF zXptSzmzZrw7VACZ3=| z1dNp8CLDkn6OeausWxY$;u{PmNP@~EBlZaDC?FrBesSa*^2y;j^%N}S+VU7T+$8BO zOG+ve2O_W$D?^CuS1}n|VGhw>M|*g6t6U-M9|;kh~dG(^cxKbmC4pNhdwU?d&5Bn;!)N{I;V!P)U|6Q&qpZ2HNqvdMg; z%S6*UOzXD#V1+W!I<230tgD-smz~gbcCCy8RO27#i`f&|JvGj9*s*C3zG-}HDX@UR zdYNwnwYd!qSM;F38X;lHw~i^a9>Um+s#Z2d2`kFav#QFoqsh0mfzUod@~isrTVX3< zg*AgQPL_5y5RjFK27ZH(~~Z#F`c*Wo9B&Z>wS3h;eUrx z#5hW}c=o}WI{&8U&vexMm0_oLvRRbgTvY0zgCij{rC7#rP!z(+kScxZBr8G~RkEU! zr{_z*g@$waESkFAePw9Zcp(`5srrA2UcD$XkHi1UdL$f}yuk&I&79B^>F*Mf^A9z( z68;?(O{SEy%<{>9)BiX9;?OIfp8e%n@#^p|U!G-D!h(E7NUdt0`1A06{(tfKiZw~h z7{wu&s)%7Hz;CN64-(ICU?cEXDBmjpiWx^^;d_cLgyWtN4&pYs zvulf1cK6nfkL=o_gtIs(S~S1P1ypGMd?rBP|C%bKH4b|{9f12$iD1XKy4>Z~JpS(^ zY^MG98$KC%gl;S|*rnWcbH%B$iaE<#Ma`o_>r9#Ls0!w~pVijCU-OfOA2uX8xpn^& zlXz@cIyfxVUXyFD&7FScDtmb6yZ66ye_=v96Io|m#YdlQX?p+KV%?L9rx%v*iNgVL z@P@c0C~XPKTY~fURBOjlt=#(JYU|0B)|28(*Wjeq5xI3lEcHI~b&7lYBwxSm>tAP_ zWoQO~RJnD&d!ZraXKnmcq-s#Qm?p zZI(8*_sBwD@`%*X@#UB6PNsCr!l>wK5v?s7nGO;X)g}ACVX?&Cj?c{VJNNAT{n?Pr zF4CF&^!_@^D_F#hzgOK5V>uIT1^IjaCR>Barc1%Gnh79(J8s&XJ%-2yT+0c2pc+@M zM-#NgsZahwp0_-kI0l;qC^siy>&wGtl{K7(ac_1>F64lc#F%Y((A$zsApF^u<@@yp z)CqIsRj7^=w)?hA%r`1^Q_BCk zQnk|W81684O;?#a24Cr*V(_ax#*723SvcmuG}JfL&d;JBGc2)XV1l0FLorQ7ZXyH_ zhC8ttQ#(M-<%lpnH5KK_hKRTXG9bcP4^R9k45W&o1Pt+0(J)#6MP}-HVYtJ4U|bRm z34&@jr>PNOU{~Dk>Af2aO;N(b9$z#6uTTWPOyE-j4+#8%z#M>Lg$7!4*RuqtHxVZp@Rmo+|$;&HcZDLv5;@#EVCsuZ!cv>y(J|*uyg`j8E^^Yc!Zm5EwU7R;PtKahe z!DROzAASGmyj82rDtQmd-b1V2?iFwM)02|7PxkidN*`J&^ND4?#j(})<16jQpPHri zKDoUQK~%!w#WBfyK=vNc@_bh95t|Q7)i27`FN&ovuDL26z9qWaAb5LQf7bTV#mBFT z`}96SPTg#Tp;OuOlCjvDasx2CP||Lu0w-)=@oFi(}hkTO8!j{iM8dd*kCZiOR( zWWFi{>9q%T;62QpPl`avrFYmLmv-6HGce>)B>dGkzBp9t9R9P zZpC$OnfoQgx~sD5>KyybRrxT!faqq~)ht?@HCei7NZr=*3_AF?QQ@G^rQoQa^a_5z zV)y$equeyC+Wm^t?|%czX)T8T9@46Zp8Su=M~4_Z%|HL=1U@0~4+;EZ0wm+|wBo9r zR}XU4MrfzCZ&3sRl2rs6#M$gGH4#nqpIJ8*o42krdeCSvUj(h?L5s=UL zTNYTXW>~akL1hu8%Yt&Z*}2YS!9D}UW`U*H?4|nCps}8slLd#pW}^B$c-mkxzd%hX z1X{PLvwiCDe@^72DKu#GgZUB;@t!KB{SiERO3TTih0-Lh)#a)9;$(wkq-%SyqOrk- z#SwQtHhus-#D}DdP@hC`EUVyy>{twiq6zjhR*sEjB|C}l*Dl+@53S5 z36z3!(m%i}kEi%9K8I8#4b1)RH`I$5D$h_Pj7^V@LWN@E_~;ajCSd97fG);=C(7Rf ztthT37(s;S;>IYBL!!J=qQz+DC!-Oll!@9D$7CQvhb3g}uc8l69q0cyfX`KE_3}R` z>Ky{aFlnlu|4F{D2yD`JrHI%GvbC|3hgbtNj#$i`0w<@%QEIZhgI_b2ReOwvF`(pn z{0U?pGs}=Uqg`gJ=5I)Bt<2WK*7R9P{lY-<&840vJD$EN4h2)KyPt%`%h$z;7_1Im zw+%2m1ccoo;B5bAW%UdFOZKNd;?Q;R)@^Ki1IE)v?1k5i84Dp6P&Qjip28!U7n+MNJShf(W|0U&U^dUj zbZNVhFg21MmQEAYsdthk60V|XbDu(7&e9z=|F#UmaEc@CB_zo|6H?Jc1uZl=$58~9 z+VUE?yhSQ+mCIWftEBSw*)z{r%Y%V;2ER2p|Dwb;$!wFzHa%nQFi3SsY=g|g)-mn0 z0VY;I^AIho1H1+gF0C69j{KVyIT{9)f;RPX;RtdjoQYyw9?hp`Ctc4YH9Yc|LCqGT z)W@a|Q16G?x^a=RFb~#%-;t+uLmt!b%u}Xahg2_U!CH|v^&y;p7SP!y)CE1Z4abry zTQ6u7rukKR@s(>-dKcyKL^ph#fJ(#flJ5@*kkO7NlUQNZ4+&%Xu3~|2Y$~V`^aD!p zZ2-jq;|WM0niS#GZ16RTR_xHr`qO>~U0_Oo2&8E#Aa#6d=Tx*rrR6sDxQZ_U615Q` zX}cHnl4UT$e%7{EZaenm&E-?#l@Y1!y4-eMVz%)PmA=2=DC;-RHmV)rZJ)nMwt zi_7)m=ya;<*z$xgRM!dd zVnCd{o!bA>@~h&FFtU|Luxb!QHo)1_$c74iOVHSY`yCZycT@XMikEJPkvpja-OFQQ za1z-nqLd8~*#KwHpd(KALS6Ej=;#pH4h_fic{lQ_IPQVx9o(myGo+2om_j2n;->;( z43eFO^;}>|@Zf?5{t1BKiH_o)STMxLaDBt0nJ)^S8-d_0Pb}(*jfKW}PcVvWRKipg zIq24zCMkQ=;|~v{XtK6!x2q+ena%JK?QS}O(=g!!8a10HNP&WiM_aqJ*sJ-Z*zR9o`%y09>P;8oyWSe~! zXBL!)dPTj{=hY8|HbNUaptV?cG(>BydNkAqPuA$pmB7`v34~}*JkN>;hIgeo!@EaF z0QR;AFU2s<{~x^5oNd0@-8S?&J=*FOn&;hN>m_n&Mj2ni=FZWtxy7=%ei@ennq?>M z6r=++o~fH^H!AKdq7+Q2)*iw~&+F;aMo&r{aiU?iRwT&>Y2TaRZm1*&iV#tG2_wYTB zdK5)>G1aT8n^8<~nyxFVV$OIZy%uWNo{q^cS-l{aUn}^f45g&2%Amqw&q)tmx+1iW zP2--du+tL?!m|V}( z3qk>L<>Iw|O{>Gl<>k{xjD|B?cm9~s@-@`=IZ>0w;F`6q=}QX8F_r%dpCA_^6HN-D zJ=s@;dtaLjvEnKiFSSU;l}U>kR+&-FhFlvV&7Pt%-G1$YUS{_q7K_@6MU#WPQ;B?hD zog?raueBW?bX;OI5qt zcJw zGupL(O{Ct9<^KakS}=Z!g$vyVi^7w%j9V8l^uHY#2dP4`p`48HDIi2WgJGRz4eUBYz|I)#vo`vrm~t`WQcQ)IaVe%s z)VmbpoK=4*rbFD!Z46H_J4C%pF^*aFmtx$a-ldo(QSVZWeOCRYm{Kw0)@)UCRSz4I z=AYSDJqK1i2PDrS*>gy&?wKE*AAK?{*?MJL@2q8=4HyhBV=d9W`b>VK0cw-Y;e7wE z)tKMp$G>F8&4ycs)b-a=y#wp!B11J4J33HqGT>@i7L+iT4Ob1R9UbfD62qQ#Mh`p= a!zr{X57xO29w>)0V4sbA&jIQJ_WuCw4R!$l literal 0 HcmV?d00001 diff --git a/utils/__pycache__/dataloader.cpython-312.pyc b/utils/__pycache__/dataloader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af61a837bff1c9e0633dd70b853298c32c42dc31 GIT binary patch literal 37393 zcmeHwd2|zbdSI0fTbC`{vL)Y^k+G2hB}MJ zEPw8ZxzC&kpVyZM;j{g!k^H`V;->Q#j9B_C3ID>rLh#r7i$<({R+>^#UY)N&E@<)% zu8m2XZ1T0prsVs>gppM-h5*eNKcppxJ{w89H5pPiF(x?-O?lP6WwLwn?OT^L%cj&= zkaE6nt?ZM0r-os2-l6)+nOuMsj2U1hlLxSh$p>g>3IJ9!7J$o`LVykxwTohk-k}() z*YKfQO0%zqu`$K)w=pHqYNbpWqk@*%o6J!*G3AVH(8N@{L-*A(m5^Q?iS@h~Du|(D zsvyP+#tt!@Of|rIW*NYhmsC#2c$;B6>ve~{j6E2zcf)_Dcfj7f*50(fy?H}>+Xj1c zV{@y)u+=>l9tyJUcE&U04H!PdGW9rVQM+fvea7oliP@d*ush@piv~1w1>KC71^?W` zVRwLWv&>wTEG|o`cO)B3227bTo?Ssv zY+g8DPJl6S+25j=oZnE(wV|o^91xz=1j2VxR45PZq{Iok4(z0clASH!G(L-*V)D&p zr=;2aRGzs@@;s%1`Z4Bj(-9rK^Wd$AcRsug@GgM25#APfo8Vmt?;Lm+!8;e;R(PA? zZG(3nr{nab?F|Sc`J9n65ql1?7jU_pnb`A)-NIS80%Ff6_QI5s7Ke-E`f!DEZJ3gz zYuG0H`-){#@|}z$$NexdmT4*_m(pZX$Z<2~QkIfSfgJKBa#7;QaWm#p zo|2124*3$fC~@St$y}gcjijG)Nk0{F`b58y-i%l|14nw8lls-h*&;Tsh_iCGLG@wE zSCE4A&J=-E;Vmn026I`+=j|MfKsMy`m}P9vSs}U4dYrW2dNszdqpt#)4E& z7wZlW2Ssh~{@#u*7A70@Mb2vZfgJ2#RM>~yA$uTb4|&gwcmv^3gY1p(FPdcUj-9LQ zP>@yjAZv%RtbxL0j^fM{)s0O}2wDNOG$L3BpcSocXl-jlupYoVbX(s7U<1b30B%i< zU}|byi(mtQrVUOtc(tN`8@RSKw>foSZp2tk%>bG)&1Oug8S`nu)LS5AW9vGn1zt_y z+JdRJG(zx}M(}UJ^jn%ByOt)%qy>{`X~rPU7z7h*X$Fs0jM0iQS}{gzBP7+@geHv9 z+T4U?Yi&d`7Phsy*{O%97_6lU)5L~o!BV!OwY3Qf-HPqf+JtE~x7@Np0WtkXjMIqz zjaZIGEJtHA)}k333JcTNif(JMqH9UxVc_+UY~y+iw;mH%kKs0eM^ht&Yr;&MFwrKd zh7hE+as4gQpqObJ3DkVc1ZJ#iV+$tSf{C=DwGFdai`I3R&pM2=4ufpKgh{=dHhf|4 z85}TkF*E#{TR^6G@Vx@ewrV$bI~s;Kxz*Y&QMl4#+|iSBrQWh{S%kiR0DUq7*T zv3S``n^5eG6*mgSjUQQ}#ci?T4MOpTSn(F2c*{H;E#5Yx8yFLiL`zd--GiLf0vw`V@q@ENCj@l`Ccog~|r7z9neN7VDbk8icxz`9A)@NnuAn z?;7TP7X{ZPVaFw*ZhT@dZ*~ZpnnzibwRoau(Q1!bor2XFwXS?j>9l?HRl}rqG8E6v zpIUy^H>r*rb0;rc-89{F&shG@lt0xy$4~O|brbtU z%f5-O#~P~egleKo%&(o;D;6A_*at4fmsGr_P%NsS=n<{HiTzN8qGdDtgu*p*tAxVM z(9D*qnR>ypYW6L`(mv4@x7E!y2)6Ziwg|Sp5Hp@5IIvj@Q&Zf${i^lS%-vwbv&VasqjM zQo&(D!RhGrNrkk;>ywIdLP^juzQRc*^^`*J6gCeUnZ;u|TYvzM*K8pIJdmBH10t zy0&C`DQMD*=kj6%-BjYVLY!q@JQpR7g2=$!#b*g|I)&KHDa6^A$VG|sbS^JV&`qLA zr9H0(O=ABg)R+A?2!0>IzeVux5d3=tuVhy*&$O!tf)eqW{bR^y8vdck>y7>IV`E>m zl^|84$Trav2U^ZQZFX!ZpxEwZM zQWO+U?2B89ULVz8-BJBDcSDkD0(q_1+zqmUyykAm`)JCsJ>>xWCB%UjJGTGRlfz0L z+bdqtvE6A*V`n5e8SL*sA0}BB64&A*bVFW6@~H33`&l^*Nxs5TmO={M;O_$A$Vf&Q zaQoc>kC$Qp6e^cTzvd4R0gP9`9~hufvLcl~(D_=j;yIEPFG@*xjdZ+5IzZt374QdM zk??nZLipQ4go^+F3V$oK@>e4;og!lX_m6Za$k0pFMTwJvSCE?!{??IxD&ouu;cp`9 z55nJh3E^)^=zpo;_p9L*fJDP_L?Pgc1;w615JoVHz>NS)#xe-J2+klli@*;6Vu#Rt z48aKmeF%`J&z?pQL@^X2H@070uDr)f(TM8p4g@GHk`zUO;8p0b7Qr$E zH3(`EpyWzYqJ*G-PKzWJ(}-S82#9D55l^W^YbydG!(vC%zd~>a0l_Ng(DXe7zl-1x z5d0p3j}ZK81kdFdIg8m}!a%=_;8zg*DuRE8;0A(e1ZDtB#EJ)lex~UumuB_@3`m5Q zK^R#Ilxk~ce-i^J63_F&7oP;py2Tf zNVQl~_+QOf2$TnIZDa+=rVCNluc=-O1Aldtwb)Oips(#LLhHqe{eNXS1hT*c6pT#l z`>VpkfYKs(tdit}K^nGr1z+DT*xK*71X~yK#XuN%m0)R@D-$f6Kp42NdZt?_Y?|v5 z3bz6k?jN`;u+=f+7p%>E+YZ6HV`6_i-EOo^uM~{SXU&3f-9+c3T*_27y??fsH#YH_ zrsoK?yhd7IBd!0Skk;2X7Gdw8s4t<%y%;O(AJUD*zY|Is!E*?9Lr^7xR1s7viOwjZ zy%G|GlrkW(t$~+>h$S&1f@l%6h+sqn8zaaNLAekaa+*^Ff0O7H!K?@_N6<8a&=Jgv zAZ!GcN+N6opIV8D>C;7k2_BimQlG*e3C6hvOWuKiAgBcGBv|TUbVF2?$o$IqD#3Bv z(RDupg0B+1nBcuhEsV}FF;Qv|aBUI0x?r7xzSX$kq`C?xVtMAlx5DP8@lJWag0 zMd*u#ZotSa+(cJ8swN?ENmNd{w3Uj9rQvC|x->lQA33JB^xD>%@{(#n}8q8X{a#LV#wn|EF$e;N^h6R zwSp9LB9<`j^5jYB^CW4rd|aDshe%-#1^rJmn*4TxI)gxRNk(V?`cJf!-#Q^hbv~N zFDbV@wcMpjPew|(63+CD*ornIrW(s7*RPc5hKZEIxGUpIVQdwXv6Rng&SPKdxU!7> zqL4?SUrM20QcC|aWoM|A(wB4PoGD}MXOtadBcU@=0j00xDl(M5@DkhKID!oCK?N7ySv+y%bv~I5;716Wa8+0hCdM*G5j0t8 zLS8R(2F&pATRsD%m)Z%MvglDUoAr+R-5&3mJC$Gm>w|axXv=39d%R@3A5>0&qAa=e zAt;Rpb#p;APz{BB5EQ&9^?_<69TW8#FHE{6>*G{QbZ63b#~>?kz~xf9)i^3SoJI$7 zK-+^3h3wbS0aika{dolMBA7&Q6~Q+UOd)s=!7n2C1puOE1e9%n!i-SB9bmtS&N>8t ziGEr(IEE@ZK;;ID*p{gA4~N1|dnR(WsB`=MlJ*U2$4=0R0Uhev=^h#$@eU<4WE4H^ z4(%Y8KjqAzii{v=kqCLkWMZRkHhA$;Xav#)-8{KUEJ>wDPSTEv7ARJwUByyQqQerU zgCO$IA;fr5?}7S;T`0_%Mwi!xhlWGwAr@d0xPpVA5F-HP1YK+)(H0;|ThJBMoQ#BA zs0(G-b5_iTmLsv~i(>I88ys*C41;=%OD+ZyVTT7mgGMkQY8cji;i=L|IzJ@6L?vk@ zm;4~BQL!+Sm-ti!Xp@AFqvG@k=tYvUWbhHwgDcG$vcrgYhT0LuS-RkANU{Rh{{^u= z!4sul^N*7Jzi2L*UM`sJF>{?@u8W$TpkBg2Rn^Swm_Eg8N*}A#6*au3h zngg=u8&R8+x7PFZo9>*tv!6F~EoizEetg;LsI7swuHn~g=R1$`+mG>v;|rP-P=IA! zP=HD}M56MNcwWK8o~0U04BEPM1tu2j)Mdd^JE?nMG+%4K+CJssOB)31ngwHHib4`p zXtDqbO-zNL@nkA+&sZ~Snd`pi?3nlP2hZNyc{ZMBnd-QnHSPH298iJc{-Abt`){pI z(QmR~s*Rh9Vy1GzR6Y&5O==$%L18{GrL@L(_r1Mu>d<{n@nQ+6>99VibbMz!zoI=_ zv+)ii)NG4ZZs#>+4?yFJ^A`dWyP$Gi^bz_ms*cRNXYyy7W{!QZ;e)b|)N^}&FZa$l zp>b=}-oaNL;Wgy|9~krbf*QW|jd-#GS&qE67;R0}6gFSrJHPbBKP3{G2B9jd>fs5j{j z)f0;9g8<;Tc{{pQZwDnT74fnP9;#zr@t9I;S5K-Jq0Lr&V@up*kD2NOQ{AlkzG>B? zZ1DTsL{%@J>Ak)DBQ@W;GupW8ZjI2`6RqFR*B{`QAB@@$VRBXL`He>wj-FiTckyqI zNBiI6xropoi5|VoAH4#3ZzAQfcR=3d@zQc$V~d;XFz;0TuLoLX=s%kF8P+rLQ|KhTXYX7%hRJSDoI(Gksya5$)X`j!Hl7Va?{~) zuV^Ge0YL-wM>G>HW{AH?u~@8|Xn z$QF~glO|tYAHmSpyKrnvHJZ&{r+hEK|xoCO;kGF`5XJBS}P@m zusG*Pt!Kc1vl$2?ssupDutcwgGWJ$Mm>X0$WnB4EdkcK= zkq`QZ6D^{W&R0bcjkN&?Vf;iTvt2IGcnDLmXzAcdI3L6+B9&``dSEV?YK>Wz3zp?k zb1ko{ea>jX^BkEY@J@fAM>CWeT(R}j9&p8NGVNjLU)MxbVP#d%q%0Z6CH-L{A6${0 zsvZ~xqzOQ&cw%A0l?)E)8C*qeI0)(nVLf!8_kxxloNpkEH01T19SwrMNNFKABppK7 zN6OGKV>DQm<>2X4GtIjfTN#NW6BMjQmCIrkYlMn5ack*~v)9kUl6Paz^*!;DiuWsT zR{o7%Yc)(705#L+UNc-ZOtnv6xvyLEbYEa=l9`L_&D1fIvGtcIvX;ooscROeVpJuN z4!9}v72Jq=W^hw_4wPUk`zZuZJczZ(eF34B9);e04ym8?T2@b{-XRDG5O=DvS2U>Y zF)fs+KI08b8x^jMtOGz=4`Hn5^iZ1jY3!lHP#>5FMWr|TuJ_HDqeYFAhImCC2=Y{{ zCu?ZjYM&_Mpw%*e(mN>BQW{qSLigMwWp z)0fL|78;SwgIqF%Nt8hw$OnG~?@FeT+!G>NPRoG`az^#*s8CtmBawO#hk-p+gKtqw zHcHSCo3gPwv!wxnYg2N#h}obd*&PnE>?#~gdeZMCtt@HMs}adEz%D%ogdQ-nYj8j{ zVBowkP?KDMLY)JG1uCT{I6W)^@JL&T(dB}Hh(s7zVN23iSjFoMjxHpa!A4>&neKeQ z=VlKKBhYVI6KjUUUuU#ohfuTwMw_iHW^)QQ=WM-TYnwF1^9pYmuN$Ykggob@_Ms70 z=;_i0IRBP@dpK>JpK z7W<4LGoc$?3TKcYJjxQ5%!bpTtj3Z_qEaSFdFDZ%F)w8)#s!e0&)@+{L4jZ+g8{Ec zj0Fd&i;P;>!jgmfn@|dtY;lb^P;pc{wNjs~Rz_|XCNRv9u^ht0amXgdtOd~}2p1ZLq$a3``Gr%xv7$Pms4kknVloSm@tL(VD|p+=NfV4jeNIeQ zD(FgMx@tjJJ+o>-w|*=GuYHnUh! zJ-u_X^PAn$pny6l4|WR~6axTn;HL`cVOaHmnLz_HLk({gyfyGv!y8VRnO4*`lc!QD3O`hUYqdNaBH3S(a)v!LsS^WOa|e?rU>_c`(jG>8 zJ4+exx|FbaX#v5?WHY)mDn@_Xpp;IZ5(6iE+JlUBPJi1de}V*9?S|(ut9T$$_+V7|4ewnY4#9p${<8DIZGjr#erQC0^>9#NY8H31^kWg6zgR6UHMEw!`B+yA?qv z0>U*sf+ixK_kveYzWN1}+OKlU=d0ZDc{MkD#!Fp;Kx^{50JQ?PkgL_z-31)|0W2USAFLs|$h z_(IY*5=;lM_4X$k&v*#Fn2@j}Ikln|K8N7HBzrkK5~K*9rEPn7v)Fx8G?IAmcrY zc&u1?&$e=L(~i3(!lpyFF7e0vrn9Cwp?FoitZp`PuWVDi+!-rx6v`Xt%I}wN20X5O z`Klmy@|^R|x!-Mw+gHczt%ALEu7AP4ExvB+Ox6r{&)yoZ zu9?mHpki*rJpH?ypBKn*4?di5X};t4E~kZZ)O~m3w>QQ&ZkpLSyXjud`o)c%3De^> zSj|Tz*c*@b(o|l>^dN7p;dM1nA87D!K@dd>&^0(cfK_@jogFe_>6wqpbf&L+r7tiF zqJ<>U3`b$M>1QYpkh>T)BAJk;!o-kk1yccxO1|nq zh%pM<%1$LEpL`jRP$v7fPF}WwG?6dY)Q5{`+5>DR&27C>dmtZVLPS4I#PTU9Xfheu z4~_(iEMq1VlT*r{5s}P@Zb54mn%$q2N%Ymq zxhCH!WIyP?q;(XDTi@!mM8KN-l3gS_(D97gnMm#Go>4mmQD?0w zC24p%uaw$l2aMq+IRv9faefU>-M(hoJ^6k+8b%=}@?`qHSEOQ$`z^W{1Cj8?Fx zMF4B&vSxNw%9kD3%RQ^5Gw1Q*?T@<#Jek@*bFJmRfbSntK0I8cJuDJDJJKA%nmq7B z%kxL)E~In}{*IL$-q~I*rohVHD9LPxAP@RL(PV%e}oZrAK$mX(Zne%bv8! zrp#$&b{$k$dLGpY_`Yn!c-eTE`WbV5;oi!etCaf%`T})PeS*3`I}3Y6+C@<5aU&dC z{9)JO-krS-;{`4T$hs&WhddvToAovj6nb2P$CzF?#FW5?O@n8=0fHjCQco;!ULa_# zdNgnb!UGglqv3IN0|48RKxk}qG{~ZGCZfS8nF)-47B_B4H3t|>UUoF-cjEyExV^cE z%$Dwe50aQKnxa;AP~|-KqJ_RfNdkp!S?^hHtpSr@KH}K@|>)hCMyK zI}W)z4(;e5H70TgVxBA};R?Z-392pXm8wd@qgP3;kWVn=G)c!1a!kk}L`*yjp&Y2F zgX08Y+esZmtmL$jn$am(aHBt~%P*b@!mqwm({Y z01hD_`L^*J8z0z8Zd|y2;Rbh|o9Uk0G{0JC+0P$38ap%~z^`Q>YV%y({cvd@e%n!@ z} z2cAUk7A{ovzzL+R^8HOWH_htqa0_K!Q@VJyGgjRqRJTN{+hWz5h3d`G>aA0`4;?GM zyXk{XbGcE+mY8Fg;MjFn6We`4*nNUO>582k5>5{BzCg@(LGWFOIxY%%_IQ3#EWcXF zubw$_FTZ~FVr%yAd53FT3eAj(5#Zhbhl=?wg#rx}TuAkZdtv9B04-)z4 zqK*wQ$9BQ7eZC{=*tsOV-h27Z*`Zj&7NKFwymp~sCq`=jpnY~I>R1OMA6o5vb$ish zkXtrFta`g&tt_Q-iqxQS?AJpGI^gG9I9p~5XjIHev*7n>De;WBD!k-?9ojxm^ zJ{w)@kF5;~YlHmgSaj|A=;{mn_~rjeQE$I1a@eeS)KpG@)NO>E{D%ED#X_5lC&T zsuPphs@jjq;OhxR)f`Yk1s5GHF~>&1vGGo4Y*V+esXM(64u8xM6dX{8aBOs37#)u~ z-r^x185Z}O~Q!U%d`%I6CkX1UqGo$enx z?z(?ad$*V`IJ96o3}Y+Nq7Q84H{QDb)=X!trcJ16n+yN`mG51-tNGOYiJ5mCkJ?V$ z&pYw(sW6f6Mdq20N4`J8J9?wGBlq);{I5qw%<_xSDwuSCvT|=-w}tw1i+=wb>OU{< zDBHhY{iAg>Sbp?I8<;=J;|9`1~{;RLPU+pop?{nGSom6~+^xWwE<)X-8os zV1XiwM9dik=_^H?j76Fmkh#dZUX5V~5a6Z*R3}2-LC%qwg$pozo)Oyz);)>N;s>Ac z%B9O)`ifl&F+v9+7OZ;tRnh!vUy`!4gT?-O6td}31bz+8BZyUI;>&T4+gaLce@ZsBz;5@{@71c@?~opQNE zgUdA%WX9mboG#Jia-9RNt>h!^W7&hS7KdA!*#fMh89@btDg?C%>JhXdSchOE0wgAp zg`LIf-jXPCQvLroREtH>f=jy>4DV3!jvf5ryrQ zWr2Yts$;})kdJ_%QAhwKv{?8uuwlR*_6$j)m)W7AvB5#V7iipUa1>?tk(}?sLb`nB~*@!{`haY58!GOpR)V2@`0ISGTk0dx@Q&1+C zbXkKmFK8J3B}lLe*AoRNEY_gZ>PTaY>Rfkm|qi^n2P3(G@UpBpK&M@D}_nzXkP8~d`3?-G`U-y8!>~3 zlZw!qVBLu11>o$?Xoy|KXd`(k0#C5!z&4C45wQ8Mfcig)J&tQqrZtl@!Fq%1F+8xM z$!!UDpdlTVoDQlHe2H`l($WEXr&OMoOveJZUJE!2`A$8of`~aun+qZh>?vrt&v1l* zPL{M);RP225YHkj06B$)$W$f^DJ(riy8M4&Ha|u11%l-IL&VNAtu?=d;m9&fmLnQR z5m~^nMDR&#@||jF!4VCpL@r@AlCM?2&`N}Bb>hrTq+;0LgrwVWl&ZmiUysg*uht<$ zrdgT2Q(FMn|0_SYW+&;de zmp}R@AGj26+sYqu^TF|W%ci@V_yHfpzX&&fz&hU-+i_Ibag=v?A%4LbdSWlchy0y$ zC;4rCeE%RHz7k*C!5<#rN8gIKzHzsU_nd|Jm+0*(u+IBpJC6xFkMVB~LVU{*J+Tis zU#8;e(wURIv60s_O7kui&Bncj%)6anK(=d&&^XLV5JRMW;FJDHB7h(;ZqeXPV-PS% zigr6`H=`FuvJCU`w13?8ZV>B2@${p;PP&+ZLPNwRTQiZ*j{4$Z$O%wakaS z;W0L#6uS$s2#iDS_CoEJ#$DPJQVTs@0jb15!9u}kn6$-83#YXOI|TtmdU(l^K(s*7 z69LK9Q~dZk`)vr>jkBf(x*f!vt<;)b^j{cv&E`+;N_=L#I%=x9XWYeWcKsAyL>T|; z)=G^>StKDQu0LdGT?7N+=)*hx0ZCT4r*O)akQ&l}q?bAqf{M$ed`bh^o)WmCublFL zX{6k_x~=^XS3r0oSzH#Nh*|DS@?bnN3jGS>5IdYKH`r&LfaY`&G{=qvHake$`7hZC z*BbAKyGKU--b`c8GX^6SP{7o6OByc;pVXAeZSK@u0mG8VS|TgC7uxL|cDyHy10y_! znXj^Y-N_M$Wp30K=ukkyWZY#1)PL&>Fb%PA+(+;;mmM0xENQ51k4cFqwHP+1l6PvG;R4XRlbwL}mr!r5v`_pTIwY~vp9d=(UU7U%M>yXq1#dZSrQw=mdF@PazMenY#}A(6FO9=;-mU6E8dLdxxQ1DbR}wW<(!!lO zyOf0+k801JT`A}W2-C_l2n1@?8EG6T!m~+SQ<`5usYv9HQr(1F0ZI;%JqlMgNEhPj zAsy67KW4Gt0SB=FrC67)&QjEWfx$}=%mR2yD!cxf-I2b~h1lDW0M<8ztQ*+vt@Uzu zJTx!k9c^=0qUK!_yB9TqEg<8l!q zy{MI_?1XflH45|29Rf}PXgN5rr;EkLk#Ouhav!AuZs_|%4LBfK4f;}py5UKOOd$}qlE^xn**o9?KPt2q;3G9n(|@DU(3&R(iZ=Y1D)=#F z`3Y6@W6JbjsK%e*Z{?3E<4>sKpHNjlrVKx!Y>WD$$)aoJSIcL#-!*(-h}l6LX5E5) z!}Y;CV^RH%i7XIpp$jIoF)Cl6^5OJI6~!slyZX2F@8-UpJIO9k1#!wak@c?eZR6y& zcg&BoHqk2{U#2$zyqE!QrkfvcQ*EHvg6|oc-cNtt3%e2BI<@O_3g3^8XH$pi<23K} z#aA`mIl><}$e%hLch=7xp6}(4^u>-02uB9^vqAo1z)%!N|31XAK`Az@4bF^zvFw~_r33RIh##u_}l-f>6PF9hnn^lAG-fCcsPZh^Qx|C zmR8a%-7;!=Nf*~BMZ}GkVsIn1_(r0X(1SF|QW9a&T52O*N^cC51|&RI%WPyzS@Fkf zxsAcnV3mI9s-#ym7Xx)9*{g#1N zJ!m0(C1X8k9ehnM9kd>@9!B1WBwZ(sfiy|0Xgz{74_hCy4k1m^deoYQ`-t_JH3#=Y zmT_Jy&Tkz~pHX(%wXH&{I#IAH zYj!<-n?t!=)cx2q8|78IX!waI%Wm1RT|Z6FxklNt72r+mE283g$@rZvws~erND=a)zdP+K#rX zZR@VSV|ea6Ss_Aona?6U}TaQoz%wZ#zHqPaS4e_ zia#Fs1>qmprkOr;uuT&sjt%^G}D-%i5K z!%e{*f}4gr40izT2;2#-I}SJRCA}2hKRvE(4|xM#2L3Gk!(PrC zgg+1eh&SR5!#@Q7C|WY%roGYYx;5#h-GQBqH@d4|*RMx)4RKS58|sQ1>WZ62+;CUi za97+6;zqjSMp|)b-vH{)q3(TNO4^5ZZI5|rwo}?S?u~Dcd;7dGZ+s_pTYspb@b><~TG=Vo8wJN+-LUJfGuw)$`}TY*`t;dD1vKPPp`i+B z%VTKF-dT!yKQcEzPgnvh&Jm6RmiTy>kB1KvjscD`?ARjUICC6F*!&!P^K(ZC#{u)l zixEUEG5#<@7Z(l}lkm?m*Zcxtfn_eRqy^Tq$kG>)a&GBpaRkmhLKj);;vAAM&LMu0 z(K#~e!w^K9GF9Q|zU(!xS9 zg{(}rIL|WK5sPf)5+9f5(dZ@i%hEi{Tv)s{js~*)Ip&#T{2bdc$9Bvu@GTbDQEbfI z62p%0MUP15G4U}JJ9mufjoNU}6{?3Q#WD?KcU7H(zW z=c~>wvfxD)a+r^YS;Y}P9%Vg8ndc~z9B0Au-t))bE-Y_uT(VW6u~KjwZn;(fNi1yE zt5-Jd0w$Sr;%(h~`+@gJgozayzIEwnzkQtT;<2MR;sm{pRBsJQjJ#}Kh{*$x{E~3rd_xE zNX@SMM!o6BgSWgj;ip;t1`4s2pF(b1D)h5%`4aqLBmA_r()Q<+y-}{#tMyfrjrC*I zx@EuWM^K^4qMfDeX4Nw5jxt?^bftmTk47=!LDa!LHjwNCWFxuKht?EulV`SQ&1{9G)xXJTWxYbO`vG;Bv~eX zw=}g&Vs!Y$6BnOae)>i8%vz&<@yW7McKzZ<8}){A%nNI_Q*|zGy49L<(ZY^glO1`s zdCebjnzgFSRypR8Idgu_T%6mF@|+(6Xht%c)|1ARF{5XUnMl;g>q+>A^#_QNB%YE+ z7e+F={3E^m8}fvF>ZeiKyKmg-4>s-(Htz0CTb8ZLVbZ8ULY^>07$%GmMhW`}V+3x= zYMelxP)!1E?Ng7^{}^G8K>kr3AS?kueV2}_d-!;7@OXq&`lyh9ZVtzesRtPQU=Vx! zxO#|>56j~`ccl6GW9kqdZ$;HKqh<(;g!>qpQB(BK66OgD1THXDAbbdbdJZ2|_Xob? zOX?saihx@q>M;FB3C9T^Bb*~VOZYv64-=jse3WpS@DT!8V)Z298Nzdf7YSzxPXlg6 z)Km0xt*i44NU00B7dm8TYWe*kJL=D1rm4S7_zK~#5q^g7 z*9mvON@n(2CG|!i$ovVEdDjais^fp}-P%w;ixNLrD503%C1g{;<^mDHngTZkYwp_= ztSN9)u%^IG!I}a$1#1f2--}@V|HVS-TNyu??WEp-Q1map+f^bf<2&=ThWfepTqPfH zv>&LVdel(w&(=~L=wo3!+^hO?n2;TIHQ-eL62r)!cDAEp{(I#rJG^8s8`-tDecvm1 z&#mrKxmK=MY%32Q?XI^v5l`Q3tNZy67V+=4K6yy=$s^yTK6y{QFm%n|NuRu@9vXUW z>XVbAPaYI~GW6fjC+9_~RV z_Z9j-PWS}jlY|n1LX+Ac{3yX9{60d1a2aqbryTlUCb)!6!jBNH5?&!(Ba{hSgcZUn zVU6Gcim5^lnA{dk)CEA zAw?q5c7)NkSS#`@YMf%qX@ZFTBIkdIkM|NNaCgLcB;U_~B|?F4AK?K4rT>mpF66X0 z5@;4?8TA-pjxbM{AuJFU38HQIv-JHHf>7x{Pv2i4+#>u{!e1l^z5X(N?sAz z-KCum__z=FxE~B3Csaddg%BPgQ9_tH#Ni<>qK64W63EAa0eqAJLe@I;MaY&A8KEyi z+k~_T0V8QC4tA*`JH$%plu*V3ixu)HbV|sd5T*|K6Y5lCVX1zlq)_D!&A)>d3;q5u zt9XJS4{K-eMdw&6|q{BG{I zi|7R!(Hr{v&Her&>g*wX`wXe*p6ChYhV@j6i}vQpiGIw z&B9GsIcpH%NiVjY^5WZRFCm=V0Jy=toAOfEby!&pd8sdIUy9PwA`DMU_^_Am3m=g1 z5pSRs9x893+@WIU^RV;SNsD#IkHfknw>{_$igiaGE-XQ|hs6G2&`M2eyZTLCJ&*eK zd3m&9EU?qKsb3idXPLUnSo+d!U#8`=#$oX>aXAvCkf+J;DZ7*P^0WYXU4sw0QC(w+ zZf|xK7-ChFd>u9_h#Pf>yO67Od-PDPei0pr|9eO;k|^pm<-O7gsuulFO<5M#0%jT2}8t#tu#ywTvaXCP87CA z;0uoD3(n@Hb-Us^g{|QNxjr1(cZH3@)S*_sWY=JT)=pWB2cphlwy^{#;k;~KWF+{G zEp(mvfk0}$zUb>qDg!U<$D)pHTa;Yz_fMbbp>+m=V)%{V=Wv@`1+{y5W?Q#3Pk&vX z)+RI;YYlT9zG>|zj4M&t6X~~ENl{lnPRH<#**RZ-^_EdeRT`Uh*MzFElz@!_JppwW z!k#`^t2%C_vC(`yg^GV0nYFD09Ts=?WUT?4IOlPAS_#jhc)jT-l-;bAEB5Nw4t(}o zFMRD6p194HZ%v-5!QugVTHWlR+bT5OLx0l>qObz{qISolzdM0p$8U{x!a5yaj07@@ z3^IjG1l$WJI7Ot8ihzp@5ccC~28^of3`h}PBYc+d2MC`dyiWK-gf9@-ck1&1Ke_>1 z57=us^>SVPK?WxXzr?thLXW~i1ih(}l9*jn+`pITQ%aU=H8K7ujh#LVlMKu}^K5yo zx?!({1{(YB3`akdX6^{KXra-7O@w2271k`P#;ezy4U~KOY$?~y6*as(J~AAC0;nwN&|)JDL&LmWcgZX|k`0K^%&2`lEYA-I)O| zAK22BCi*ilN{SlUBgXGf-tMX+7o}|qwfRYX zg8$KE1W#hYlGO8Xqxu0o{Z5oWBdzBm8Kj1NOJr1!>(O`OQChG>jikhhALTI}OPqr% zrG_>z>R+Q36*i^&FO7#&_&Hn?wgz)%tz^J}*RWt}YQ-!Vd%}D;ijc@k+Dcj}nCQmH z?!Y8BZe^@2Tw!zIf~m2F;HIo$YXolE8nyPp9k9l%akv?4!kUDeB{KjwXU$mq;SO2{ ztb5?*tpvzd!Ox!Ablk>9XrOyR*6i89c;c7Q%G;j?ELU>C)*t;OIcM;5E&)8<(@f1X zfH0s18eS9_@nXQJ7YD}N_;rv|GXbVdGm|jf)J(8sz%-0CH510Az>Jp#W?^conK`hU zgS#5cu(k8rEqxirk13ie1;$?kvMo5lLsX2s2At5CxVmNg1CkUJ5A?(z6st56gdmq9 zOY5gGo9hmfI56^b%gQyY)~kM|y0L0j*2?v|U2|Zkh9`$5rme;+^}qnsk7FUaE3j<4 zRIS4R&W~AQ;D&lul>!fqShWSXRDg5tZTxrEVhs8-U9IOb~WKiT#nBoa5a#(IW&ww1kS7sL`0 zZeIp>6b}K%e=!(b3@wL4{W>CGpINY9MJHBhcP19DX1%;&J0_@7*wcQtC&rHl18_@M z{{4YSB>t2#z0t z>74qX@C9%8CVjsGcoyZVUxjDuk>1xTTb7vH?X~4&XJy%b932hoNa-PjvGe?7-F`(f zmIeb;NwXCY)GTda^?tHd#W}RW;pEr!VHoE1Pl#5sp6=j;K(=7DlYqIH66-MPZcPIt z*uMVbPmE4!7=lwZ7+wXq~58PeWak0XsYh2gIG+g1} z96%4?5(j32haT9Dy4Z6p{koyfd4{BoUDq!|0zwMR5TuCJvD56ty$CLW#IQHvdWL1Z z!Ak*=Wvmfw52V{5Yf-v>OlTFXoV#dSF%fi!Fs$wAYGT{C0%nL8U-~_^WqTqxp&A8X)71j@5Cm0NUq0cmF-;6AZ(2HcDX)fkj3 zzXr8eW;=@%$w_TJ>t(RUz_qRC)Ngn>D|b~>U-xpjAUB8s%;K_C*3+?h{<^2{40>4( z?3FjdbZ>-lZ-7^fKv|@#3B=?t8{%uH$MLL`gnV&PD(UJJ(xoKbsZO~)>0)RH>lwmj zt2esRXzO{DUbFJ;GTWtJIf#W}tndol@N)6NyE-?#t5d@t@R?!pp1V4ey{i-1@BcjZ z;P-Rr{rwzyzdr}wTT|u=xc{07!ViJiTr%g)CmZ$ES=@ekTR#L{Q_8Yy)eT;B@e|=D zA~au7u7WNnQ)$#TH|hb?3=!mVwQ2H745G5oE;zQ?Xi`vj{EP@N;ZqOA0F4l0dLFJ2 zAhPQgxCQbD!Oo`71{*!DV~VH@e)S(Cb^kt-JXtGmT(ZiK|0-x%z^I<;rC#5O+3#jx zgQCEyvVro#>Yu@P`yKoRVSFqO1?;WSaQrB5`S{Uwa9tuQk>`cB!U--GWu9k;JQw!I zz>BzM6h3tHOMVP@h-%mTlnq^ojq7=~AIB;4rtJiBmQa2QQaJjO4X$egZ{5O)(}Vjr za?vNac@rF)rvqZo>qiQAgK^sc;D@2YFN_2!y4eLZo=LfjfGeOPlgHkDJ+J;dn)Fp} zAgAEb$TCKf{Eud^%V&VYR72pBO&(iH+!*DRn@Aom8LR;F@Fl?yWQ?uZyBo9~27!Dl zS$x@lj-g3zunlwj)pwwm2<8=b7H-Kgbs~%h|7}HJIHRq{W#5U^C%R&=7jRZqA z$%=sguNk+l;e^+QBS%PR_?m~8S~B z`>i~8RX4kyqrPMPfHic(XupM|led`hg0i~aZD)q**;2Uopxzy<@dEdBgd{$JRUXyc zdv6-o7IbYJ*WI^V_daXb<6+l`dq4N;)*%F26Sn=^c((`e%zA=Wf-C+6n2#TZzGZvh zQv>Ih>H;Dw{l0_G3)>n}p=o?LUD>4fh|9IWmY&IK; z3U4k_YP+HPQ3IIHC^ZXN&Q{GvtxSpv{lkAk_}0`@xPzz)(2!Pa$C0y#fZ4|(MRvCE z)T73eiAae^GY?m4XF+=_BBL^`rA)JB@4>b zycS4Y@qpi_KHpg@L($Vw0b%7qB5UA6q`Ea@JEsGj_MOH*b#I4{6!`>FP;1`2s9 z>2X!lhNVt16A()wVp-vPNvWw@_md!YA|VF@BacE0a~-LsS!=i|&E`g+ODl;&jcH1Q zoiy@#zHawZ#w4{~P!g zIhr%@P&t`@C%q5*TnxW-GL;;IyOcbdd`vOrS)mKJsR{`Ahkpy-g-RaS`lEW1B9n|* z6CSEcxJlt`U)zC#kEg$_5X-=cJT|hOD8Ae=Yenz`@rjH+6ag^^6e$p9s-d&|x`7qF z!L#!?&cpF72nt3Jog&_loAB}wq0;>B0meG9EC-8cFSHZ`oi$7~uKKTp-y)Fj5Eclx z*wdHFZe=Y{D<_<_&6Sm!4Q^32noyhKX7B*0g!xKCUAC3u4>qCqw3&Vd3WSC#4TDtR z_AaQ0+fI~j;y&<(z0pwDO1U6spc`JP`>Bwa;+q44Tk^T4ei z`fn(3ua>J|S1|WD5{hHs4={CDA0!{POvjpnjy2#reUE}Mk z>NzC!6Qm7L9zfp?4_E3P3&0<5FJLn`{E_~&t$VmA#!#0A>6;_~XE}1hlX)i&QlUPH z3VOI_Ova8X;vgP4fp)<^L&v1M1}6(OZ$+{_JsEp?^BRfjy zcIWI8EPbjqELLeC^h&~A6w|f1FQAQ&@ja*q(B$^CVpt_B)dj$bKZM^I{G6` zmPe2-SRU`5Za7Q_e(gQW8}VS+?Tv`31C|Y5DL5a#%@E=)2C!2y?3xKyrO#uq0F z?wVayg$js+?KB(6LHmq=M=Ho+4?kMMUXs1*&{BBBE}5Im=jR)`#c{ln*qTTIMWZy)9`U}h>8i$L zg>Rx7e`Jr+pXJ1rDL9~hl+%$@uk}YZii`OSe$Fuf*dyxdsWlu0eTqV=i-4^H#o)Qk zm9%R>-UEweg(u-R;`~?=CT+ZYtu<*s-o*H0zH(y7+T++hiB001Y!U_X-391vYu5@Q zYSU&&KIXI6+^lT!dl`$#Z8CKQm{1b)K1zKmw2U|Ar zB_Zcf!7g`ZCfi(=xeIeAs{7@GE>##nq2)olTP1gxgcF62kYOm0tq-ht(3sAfhfA+- z8h7G@_u7>1v%2yHJ;v^A*WKM`Bor6Us%JP2L|GFHh~~!_(AnFU@d&O}rl)X%oaDs> z3`F+m;?iJWe@T(J5)h`v2y^p?5d%D}0}(kV84cSX81IB5*2QMZM143&)RV^Kyar^j$8DP6?1qJ*E8e!@4-5KtQCc|$Li460)5RB6J&*{oWk;Owk4rr}L#WY32^<#mEDspE${DP0sgxS^e&zw_S@Bz1j57gk? zbXGngGt1FZs32q&_c@nqc3IVDJDVNmy+}f(G0Ate>Oc&iZA(Osm}IDb<)WN*QUz zZ14p*vG!6&&{{vyyar(rpYcf{l^?h4tJuSYaaHHpK`#(~iN*Xh;j4t7CHx%W=LugY z{33xiIsu3KQD!51D7=>JuR_&iQb<2_b^R{0$$U$m3%7w(OI qK4l=drr}FRl20U0hX1GGuIGPD+n^0m?swApWZa-wG?C0Er~V&&gi-GR literal 0 HcmV?d00001 diff --git a/utils/__pycache__/dataloader_batch.cpython-310.pyc b/utils/__pycache__/dataloader_batch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b49db691bc8ce02435566cafec49a67a4037ce0 GIT binary patch literal 11546 zcmb7Kd2Afld7oo5)9hjv%80{O9OYNmM zvy`}gZ`nU7Z4Oh(>Ct!dvABS zyQI)|7xTS0-@Cu}z3+Nc%VZo4zs*1TuK!oBYT9?{rS})Z%QBu|R@XG9t!qqYMp<9i zWooROGL5ark(y%bjMv(%xk zYreTNd@$3_cXKSwGIzE0Q_pKG%ZBc1Y{+-^&Gp&ja8gei>?|8+6LOWn*}z{PO~#eA7YQNJbMTwXPI$LD?Hjf?_B1-7y2w;tL9(Cv*KUR zFFu)HccmXif4zQ(%dP|EmW=v}bMra=DJLW-*Atl^JDv3=Kj8AKmlv?dH zA<$#YzM$3xK zRw9Z;@ky;^hZY+S6OpyAe@SB_k@Y3*E3tiSb{n6N_|YgafVU-nEV4Uzj5CgLl3QBK z+0s6XnH|`N+7<2fXEuyhs+Eq?M1fWYDdyhFhRFj5d1qQX)cIF2+C-GW3?{!~(&#te zh%%A2tMl(5O|;XbO4-!3#-?u=0m@{4M&_M^RFok$eM>_wOSzkGgz23OvDiJlMNVrd z%wnxW`#L*4rtRykTsZU=($DH|>YF+{!`^q-ehYI($?%&Ra0ZyM2N&j|p(qy(vxnH( zd$HC?I2?^It=s3r)aR$MO0*x1Mx$iAEQ#?RIn^!$&Bs8a@n{UR$$%c2NZ-}o*7$%6H!u@b$eE2=z9`POv|8s)&h<%PvV;6@y)2zH;@-eLCR6T)pwlT<#^cK3DrXb9v`9*+*`ND)S&&@9o0OH*I zc>+%o_%Oi2JOL_Qpwfk>2s{n2NCk^jut)`qO9ZIxA~jn?m8E%9S)#6%sH>%U!YwQn zY|5kN(!x0cGDnrD$0h1NAjT`g#Ea&e4|?SuKQ&O@NUh5CCqsmetrRynqS1o^XD<>{D%qm)M7L9q0SHF z&Nolzx4j@=t>puMtKwI~V6Ib4DTfl(gnmxwOFV@vq;)LteYS->e(ye8qzav%feh*2 zQ9QvIz!pT+>sCvT^tT{TzhK;i-g!sk%P4xs5XRiR&~Ky51lYmFmdk=4zWRUV#48@> zes$xcwQ7wA?%K8=l!A>$SSkk_3|v{3ddR&_Y&uup6VAnQDF}ucGpVFqFmq_X{MOv=rSs<~w*mmmMosV$Z3PNeuZslt>t(O#Z++{`r@#N> z-}?5m_i6sk>C0uX3MSoQp8)JoB$W<6!UhNV%^E*qDDcq{vdvJ!oQjHN|)S0khQ!tkfwB-fiUfmaF5ONW3ZkEbr zVKu6yn+;#Y>%3GA`A4ZYyA=AI*KP$OR`J4Z5ibQ*uZrgNx?g2HfjazIsu<@W8bg1L z3bzRnqkSsO2#0NUWQwr8a@nO-iit}rt~^!linm>=_}h5H$r9to@t}p6J%(l_28jda z2>hPT>K?D%-U}*_Z4f@$ZY$M!=IEU|9qGPyX)9)m?M-XX4plnmS7fo^CIF6Qmcv>SFAqB5|w%DADd~YOnzn$*x9E zC=r9zyF^q6<|?rUof_dB9phQN2X6p4#*98|r0Fs2xUJhUTaOtr-PWh%-;Tx1n4U9G zqGypB66@%WZrn+mF(U{3h%u&HdQ7+8n~lvPwGGE`^w^zQQ@-iyEoR#KJle(XOqtZT zjdvSmyoqV_8M1hnM(i?DIZ0oGZ=$UmNi&&%g>JJXEOe7OECriA*7gA6%Kl%sSdI+? zpI{?w6sgU|*f`Q8n_!bj9X7?Lk*3%TJB2jOX4z?^8EJrN>sjdmXzN4p9UcZg#}ecT zh|J|i5Y{TB%wfl`!3Ym_SETUg&tszZ?*Ob8Ndt6#lqU&v8BajsDuU7NYLNk`M<$>F zGgx!YC=M8lEWmhZ?L#BDi6{YR!;DQHYA_y=bDby&n2H?0G{mXqW|#(Tp|2LQB1!Vx zt&?m>NsSk%mzX@}si5s=l$yRs$*K@~kR)PAiXxH+sud>}LO9^xYCu&2SP`MeL$}0k zQ{_$AJKHYxK;BHcRM~Qi+g`QmmjehXyahCb!)mvx%H$BE#Re&%P^lSL9@avNlw^66 ze-fkdI|M#OfLN066Szy@GX&lu@L2+%Be2{BlBsPMJwiFTcYqJ14`f(iNA%Du!;`G;ph4#SdHv zn>IN`rmsX;ipuZk{HM{wz+(8D!2KK^rFiw`>Hhf+P`^mb)%nqsJ5NBq;hP$K3OM$6 z?MRnCh4GfQ1CKB=+Fpu5j75tnn5%shpl-W13Q<(5@T9U)b~Ly_Mx-lCMJ}MA%grko z{3EC=OxOwh3&^R}brtq60wotVqctDzUzo?(1FegKDSH8wi1FVcJzI{!w7~{()?#p z)U)zsdX=LmTo-nY>;4KbKcJo*y*b`bf_9B#dm2Y=IqqY~U{pCWQn2F=0ySTbwTv)! z0KYoY4~_PI8monvn#|j1q)rwcrGj-5R>Z>^{1+TZZ`XgPP1zyDF^V_pH5edDjyOjM zMcZZSTCQ%08bS(m8hiyM0V!fq9rU;UD`ySz@<5wo>m~-2>n5I}NCwuYw~WYOIz=EL zIt}vZku#AqWiA#O2XPi-@k2VS5T^%;E{|KQC#_w|?}E*L8H2!5f;l`;OPE`JsN@ha z9{&=7q`qoeE*V)b027fFvWZn3_enZ2XwgV19jd&HR<^T~G&@vvy>46F|nek=v zam`g_8Ip8|aEnnY;!a6Xezf9?sI?a!$07Xg}& z_Rr-3^^XY}5;l_Yi3_lC9c{yZmwG;utIPB%`EL@7JZUO_ff`caTKeE*9?;@}MB=|g zqe-QwtF8Y%LM(TIoCtIsG-1bDc$W$#B~BfuM7LJaM77Q@jtsra^}mEnN6@w)>LL^2 z8lwdk-_Z|XIguJ8TMEn{yQoRlH^BO5iP=#OaiPRvq;HYk6>qCLkt7{KZ0Slqg-|?V zva`hh233MA02#ty zRahc9V%CTH$#O!=KIR}kP2iEl`CDWVN7GA65+G{$`cJmZP~R~QNFLCc)+eG^~ zPGN-Zr1&+&7n8f-z(|Jbu@N>R!LT)89=sgs2kFSB4Y;|`uE*MC?5GUptZa32;bsB{ z(AcDqdup{gZ|ES9M%Km`=CJji;4-rqX zH5Y>nP>KHxivJ!@@E8Eu01lkQv_7Ga=s7r&mN5pGlAOpK&I4S|JGRxF`~MNKPgv36 zphq5#KL;s8B_&8Y=(zA`?=%jC(Jq06U?yF8?BF5-c(Zwr@+@}4;1?tFz=9c}^NV5P zzz&mOeUssG@BY{3f81)&ij>(ouv@FS%xc%ZK)EZUFx*7||SZZH?gG_*NrZsf_#c*P0QuY{^ zGf}fJ4yIW85bLFS^lTcaH_bBlOkB6n%rU3#%w-Llc@#gG!37S?gPl|S8#o=a5!O%m zkK=sAwD9!aKj=98y4E}wo?+RDE^LOv_eHn_=!DU`GkX`~)OI`O-^fK7?>VA*EOTtCOYKZWOWTP4&Cg?YP{;Py_UZsOAXDZ!dgd=tr zabbu6FWhee_QJyO&;m?vjXa!WIjqcP(I1qPUx|Ar+KwkUcWq>g8I^`(;Si1nmFj-9k; zkkWt1KAnD)Q~edWmiuJ-Dpg2JDP{mVzhPuZWl2H5j!fs48+XRI;f4sqou<0?hWTgU zrWf`QSr>HUj1XHA;j?G6gj>Iu#~h&RlF*BPjtg1lFxUFCGj}< zj3Pxv?coZ;g8{bh(>mTDKmtnH1^-03e)OMG1)zD zu8K8F%J{56JUxP590Yw40yeS;Gh6S?z-ioW$IQY?$ z9BAH4fhScd`1OR+T)Px5&k3c&>iYnBUmaVO`f?4f`oMOdu*oo>!D=ef6Q~WAl0A^S zQy5Xq(!iSdZ324)BrDyeTz`ZXLO*b!SQ@v5h1_<%sQ60i#yhAF7WHl zP{M&N>9+GJy3cLD4D5(JMr{o(-K}S7cg~>mm`U0~Mty&jHf7|un>r?sqh6mb=x`gc zj%3lExGB6Uit!apuZ(6+9={j8dXPs58z18Y`PwF~my13RaT}Fa(e5CB-7DV6hqZjT z?U#7Ih)9MX)N5!#m(|LU$V>ku4F38-6j(T<5*Y@oZFXXQt4UGYg zDt3p*OsJrJSm)XuwqGygaRuBd%h%-1dH&XRskn`aR%sc5Y+ZpPLG#4|`=^Y+cJG{D z_QM9RcBlOU{1IwHtD@EQx9ul|qR{aLs2Lv<2qfC%mGT5KG9I01;l>={P^jttSTwmd zomjM9LMZ3ofl&M-5AzCM*NI(d>nF74`?13~Xo$@{v34z#S)t-Gre6bw&Z-n{kK)3W36t zRc`$~3!hk7U|Y-EL+6qMUa5lvz_G*{^1RXRUV$8cLCWzw*{FGhHOhN=>C(~X67p-3 zlen`J_3AyiinH1!y*;))n9R}2+lOy}k&nvNx*C#vtf!r# z7?HC(b}!9HRIIJsC$3diLR^WxeBOsuWqUtllc)&>44(H7pfZnc}#`l`_yA`oyQ-z{=xGe?4r4% z0Nr%M1%Vke0$d@f?AMhQ{yiE7s2($RwP39r8MUh`D+Nt7UZ!d|E)C sD3BM3C=%IWlqZPK6%5BrL-X54uBd&|_@Hz*eq`s21l$g!b|yRXKST!BaR2}S literal 0 HcmV?d00001 diff --git a/utils/__pycache__/dataloader_database.cpython-310.pyc b/utils/__pycache__/dataloader_database.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ccea8f098a78124ef6b9767ddf13425b73278f8 GIT binary patch literal 6203 zcmb7IOOM;u73L)=ilQF#@I3vB;xuiQq?35;q_N|=PCRxpKr#vJiGvE060|NgLs=r_ zOO78Z)ub60C|VdrmTg)<1Eh;|(eAtIpXlax*RH%P&|Qkg{m!K{GnxcVD}m>}4lggy z<2&atDwiD%zh9p{-u%mRn)Y`Z9Q+kAxP&V{&NPi{b&WHww^^O3R<9eXHS0OFM%(J- z>v@*(*mWCoX1maF>Q1LvFRJ-myVNPy%c^g+E1ik@#E^foJ_-JOTklNOr_`L?p6(o} z9~rLC)Mv0>XwP=$>T^u>3&K;sqP*)i(J`2?TDHHE#W`4O&T&l7xxPc(~smd`z4^^<&_FFerdPk?L0QRO-~ z@_dmW1;>+IzpmAe^=ApRZqWse)-(_7td9fUsyitIG4Qb zWGfQqtGu!0ht4As_gam~+-k?$^lQ4zuXu?U`-ya@^Je66Utqm*J@G>B34XodwbhKh z9uW10q>jR4lhC?7BBHK+B~oD3A8pOIfb@Bazg7sjYyw&K=srFH4<}2Lz_!;pjK*u=F>c=o!X#<)B$zU0%$QUf|gQkP~O#GYT9+}GqzT% z$O4D(-L98x$zt2P>9^zIARekA==rj!n3|F3c!{uZBIzVhL`*ZWEPz!cZou!!Qqb9S z8(Ur&Vk2qOD5eb@9^DONuhVV&(h6cXX~r_gllxr^Zw8?!?#l_`H-#T>x%4cUMXe;w z*l#z*G@d7p5Sbw|OJttN0+B@`M?r>fLVJp1GAkrzg)fb6ZIHf4Kk6H}l#c1Wpc!pf+sJeb4U>k%6c%cRtorvL?ndIq{$|Gy z6F2lae(b^%+)QP%JhmdOZ2B`Mp1=-D1Q|I+!;>J-;LTP=d8wW?|pPld^aDCVH|!mZ9vTW+}-^gN)vRYPP*LIe_>M-GKcx2Bow>F%m$ir-cjJ?H3>+y-9J*S^Jzlhu~4*emm$u z6=Gk}Vghf02vurC?d?vOxf@{aR?u}tbT`IQx$BA8ccX3+L}4sTh=_bNdPF)=5ML`R z;F5Y2OH*O2<=+e9Bo-8oWa%B+PF;n0K&)s6;De{n1=hhBH+g`($xwzX75^2L0;=2JNy3vigfwbtXDa6e5tshmU*Wzn+BC8%%U{u?0O@a989HTR{3Ola3sy6XRAH1I+)=A72G+a?sRZxQ{9QgTr|NotM0Ty zGnwP19rh80sClk!Gx4irp|z-XwA4wmQ|L(L%e!n>@eyxR2j4MX0SFe7VhbRKS)}j} z^}X}Y6Q4SN1u%K&tp-PlpU#N(g!m(Z;K|fdW9vPbw5cUe-2VpyXfmyLCaLntlzpP{ zspQGj*+K8O<_>}8=+uA3JDtLafM)x?J&N*?fb}|P0Bo^bAv!%ojXf;jmDj2ltjp1 z!J{XUbXM2auUvCqy>{hQ^_D7SmXo7g-i-kycvV^0dp()!3dlzqaUx`{=Zh#F#SzM7 zqz$R4iXhX18mh=KEF9WVB2E*dfn-0+_|>vQ)XiRm`G;^uEa8qH;fh}dIm&c>hLy|; zBFJ&{C=Hrsm_g)l^kw4|W-Y`KORwk)SUbf|>Gp^Ezk2iIsI!-yjUf!7cpjHRn3vE= zv>i%L^Z^AAlz=H~AuU`4#=y6M?JY_d8Ko*J$+VEJP{LrWJp$Z57Ikb<%L~eQWVwl& zV6PGqF3sgK<-U);_#qLpwh{h5OB@zy0r|#wYAjk<`yH;Bl3j~c5lMR|$KUX<9#Oxc zGNEd~b*1CForrI@sb6y4+ek^X6+vf_`6lF(oLLr6VWesRQKQ~6k+Vc7q>Fcnkb2a4 z4xCY)mSUPHgV0OGWWY9#T4By(d)hYfpTf0Z&zGK2TvwH6Jfi5WB9@@wvECrp9sQ`f zqYagMC`2Y2B09hj5nCJR0Pjj#8O?CLKY;*Y3`~3tndb&ZD1D(O${O=1da2|srlqvZ z&47l>q=;-|EEp1fUs%b^ZA;+O1O&hPoB ztMU|ANzv!O#|H5n2=WPdw+T--kx7=A_4qh4NbyT7ePwQ@OosAYGMQbZgeU@8qLX=~ znE~zfDB(lV*EYaSKT|`Ar`)1Mb$u`trtq*P2rcN_gU^xH6-L_NvN-J0$^pkvKJg3O z^Cw)&d65XiCj`wkC^U?IR94WX4*mypI71bqQA!gdxN>9Inn)>T(|q@9|2bZt7p+tdh!ZZ;Yq_)GckCP0z5DE@nx%DMrKT2 zgUX0J2R@`~FnmtR0|ITNA!YJG2ykLeh2FZo;``m#@C^{(!qB^^dNh~y-dcTY_44ZK zYTfRjvT6iDFzjJYT59f0J%2su+(cPZ&&OG(H@O<3&IovQ1tvrvUQ~$BYBA@4f&P!S zS+itibUGElP^D$h9*)Yx2WLY;*rRMs`Fm|b+GNV=9Qz)!AF}Bikb(#4P+`|W-VT#L zL9@zl2-mW&Rw_-PL(5{~BfI}=p1-*i3P(eJXAR?OhJMars zSu7=(N@^B zcN0YByZ4bAM0c|~*y8>je9Be$C4Np!KO^!n5dyGb>^l8YrXTVv(I-N$e!vs=LX@;G wWTEgP1BXg7Ag~;X3`qq9Nghi5j4q&@DFJ#Wb?pOlLdCN$>`BXVSmp750QV6RumAu6 literal 0 HcmV?d00001 diff --git a/utils/__pycache__/dataloader_smote.cpython-310.pyc b/utils/__pycache__/dataloader_smote.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8070ab17d87062e46ef2eb59bdee617ecc83eb8f GIT binary patch literal 6462 zcmb7ITaO$^74E9OOi#~cXD{}8eaplNA(Mn9-dso=%Op5<2!u@(8zP2{TH~pn-5Jkx zkE?rq8G0bf8YL1kA`*xPK+H&h6!8=C#54b(-VhJSyz*us<~!9hyF2SBQD#)9tGZ5A zSDiZNJLgnXDtQKefBU=J8b4_m|DeYHAN9VB5+7lPA&j~qnBZMjXZp$OrhZy=8&9+A z^m6qa%Xr+ni+5Hx-}CBTuTU@O_jb3~E7eQ7?sUt&x%%9Mzf!M&KiB2G`TD$m=XMu* zi}l4x|5AMk{rT>p-r@RTW(4kNIdD5iMk`}OzA3zriNcuG@oscf6vqr4W@mnMY*OE? zy=I7#DBm_jIq-JO`f)KQDk$@qdqFG;jyX?=rIsfSiNm*9{iIkHM{XPShrqMf&s&aI z5l7Kq75tJ>J2rUEJ1>Jq5{POPR=e|z5&wAd=jqPMB zl4q)-xfO)oT@v+rjVb%}USl(;ag}?uku>5UQ65#^h#Df0=r3PN8lh-NajDtp>NoD? z=yDLo5%oF}ol%c!&8?{23{W$h@pWZ2w?zZh{`TD`@TDewZvW3k<7Je13nVabsz#kd z5?q+V61H$4LrbuhD{{icZf%hl9;D%jf+*sd%Vdd@h7{4xLyil0dXVH2o&_yMqh6dy zu_PQwYekjLZ^ud0o9y+n;{8NIzl^(@yocI)vj95%QJtflM~UA8Nm*+6Mrwkx)B@#+ zxntmIr8cOYI-pLP1I?u_sGEAA`ILitX&$tY8l&RafOZ*|jE~uRt)#rT7bSt;ZzNl) z5KkoD9t5f&a2S3olD$SE9eiJT31lAAn^@(+DigmgZmMFtx9K;x8etf8W93pSrU^U| z-3a4Guip)n(~kY56)R^`M%(>Z*&?~s57e9tS~7^YeEJqNpjJ>;9CTZ90pF9yh%6FW zB666>GLa)hRzOryS;%>G)J#PLu}VBsCM!={Vn%IpTA zd=y>sF(TDnLf=+7-$%diuc7g8n#p5>L$lwXa5SlR`Xf!VP_($tp+aikF_w*lo_vQL zHii&gho^kS7{VAjW@@4agBV$vx7=$w_UWu=PyC0MgFz4|J? zlJg!BN8t+zy42xwWuBnOb zSScGr7?P*#BYq7N=|id6ibt`sH0s*zShS_GI#D~+zz3JeSD#ik>|fs6XIt8=ytCbQ zoHV0e|Ji_!x$sNZ!&Pm(O87`3;)X z32tJZSmvowe2Z4`=426tx3P$6!__OI*ddQO`xJD*!3E)VFOUdtlykk&-45Wq8vT9{ zids3tT8(^!7WH}U)pi%s2CuDOO}(pjcAl1}-)bY6c^SjxE3~f3TMbZ)8OMHniY4K< zmN!6&T!4?&WR|T_SL6I6;Q1~}dAd~bMWMsSoO@|6)_umpz}2KJw{%g;7WT@ zHbU4t#__EX>ShoIFPj3bfycmbjmGBSk5m`dRW2{2=ExB?j`c#4>$nqKwouO_Otpjy zOolfed1GT#01i7zaa1B4AC)o397=`C*c|4<(|XoCX1$_k^)Ty|WFCl2KHW_gM@y-T zT3*kQ#~gRw{!16l`SKw-uf>yhDZ|!&m<3vg_Ql)5R1vFw6KHP@2wrAbhHw^>VINSkDx_g z$kqqRykj`V51}c;!cPj9jC~w>PptY%h*y>PP`X!Jf)tv#x}#vasX`9hs` zqf><+QH@|_c_a$Tm7nn$84+0s*wPFSw6n56=bw3vw_1_vIB#QOik+S5>Knd~P>;l`VQ_ zXXi~ID+3~=gj0Baf;b#H7z8_2>eWQMkZWlE86|c=99D&U9h{uq;lZP=l_{&FT+a8E z=li`#Yy9^^OPq8L z?%zb-25*9AImxBCHbGw{t>}Xbq`8${L*(ME z?N$qS5P2b^z782x0T|%lh~!m7m1?e!JUAf!7Sg*&ss+FRt_Xk($u5)EZ-;&_=poEh z=L(xb|a=nOVklN@)&waO~!~1k+!phlS&{XBBj}_ zQS^s2y=#J-ewKlxT$=*o{&v!y$Wd;gPZC1V9Z?{VS>&e&Ayi>frJjAppzYuzjQIpb z8!RGX#KDr@HOWIvf28r`!Sup^g^EC|k-8)JUrOPI9(1|@O)lad!lR4AO^S_7(qd>c zUBT(Rvx6}9PH|Y$T)05|T&}X_LJs=5Tyw}q%jsN;ZZo{~Jzsm*#VYG|E9#2VaA&g7 z4r}>+dh*38k_}!{E>em%lmjj;3E~7GsPEtCaJ5<0D)KF=YX`fhjeD+=?&7o+Dvz8} z4`~^6Osl7!N$V}Gr=H97@AR~?Hmy8!;MBBPPGSLl5%X6J*O|7(i%>!fP6n|s-N^l~ zY&}@>EcpkFs69*2o55);r!O$11%e6HxBi%;lV8 sXvLHpL1H$tLUP`4U}P0k&*%~48}JpZ!i}F=740`Zbt{hJvhw}^11nQe{r~^~ literal 0 HcmV?d00001 diff --git a/utils/__pycache__/get_paths.cpython-310.pyc b/utils/__pycache__/get_paths.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18b2bb28bb8d494fbae192a5dbbc5487488af298 GIT binary patch literal 3278 zcmbtXU60#D6tx}a%lX)J`>|~)#Vw#!q}65r#uTs$@Jiq?_ z3;pbrB>gO$=_do{8eaY*43Rn#k%{8S9hvAvCE0J}jslb;d7um_bW~EL>XJmXC8?7o zr3B|lImQ)~=MhmQ3y3HH)yOfRBG4i^4x|B{ASZ!JK&QwOP#NenIRjJyI!j&wssf!O zuL3OqohPpW)rj&ys$UqcYnzOk0VTTU>N_FTx9G0kx}-PT8|&*E?YH#STC1&T*UT{J zdu&4|R-d}sZ~ujQ}gQErqU6?z$HcYJ$zwJ~sm25a2fe(%1q+4tPWP0!tHynlV? z?)4kS9(5^$v<9O*bOPSkzH{5S7do7NVg^=!cx5I!Y4C+;VB9nVbJygwah*I2iEWvI z?YYKBo1W_q+cUv8O~$C(xb3+fp5H^Mg>hN^9&a`t z*;cc?w$^G8uit#Txzuz$3*EI>mcC zM$cmdGZ;4SxcjDK<6L5~^>N{+w%L(YP17+9Y=?LEXb_KpulkQ8En(myhwv7RDvMt( zw(&P3{DNG?uQvNtG=(i7%u-;UJt8^Dc;kem1B@h=LqbwUNPS37WhAB;BRR$2y1G4F zd9iaf@Cbw(Ui#1HU=q0bQk!wyNvwH2!^E}3N-$ArToFqf*N#n9G^oMU@p1uTd2{NY z_^Rq(suI_B)Pc>d7gb=KS)R&`{~^LScKC84#GQ1X%H5eo7n?vV`X4JWSoA`GyLd@T z7QPn9hkl?O;<~ckDLNkKj?Dx9Boo!*qhrKWv8quOM`+sD4(pCAMK$Rrbu~_7WLLxx z$E#70RjdZAS`e!tYM(nRb{w^0q!kvrfjyu&-xw`BA&RGW!{U+p)M1=O%objDMi}AE zrHx8HqqvO3U6evvo2pHvJO`2w6H4LDr=054@R_%3&QA8^A@8_JD*1WPeVUL9Z$5c` zKm%r5DRV7PYEr3mblyv-gg2L}vxsY{7jkjS<8A;cO<`S{ZIn8I3!wZw8G!KS4j?Mw zuRH6Jk2@u8L)E5gQz>5u$ta-|-dxJgLjSQ(nQt-^YEqUm;S>L8kxKmr$i7Xeg*TtN zB+jT|g$#cq&F-k$RBbBdn;`ivp%h+S+1jcfi}Hp+Jj*axR@91nX`ElPJUFt{nR Wnxg4yshlaxWf{1HpB7tutJ>dgcQ}0j literal 0 HcmV?d00001 diff --git a/utils/__pycache__/get_paths.cpython-311.pyc b/utils/__pycache__/get_paths.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e0bdbe3cbeb7e55608d5331d4891dfac70b4cd2 GIT binary patch literal 5898 zcmd5=-EY%Y6t@$niR021mZg-?)hn8+YOHBO3kW7CUxjX^6ewe3**w>Fn}B0S_d4Ck z+6hg2$RnC2b&ou;O?%lNu|L2?eTXa}MVffn!=_5S@w9X7G&oI|NKJ<@8HEc zv&xdN%B}^~z#69p*Fr|`Y5*i0NJ1d#0!fz|hFSz_{90(0QKR=<|A}m&z1rt4yIp#; zjn1~f<*Pwecnq3f>)N%hw)%5w_m2MIw)%(EeLMO`jHlpD&n~^`-C^y&_htpW>D#3@ z`~P>|^jhJU38Wr)4EgnIJ;%OfSG!ijtC6K3%#JtaO7AE0hLt4AOLLcRNEfoEk-TUc z%gL+f7G}?#mzFg{gKX6#qS2ym(d69B6=}YxQ|-2FDcRELrrz}iUt$C4qHM{FGS!mj z)VoDBr^r^$G^AS>Ov5NmGz78U^u3!#SFwolZle` zkH!Tu(*R_~1=0yBbj1lTOy88SFtC&ZU?thIw0uFDkWvylB3;Yooya!iPOxAsJ3MwN z=%z)TK*4hO`2}goBzf5?rDlv3SO4iI6S75%S65h4uHWzBmy-KYmRwxg1M%@0qxYiNUMJuP%%(7;=W29q+`%bhz z21*XWtMGO76PT6{m`b>Ji|G!@Y$bN1%vTa0+mVBn#C4!p;6WL2l_ybo)>URa(<)A+$3me=*~b^FLM zdvbmwetk23y)5{}zuno0Pi@Ah%0gMF@Ub65PxwKbAM~61a3gkdGX^8__j^rq+!Ow= z%^%(ibI*DX+5@v2Jy$n-uG*2SRU!89-lNmDFlKXO&v6k{>YFB1a1nGj@VB4<8#qIP zuz?4RFl^KxXr~$0(x|U z>E!ES>aF>@NvWQr7A;M+4em}0vmK%ATFcbOLk0nc2E=>nx)z{|(3&8bfTt1k99tQf zFZYyt+<6`hVf|CAR|m2{gaMly=os`9?SNh&8tl`!P1Dl(UBd3SF}DZ6`|LGC-{;e| z0Dl<4M{*XP=V>7IjL)-`#Dcey*5F5pQTdXsh@ab$LHO08f|8sksfsI5s0yOZ zi5*AfSUd9~uURCg(9)S*24lPb2YV2MnyqH&yK9;jh>sxRNEQ(BhLvAqEB&+O?s9hx zZ>-Jl=0+_7 zgF7)nB4J<}roz{3Wq8q!48gA!8kFQcNp-jcg{m-YbHg1+r>!jl{vlEB72wnkJM(V0 z2)~WJ*dl!Pngh}2)3%7fyBGl^cd>8;NvG$W=8N&a+-3mPfNM26X`5n7twWvcJA zb7K$ke22|uY-2lGw7?e^8-fJ?c52M@S4B6aJ#}R2O z!|^mHDMbSRJnTBd{WkU@!+rLeqtNHmwunFyBY@-^EYvf6G%!6q+U_5v} z_#gDhU-A_$`~ePJ;5liVju628kN&i46q4Ty9cTqdjf@XYz0qrj#QJ8+AH85 z@O#W3B7<3QgFoY_F^B@|8&z06K^~yA{Vwrg2<3QmMD*xhGN8(B_M|WJS)8oKM>6Fs0FM F{{w*K!=3;D literal 0 HcmV?d00001 diff --git a/utils/__pycache__/model_func.cpython-310.pyc b/utils/__pycache__/model_func.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94b284b511b7fe4d1a23cc1db895c5e098d33ebd GIT binary patch literal 35849 zcmeHw3wRvYb>8mmd$Cw7fW`a4#gkkT1PSo{kVuLmCD|11h_WpqR8~vOk^}@Sz}bN$ z(z9`EDs>)~t;BgWiR~=YG!4@x&9jfBO;YF4^yM~j(xl@yP3$;{nlygIi5=LH`v2$7 z&dw|VijpP!%hv$kp1E`HojZ5VJ#+6l|2_B4N^5H}fWOoq4PSoydxC*a^2GZW!^uhf zDi3ObfF76&=s{gu3eE-PsLg3|49$h*7@3R8F*X;IV|+O=mk9Ej&{A?aIhT~Mw$w7$ zBI&8QlpI@^({pJ_3oo@TXXY}??Q`vN9$D&G?wsqCaC9lV+%?xN;n-5oa`#+sFi?us z`bx3I{@TE;fN?>OmxFrZR&Wk+wLv|3D~KH7#g^KT6K;fJi^F=0p3+HRnxm#b2| z=jE_IpbsK$!WTEB4pO9rdLW?h(x;va=u^0A_u};8o|~b$z4~r_8mTk-9(^y4v-*rai{n0hpS~Z*{rUm@ zAdUz0L;7JH59&wsqc|SYkLky8JgncN-;3iB{e*rJ$D{fw{XQIz>8JJkaXgN@p3%?Z zu4iv*bNA@lg+TtH8wZl7jZ)Dp>A971?qU4SmM-OH59Vg}A3bpR=ho6 znt64_IGWSvua?Tm7b%+O^FcdoUelNJnjJ^_%6ze6+Q|ouqJCk%xKuKbo?cyAF$!uR6ryu2L<@$V#l zDf~|3SNUY-|j90BT)r)$?F2<{!)vjt! zHCyej_E!6<{Z^kI?g>-}tp4g?`H~gT<9&f@D~<^q(>Nw^Y{Rhy#|(}s9NTeh#Su@Y zhwuDkcT9inPu8ZV@q~kV8)_aiK5k}esFRhrr3C|KhZRNIklDG2@@rjI$E_f_t!JJO z>g~@5tHWkzEqg0a-KKX`b5{b@5&TB+8^doLzX|*%@!M{8>zy~z!H|_dsa_&qwYLJaMJE)7q&I_62G=bzB_L+xh}-IP3`z`NOxu^(U?FLjK{t zfO|d#S=a^13{`hmr>oPr{~je#xMNFoueHb8TN}LD{sxzCT|4}1*&Co>RAT@ z)ma~pwm}ASH$zQa+R5wDe!Ff3-v18a*FNw|Lm3+QHR9pdsn>;H`?acdAG zMzT8StIcF>kJV>|^>A;%I)*!hrGMPmPpZD~9_t=!M{l5dueBZbxmWHugzJyr)LsY~ zKZSQi!`qX!VmSM4eS7r;`tiNih&5`RurQ9^ilAPZ+Ke@3jaw7eq@*Nn#Uyst+OGP0 zJ#YbUAGD4g=Z!dqZX*G5W%ud9KmEkP{u3_-adtaqrxup46y~oM%jMEi#g6EuRr9Ky zC>Q3Jij_*KVz*o|R-Q4h77QpYcFX+A@~Tm)R2EjscJk7SX)cw@rTM4qgl?>?N}k)A z9o0)uFU*%vn0bA*bUSEkONP)~fm@8&+$(ZC* zZztKoM-QX?;)^fi4}}#km6q0rll5zJmsgD3*_9Yas$yB+Ja2vbNZc(n-e zjh3EWs6dvoODjdaa1}ymtQL*pGT3EQ44QVfX0F(gl~r?L*$$u5i_3Phk;86(NH3Mm z1@rm^Xy5u8RBO9Sio)sqQ>!ZrWwS7Q8Es=nE+gJFA}lg1YIoD!?5;CKW9j+@b7gfE z5^=h?v~;P67e*_WZ6#=s9=to($qzefLRnui?8qwQ$+Sb2>xiqE`pO!_?efxE<*GsE z+YzH!zEXlUu&`W`3?%~>nG`!agjdpz;U&FVT*W)64x!>DyjqpDWjiJ{-ZyK9msTL5 z;RUm_Y$tA%jFrL_SQ2)e)qAFB=ytebmR9X>>FMH<9VpOWg`6s<6ZdRlH(wi7n-Dvt3Fu*1-GaQh_P?lXlngie6f3 zZbdsq{?DMaqK@W~v)TGtQeDE{$Bvf^^7dAY4stFI0a+-O(L05CGrJp%TEopeH5Ul zy;xY9vD5W7a&mUK=Z*W|HB;@V@;G&GC*CaQti3Vktex^oft^&(D&n41qBOs@ zETZI;oMcy8knYxwJ?2z4_P7&>F0EWyK!*@4U@V%kgR6E%vZ@jk4>H0%Rk*5c2P@um zpU7jv+Q{Q_5blUnJI}XC-Dd6PlJUeFN}g4mcFV$5BDUTdW<4H#T zuvEF|NH4I1>DB9YuX`Kyh8F7h!59jy9KCtB}36*_~mFY`f^gMhqB>p zFs{Xe5`@lZTi|G!i2xE2nEwUibQ_2UYb&vWM&iieWHL6nn}|2We6gZFEhU>f;S z!EA6C6n6@x_|uYF3i)`At9SW1UrDM`Us{K$lg>jJKl1+vh< zS<+UAM{AA?zS1Gsib7d{uvoos0G{Z`cY9S2*9JJ?EuxQmOevfA(Y9B;ISwWpce)-iJi zyrji{kK&`lxnr>Svm07Zt32y#y<_Sr!Kr(FoEqJPQ=P8Vp78MJt0%RZ@q2fW+Ri&l ztycNG@3}XZT0ytBklN6-_$x^530G=Q0E<*oJLQwwDM)RXNbQs(wc52G0ye3n_JrFO zCw*-(wrN{zl-g6CzGTGHm%Oo38!~QhBDI~bAhoA}>ux5sr(OtMTiGDBp~}a6t#`_O zzwh&LYJ3w;9kot6*wkt3Uh6(5e7}{ejajFqEk+Q!UqYh@2@++EA*8UPafD7v>I6df zF=WpA#@t@(e&F$O;8X*?+Dz8i--ZOII*mIhoND8^vcGl!qyIq*c(}g(d99Y<3=igb zrp)y(A|{2&xz-Nlua6 zN5c6+<9?DeBxgy8lo$_?JV^2o33XrN%_L!xb0iOwoF{n;2~iK@tt4MZ@(9TVl1EA2 zM)G!&OC%wZcaS_r@;J#H$=8!y1OY4}Gp0{4;u{zdU0a0(3DcZ0PlC*)8#Xa5Lt~yP zoF)Z!vua0RLKUxL0u|^_iIHtZhiR}Fji*Z$<1+JK0m;9~xXSPX$s)-T3E>E1nWRjz zLb6KoYTyRr8m}-&DkLV!8p+cngbR#kNS-CRPI80fog@}XmE;>q-bM2FNd7*_yGd@6 zJO^TT1NYSz=81<^1e$S7>+FVjA_(24u?7T(n5IAXhIr(jOR3gSro8WL*7zpgyvKVJ zWd$pvy=!9v3N-!!tNk95ZzlN`l5YjETRi!VZ)5a(NuDQpAIbYkULg5)lJ6k-0Lgcf zd>6@glY9@!KP35HlJ6t=eh|Ce`-G~JZQdwW?}N~rD1?4wGJ5j8EyEb4uAp;Q#=iGcq?O^ z*QB)$ZCKO(5+-?Dvq}E?mASQ%-XH20)7@*V_lNuOyfoOuTDLOVzi9k4+R5h_F^h;4 ze(dNfKZ9;(3#0w-93%eDF~a8<@x%|_ty_dS3?L;4SCt+pr@5Qai($u6g-!#V77Wyz zA%FzKaA(or&eH4VQjRRUkSmMpnQID;3A>sp-F_M7&1pZKo6Wctm#dsNBe7-(08UscjFFso*vx;rwdMx?uAO3 zv-ZjzIXyZJbunoCtY>;OX#B1|P@RG5Hzt#yGoI^Md(PKK70g5|10 zqWX9=o!Mqh=P5ql-$Hdq)DEQ#$mlNv|5juErsMh0mg6}c^Ugt`t^6FA(>$Dib@20k z7eDXEK+RtYzgLN2|17HS8xHp&B8i_G4x!86X*hIz1TT1e1TVNgf8^L*rv7+~)lqBI@yC;+5u0Ld;Ian5byQJ z8%LXs3A#8@ovcoRQ`RsUBcfO<9Q!iNg10TLcRRY*El!ramF!-VH9WOt|H}b>W?V?h9 zpq#+d7V{F`YbI*bP;Qjp6wAD`Vz3t|d=84ijI|9lm=VRG57+O3VxSpEp%?_O{W`p+ zG~+RS65f;x{pAlsVP@}_VrPE?6xNFoV&rAUX23O3^Tn0tZW9Ch$|USkPuK(Sd+j}E~Mmm zy)=2s2&SYkq_p_I)V(Px3O(83=)ndaBq7 z*8`;qOtM=WE+?qWmavkYsy`2X8w5*Im`)acH_Cg(>*LU(9#gzCn!uY)@ft!fc1KfD zxRIS8hg2~xl4GY8cElPFz1d~OZM0apUJHG~>&4c(%o(Xk;d&JA=mP_7MOafGNlK@5kqF91nN*VD7nCT9e= zA!v>>LRvW^d=!u3b4ECh2v?E&A|Tr>6ghE5a2ydH@i-zp;yNOZ6pn9;hbWa0K`+M~U+~0`$5{T&Bqp0P3AM0`$WX zU;xUSPm#m9>lh$Y{$pEA`FC%a@((`oNhrK(x@Fuq<SJxs-<9_Ez4C+wN>&m-PD<-Ze{M7K=&9|ZQGru?xeqiM>& z-O7vdva32(-Mxu=a?YooU}6bUIzkmlm4nuq#xvKa{PzHv*h96)nev~8((@!Rgbten7H<3 zEFw{E0TY-Q+#!n7j5P?HLev&t3ynaMl_Kf33u-Qtsd$Q6JQv*nu0dgO(>;^jW4>wC zeQ=&oE*f%l+N&OBd)32iFHhLhUi%U6ZLb4Pduc`*4)lu8u4@I}O|=|E+KH2Z{&s-! z)ekPy8Q_5944`;4ix&8;4ehE`4*S}5)}4Sp2sM06IqLhkH@=a3a6eEzMi$EedN2(d z!{RTm8_Yy)7SE)nL1$3rR@0#SYWvYM9k5`A=#!pl&`GnNvb`x~J5=?Kj+#vAD^1&m(RM+-`g+M==Si8S7Xm za14FXL6|*5ScFiY1l=LAhpZ#kQ7F4PCq+$y9+tduM;Sf>RoOrJnZJWFJd9D|)nQ{z zu-Ly~0i2@pPJI4LM(!e+BAF&}Cq9i^IDORkSBxZjqUJq+o}qt3@^49gkmMIY)*H^~ zP0FV`!FlxcQ9zAfe9j_WM*$@3pG+r?0qYyjVgbIEBwl&n#|i;nXIR8u1q$j?7S0pO08sC zzed$^oMD{2y*tC9rOvU_UWBf;Db8+a53Cu7rScRR7uPp zyJgteoBtpU@$7#I6CI;ETi|oc4sFmbmcPC4WCa`2p!ss+TmZaX}Gw%10WAOc;4 zPv|7nlUW!mW3@P1YYaZ2qqT8C7NHWO%^+jeq$r6ytn}i9KCEvObTLz#w01yAR2XBX zhE!1oGDO>;Bn|<;gdZN1#Gz{+wb~rCaR|K4iISM7qOPXie65tT2J{g@8{O7L?G@VC zi7CL4pp8@#-E>hB2gPG_)YE2N-ktyn2(a6zP6MVc-H`N0aiN{K#1WpP2WlSTLzJuT}k^T|7x zXZ$9H2jgQTRg&K#;SeG#T-EHYD5s)t_A_CKMCNBl8OoEmn&#QBT-7xGD@x2Cw6l$; zAQ)RQxh~Y>^mddt?j@1>9z)1(t1qf4%&(L$FI+KBGW{WvH<7%VNrZ11hD3L4D25F!^1m_l8%chhWE1UBsg0spijMg#^No{y6UoJK!&wmtrsvKwZvgUl9=zs`Xgw5jUS=J zuoefU2iJ$0) z9|1Q^XlkPMK)W-8i_q(!(Y5LtG`bLa;iwg?qMED%n#4M)R_J=I z&Tsah+*(3lljO!q)9>Wb zQ|G-HR|ilZuG{KVdKjLeN5*vQmV@19I-T@>H$Cgb4ComvbXjw*>asFcA9sYo?iN^| zbW`JuGqjd&o`cY)Tn+ zQ?gAdzV%qp$m;wbbNFvH@t;~X?TRk7ie>+dG8-UjJcxSDJ`=x)iu^$GPII)=}yr2wK7B+$7vxu9X= zjmFCQ_0aUHxju6iZbbCP&cQ2dd7;cb#in!TSIk^7Co@*L#w9r}W}g=Q=SJJ3Sf~%B zq`tR=U~L;TAW@(8vLZy>1X63hayF0c$cor#LDm@D=vVW__twjGHh7r8_7CfQvRg+z zY1Wf;quWV(xKu2A;xVC};dr;kokj}SLBLpec75b@)5aXR)AcO5hb7DH(2dwxwP!{> zFgq7Ny|R31p}p=t(9p+@lOy+l*x^NN1!2c8FDyZmEnbo3 z5M1qzO()bkYS|>Bv8JF|J8NiSqMbFVW1`f}@n(YF{V+o=Qpc@W%2<3*V#8zOfTu*o zp2ijz#=?BX_}|P7>;xX~WTV4DI0E^(!hTsTU>kvyMu!v?Q?@Kr8=Qf!ju2bc>!2{4uGYTwY650VJbk3uMa??engknQRC_7Mm-am#xuVYc>s1fSI#*COb z#%yy~JkP~_60sThl-FZ>)P{*Z)q%>}dwq&Sr_}>cZ2-_hFOGw7+Umn`2tMom7{HSl zz=y%b7OUGGv{P2U8*a6R>jSv)C2S(vw$#1cQ{9H)dmDz~K@8tHJyRV4gwiKV0PDkd zZ>`@Qrh98t7u1m5TjLyAJv;=+1GD6CP~&w`8M@msbdRDu`rr3p&_;bhM>h=J9j`ES z8~+1>V%Q`fCpiRiW59K;I_vE4bWWV6I9eM2i#dAK23Pmyng&J%V*DOU`hAi=0LgFL za$GZ{+daXYF_M!cn+#>fCzd&(mBYdKIvFt1KVsx+WhFL!N<#>$Spw#;vu9fok zJKQow`kj|i^gIXg`du;#iWghN!x{ZWzwqiZY$#$NPtQm<5V_^#a5uw zrvZp>HhJqta34%s%{6H|X$p%;+wC)HJFk^D^ok$%_lmAL1>>fsOxjG7X*A%bv^S;n zx+%?Oa33V2tKNTFA*`Ay)2vz=T^_3rqSqWmuZWF6szZ7Y`dzq8qcH{F^p-HLWg7bT<^USgsb{Eyk-Wj1ksyf zRg&jgP9LbUV0O8NQQxcVT`?91?3iQX+_h-~f7yC>>*nt&bjm!y)4`uCEUn$#8wI zET!xwx!cuUE4!&sv(nlT%)qHLibbp0VE8je8ZU%z2pPq_TksRr_0L}Y5R&u`A>!Pb zqWbC|O3)p)VmRTfczrl&#z~up6E9AB-ZPxE$#CLFNE-;uTdgbBdcEYo$B)6RX=?2M$Bw^YM{LsZFHS(GmyVwfVplY7 z94MB-8!!NvTg5OJvX@NBtZc)EJfq`T)IA2pw1h;`&@fN{dY3wNB(wn^G$frz-B$X^GN@i z9ld^c9sOOw*^Q3A*MX%~M?a{AKWlWblgAz1zYOn;pJ?|?wF0$(coV=6!VJQ5Koida zO~=v)Xz1#zDlvF1z*n^vQpZJn(+VJp#34+-+IpDYwVp7*m53SD6YzV5S1r8^-17ur zW9~(Z-D=@!fv;Y1K~^m$`)?+(C<}|gYH|2lCdAh=b?r?6R+F`eiSPNq(=x$jJJ@|Q zVx?sF%{2Tm(zRCc6M$#NVwzvD5^u{i7nVQ+)oK|YuK|=+AuFn$6~HnfYz?|$+vPC6 zhR}LmVW>O7rws@}nvr`*hDr8d)~{_=SkiN zVka959-ClId7O5D!WEi?*k;+cjWS)kxhVwP$MM5_h$i{xL&#hIOCsQ+k^d7XbzuX> zr(q`H?Z9z@)sPv2*o8NU-N|Yg%f2agKq+}##3$4sUiC1=t{$e?dBPC87*_95?07B3 zu6`axQ`5;OBxWkzdNeCWA+w@%yoR$$iwIHZTK5K1s^~-JLaNRXA8b?$v1xI{W*;8( zZp0>^Qfjrxz>~ZP#`XrkyOWF)(K*YID<*J(6txu}fOEi&hMxNE%=sN8jb_g>pTD0n zeg;_%vVmDX_n~g=fIWyEu%S#Q!kXlo4;{>DeuEwR=6Y98_lxCMWgxT!%Fq_>IMzgn=DQR|I%Vw{i zCjoS~qCWhZRf_}V;;UC82(kp#ia6;hH$CdawCWHO7s@3duDtS->{%c%=vUyK6uUOE zL15nJ6PTEKZU{`;Y_D~|JE;@G8ALw?p>sTf+uW2$Q%cHB;gLs6h;wW{+bos-`5ljtJ_0%Lc!-|lsYvQ2hWz5%w2a8<)r^MpidFrUSYA!C z{SxYMma;vK0QQEzd&EpLc>S&uT7$E+MAea5-G~I!1PqERV+uu>&)Jv6r%h(bs|>oC3IS2hZn2*okrtpC^%$ z?0BJo@5Y#g0^-A!($ZyF9|bfR-=R=T<>42TD5!7q6bkBv|1fS~^gtZ0kss`2x`Q|R zH+;clx?c7?vbui^^LHA*N)*HjEW#5#bvID(#5PvD<-=+P zVi^^Z&s;dAN&WLtH3~E|f^lS18+!gZL;kk{Za{SJX}HZ?Cr|k5!4DY!-9Zii;V)^? z__{F(?)et|v(I+{0*+c6`6Ijv2@MO42*nx#a46nbI!X@3?QX>(@pkb%yElH;8{gmH z--c_N()`yoIK1&Xf139?PaA|9ZD34*FZsk>@2k35GblHzMl&eCsduw6*nz6XsJm`U zlYCK2#J^zdW|JV7`^P5XxxAg{5uo?MQvqWF+6>fsxHJd(5l*t-iZ0@FmAWQQ$8jrM zjhGSWLg;Rw14W>U17?G&9-?!kh0cq=+u`#K81u%i9teX9~WOALBD}A?S#9Fh;sCo zVmD!`@k=P`ePp57IM`yN7y1k~9OaimG)eXCMV3N#DyYJ`M-jkOcO0zS0o#JCOFFEB z_F*MFtiyLTpsO@k2ZqD;z&A8*0+DnX7zYFKGo~plci;tERPiEpHxjjn+4qvHqQ;8F zoAF{EsUK|S1%3R57fkhObQGfqe!`3IMgTSvc@TDZfd0*b2Iq=TMB>@w1&uRibP*oy zP`5+y$)*06u%i6ZgREVm#ueo%OgY0Y;W(33j}S7<@L-#b7>>~6z|$ms-x05b60LC= zB8b=|W2Ik(itVE680b>V#Z?H2ns78m$rd;#VX=k~Jce|Hn7cjg`o=;|u_cZ^vObzT zV;C#ONY0a{uaqjeG8|;(>tG|dTeg!NY2p2au$&@ERB@21eNq^61lkpOA8u1 z<%ocWo(w{VmkXXJm+>kgt<@C-D7mg67&C0{?Id}UT_ih5c9KkyG)BdJJl#)n00cHz z!>6>uHZ@7sNS-F~x0P|27e32`%^K+p5`KY=G>icIpGG4&O@&@5u4UO&%wJE(cyLG^CJyhT&(lcowgO@+P*Opdrs#pa0An+j~iH-6Pyb{Sus z#(B(TYkYIIl9@N!lH$uyslP!t2J3H-6R)p$-W|o=sE50oak5FsKZU3M9rBMFiI;!) zLKJz%^m*#9^*{2<<=K-sfu2U5ag`5;Jqk43*<^bxDs@@f-lrViEs4gn#pf)^ZVQ(Sg0W#>*p8ob!x7R#|Xd} z^xtj_xF=wYKeATiVqgzd+K@th4a&)CO&A0Ap`Jq=G&MeFY2p}hE4*N1IIoRPxO6T5Y zC(f}t5{zA_*XB{F&4qex7*Mrgs9qaMMs1SkR<6{uhR&eK$Xlhj2q`M_h@6E_GbwyQ z=mRRefS3@Fj=VP4-Vj9Cw_E{kr#(?nYn$cj6r^R4a`hktLqj*By&TuHm*OGqZ{lI? z%khZzrFc|ZkH@sXj>okx#uM5X;z{kV;w{==##7qoly(=|C_h>#N2v00 zY|&ffn9@^nY}H%knAX#BY}4E1n9(zGY}eZx61Lc>cZiI2o;U78{f*Nk_mi9BOrEct^8DZ<(V>&(1*YQ;{sD2C3zbOCM__7 zL-H8O<0Nw=Ur%z8`VCqyZm>sc&o`UgJiIiA^2LZ=)&) z8UhgI7L5>qCUztB?=>?#!2ffjy$06vh2!7=h7(JskB9arX|ccZ>)P3Miiq|`E3|0-{;2!oWn z`{r(Ras6g)0bf{OdAhU=h(tc%eIu^YE7Omj1?L1FxIz)SN+R4jhtl&Q#V)ah1W54n zjHoL=09+`%jf|uALbmZg98a%vxf9WYt(mAUJ`Wg|;((gK{{!E$BFT}kO^gMS#*HN; z5bcAw<~(mqrpIyXR8(XO5GO%LgQlz$eD*xh)eUj^d+k`EpkvKpL1_jLBflNNMk1@% zW%9v}uEGhfSTRIL5Ys&_p9`NZm(`0-d9|a;RZh0Vm`ke+#Wiz9efqi{s;A7a>BZ?H z+Jn*}AE&Y-@|j#&S~=Gyie+V5=T;_Vti0s2$vT_4!KNfR%F;B0>qNxUSG>N&D?m+F{|pme zA~^+uop_Cci-!VlKljK5fD`h4b2XD8(4Ambzr>5?Nq&*!49TZRXpX3*vD9#6h>>na zbrhSlyd6IdUI5~!%7#T@AXv&56<7lU9T(a$oZ_-Od`A~@h$&}T5W{c(29qxZ-laXO z$uaP*@PT?Nrd0TY zF741gj%|8^h$c2>W9V7$t>(7Zm<;%>MFJET9m;A7}*iOO2ANz0N$|Dr@-!YPX zteb%^T1*u6J&L17DbB>1qoz?4`I<+~?N9|@RY@K#wD}peS4X2B z$1h03bVT^#YrwnjQ9_O*6g8a-3xJS)rgN=M=lYt>!|yh3n}FM9BezowSCA?eZ25B2 zS$UyEyXY`-C*?F?7G_KBfLS6qv$@cIIvY;wYRaZjlvu)9#KJj9;H(I@)anu{^0+!$ zImyfC0xhSphak0uTv3#nDNdD|D3RjmsvLT%4B|Fjyd2_oUAzL~4qe<0aYh&S2(>w4 z)O1NUY~*~Zk}FJk;jMzV+DwV1T(MXtTEud(LUfCsN6rUDAcOE~PdX>t*%bVWjIm#- z8iOq4W!FM4OQU zUnA_*=ZZC=Ppo}l0e--%J=Z`YQ#C1Hsy0=XsuSx{^|2CRUp5zHrqoC?RWEeqb4BZ; zvMu@SB$e{R+aSD>$1`%mL>XkRI*XUC$%gdbZDkMVbA?y)VX;aqF!Glx2TkXGl;@oHX7@NnNX9i&r;NXxAp~Owh7*5LtB0uejLx&CYp>R zBlNvEkMnX>z^Up7APqmx=_ufo>H53ZKoh1~QmtYuSCgNe>4rSDe8`A1-I&ec3iBmj zh*P`^9N){~7)O4WbH5E*SevhBm!$%rduqfsBhRnF+L79mYEN~*DDMTS*oB@rNnLMG zb&7k%&U90jfBLUchSJUYylmKrgIYhecd-LnSSjw=xjzPd*OH?HYh!-*AO_N{*&O}X zSetP!r1pvXM6bAa4y0-h)?SGF(rxod0gGjdgc1RZ)%lS#8-BqRSY;U&?+y& z(QB0#;pnx>E^vM$t#SZ*`tn+(hiH`pMw!WL6|sABPFLpTCLc1&gV^)(vv=3Cq+9IT zoLyau{|inSxLGw@- z$s1@G=5gc5;|^iYI}_)^(I6bn#&0I#k(dyin2km|6N2i{rTwY^Qg}Q}eQZ+cli^r0 zJRBBOD;G`(Q>tTk;*F zk;+77CZXm~EEbL?c|01ZwpegH8cHU^NglWOJnq?f+^+Mu=KhcGe~8ateH~h1j`C_j zBsKvjNjeb{BH`G0c&_piABw~xu}L-*uSV1PW9O7fz0=mf2VY z+8+&vZiVMu_Wqa<=CN2mq8*uY=ID?F)jAdu#-~2Es8%5^grYnOAdh=Po&^9jLqh2D z{|gV042^}ObFO^d8qK+0doOuj+pL3qJGJlaR0o@@>1;fiJOZg5x^}bK#JTI6#Wf_NFxZQ>#Qod@B@Hi(;Ed6-kiW;8-{*=zCK*5$9*1 zJI+KjE(8h6#B6L_b&}QvfkH3}>TV*;>sZHVh{z@2nq4bE!gMT1W>AvH zHH!Bk59o>afbLQ{!x9`}!G1zLHw zBJ;Sd<8eQwmgL2U&>><$bj3~zLbZ+az%L%Bye%4<8RJ61XqW0p3L#$5dXSBf8Le6g zBUBp^rU;CJYD?zC2F3=2t(dSGgUCU(GK{hkOl}HGa6Bt5PDF` zWE|C9Qg!7c!2};43xU4uQcVffrBNo`(du%Mfz>ytMBgCnbJ>=5W10;>P-~O*D&dTE zgnXPjdWdL+xfl&6zXl9of5Ycj=dWFZHRe{hGsJbyj>luMPMF!(K+<3^#;30J4-Z}l z_D#iO*G`6bJ{-GtDjtjT$>61_a59p-HY-G;$!nU(}^~pLtrzDwSO;ZL0%vWuH>nCs`^0t+ML7li!Q} zK~!pdW##(gt8&ll()l;#o;PLBWyN!O{v4oK>3gT%JNNFnr8BaHRV=JzVKWxLWc)$R z#SQ6Jh1oOTpMi>tm9m5LXLO;`mb9pp9+*F!aW$uJDz5JNA)sAauDBXySF_@3o*x1l zrDfz(UV&xH=g%Nn_rj^P|6$MSkaY1InYzHr^pl~D0#or7({l>pmjwRnm65e8Bhu&% z`ASH+651%BtoC2fRH^5?j&&Ei=3>+Cb${2Izf1PNqWE7~Es$KS>^iErj?ND~b@|f) z+0~)AI!Ntwom3R~WbD!8PhvlcNxc{4&Pz(?B}jXE_>?phltL5I)HkK1C?8HKhf^|r zaKXB8cR7%LYo#gU^(_=H6#tvIb~*eZbDvo#hECQWSiLIM4aju^O5K2Dso9_@k9U4J zQ|_KWi~O~}Gk9-se(}*DCA#`4T`W2M=^mL5D0D!g z1JCH<4;*bZer^yqZk4Be*nhK|C>O-N|VqIw1-Li#_X1 z{Tfp*Gk%5fON<}WYS)?OHKti+U`n=1Oe?|rVE8-GVxPkJHcQr}OJ$}(VrntlFEt%l zZ#ulzbXabBRRJBtyecuQ%&?GYEBnCt9j8ot71}G&UP$rnm%97pnm&kSXwN!byGGYa zb)7Q3Poeio^gb-){J{Pl`}@oyv%qAkYZgi}wRIqnB>1$rZ1MX0!NuSP<+L>u@RTW8 zJiX5N))=46)G17z#MC|A)A`BW)oOX)No5}qztR7C?jO%dW7~zVod_R57+!!I6jKaj z0m!bFCCBo}Gp`>+&l_0xcCC54RsyR7N_U^qJ@mw{bf0|^k*BLG3- zswPq500P#OIYzSMBs*?ivSy%(t%`Tgx_9rI7bVWyqj-CktiZ~{udP~F!+*v6H6y!v zmI{|0QgaU^mo}y&>BvgqUowBeeCqtn`Plk1`%mqE$NW7bx4f>jyuRLYVXftY+;T~2 zxg@(TNv;Mg#HPDsSJP6-QprXYa><$<0Wu8FaX`MzjY@uewuFB9%U^Cu(V`7hb;rZF zT-Cea%+M9|k%94qT!u2S0b2_#X zecb=#fb_0=3=xNoQ65my;vmDMpHn_Zon*rmXc=&Ob0mP3i1Z<6a#UKE?x|At1j+@ zxX!?zDuH;BF5mS6CzhUj+2(*_bgNl}qd8mDKI1?aBbYf#IlIV+X0cdwiY21!k;7>4 zN$cIdd8%7nW2rJh=WLqJxl@fMvsAf|qX3T9oC|Vi6S`QIF4dvuf<~Gg^Qd^s$W#bA zBfeN+?0?EFwB$(XoxK(L)O<*Ea89uR4gOpS!YwsIn!FUBz}#h!N7PZpDP^Bhqt-a^e)IPE;p`}E91%^RqWtadyg2v z!QE<@8+r5OuG%QpbM9QXsGVJoMx1Fj+Y`O3{)IR?cl`@-Joy&v2FEBDyPT~oP_E3^ zv&&L`v1;cl5eE6mIch~e=QYk3qwEvyQzmXP+Y)1ZL#i>=Bv!vvZH>RKHgFg2s3Bj_ z`X^6e)rvmOPC`C# zu)Y7%E(0h}Tn1iYC|e86@thtt@Tet=Ay#qpqpB?}-f_LydX>j03E;4QnOduVN3}M- zRIN3?qgtDUb6Nhus!}a(nIqu7vGU!pM%P5!W&)`eGD}*-=B<|^-~xe#FW`KswwRf# zO|=Pc=BZAj)s8%3tJnrkhyt=JDFo*;yxIfx6!M*hQaQINI6kxz;GM`{Z2=vwuOsIs z1^4f-$L`G5i#^%7MX7)ofa`2^I;Y4B&5OhX!h`>t;&I*Pak1rb<>FDR@#re%Q4r_4 z2EfdOmN*uSku#AP-jeHNhvNbpVu{I#&6_n@G=+DnMHj&Q7Xj-r&glUj=PZvm2Y8$f zJPLontc9^^@Ol(*k(oJX&Z}@1tdB<^*#+jYiCOf%1i==}NAApdFM*vHog01G9QJiB zr{;3bL$WP1u#d-U0>nzI`HKP=MJIjUJj?ILC~}$aLePx>mutQU0s6`K0|*Wxz_pl1 zJ0O1;0S+aP&N%)E0$hRl^9bHR@O1>o5a4#4KZ&3p!3hKd2nG=xMQ{cI3OG0n5~>x9 zfT6o^sStYk8YY$`Z-S?Xk9C5JfRBtP`S%e25@vzBDmJmnDb+=6=6Ta;Qgv@h*)*mW zZ4wIX;V%;^=l}u}D%#353;8G}7lEZ)0Os>=VSU#Sd;>uc0h+&coWyPEP9|-QofDWl zhF}uG6aw@C@*IK)0M#Y%v*93dKcS^PDd>&Bnm`@;C zJL#Hc=rvtcO(VR3Rn8)~g$Q7t&ef|ib`kU5LGS^9B#O-L4Zr-mfWB#T&lzR^2+}@7 zTHm%3qb$8Uqb!*Cc4Lf%v*&MOLEV*gHp5i>HbzvvkrsOM@`G8Ar9`X!)f3 z(V?Hb_M_KU_sQ+2mG;w+guk=jk{vAy2?MEv{ier3VgW!ytlYyZJ-lq`p6_2cyc9{- zJmi3t-fFOKXTKR=on)zknAQH?>32`RbN1fZ`Lo2noGEe5pCX~BHs@mP`}K?U8&tWq zmVlis1W#MrKN(uNDR-PuI!^q&@ySj3^r&)rR0>T?-@LOyQOC{1Jk&Ob1_*#(61YK? z+G?L8keNm>g;wum>w@-`)+`@XTn+24J!`H#509)&DDAyUd*9>ZO8dYQyL53xIW?-B zx*>(elv7}5h)aoEO8B-CPC?Dz(#)h2nzibrc^J(e=ge9NMP<0#?{e45p>TUIc*a$|d#mkHS4;Yq?CMlposiD>)|tjN zrV;!Tudn!5F3U|vl_s$I_9{%T#PmYPh{e?Z>8a073H8ev#A ztl2>)BmW#_;LDsKzuWf5J@*fT&2F_zs_B($ zdX<{q^_pXAHOC&i<(fgIW>79WrIekTKl764Phnc!`7`TBHfhh-AHN~7gEBj)u!HOD z={5HBlOCCULt)>LD=#RO7c_s5)h^j=NZqIM;fpt z)XZ|YuKuCPXqXesh@w5>7Gp_TmzTvslcl)jh^5_+`4~D;0N(mp@Qag{sB|z07fOU+ zFkk@(cQ`sh7GTv5msGM2Wm{0Yvl0v@v6-;c&67opPeW|3dfRn`&Me7V)8^{<%Xtb+ zr2p)!Pglrvi$b@o)BD!weJi%rBQiap&;wHT_XYN09LnV$3?rw$Z~^!p+}@wgWnHBB z{{RuZAu|?A!rp*S22A`9ASsEZSZ%2-#ETnIe$KnSp;`_L$Vs>uMa@&sO3D{g;I-mE zB6P`qG*F04iox}ZU{JLOgWzX{OBWDkg28XjhN4;yVFv$0D5e%~@@qUCjRIdy{GUQ9 zY0)u_Z)A6fz=t(ofB-Nk`?p~>SqH(Q`<1ZJj0?$E0n!69#N&lfLH3~`K)Ctu5HLzv z{y$7aYG0A+rhr35mzD=af#A)0rC31^9-9C4s-Jk0Kf0vwp&|CjDtMXT}ftjT`B zJRGp9F3n927bd_dj@Jp)irn1@xb%=u&0UkwrIK3?WH%bHXr~9<^Q!iIIl}QkAy4+R z`K>J8jv(d8rZ$NqqU{(p5iKFR@cfa9{}d3KP&jb22$RQRyLUR{^RN3l)_fhZuT$}L zX4t?wdti+{AhQP*_F(2npTzQMn?6lq5B~GesPq;m4~3PX@cK|rg7cqP&qle0S$y%X%VeAAq2gI19Zx&H*APR z8^PFE#li>BbNyji(CI*K%ZTHV{UUi`w>W|zp#|fNwcj~!3#_2TshlHSqzRHE3{1S*eMk7thxDcMcYcl z+Neq61jv^Gm)?qtcS-{`{BJ&HCX@hSfg?9q9{5jyI{)Vgeu5x3z30m9eYu;aAeL}m z=KOOF_I{p^^L|#B$Hl`*HU_lSec^*7-9F6{ytqiz z6}g56s_qAfg{Voah~3m!X$a5N_3w1=Gq*o`9f!RQ5a+3k7x=wc{IsfWdGBsOVSazMxgvnUU(YQLc?UIX58-5vL$O{F zcFK88$C=UiK5o|G>+~Shm21mQ)SNIA_3mS?Ks5akIED3i5V!vEmj5?~{Xrqz9WR3M zr-wBgXax|PtFjaRvDv$jYzU?+UZ7^@+-U7&BXAQOZtcy$Lc+q`RCbQm@{^sH24MVf z)cC(f@HYs^aFSVDpbZ*P8~+KilGqJcaJ1b3>^AYzWia^S@ze4z;`k+@D2$(@a=Gb# z=bEEUanifdAGO+KrvSrVR=R4mLAQ2@nArV8}+A@=ZN1V2KM zM)0=)KBfq!w(%u%3&B}z`5_3*=l(X#7Tj_e0mE3`aMflT&}bs1fQ`r1ibp{tZQ*SY z8Wl!_(urd1@J5S zsCLA7jsGtY#hH@43lMH5noJuOi^&390>Drv`@9YRGL(Iu{AH*zDf`P%%slzaP;SZi z+bP8*QJX(~E~s3)XTiPbOIts(ud|2N*h4aVSYZ!Ko|8)xOB1WJvaMgS_0Jaqrf525 z&g|K@VZClP;hmJ7KF6%ob<>zBb9`WniNp_rf)krfUS|{<$?QC{7yL;W~&i2IjW_SPp zl#Z&93xR=t#s&K8RGq)hIdvX?o%6rWAEl-mRdAU9V^QCUuc}r54_?qCB^vl}B1xrs zUB#%nRg9X^463`;f~UokGs~Aw#!8jn5K?lLxIs+1)l0P8rM@vUS_lDvipM z;%%Y@md{!%O z8Gu=`D9lX2ERCk01>rIhu4k+eE}tg_GnTGT%VYyi#R51sz^PmSCkJq<7QnFsPW1vf zxq#zj@?KVT*D(12YncLot2C;;DyHyd6;lMychh>7&fXp=OhewYG$fthfiZ$d_iCmX z(rsiM0GpT+fXz%Pz%@)6z_m;{z;#Ro!1YWezzs|lz>Q2bz)g%3;AW-<;1*J{S}577 zcQnzow_epa8?LQ2wy++z-@`aYh8+js*XlX%XjEA*Uvyhxn0Rx6dCj?q%J~(O&nUhlTjG(ZLbF%jdr2 zarKQ2_re{+ZqKmK)9LXy!-bw+M6|tQj60IlJ398W8c6$R3<9i%pCL_Y?In~2LU^RY z52HSuf${RX${!tNDwQ|-Ziqw=KMkn7+E_3a@KU-IyQ5OcMzI=3v4)P&fRRZGm>6xq z%;|5bzmn?L(}%qB(nW82v6OKKr>AML|IMFHBe@uz=t)+o0;z$tKzhIu$OvTmGh#Sq zE|t@9Xb$eTk$(wWQh=1NyI zL~EBb-Ajp;F?$KATNjE~C8$j|K9%ygznDNXP;q zGYMrtD3yd{ZO&*{qRp3)%0HbpFQ#qIq{Vtd0h12xnZnxqB{bGsN<-3_Q$gQ|(jB$R zUqNHIN6evoZBOQZ*VkR@-=vM_w z;8zO2GWeCluL6FR@T>CI(c&;!G2i&pVQJh zWUNd!W4o90PLx{jR^3WD^N%o=a{U|WlU{|H6wM(}Ei0YEyl8>aN$+dta%q{C_r^d6 zx6+bJXIfsEgQf?ZQmt@m|8|PSscD@vxlA6^O8z@KF%D`)eP##LN)1=T6d>jB?xasi z=UCh>=^hO+>G)Ja4DEs#g^R%aFhMN^yST%4(T%FRJ$-ZfA z^m*w_OU|wGwo7+uh~`6kAm`v3u0h+Ad>c6@*XTV!aiueT>Ke>F0!}X5dr*p_A&R3` zK^lkYUFj@~&-Nab?$eMIe@lI{MXTa+5@T^rrT%=)1SM=*p^;h)-(LrvGcS9r9;a_A zouzk{(drXYJPlF2wJI{In`NU~BddkjjFKsdAJu2>L%n>t#`Ky+kLk5!Os}CMLmJau zGsG0bD2sikX^l&>b#Coq#?gYKrupXYYHsF{2m#$zhwl4U^ zS-)W4RQlqyeCc}Uvwd-_^L}v>u0~e&i{tO2z5i4H?yO(LcV|6kTjqDimiUVT->CEMFTHrrHUfgDG3tel;$|$XAS85Hvk>;LoCBBv0%2i{z0^7JM#hTnE zl@exnh43a@=$y-^xs`JBf2dW%YM}Kg>;a}KuwBN9IY&z%ow*XOl-tgwK)pv-$*>lX zcsJ)IOd8_KxN@$7t0YgxIDB)SLOyid!BxpQK37#&Rb{G1l}~+HqgHjn%wMg#rMsDX zC8_N0m5&bdNJunP6BD(W# zfA596+KA3S;&%_S$dhB82xCCx0Z#cNOmW{LIK0bhp02G2aCi*iWj6j&%?Bdq5^OFytW!9u^+*W4DVW!H|rk zz#zTg9)&R|o|4?>VQBfrh9W7X#LZ0+?cfOXQ0;)ZHZYA~vX~G003-a?*vMJNPdQftV3$7ReBc!WuRc z!F-gAEy4^;V&0^5@{6WOI_Yi$P}MFmHtq&s_-KEl7;|qlv6Ywwv>a5S$>(>oe(~iZ zS)}^$zN;6h7uIwn$AewZ<%{+`7l=y5&X*o*M3>lSQl%UCpkdHGbe?g$1{))3w2h*e znXzjr9~fq0JBm2bvP+oQ>f#$KVKzmc#%zk1<-H(c6yKF}cdOWyddG%HSBj==L{jP^ z1iHVD)&BTJo<)b@As#e!U zPNa57ua3T-LfM)k@hQU_Kb3NmSktkz;3?8=Mu^36i|=1}?8 zH!AP14W{lJZ+)DsGFrxW!O5CE-T^0lk&xoNcmCb}yZ!GD-yP<+9u3tWd;bu_dq#qX zMg{$W3EjktciN}&rn~N^ew3`rDW2-Muj3DQ^A~)x6{}~u`J+7_C8=}Hs{iWjIpOR% z{=)EM6->W&YWTC~K2)h(>Rp=0Nh*W&eOu8IpM;mncf48qMr}B+M#!s~+8nOkDb(%^ z*0wy*@Od@Cy!MIohk3;dV7#AKJ9RLaw*_%&=p(IaOub(N1#Z`zz|$#BFP=OaKb|gW z_9Ve+Z_)`ool5G*(?F6h3Epc=?N1W)FHD$jxh6-ZU%Y>iKhE%d7x|$VW^MU*+_wiN zOtVF$f<7PWIHPj1U$E3q=w@xTQwIdwCWwyJgzOCy##yU#s#dVBn=s7U>!wBo z`?iT_jqjX3et&>JbK#jQd`S;}a`pXwQQ734Zw&JK{8?Mhgy|=y^!LqqVRM6EZs1q% zxqmHWJ_22=c+GTMuxKl<&By+h8@4$GoAbTw?-qQ!U}h-T*b#0#C^R1APoCi$4+b00 zhHS3!gAc8?u(e9CRt2rj@dH@r+2i}*WV4TV&gSOxTHB-iLSAcsIt)lw+js|Lr8Dxz zfsoewP@D4A{a@K1)>;LvHKesY%*cMt14Fyu2)Y_7$53QuNZF_!+N`*w}(AW}g z+$S{ddr%@Yc0BO%T`r;XT)4AO=A2=J-rTS0(ZB=yHRr{v^BC8w@rYM_8tDHy{fLp{#ajR&p4D8z6I@{Q zRn5fSStxK0v|ZS?O0cb(Zsl#Og0{^Qdp}546;w@L4(2!9=$L4|we!{sP#Kw76T4=! z9pUU+A-gu1UH5k?b!zo3^=x*|osu^z-l+IS6%O!k)Ct*Dw}GNSzELRO7zDa# z-prdFkZ)>1*jyo)D?;Wfyegb?1#u3+?D(;{^l>A!!Q*u*gFUP-5cCBhebJ-*hWE;* z2ZK#5LVgRcx4*B{_zlz9{DuP|edj#wPdiF#<|}7zocX%z167KlOFdgs%A1QG7L`tJ zdCxbqCAhxr0mq*o;9s~DD!Tl4DqU)q`j&ClUOKr>u-Am`O@h5?decmw(7Y$y+%7b? zKiDZWck+k2`O|LU(E0G83&Nob{D4Ax{tTl2kl#L>3&jNGqpBY+<4o3EBTK4PUEbl^sS-k*6((FyW=~Z_xJo@ z|L^Soen+tKaJce>P7ltZ;wQF`N|^A5VhG4WwW&KRplV38(c6X}vrHUu7Tf z9}_N)!IM|i(GT!hB$Meb%||Mg;hcud9EL*r;dn{~iCgWjZ@j(n)y)(7SzFPaYw(4I z38fw;6ZtUTywe`iSHl<9TsFCuui5rv)Am0%XFTrL;#}o}L9I$}p0%ujNsHbV)>jDn zipgPqUE6~S{_t`Bq&uWP|2{_43i?`p)ee5wG5&;?AAAYQxl?lx-}f+`A&-!PjzU2J zzgo~&PZdmG`;mUvV-sZY@!=#S%HqfD$;ZR$6vJ_HbzJ?#hwOtNnOcg6G`gRrw3H5| z>;9`oH)J;a*L2-buAvuMv-ALM8d2YPy5LjQH#UI2u@Ur*O~ zf1Uj=GJWHi7BbN{HcJ}EGEHFKbPz)CM-nbd6VDt5u$QwoI$o8mpE0G?0;fMGh`gCO^{8(KXH_dd8y0F=!ie_NcBg zXdMUghyOp&92Rwr%@7iGjZy1Z)HOz}V^P-_wT?wyW7Ikpb&WHm_69BCWw-Z|)A*;; z-Z2TXOwTwUG#2x?#1xEBvci#o}-KPvVz1~zcSr5lyk%#TuggdNtcn6jd7V4 z<^r0{R39!-#AF1DL7zCGt}(<{!_Nu78cFXsmPW0@y;c4O`Wj43Pa0}xHNC53&*hD2 zAX6+!iwvG!Pwz_l##|nkMC+c(LJj46HY#TYOPc(Z$hb;bx#qExd0WAViPnkZggdyrFC&C~a(g|}R7_?0n8UcYONQ>w` zFNoO;wj+5LLBhyZa84zSV%ezN_4M0w%}fz(WjfBGwf9j>+B&Fl3KH~8E~WuNv?tdD z)&|za>oRin&t8t}Yn93Ylxsco8f@0MgO)3*`L=;u&uvJgpP5iE6VeW>kLp?%USFeB z=du|?XIxR>lGYs8oJixD)-!0Du8-+jZ{${U8xv`o5nb!moYk99e~YX2o_xAo+(ue5 zZ%n68!lHEu`Y^SM+7q$)1;L1Eq9~;RI&+{|1e$pYnm4k1Rx6F^sGgBTQtCYU#lBF}u4pw;=GdsvpN{4|m#ivMH~7 zN)*;TmglnrvhG>Q*+CX)2uFF1pTz9o`c9aE6n!f5f}Jog$Woja?39hOx!V4_R%o1+ zui!Y_bbW_RyT66o#_f__@8%pzGdHeFuZaNoSaKCq=dWTe#hnV<9V}#35QK zj@SUr-Rq?x=`34i4N7-uhy#1gbax z;(g$*l~4>XOGDBbY)nL}5zI3tshRVX#EgnP1~w)#?!BVTh~eg+%wsc>a?>_XjPB;_ zJTXZ(%cFKCH8MMszxn^~e(=Ks=XY#pVQ!*gGY~isfM9b06>1h`7sLdF#rk1U-;*sz zfT9SZ0^Q11VXzj#Dg!)whaN=6tKvCW%na!N6>-b00QiNEHWTj^t`|xLU0&C z5`rTLjv_dQ0G9+TdJ$leHOQVqz#z~fK!zaOjR0A8Y!3pQ#Ixux3=9ZBZ5&kA@y0p4 zkvujEy4D`njd#xDois^%9CfH!561LijAnQ=qI0vX`zkox0y~L*yqWHfRvPG5vzI(R z_5$J~A$IO(k?yk>5g?6caR?H{eRc#xqX=F=faB$v6!TdhK7j*<9Yb&l!DR$j5TIf` zdldmrDcBbga0mhjUP5pk!8n4iAov9Y{~Lmr5&R;8e+D380}Cl;092qyhJD0{A*zIK zed<2y+S~l>7}z?X_0L@Fr|u)RIocYcy1D$SiTx^Oo+Hmh)E5`k+q0jJfCSkIEcOiq zUqkS91UCUhOfr1-m+9i_a!5cTF3t#)2wDE$(toj*#q-ZA|!y=>M} z8Mf33mb!bMcZco{&FsB@B3R!MvK)9nBlp$34^*1eZ9j3YnLZhGZWSz*ppk5vJp8c^ z?mpJ5Qp>~U8o^w1Z|%EV?`{Px!MAsu#?bI0$biCCaA?pF`ZpVYp&9UzG$WjrutP(7%?j3*k z%-u6Hn;#?x>kfu2haOow}$lFW+khKjUoM}rP&Hx z)!}sWqnC85-26B7Z|J|3{dVqKxqR);`!9q__JvD2gp!U2$AcwDgZ5**Hv6H?{`!U6 z7rr_6=G8Z@^3JXI8$-o=!o_VuaodCJVDZ7A?a=r^Fi2R)@ZgbUL)0vxQqWiORU2j+ z_;xqndzrr~+8;!XFb)&LgCm+{8!BX=p~4xQ*pWHklaD*p_`Uo z#}gki(gbtrmGYFZb9eKVM%_>HQ?G1G`I`;9S6?;heqOG-Qg8ZsqwdOP!_T+ruIf$y zVA5U9HYht%FEDjzW4GEPfo;y+xJa?oX?L&+1Q3>5u&goI} znXcs1Zp+D?=NHqvG6RR_j?_Odj?~p>{s2@`zOate{V2U52N+AKVQYpQP}Oj}gCZF? zdO&Ov-Izh{aypCCaK+R)G#HjhX;>k~2Ex_&G{kKkYGp1Lt&p3P#wu{XU5oMvCYLB{ z)^Ro5D%o{CC^Vo8Ib;TE;2NK+|3n-d<|#R3mpM4B1Ccz6$Ka5a5#@~JSg}nCQ0z#KgbofHz=gc% z;BYl45Gb|iYN?bU%0~x>^)gYu`sN?X9318c8fBvAR$2n-OxzSU&gP==r!K?MSo z_sONm*bGGaOT@m;%1>FG`z(^&h#n+k=$gNi?`>=`N@szjpN@HGk+de+GO~T!TRqPAa zUeVVXzkct-+>*)KDQ@OWFt>doZDmpcfz)%CAK)x(b68x!*b;qr*g<3D)7~qP(|_exm*BM{=AVQ8y3A|G++6|{y><`}47x>n?U=hu zU?%EdiN~Uu2nj8J=P(hnWL`01KF9-B*|2{oE3K3vvJu`pBFhC z?6@~P2_7s~%IC!wskW4uD|WEK&RLON>T*9p?-54HXM}8*y1c|Wz7y?_fqdAdt^hm~ z6vD3ve#Md}h!tD#t);zRX6ciYa929*8Bov*WJ_Km5-t@TE3^ZE9qhVbmWu~0-!m87 z=ch!L?`rWmIb2{jyjZO~!>)9@p;k({(geHG(YRgml8E7`w}(U+qPK@c7~-yU3t~`O zqhyu9$$D&{4BlpOyV8{t`{gpa=qDCR6@kh?C5+OlMR`i7MvZhOCI}r3N)`h&4w+Da zs;H-gJg!3N&17_5fnh*P4%Ra5Pqz-uhxS0&W6lX|peo`mArxgzUQDR zL$7r%us={%BD-sNz64S@`gAU|bi^Y=30PO{1&;^Th0>C-`R3YM)o_JMR-2rR!6qA9 zo1QMu0%J>JT9v@}t(3AT!IGe!ruFQ*YeFA8NuQR^v}TA8ms**{&M66phA1xjMOaQr zWP1=P?|oOMxcA*EZdC%xLF4A99EmVQ%8>{|q#O%kEShrE!P_j3a?}&b5n~Zcl!L2< zn0y$^v9CXEDamUQcywu(?$Qu#Njh2=95c1bd*5+#R(Q-G24Q_Sa7Bt^L~a!&_2%V_ zk@A%rBmUF0hGJgV8bAyX+y4bTIju~YE0nCRL2N;H>#5-yXit);<0mk;SsXKaHF&yl z5WBQSu2`{E8>Lc0pUj0fX`p=z?9#M1zZPR=?_(-qcTeQ-l0#{adtinaaZU3wv(H5z zm7T~F1{s}{(myXliw41qk8EFNMT;x-&EcqxRMjWV-hPrZ$71P3Y!{n&miLpi7~&q> zw`N`&s$*afW%7CaF8_#Ycx2e~Bx~*_HR982pf;XZr0k*_)?~-;M?=n>DD-O%qv@%TmNP67fFN?9y>V)c8s%cgiKHH4|>4#{Lg|=9d5teA5 z#o#?R0_>I1JyqiNVIg){qTN***+Wdur^yfj@a(wlPD- zLOWrkfr(6Z-s|nR+r!p!!CF3fJY2C>s8}1USUa2S4jP^(R)K^S5OgB>>iQ78FZw!!MJZXNfWyTy_ zvxo2K;d{M2dpVSUxDR|we^!R)H#@A!bbJQPeCrgpuRCYbBN z%{-d9*urKsbD7%Bn~Ou{M!Yq5lLy-?oi{o|B63OG9uYr;Jt8(jt32MOGGxAzb|WpM zw?hgAtLNHMA{3&1p1~A z&FQb~zp+1TE)dKGA=nk7uw=3ZdXMot2l@T9^AukBJCy;o;0=%LCD3>5&ak~vus2RS zVV97mmT=QPp$T^hY3g_|z@IuRbh^Tw9--62_w@^%7kDrDBflniUxfC(2t5ZH7*Ei& zEt);z71<)9O>-D;9?_h_n{Apd>=11lmw4sExO3v3A#LC$9Qs5qgBZDFp9p{uKcX4K z6I@{QC9)|5b{X8pqE4`5m$BCc?OSf?KxNobF?swO9NiTHDk`IFL1lLIM4yFK@v#nT z?{R@>{ZjMJh>*K=B2Ds>-Wk%@Dz;SRl#XxM9n$ZKZ>f*Ws@%G%B+&XjEx$OEV0`5^9@Bl>~sm=ke{RT6uq>{D_GZcFG8TzBRpV1`>4=@6)V zh1^Bua8^3*fJvUQpu5frS-z;46?Y_dz6J2E4vcNu+b9hn;)QzXdAadzCb6ocE6?5fYuT^Rt*;fsjK5+*vF;|mFJVf6wfUMDcu0AH_S`A{Bqj5>X zWt^C+Op%Ei(+Ey1r~!v|$Z(1~xY#!oMtIo*?m8>D)k+NLSS`ZrDNq4ovOfSZS(Z|6 z6ch5QkX#a8ucoy}L;#Ikfs#m)lQD~BbY@H33&CY#IwPa!B`_P1i^*2gw4QyF2k4_N z`m}VG-WR^EO|t%fPQsxfic7>~%P5JAek!{iFPq1)XftdLF#m=MG;V(SnFvFqpNTL; z`ne#+qUq-vc$>v-sIXQ>KNa0V%GyndekwWz2XRIQw}$p>iBc-N1zaEpMasxEK^}SF z$UiolqdiHYjojMBP@@IMbklr}MY$$v6mxS0dFw#fy;mv`4RPyeoq#Y{>gCHB#pNqE zie--6#Z`orTL(5lF{gZioE0i_g%`E?;|}PD)DdSR)RR_>metqFJL2r8MetsrA?YmZ zm0syC4RN4?a<7tp(UiWJ3XV7{{maR*Y6&@3f#^}GA682BI*&uyDhXqe^X~PqDM>jI zcy0hOwojl+M&hQ(HaDSr|#z8M;&o4r_HT#&Vz-F0H z7hM6y9iR+U$Zd{u#92M(so}Us;F8;t^)LvR6mD-b3(11Ybk&bp+^SlKr;`ehI-ZBaoa+qTrPcA_yUP6~Su=euO|0to}Q^{4RoD z0pP>wf?Rg`tl_Br*F3V;sB6dwWb(Bn$jxN<%q6apC&DkZn04u|@mb4OK}eIx6=ItG z>pkPGi#BwU3t%s=oB$RIDRYVV{=DMYiZ81;cJZzdH~hp|KXv(?qL^zs^n|!!F`f`V z3&C#ID;+mF_*`euToX1o3+CpKc`Y<{YSt_5H`?zQLgrHO>cEWyHxFV^N#5J8hIW5^ z2%I6tOMEW|n|2BLyKp<|PfFsOzIci6hpN;sm&A8Lw`U*{u4$MWd#51Q8)xTpln9Nx z2w^dLKFh~i{8)z-|G4~1AuRsZn&>$xdKMXkDn=K*PL7^aqGw{A2hR}^t;CpsJ494Vm(`5v zo5p~ih%qr=a(JVR6nc$kPS3?e%CIbqE)aorEH8cZ0e>=mah zU|`Y$MwuAi7^@NOBI<;^Dd3U1cn2n>czB^^;|~y%rpNL$`Eiv%4kbHG*}LN!BJP0P zNK4Sv7RpT~lr<;Lm)21rRaU>~+9U4p#>D+n1ZVOVN>?;QQ-m^;UITW4pf8C>Ak80R zl5%PET>|M6>w!KSuz(0M1AdwC%YvU3e%bJ|!7s;OCZ-4dN7^IXF72Ex?HC=%WwHV? zYeYu2v#P(ErsA!kA?fU`lfpDaO9wqQ#sZY@APsvW>&}2eQXt>oNS{%%^r7hl3S?za z64WLxLlw<~$>#ED9fL^6lvwZrHBurqqCKS_%a%>NvaVV0$asG+RC(!p(pO7@w8W=;0l478q6nE~h+`Py-nF)?v9m!>4*W{U0=)AV5+T9icHTujKifUTrO zNn|8ZO_KgmspK?7-vHtZ!QS*Zy)T_92C=3r1d-AogP1s7DL0CVX-Omedd%XJh>4xb z3MrC$0`yUd(k_R}IkDw+sGq@oLAESie0g11>56|Iv|QQNaqFRo6B zdJ%dVbB-s`OAuOZr@ccu%X)?qomQleTn*%r3aT9yQaUt5`yE%e7@D-;n68;uOk5+4 z;y5vJJt)2?ys3PjycHV7Wh*y|W%iyXE{#h^RUWWjOnE1|jsS&3X=E;k@{}uy<{`Hv zCJn`mU@NcWyil5-t<`{Xd4rVK1&S-3NnRS39esN>QcEs#JZ~y_W4dVA{1g}xijTzf8)QR9A zf3<(IngWyvW7_Wic zo>fSU_@bb=Xg)!4v~qt6sWZT3^!*DxN{V5lkxcuYl-tF#IfZwIgq&5ev#sDx4_Fb- zX6N5&7qY8obBia_h1}J#bDo1QX%zCBD8fTq&D6T-RYL9dpl!zo*``7xug(1+4@{pP zn)AZu2Ep9GuikV2TF89leXy?UxzY2jJ#X)SYk#!>Z(#qMlN+`Ks4t(31VS>9#3saR?U2 zWOf*;p`wBCo>l&|jp%Kos{Drl`C zt?glk?beHvZNZGX4^_VB0^1d{pZ#JAY?CKn6dcXd=LN?G!MtI9d)((@+b^=A?w3n! zoBWyC_hhVG|Y z-T5@rPcwAq^9{W*5hu)KaDtEX*AM?)+FUDVCg#p=&JsyMyy}k%-aynMPZ<)$!;9;P zxP!8yKxC=`k!ccLb_bFJUvi_%LDN@C!$Jg=3t}W> zpeM^*?t%klCKJ)nqNLz$)eJ;E*>;O=znSrdS%6L$3+jJU5UR z$Ok6gv*(bflmjsPo;{^}#=<4_X`U`GmkmsK3zzGS30kE0r@fK0D~Y9PEKM_!dtM?~ z%LXR=QLW0LdP?M)t<=MbD6K$pS?b*(m5hdDEwqc?lg?ZwW4sr$6Ibd55;dnS28saN zX$nioF>47q7A-4B>k@J-2AUW1EC7Peg=k)xte!s|nkQE$iS2&GWIv~Qv~No1Ih5>K zXr5yUbYwo7m##?jN|um=GR-SpLQ3<~ys}S?=9SG$^JLyTxL6M-G_QPF^uVzMdQh>f z97~pvW971PEL}p5Rk9p+OQVN|KsBVIPXp=N==Us8y}%PiTxHx7c?&#I%vHrbkq=gc znA>DJU8QR{7g$>RuE)#2s8Mw+ehN>FDeWN+aQS#v#Kcq_3mt&%uQ`YWx8TgUYxl zzlb|P;HZpyEDjMau}#78ZY9`TR*|v;4;&A*s$azv{wsB$PWBEb#5g&`=lfwtMXm~9 z9Z3O&E^w$-isCZu&AxfWW$K&%5Vfi->J)`gaS|(B;;}ecjhK2bczQ36jtmU@T~Ctg zNBvJ4TVc~L6pcAx!?mG-VUVIR^^VRFzr*by?l>LNBaTtG|3Wnhv*TvqJB-^liaO% z(5NO^)HPc)>VU^REV~cBeJv=h81um!x`x~!G`sSoq=f|^rgbM`zJnbt6vc6XpxjNm zmJ)q08fc2>TSkV?4-9)4=PFSkrX+E*NVVm2VRf;nFvw!(h!?(X!+SU%jA*^kAQ64v zz#xb%-TmwiylEkBd5KS1S5qHUvdql#AU=wnT@=OPqEQsPVMk?A)rv(ODscn7HVpM3 zID_CWf?q{|lWgb_Bg4JW=-BX)bl*iHPpu#FKuz`f*#C@pKnp~?zGw}Czhw6%k899< z-ZSWnSjfdF>tO~+NAg9o8u4f;vXvFHDFVKUW0f8;kfz5f5PR51 z*jQ-d8Zq<^_PK`LLmr>69ongxz`yV!-*r0J#i0bz@_uICGvR{_ zFe58$DHAMZK}!Y51i`~>?(6-x`(GctJ;;|ePoKHp9o}~(Yxtx%?b2mx?9sD zUiDzy8PW6UZp}ILj801)B_H7uUs0;7GJiJh~SygTcm6~dNEFkzeO<1Ljz z%UZZ^E)AQVg4ubm;@!Htbu;Rjb-|ilA#)2@ysh1#zOVjHr(kZJXuZ`Aa_@L#)BO{{ znvRh9z^u6{Y_12-xA(^04crazTMzJuj_^l&9=sH6I2$s%prTUq-x!!me%E~0{7%}8 z`g_U0k^GzbVA0NSQL9kY8Z2rP%xz!m{Gbpk`ePk7&f`i|rsG>>lY_z1W+9__d_OdO zZvLD4H}v1iemnQAT)uYa{TD(d`@$t1LP^Ji)nT zyfwhDZs!jl2~{2qSDp|mPw*#uf|X~2g=cwf-t*oC{qK42`hGzWx<$}JPuUN-#*oTDmtmurG_y*q(L=0%vc}*XlE&%0%PprYG`Kfgsc%w&a=So3e z$yaaWH+S$=Kv??t3or2puLJTfO*=kwKy(g$6t)iidwBG$=5a2(&&O^Jk`a;8Kl!** zExYpJ{OtEFt>!c8q(3vlDJ|(Q)VedKl)p&Row29P5w_t11!t`vKGA+*g>5h`4#b3Q zfm9H;$pvL;OBT1K%fxLhTd&(msNj0`^;{Avyl%UmhatvD7&=8Vrj3cS&1KBU z;>=v$ym6^q(cCy-=)BFOG3kT0p`1W5VdxYCW2AsGTmlZp5-5@BeB<&y?f@55C2@h>m#p5#x#V(|su$3cI+4jZaKZ#<^n2Rou4;`OrzesYDo};#49GQE_TP3_1f=QiS1j zGR3KEa9cdT;*=cI1{VG6r8cA?SxYO;Ji*$I&Q$4WBy-qQ*zX%Ysdx~@DKh^CTl#0u zK~YU%NlIS1pg099^ry?qWq}d_sCp$-oU*Nm;#AhWN(5Ol#i^K}n5;YGqUar@O6LGs z3n?j1S(YGR&zA|mpFPKnCFE#dR*p+i4$GCT*s|kSY+Mriw;*P>4VDg-1qw)yE`U{& zo%WqTA(I34!~1AMc=ywgbcWSX5vTxb=~#u8tc0&5md}?c@*G?xkvy!MXj$gq9+L8> zA+Q-&#yvtYq%*!F;A5$ybA&mmACoX?h&(ahTPg|SF^4($jJ;nSD3;X?%2MMFF27$5 zt35Rc@aHRAcibaz31t)C1^5Jb%p>Yo-~3*5eVF@<>qA&8s3Iv*(fp!C^04JCT`)@0 zb{3b3PxAMG62KU-oqbA59?=~bVLXE%x5W;j=0usfJZpKAq-=B!o zdl5^}NJ>;%9mylo?A}4Q&o|IF0J3saC17DQRMsD=I5AbIp;f{K>B1$!pZW zbH&B2sX4dC?s$V{=lC8FG1&9O1tK}m78HTCE8Nu?(H8UNj@fj}c-tHyYm1_g^+njk z5?z`;$c^rEng{!wrWsnktTNVuu(?bymx1Z;fwvA!mrS1smTw7}x1x-7(@e>CK=>(% zSm)x_&RhzXw}#AnXU!#HbG2ZuzL))O;oU-h{oV(q54s){1)Zlt<}MVlX1`&WtbBXb zTdV$c{dCE9E5BX&o$8={bJ)H^u4B7V{(mPtO{wXl^F6HHB+ zCIiqI{JJ*WD!Ge4?Fs4o$fG5KzGSkNZ`gj{djHyw^hX|-pj2un$e~(@RH`Mqo8|9S zx0(+(B?(Qs!&~OE7fe8(2z$W@jD{~Ld%*|-0vUUO>|<$`i=XDQ8DJ@q3akcrUp$nT zXmeN%pQNqUv#=T#5>6?x8Wv-#l`c&{<7~7RwpaM{ZM2?^)$mW-MoZ2>NNlel$6Q$( zEoFO!&)!B$nbq)lv(cK%YWSyMqqRJC%2I5!^5?J`;w%B?uo{X8tDy*32a-mAF|ZmO zvgNT7s{wZ*nzN{u;VZEk3MGm>XMwN8YRH=hSBcdy9}7c?)d24Z3-tMx^Gd7+d}6-0 zRAMzeg-=)wC9=9vVl@;3n*rGkCG(a|iPcaTeV6628txiI$!Qi$97KI=CAI6n#8~1r zD;GoWBlv3sL}-mn3D$-SNwhnD$|aTqqmh*>*$A*Xh>Z*g){dcE01L4hBBg|BaN+8x zhaGjZph64=4L)!SHq49+xqRRztoH)@2$MvnB-@Svdm(e9RmeeJZvP0m{Nz7>p`sDGN#&OIDC4vT)K;q|N3s4_F7LT!Nq!K^cM@nDY{O z0j2QlLc9R#ZOS@V6dv_4RSiWr4%!&LBP+lZ=vD7Q)wWj6A(Z;|st;=ftxLun*g5mU z{d)e$Q4p0Qo1kDSW4hx0M!wqw@1sjrt4Z0(md3XhX z`NdU5c+M+;j}UM@p7RQx`*R^Zfr2?|3D0>2v|2z&Ij>s6b6x?dC8|jB8TfPg{P7BY z6jMuBCa-|8GoU_wFHcd>|D0zKTgNZNEfClJPuHfRR{{0SyHQnxd}URGN?ZeeBCGmq z$-^VVbvs)lnw_oS1HeQi?4s?jvu**Ef+{T4BN&XI_r-jqK+zW71&##V?A1sbHHjCm zr3HT^L(&$AB0EdI5G#LQ2{g-si+)i9AYW(U&#!BJpnc%w`$xcJf$zKYWBujF#q+2H;4;57NhsAFUNzSm z6?M~yO28I?xL>CiNKgDhb;!X{r#g@<>9~Rm&6xE97jqTMXqY5M3qtp39C4uuf|5WA zmm=;h>5cBYAy5ML5s{z4O zdY|^F%9+|Jni#Hv44;zrSnwv=YqAamDDpYR-UsN+V2|S+9E0)OhY(>+9!Bgz(1ifS zU81__X$(oejn3kw3&A-6P76_vj2KbX5`{w^mi-fq{8I$I03vCur+;kF%??~6YMnBP z$JY?&>j-WlAaavm#?UJWZUKlGK?k(=0&)K1!*`3H^pH7@_-6Qe88F~|0#KH5)vVU= z)%LHnzkEQ{?X%_G*>vydbly9z`M#==rPSi$}AN0q>xz=&a4(PtKUoJGpmD{>nBn``OIE;=j5v|-7rri-%{Udgt<&w z`h;dS!xqjc5i&}G8D)Q`QX9&_Z%RhiE%)o5+n!hZU%zU2k=X?ofgiTW2nRkNXYoH(v_ock+i_{CP$=%n11m+0Wg=Tg$^%r(kskt+j%_ zc0v!?8ZyKBd_kW-rRDYcA$=X*$_?vLr!1r|BM7gg-AMbf-u}1|D*y31l`ie8onPq; zX|0bka=+O!Y5c}PA)|JD|09#-mh(>6l#w?z@!BT#x1f1@vG>vgRju&)!}FTOd{LyY ziPEmkbhM)N5>Uq^Y3DKc_C%GsK({=>x97|QpgLMk8~yT_XfdU8EG|afrMT#qhd!-@ zA%1xhVTfOz1uHY} z$=+;Q3h7K+h*EoRjub~j6o=6Eq?>=JmFE=QUiEC2bwZEQ%WLmtO?Q8%_&&#X zyZM3PP~OPjsdR?DpyoVlExL0_uvUhx^@6p2>e@`D(6BSyuvci<`#>!;?0>MIKY2#z zI2-O@gbs%H^a&mP{D6=5UlRsi1On#;dsm=vcmmqiqS+%}wc=Hq<_MmSYP#@do2DBn zVw>iictyOexHY}v74Z<2?Lmy}3wR44(9BWIB}@V?fM#Z2)=ccd0$+x`fWp=q!CLcP zC2y?>S~pJY`AI%(2K0?BH#)#a)XrNk%$lt-HROu#)_l9>JF9~B?P2?F!M;0aZxzg~ zH`+chVJ&>D!^(P`smjcoL$I>JiPOh9Bu<6!i4Vu||JhQtyCUiLDs;Q+Q+lHuCb%s# z!mWV`?CapuGN6T@4t~jtm>#Dro}l?*g)bgTo6svw%jiG?J)ui6K zhD|^(%hf{Dkj{y7<@8Z9nk79!L$Nq3y(68YWRyOwgdvjAL>M9&T@WKiMimH=-%iUU zBclc=xe@2FN~BBi2_qN`E0_|?T0WEJBb_-TW1>V-n#2-<99v8ff(-jt1X5v7hiA{h ztds+g?PpIZm4Q!!qIqTt`B0dYrsuSG!jeKk;z(m@9VD7|lMh7-Owdm&eJZ#Vr5;Z{ z6>0u5TI1ew8j{Yk7OIr)(hz)3)v&;TX$u}oy+L8)()4BJs9Qpg=4IuWyo4N6!PyTi zIJq=j=sfiyeh~c-r$s-+it27iYT_0^GMzX4Ksv4Q-{ph&%pnAa5lEluV|aNI0qP8j zd~mW(V(@bA2RqK`Yjj~G`+g%aT8{Qtg7r@pP7)Bb;x~R6I@Y-AQBLkkSUl0009b3C z<@+v%@&>{Rle%&Fw(nwMGg>WE;f_-yQ*9+_-+Oki0x?dMIKdF|$pH=;Sv zV{k``Dn2;R`L$N-enZk98g%+=lu5v~Oo#k#>&G2uBx3@(=_E_3-X zYv4SLi!uHiN;2^{m%+JBXQlA=1E1t9p`i5!4 z%%&gdTg8PeBoR9>OkNC03t^4MqNKo46}tftL5Qb6?}n{+2U(v}&2ENUK72m9ooI4D z>_a=qUb8=4Ixz;t zREeHIEpkTvO`Y5|+EtZP)9KVjETX@0pC7z|yU?F^#5ywS9~c_A=3!yWVd<`q_AJ?5 zaj)B}@QyG<$jWjQ!V#nN9H~iXvbcJbSaFqr#^Q6tgxQBR4LuqT(n#Qy9KDT8UcA}; z(27^F!BEGZSPG-1>})|vxL}n~uqs$kKkIOY9cu)~nxJFdpKofJG>BIr$GXtIZvHeA z+~*1J8x{7A2KTYRh?=!nKQJX*jE~JKbN=6_ChNp**BfK5Kt3>Z#Ww%X5cp)H_7kkNxkN&XX4`KJ}337r=Ob} zCx1@X^OB^V$L@oE;xx$o!H+@Hv6m2BMsNi|T<60kCw))qf1*2kssBaY@gITIe*t@8 zA3P%U!qU2M=?0;6L$GwytUO$~He9(wsN4~(+%>zHu-Gdz4A3hxu~%jpXs?7kV@He< zAx|6VGdM522(8DuV8sLFi$5sUf&BwxluC)^IFE2j4XI*|amO{dW;6(C&jDtdHlXw4 zm>~xU3E;B_lt%}>3&%Gc^7Xs4R~CpY;2Vp-yV_34lH^*gvD0%|z8A&i0sjTZd5<5g zJRImr-f`LOb0DQ)N}MU8=RzgR4x}AKxdb`}VPefCLlHA??ld$z82BM%q9M!W> zWHH$%5b{0UPuK^Df#8?1S6qYue8^I=ZVmh}rFb^KBAmZY$X^%C-vA8ihj|5WR=rX6 z=BhVV@m1?*3LoT!_w@+yzp;mRF~UYBnCF>DpS3#BC_3kk?#|UI=k)QJefO`2^e5rF z&8KY%>9;;gw?kSY6hGa#hM-4`V1?igbd3~FPrAl=kXR+^8pb7b4I`t5R2SDZOdN1q z6}kp=U670EITJNhp0|VOp@YDWJqE9lsI75?ifwIP3=uev=qC`wwKdKlV>AH*;@rUl zcUc7A#OD4*2&_zV7v8xzwRtA}K{4Mwz+b%f82D8pLvpv~0P+tHf}jRg4h9!^H=MU4 zXrVatz}W|xt^78;8+3%Egw3yALi59X0#aRE^XnE!KSncX@nHrPW7e}iiRb6V=~~>u z>10BJq!=|qo~07EE}0?8^;f(PU*Qb-u2eL(wUI7SHvW@*6ae;@FPbscPOV;*vk0;Lk2(y*=AclRP_LlFD2 zfq$Qt*uFnNEI|^=lsqXl?qS&xw!|TeV@5na$1rUEVFlamXS026;hFiF|vGhc^3S zw!%m;eD(5IE`RlA%$%%H9J~SAA+&UnuAc?|6CVW}yhF z?W{gSlod|a@O3Rh>F$tzkK#R>V=_muIle`hmhqEzGNMRt38{_vz=*jZ`Ubpc}xKjL{9Y3#mf~f=&d75F9{o5W!&t zG1|eN#>*ZAX8?fU&bu;EiiWFR41APmjqvR%_6_*3(XpozUnb| zx$KbzXoFUfHt6P|4Z5djgYs8h7f%~x<1ASkJ@{qr)65xLDN-zqExq#C!Vz^X-FN_f zPGJ~BeL(UIK+}=Lrh^f`EdOtPDjOwCj>#f@N1WUr=3r);`LSJZrzXc_2PFfw*G|j#`-dnBIwKNgnb!QkKh?{EPCy#fxfI> zWM5urtSI&6B-$g`EerJJB*nh0ou@BrNPwC)OKo&OhvWgwqhQ#y?Cp=v3em zMKq&KM9+G_2`MBVNrLTCBiaF2XMyGu>HGcA43RvMDH@+BZsW;%u_(CYoVPrR3Q;@p z2?RgKQS+A&fKh|2qfJ8oreOXSUTX(Wfu-ed=e(8kcEMW(eElwd_X)nMH{3NS!2j++ zeq>D8Jr*pv#A}Nl78JeN{zf}ry(?JI0!}4it3BRQG1)Iz>cf_1!O}bpn_8_4n%9r- zd6=2a+iIr{2)0dO+jhaW{k~eT?FweL;D!cOQ>jAcx^U)ZA#?N0P9bv}+{~z)N@Ya7e-;wIC|H`p7T6VU{ft_$Y#QG;YjsXF2-Z#G2T*IW7Muj`pE2Cu z%9{@H+Cxu|N916U@z^acd7h6)sF1i({8^31q^OF}ymTgDoI_v8@@4+99;0O@81v@D z6f9ttc0T~@)Wq1alwnT+iH6`J2~{}yj6h3ZLF9o!%=RH512rM_U>`y=h%SzV8j3doXW5l0Yt2@_YLfr?DCJe+^|#bC8BcfOp_=IJ4hbD^b?W}rNR*5 zr8q03Q2Y=RQ%MAR_yqZoz4;{~DU`T`t=k1F(k$z)@dFRD?R;)M_y`qpcZ75I2)U>! zpW7D9-Z$QW_pCG{o$&ELe)Y3&|#tKNHF7=sOl{dts^ep4VFeCQ+6+M zW6cEW_G+FcWm;%RLdrfOq-+3`wTGlf-2D&RA_rJE|QbO@m^N63}71*5b_3EV4&2jET;D7-i|kqqfG0K#+pHbl~7w}7|r z!;cabI1S_10YL#$a%3-zs~&5zY{|3wwAVDRr`%3?)$lk8t{>^Gubs!upkEz+tb=P< z1sO75YkGat?M<(4d7OfG^eRJo>=^^zG0JgFc*m?VWWBcQ_4eECuO4`uig(gfhHOZA z==RX7BahSZjz#`d2Hwe38EmhyuLo`iUcLS}3-4G}hP0dPE0=Cu`udf}*?7mMGFV>A zeBFNA{%YRiod4C=)jl^-#NpjEFKsSqX(1(rq@ZwFpycYsN zt;7H)04D*b0DXXdz-a)QiIqXXM}Ut3Lj+k{EPOm1ITF6pj?u*ibl{opCZA`c@!)lM+%~ zO1>}Dtw8D~C7f|*azJQXfwZmodr-ClDVr8QD0Hnrx+W#8acj~dRINa&CMC+_<;nR% z(+Z?%#V{1DK#EojL(dANXT|e_niWXRieYG3fwZg`hLROX$%^5ZbZjJ&u5U@zx7@CK zC{a~^ohD|bS1Fz(*z&kTO8i`@@BS<=`ndL$Ewm_YMCd5ekpwTX;zbs{xoDmpU;)_0 zHZ`1JYJEP6pG+^CD1_8v6QN0Eq{hiIzcPnn#IRvHb_DyI0H~>CZ5@$Vm^^&4cHxL4 zoL?83glnmw>aH62efHO{4=P=KJ9;BMJ@hHoSF7ZmPIssy=x!{!Ul zM|Jfhv8x{1t<21H)0ZPNYxc1DVwIMKlV?t8&`Bcl;VJE)%Z1wYXz#^Mc-hn9TVlR4C98z9$W!zYRfgFre=B`KK^kIuv3S#o?oWutd$M4?4lGHgs*^G!^K7f1O?AtuOjKbA3Vb6TlE?U%#u-=a2Y~b@wQd6QOQ$pYnA^ z6kmV279VZgxq-?SC8~S_xC#(OE&Kz@v;uttM}iYh6q!u5SW$t6%Wnr2V<7PAven{u zp=`S-DV41>bL6$_jaM2UNDDIFg~P2E{F&Jv`fAVA&!exc^x`tj^~}kf)rt1=?IUYb z4(~-;X=Qq1AU-hCdwuZA;Df3+A5C+q?&e#|Z_QU%`bT?zAN*}_G@>2|skNPB(~dlu zIr~Y~3~`nC$+Z{{QTmf>@z>q~tOMYhRIU-+WP+*H$#I8?+suzs#sm5+%bUy=Pu}6M zxHKX-@Yr0XF!_OZO$)!Zlf2ARtFN0s*Zt0)s}fmT;`Qr%w$1$Xa`78suP9 z5zP{-5L@0Gp$U;2!(mPIk>gpbHd(T=1A$775RCOkjwnHD4xQY49@7z%gN;Eda@@`e zn*XzwSnsi}sDf)TTP0dYqmAT>9u0*LOm) z@fZ_jt)Pct>LU2Jd=+H_`LC~0zNZNKuxJC_Y&6`R^S{hyZq7Rxw~sUW)Qq#~5CW ztXMyUdH}B(vZPscid84)jxkq;na)+5sW?}8rgB&rV>KBj4O!1gXQbhG&Q=k9KP{t_ z-gh*$?`Y=r1-I7TGmW}3%UbTW<}J}OPD3ohHrB?s@=WWdyd`3WrL%$sc_&XF>P{W% zelbzI36z6owwE8|={=#;p3sYwE(2WLE z-Hafa&cla+FO@mhobo?^feM((*2iQ@3u{ zyKddS_q&Dm)Rd**`M35j+@HOwD8HbWlaGOy*O7w%0%1yBVJg!+wXVust7|gX>xRrz z^_0xhtxP?mQd`|KTV~yqyyj)=Sy^w@Et%))IXBhMyQ!n8eqo^S+syD)mKvyal=Y`s zdZ4O`YaHeJ#UVfDvRXFBDlE^YK3D5A*&N=MS%FRCZT1HWE3%o-6*hyB=Z;E8^T&F< z%x2l#=SqEnt*{cCN4rH|??vk#cb(8>)wW-K8)@6! ztKNF4dh^cR7hk!1>qhm~jazpt>vg9a9=7@2Dr+3NzV!@ZQg&);Y=jS4D>fQk=EQoV z(|vXs70(zEod}5?joemeyO9S!dR_sOL%N3)Jb?H^rLTs{k=EBCAczkF(gtdjXL>Xh z8vT^8L~fu(1@P%1KOJU{%zjo(Gee{o6qZ_0(4rVxD9woi36-T;#>=;+q8U{Y3Nx3K zXcl=E`5bZ!c?o$A`8@JG@^V<{PY;x6fu*BGq$Q+^m|;_5Rv2PVl*GI!v%=F%zbNup z;WAb|b2KZK`*Q<@O|#-tEm{dnV&zzkRz;R6U6uclTD=WB=lpA z&Cy}A@>AnY;lceUt+=%P3;mOj?_&&3O(^0C5y{_tnwB3oWk`Ws1pbe^S`lrcYbf?qk!4~(JY zE}``*Mo8pXgG??RtGjx1Il2;E#rm!#GQ}El(U!O-w)&d`rGH_dEGW^nFV%+yu^=w@ zi;({d169=e7YE9*7F`$D#iF=YQeIZTukYwUP39~8%QC;frci%DY@+7MfZA?i zN2|lyO>tpaiEfB>=+O<*Bdp{q^~0)eBHf}K8hdS^{H1s1M5;oQas*{-C*=4$QjWJ1 zIew|~PZLQJhrM*f?V&VfK9uH&Cuxp&lI971EX_M8ok;UVNb}A_n&Os})xtm#FD7yt z?)sLvDJ5}}*6<^kJ_EmFT~++-O=wc^ACIwn`h7rBL~F?9C9wgyyhL&dw-Qapu3i%7 zL{UJli^{PH?LDb~=la*tLZ>=$13isn>bu0MG9T-+sM0vc+Ls!)umbhrPelg&GFy*c z4sY~tigmFeUKXZEEC5WeFi~Ga-7Q&{l&5e`%DXbvHg-7KAx;Lqx7cbk*Rx^48XtGs zO+U2vyPmhz3FF*}vafZNa1|ZCQe*Q?*ALtsH;hxv?SzMM?tr%+hle(Y%|y3$tHa$O zXtw>>+G~fQ=lU2t&NANa$TrX9K$&{J**d_?j_^x6)8B`|H9Jn&bp3|gTX>H)SeSI4!<(uRM-w3kFU^* zR*9^EU`!`;o_&J{Lppn|*DFruPR8uLYP~9-*M@Uce;i%^adfphp6RW25Zp!S)f4Nk zjtYCb!|ur-|If6yGG4`Lxyku!w`tqFaTuqW`>5G)rTWDAW?(yyoF+ExxpDTu4Q-sm zVG!q}N(J&QE>5&)SfaIq*r1lNaRh~n_25Ymr=3p6^;v8((k-ywmJ@XS}x8oh-ZF?A;M<(i8)l!Wp8eTREb-VNKG4s_t2<9T}y6gJg?N`!Gb2wA%eW!d+-!6DaZT`BJP z2d=DgIfr;i?c%KKcU!Lf3UnxaoWW;s=ycpTm1MfJ7ht>HR-BO&-@X+aUfVy2vptu$ z?E~&W05tVuhqKrSLbnqeO#p@1a348dobiW^N&(c?-gATSpYpcRz{K;0`+t{sIQ6mk1V}ZOm&l z38AQXdZJEp9H}PKbUe>q#MuUi{5UK?J?lBGJ?7Zn%{Ui?4iA$Zj9%mi9;c)r;;dAz zCQc>J24gkCSp`9TN{XFWl4yOwCEc-uq4L;`k@gfu0=5(7$dBLAjW5g1LNduJoamGcB9>eF;4pClA@;1+)wJcv?bJySf z^|s&Uf&Jd08#II6ZrJpK-9&jw)sqEmb)LiviI6ZMc>vcQxh&Gu;9o!%Q-+?`^J-aA zQ#aM&PdDhNT6$VFzDcXvHzmVVOL|33tEOh@hGD2_-BMRn3qMmcG_+MW)S{}XZx{u& zfZlnvMB}O$RsIT+qBX{)ANAZo8$*3VrydooLHnPzUz#OLt)wk|5o@v3JX&ktq%mI^ zGnTO$-p0ZkS&-3l1q{8*kf98({v9DhR)z~1VIml?WVn#$g}i_q z&^4MyoMD{8LpHOOi+l>lC=me25Ru|HxSt!tAD5> zR460tK*%9Oh1sYS%|{E-Vzh*Cpn|lFw1U|H>;QVqXccLVa>!;8;Pf-Vtjy6;ze4rl za=0Qv{;CA|GZZPZk_7qt0H^1oD%!3kZAF#M1Dc;p@}nw+PGcTn)0i)x<_E@5BI{_e zfDsb;Y=}xfi#44S8*Fiazz)!DR;){adOq3&5MLhFZ;tC%h9wtRS!ny(u-6(wu=D+O z0NIWH1_A#5c?sXw#457@<5wU4^f@qg;S`KTTfo?*aI=4bf(KDyCR=){_45>R;xu0v z*DRl@*&NrboT<4GUSt)4$QnCK0n~e_U-mAwYSBfMTm;zPL~wFR7^wS*twc3nW2;du zyxhMcE{Pf-tu9DU(`cPWo1dWOD#DYb)WGRo?q3rZPQ7o@sFR-2WpNoX{n`o0`<>oG z;#*D;IiDj^B2p%@K!n0}PVsr~<~D2%8P+O%ORMSARl!zuryW+Esg_8`V_bPHL7PjPdWcDr?vI1)7QCvA5-0^m>)Zbs{&25H91l ziQFV|i^v@!FA{l)$jd~?H1TUh=w5^W3XvTmzeNNWP6~gG2;nLI1`#r1{BU%|^f_jdn@IN&)pTTKNwn&~f;eO$6J{qt;HX9i2* zj?nl5;eR~^E0Rexn?BiwPMwDj@iZRgK^Y`eLRF9*)`TixrIX8h*>-H%HqaV^GVpoZe%N)qq(*Lt|1sLfQ^Wl>T+bWrS?bp@Er|3>3hyed zfp%aj1+}QlvPK5)qlfq`lH4lU5yD17>1*(ls-zk(2%OPC@l&Br{$G~rxYER3_CKXX zdUSF%Z3o*ZCO{Lx0{|sK>$o(I6A%DQO2;9?iBgI9a-dSv)XD!Ak<3nNN}P#e;0>3LHkeVl+lA=Cok)qwh> zg~7BX)M3M#d;^S@Z@PQ~n3iuaA%M>*v?e%By$DWc!VEK|uo59cgd($_eiy)$WvQI* z>S`AKnUHTe9hho3JNSJ5XIK~ib0SZPoK}P0oON$}jR*0RB^)5_C-Hw6|o* zRaQrr9#sdLL>H&k-aO&pY7;lW2uuTOv|AQla!j zTvDz7imFb3&g4kwe#N%;NpNJ$2tLryFbiMj)kXPjf42~m&iJgPI$$MV&?J9CHUIzs literal 0 HcmV?d00001 diff --git a/utils/__pycache__/pathmaster.cpython-310.pyc b/utils/__pycache__/pathmaster.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5411969b3aff43f4665c0d3b1d592b6d639cf63a GIT binary patch literal 7890 zcmb_h-ESMm5$7F`M~b9AEK!mzD+!|{wwXAhB+Fk)n%I&oTdu7tk(f=}4VIsR)Iez5SS( z-_GpL&Ys-C!E_A2-&~pA{Zla(`zuw#zc?y)@QAlTSgaId3R9~}Nx2_mai)C}W7^kh zNoDF*tdM9frdPOadN#|uPJX*#=O3FqznGuDwluf6v~V-OFt@OnPTw&b-oDG1^32+| zo%G)!IksUD4gUhoe@3evD=E;fvfE5&$#0ZWoTb>nH?fk&(kz2A!Ln=+rOt-fFv=tw zVdqe$*eDxAIlyvk9A%nKut}5|cAi~8nPpS#BFaHF&E7&e#NKA_pd4m-_Abg1HpAXS zd5*o$K0rCjF0sof$JiBCK$&A7vX4-Xv#ac5loM>06;V#IIW~{-JX>JbP+nk*>^jOR zc7xqSd6C_M#g_c^rs?fJHic*NwQXOkRfOeho?WZ^LyqeirejoUdj_|KH4rimy{mXu z@Q6Qv9OwrzBX*DkRSpzT^*{xUAH+em0}V8BkO0*WQlJ?3MCA*h8o$CY{z+^y#?;9e z`cf>m1>YWTLra`Ll03@{5|8mfes7x`{S^Z`*^>-u0FK_a!oWszu2^%t;+MsH8~497 zR`y+|xN354JH-dC<8ona@7tmxio$$m3&YzL#WzZ^dHpO*b+_VJCbx^tPl|hWNDvk` zJ!^kg%oq1}>+_3qa|=agS_>!_PE^~x^M5iQslQ(??{W8;J!`VrhUGfWtj!u__}VtP zwOKgrC~2RBgjE_xw~v7}bi?c<$?X z+pk-)gaiBx>JDemwLkt~rxB11${L}-Qc zVUSlbK8oVab-vWkA`P!n6=hOVlN)Nb{=!cgz*g}LgVc#jAiC1?7uS`Xl2fTwmMYQa z*wMzE?Z~ib&vkzE-r0k%NxIy3h>=^dSIR~~ky9wB{2aPk)1i6D$?!3}`B`CmM#sK{ z>@|r?E6vG}>hPEo)$?(ZJOKi=WWJ7CY04h815zY;_e0`357Nx_YNYNibx90M;E@q$ zS)ti7<*CY-XczM=Qx8=ZKa4xms4G0jG}IHQ=TKMqG}BSnQO}|t=kK#5>Pgf!mcm== zkm8NTXW77^=BO-vsCkJe`nO3$F^V*)m+`}dvx>JEUuGF_kS>y?@P~mejb+=i6gCKK zwv_qvjwCkpQiYUvgZi+nU%*U8FuF;j1@{Knxvv#P`Y{+eDo1KwnvH#}{7PxJ$5{^T zx~ydm5-%0;TX2teac2&6wEqt66P zujV1`a^1T;TN=F2-G;-AZH|OVG|#URC&jlsb8V4Ff{?eLSA^G`f{W~`i0Uh3Ly`6< z$WqMpRaf{L$@8@*@PmxEexlA1&HeOG&1%EG$GOXWU1|}9qzq<$hGt;2X5bI(n8G$l zNNEJt016{$@iVPKvfCe!Y*JmOJ(4P|h$w@U42$Vtn3x4fQ36RRx~fs!4pW6%v^c1S zQq9G6dT)=R_7o!fh%%yrTS;N`h>}u=v}~N-v$7?nHpe%Cp_PEm9%;5H`|SL5ywKB% zeRz>^NM{OS5M?onjDK;&Ekq^MRT=*%jwKMs;!m`16Uf6foDb?t?GSkw_P)NsFG3QZ zCh`^$O0fAmAg}b$VJOeW8R~kE$on7#z2%0qsJYk<-ix~JCq%tk@pys8P*}%_D?1|$ z%;(Yn9FNF>bk0Hz)*~y9s#>>Aj^tQ~`oWLR0u2w@hjE&m{*{UPL6#%|R$z4;pyF2_14TBxyk zsWWFo*n2hG@eIeTAvtdEv90a3?OB8uPP^$2ca#C5|LAg%yEFMpq(H#hGao3{CM71`3`^~dWg>+93bh1igv;gUX5^`1PyljkeVi8y4IpYEDds{el=%Uk^s}Vi~j4p5T z>k)p-)+)7%b*<&Kmo(reNLNxB9?{=BcTek`tx9djbXbIQ?w-~;CyQ7=t8Y#e#jlZ3 z7Re}i(SqFBInkRY(#(S$RtmA=_ko?yhxS3&kS;@0rcdEi9w~S5BEx4AM+&a*TU1l{ z5R&RnP37Y_TspNl5^Akg!x1A}aiBEs9nYUySLYy8lF3`K9T-HTRx|kvW5Jleg|JGO zY>E@j&13mvkQ)ZK;?*PEX(6oCw z`=bacGwR&kF~igM$BzWIdL!}BG4Nf0Cm)F?#NHQ_pRCy)uUOq?$+jCKA>4y{4M$RI zC4?hSn5(bm(u7qtg{bUSkdwnPxUJV+-m@^LBF99+S_7*X@bO0v2HG4Vx8)$eRcVk{RIqpOF zB;9SDTu)~tnjZtrOn~N{kZGd0`yjvp*#MZu08H35QQW6MWLofx zY2ka{arfkUIwKMN1c;U*5k+xt6jEzz+iPn)4mM7?0H=>Cg=zYq1>N8c!%rIq5|jr1 z1!MSG!}vwRthRbMeY@e4L@p5dmFI=MJaH z&neDwiddYY0jFKaY4LMfcbpv0$>5yk7|OPS&vu-WCHkk0po>)sF*`M#((v~%l}nAw wzmZfXrJwYd)un_&+)o+?!}2uv45=cqQTqJUr!C8T27|{0eV)Syzr?8eKXQFD+W-In literal 0 HcmV?d00001 diff --git a/utils/__pycache__/pathmaster.cpython-312.pyc b/utils/__pycache__/pathmaster.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e70d54bd4765de88c6f75ead4011b07c0443abf6 GIT binary patch literal 11785 zcmeG?T~Hg>db>grLJ};$0wjJG@#DzXK|o02pTtQpHV|V%ViGr4na!eIK(_vJR9^*)OF4=iYpIrp*C1s`jaKuuEu zEttaeG;N|rD530IN-+Om%2G|67EC@%YS4a$hmT&S_JxAdb&(7 zeM?Q7k%MWAUlr+mwi)JWU z@;D2Wtof1+O6B5%&nkaFJS;_x`_NU(5PosD&6b+GjdcoQj< zSYTtb>={t07)*qM$wXin#EFQ}gcKMFCFYYefg$d~I5$aRSAZgrOoYNx0NIPE!KydD zpx8K&DwN>3RNZbA>(wfed#M_t)SpwC2KS2jFYTEo7G9Ow%hduUM<)03^sJbl*qSoh zyRi+h&SzH2S$+H6TkmF=y4yEy-N<;SS90Qd z+Iw9^Xaf^Dxv-U7PQ#yf`58r>F^!mhPf?vCrfmD`a_XBuQq&FlELP`Uoo7uzI%*nM zNyk;v9Z(Dv%m-{J4RKu03=S)d-%6OU<94JDt6)WA#qmarAe5^SSXNn(QA zb)i(V&LBoq50I-K17>k|t(d=KNIg{ThacjC20mK#K#e2xK?6rl99Ekfsm20+G{jLI zH|QPTRUVidoD>>2_-){(LgfZ0Y^uJH6|C*m@qy0zoxPY3TsYMw`L6(?l?3|gYha9D zmk!~?BAAv;LfKMT^lhNf+Z#H<2Pn)yVF8LBpeVKAS_CUlSb?GeC`v7X7NHy{%7MZx z*x<{ygi9+bpx_~8LdB9fY7*>A<^)bo^|^wfrQo@OQKjI4v;vn@bi#t?hg`C;WQqO@ zP!wRX2y!}rGt>l1@p4U=1!qnQTBua@JF$m69S4y@sM1nU;P=XbdSG-tjDZ(at6ZKa z&86f_)M+XaYJSKiZ8Wt&9cb)Y-Pn#qL%!!at>?d>572L!P@nJD`U~wFwJ~?!ixzEk zgR~7=?JG!Is#mMl-q5Q{tHoXyEsNG!(?!bHIH8#0iB!*CSLa)H}t>h(+i87e1I8|6qul6Qd#|bs}aT$#6pQ=ex|R z=(urMXc^d}<*qntOQkF*T~o@DBmAh(votJ5G4D`o%i__wRNq95g*O|D@N*)|U*khz zekLrkvoXTvh*_;Q8xO}4N`)#Xd`YSPvFJ5E91>W8Pw?0z!Y6uJ&~Z!>*&{IWkz?!; z==~TAy&Z`sj`S*(BP5oL3fvTd{iKv?8pf7!NNHFhCQ58HmSDxNLsBB;1}lkSsT9uV z)TL#+85u@V#T1hiGxn#LuYeuY-|38a3C{WCtC6weY8BZ^fWMe>!ai@D-J3An5T zO2rH>iJaO=HE094;C5>qS)weuLT#o}I&zd)kZ=!|O9r#Kfjv(sr5YxIB)i@0-K`#g zCeqz(s^)Or?b>j(ZA03#bZaToGJK~Z(|JB!&2HB;ta-loWLnL7{h7^RV7Lay(6W}~X>)m*NvXVtOY_NzONKUk?| z&;9ay<@cTUoDV+#(ecyNKL!3Fu-WwC%`s>UKfNheShjH>wQ%Iu>6tpp>zH7pDH%lq1CaU^>y;`iB0yS)v-)X%LB_sO?Rfzvv%YA z8&B(-W%u}#`U{zQ_tTcP`y=;8?vLLaUvK{L{ASDhs}(=HSh-_r)Agxb{plZURAckn z-0Iv~cr`3{zVRcs={mcDvQ^jKzJBxi?Qd>=b2s#adF`pQMy@^f#QA#0S^Jc!$>Xj* zVUDV}-X~5!;I^5jyT6i|ZrRqIO-Ay&dN3EJ0%|hyckltpNa=;QOrT-Wt=VJIg>wTwK9zO2GqX4`xVZO+Vy%Ua~A&AXHl?Qf>%``DmDr zJK139hG4h{j{28KaW1hC7Zr;X4~G)O zk2L)NVBe)8)tQ45W@nB%amWcM8)nQd?INJ0&j3Ic?O?VXo(+d5)AZ)eG1zWwRkm(a zwyGs?#OBj0=)-?=N)?d|E8BykRR z@?4aUz!6Yxakw~jZmJj7Me-JS&|^z>BblEF1lIua1^@~7IqEdXZ7l^*@C+zeU3MhH z*3yW&gRLU~43i>km=sXLNs*)tv*IYu3O~$>de_>>>d4yo>iC1^_4Awc$5$$zx|+18 z`rw4z``M=J(u$)fHvVa@U>Xc_W$0ydWpZ+an-ux0!>~8MFm_?=%-GmiiODhavdQs^ zeSG*Tv*FeHQtEIL<58y0-IXS_veZQI!?i&lH_Au3i;!~Sk*Fpsr_TU6!;B#BmhgRN zBoqk+-^h8s{t!ct_?wb`|AoQtp@e=vYxH|R`u*T5`8{M4U;fGOVY|J@8u7ua`u)XF zWQLClCEWglS9SZnyBOQA*YDXCIR#!t-T{zer7X)@TrVBcOFD3+)>KV!Qcb$_-+=kY zk}e!DxG!ME|OIcEi={5L>Kcs{A7C+ekwXJ%#mnTx7!k!_x54OCUFdR5?wKy7WVmVxc( z7Sq1Lv}f8c-*G;3=IR|(t!t~MW22@cTMxThwHFJ;F4K3M+l&YLEY9Qe4*nr9^8a^m z&5?cPiPW*Z9Mo_t=8wb#I8{i!{R)0qvf-J18s5-JAUir((hl;gms^@u9a&U%t<@WK z6ev?JUF6ac{yX5LlK^&Y>KE#W;$eB#~}$}&Bi&s=CgYly&vJ`#^6V};E1u|e~{0du3hf6*lKLu zmUi{wYITS{;)oMb2*G`8ENb+UY8|3j&liz#k512p)c*?vJ2@?T+V8)2@4fXe(w>vD zt!bNSmu>Bbq-7nKT%5#ac)h-rhFz|**kVC0p&)EYL3B+_5F@1~AO)XcyB;wQ8F^qh z%?^XlZ!=x8t?Q7)DUXN=5(*kE;mlPQGe3>Y8@2%1<|?HPdCvI{933vNF0VJI8wXbQ zTn7#mV_d;7FG-=<5S(*>55duWJVpuvj=LmHu{d9#INds1?A@+PXOM2+Bkmz5zx`3V z`y<)r-DZ5U&38x!=_ttp{B!~m_Anm}ih{x6x=WJlsEdXE5{2&jMWJ^GRJ*9x$=^5l z_snk@c}Pt6iL&kWZDvrm4IYyGwQxQ)o0QP9aG3$FDH?sVu^3-0Vh~02>P0N-RBhze z3FYm1-r<@Z%h#=IaZP#r%el-*%naOSR|$o_BJQt-bU9^1Wrbe<iH znNHc(c}TK2Fp=PbNdmtBGfbIk9imtUJi0UJypa0XA?w^uV!={uGQ0J~T*i@0w@%Xq z!~^KqxlrtGyYIi}Ums7qkFPM>Oq*gWwZ<5Uya0U8BALSEC1DwUtK#0R98_s^OY&(94EjG zNDgh+{^^X?Hz~q&|cb851k%7IrR6IUjPNgVkQ%C@D}^^ z_P1}}zV~)%j*Mgzc>dD7uzO)Xk@yE?!cP*J+jzxmAS_W%FomgArL5dbuq4x-C7AYw zT2`6*Ffp5IE@fA_ZTdDVdTw#MVHY2mytq_cxVAFCw6b`!xH!MKl+E5Y8~(n>SBlKq zx83Z&z&WvL5e+{&GPm)Hzk%3^vVzf7_8v>K%rm8&WI7vqmMCj1%W_ClY?zH8O|v{3 zMVes+b_%J^#@IO0AvVD#k!IOxHib0Drr8;!!)%6~MLNRHvGYjt>^1f}(ot4q7mya% zMfL{LQ|uCZ6X_Vc%&s6EXK%4tq!a9I_72iXc9p%0^fa4eC8Seqo-H7qW{d0^(lcy{ zT}L{@Zm^q3&$4B93+Xww0?EA}WVcLz|A8rdo3C#NTFntwp!s&K9^_rmH%!-XYI_E^ zg*60mbRPOv@vh<(e*!s3A0&*#K?YPgP(alK6*PH}1l0~S(9}T+G<~3hqTM5v{}NQ= zn+)wAC8iQgol2li|Ll(py}~8#&yi2^FJ=2*GjI#nqCGf=OWe0)%RexnlI58eXScvG zdboroAcdjL=6tE<`lU}xA8y|J)L7m3+|ruKx$Tzjd#=ZYvAu5#N0fy5#1@9XD@rew zQuF#Co$8+BS|+zk&G$-sb#M?CH+^e=S1gqFck2sF^Ye=(W?G9#7mppbdHXwd?hSvh zQrY9)6MN2Na}CRL-8q{zDp+CL|a7 ztcD2mU8iarzPT6Zjxef@+xR?4<7>Ze$rK9^%Irjppi41Kj{N z$2SZbPFw~_DCy`gN%=8#OqL>5Dc2nDZO6F|4~uqM=b`WJ&Li$}x4}X_OuSGwUnsnQ zvRRd%LL$HAJX&NNWkFupzR?p5VH0uyrFkYac(m0?A@E5WU6FF^fRHjJVTM2q6Z_60 zza4@w)d!)|W>5gMnjHXgW%~dONdTm5bD_~E`bVb*k)bnaox&>(-?~ELXUb!h`(#%5 zG*b^%mOM4#+aG+tpthnfo) zbEx^LN9nIJFs5Ym^ykULl>1wJCHOOzLk}8@cq#mM-LZ5w+~%dQ5n%hXnVE}{Qqq<`)pEo(kN&FM}JTta%6T=>*xpw|87<>M^s=Ei+vt5Fs9&&Xz;pxv)z zyNgGxwt~svq+9K@n^_MfND;4qz#>eJPrHe(5@p5ME__~X6 z<|EoaNN=w{H0pkk+iH6VWSZ|E6@=&Rl^)#L{`k&aW6ySNI1iira--^tQkP{*Jj>Bx z(X(<)&)#HY&md`uC*=$wqK}C#q&3?Wj{lTOgJj*^YtC6_H0X8LnQ2-p*L@e0PcaPBf?2sAnBUx5zwO6cMWw?av+IH)pU!c2&gH%?^jmf6wxh zBM4Ma1RC)Rv`1J6=>&pQox{Zh*`J!#hW#Pu9uLxT6fv8T7BtI(8mn8JymB&jKy2DhSW z$qBfZZW}63>S$9?3ksz=>I$l^<&$~&Eved^+yaJHEI8L2*`iRq^Q7@Xue5)6@gZ%| zFjHWM3epad4NSsz!7?GQN*hSFECpMZe58GqLP)5=X{pb(LxhCb`{_-77M%DwBIk)v z%+Fs3d65ozXz8#oQq>zoE`iLZc_)qJ0kP`K(u_r&JH@!aMoVZ-(Pu>3{T@W>Fl^1ZMQ@(B8j-KuB$3pdDIj(6h5P+{|OXU>MO z_iDE58?ISH#NOUx54YF1=U`qqIsF%@R*=+H6#foM`Bfr)(L$a@H%kkq&o1>sqL69i&23)Csl#tS!@id0mri93$!-m5t~KPg525f@moE1 zF-JOG$uWqpU$L6gA>SD37+Fp_NRexhu3#A-Ay+_007V`V6!)BT1DD_=TP`BW=Mlb> zi%1>f{zN*(=7&dPz}B5aH`p7&?Orzw z#_?8IpQ2qWuK;r^tVe@&tqgeAFc9}yEiN={pCE(bj%?#%h?`jEzv7L?5Qxs?4^c!j zYze}gj#r&xRCRihRVRud711`ESC3PX$fZrG<}n1L&6_9CgxHR&1ByPvq|zE4vZU(@ zdH2$hYpzu#?P;IS<+OYLj#p6F*LAcc&eNf-7-DclnzR|^Mx^^4+K5b6^x>+8o zdO}ojWDuNuc?VOzTO14P7MlJWub_x8p=6`pKH7IsL?gW6`cBQpjaNUau}D=cvK8R@ zPXyWdkf4tC9}r!TZf)(bu;PLxS@ypOj_`uD?E`J3K>_6I+-8pDBX*50Ss_#rE1Dkx z&NmTAXG5at?>#84(C|i`dpl-WM8RZ_eXBMW;g5k%h3Dw0RWTM}h{d2VGd0`ijujQ% zaJw)T)E$7*BcQ^TEZlQIEalTy)fB?nb#Pk2a^fDN?(x2jI1}3@7APG=^AVtLhNO2q zpFv??h-HO)XoO8w)3t0CS^76)n}=G*PKHd@L}W7aubfN|fMhx7yvA@U+-=aI(7;U$ zl*Eo|$<)#MSm(px zNB432!0RT*i4*@sL?t-UR~d`%9`Id`;CnqJzBmqpV$6|)GOPy2-&hetp-(FwmNp$5D2XVRrn>|l| zd(nN1VFX#jKxo#$zj#J4Y#5(4%xbHK(@h(nA~H?n43T$<&|Mr~B66L`4I&g<%Sez@ z+{G!xX~i7m(K$IyeuoG-08WOJlL_Lq(K+oVPHM|ZOE|3sr^$u_O?k~j|B7*n6^HN& z0-!6|bGjyfAL$eNr2G~1oSr`ZCppX}gN$J?=(52tcG$*^7wGS|#Pb0SESEreHuI^^Db^@R7H*a6R@!>?`A1E{aF_3u_KkuxPNU(&HUaXO?p}pBEwdb9AlqH$G%|4S^(a`<0nGc1q)L#)7x_&>~M zSqcBQu{k!6|J&IEY>_QJr#N>YwagA7RYvMCyA7!$NZrovK+b zQa-S1wb8y%Yt=e+m!XbU_b%5}8f^3-CNl9mgP->tNT3W7yNSN)D@}Ev1u9blrKv9@ zn7WYIRr_j?VA>McZA;+~1c`yMn|M6&%+h(aul4mnW7&h9Cp(daPuY9$kvrrY&u{${9ly-*h^z1eMaLTkOza;tuA z0}Xw*?S*Qm7iyhaC)8SQCscbZOg&!nYiH^8q3L(4cz4AJXQp}&ZR)gL^DsnSz*s`< zqPra$musypcQ3(@py8jGW;Zw zpiLGVd>M&A6&tKY8?0f2)v*oMd}CnJ#tf271GRjknc7uEsWM%`u@O-bzGZ@MK_wWfVpVu0lMrR|mv1$z__aPOq$pM#FM1oQadw4tQmvz^8W# z9+2cHwatkwja!{SnMbQZn|Uf@DLEyt@He2KPFpwpq>-@j6WjQ8Bmz@xV=~&tB(~8U z+s5SgvyGyzI=#7~lvv*GG-5BjUU6T z1YhvuNL5THx87~B({d}oA5LE8M_eLntWMQfrOk@?2%^ocbw(9tM0WI?)ACo+VQIJ@ zvT9YlDJ;gNXc3hiRCD!!nyY-3=5s%hm~r?6lzlys2Z<0ojIoA1?ZcOP)m0kCn^7k| z?I$xQm6MA8T}u;e@-Uv=1n2Q--^9~bhjZTu64wCNL=PnZ*~z8_D)LN|pw09b0PudQ zX%Esk(@J0WGlMM7x;nOjPL6MN{6^cYj-7Fy!yqcElXH9B`es!;_T2h#U#K>+m0Vby zY<#ODA8OuKn=heLarpTnrB;aC2NIp{-ZpF0eR00|X%trsXJ+jEG(5Q{Z^1(+Gkh~O zaY}2W#}xB>88xr+M^QAfH9880Tl4bM$Uu4~==2I1 zs0=hAb@e8}uavsUk<|6ACKewp(geyIkf!(<*J)E|BOs^ajWE}oq|gl_loh8bG-VM< zN&!=q96yOMNCedz9siACI%;*Rr5`ObKSfPnM}#m`uCdPFMrpz8XOVI;QGW+6WLlyi z=`damwO^)=G75p;WB z{a;V{UFb?q`Q1o4h4D@goOfQ#d#qZQdUz+1_kc`HnYd-hG2Ta!`F=PCdKbPQSDCi1 zE}>oE8Xl=fvup~H3_oX-=ahA0fj!m%I^6N6g0WMTKx~v;t8-uXPJzhTAj7_fYmCx8f-}s$*PrRQNB$3LBl!a7Iq*6%Dh?I>~fl`<9&_$C0l#jq>#tTNj7-+l| z=mQ;~I|~ih93+ETW(IoG0_XyGp`^@4@~#MgE(w?|1+zn#6=sUHwx@+gyHV#|$7t8^ z7h2LVaT$;{2~D2YHekthFY`T>{~*=_HuqM`Z&Za90~WT1_F|X%E54^j2y;I!$w-`5`DjThO0B1jcun zlcawf@lYqu5J$_gqA7V!b~tb$?VVgadYMm0)-H)iC%JxFmRe-!3h_&8!K*}mh)9jd z1&~mw$|)zKsql3wg&HRp;2==tEmZjp{5(>sEJcsMmTD{WIIcGS>6=khO;zOA{I+fh z1IoxN`hSZ)Zer!wGX6(&{Hm<{#@EWq6UZ!>{r}9=dc@S1ZenU9Vrug>GIer|e-N|f z4iSQM{$V0@B4np(A4B4r~F_dDw3}h6R9DE|}fW09`Xd3IBGG z59aWn4hq3M{lHU1$iwd8>FlzN5;7CZq|(`#27-V{$%W~LBH>Xqqww*$YG+rdwq;{%QO z73!R{cRmS8l|xQOLXtGSCUUEkBbXXJi*u_^HiE9`;Y4MhC1A!q=B7fqEtf&12Shv1*e}85Ou1%>f=et*P=VYW%Z=sc*$hBu z0ict|55Xo;Yf+9+tle$>44Z|!DzlV$KEXxUpK^yJRU`E~r*;DsPaoZ>Er26ZP4G$rs8`{Nt@7>c!%|@vkvuA59&n-XM z^%{QTvRi%%5_~@|+gt11jxRbWo`bzGS!pNp)cekq&rkr%E1$TbR6_j%WG7y0Q!{BD z?buZ5z2y8U?^xv%kA2`w`7Pxe+RFXw`()JYsbQ+s=(txJ46z$SU}mUX+{r%ya4J*V zVUuL}*!f=jSb3ks72FO6Aim4_=aJ)|0}1sF?rv}V?C0t8)wk{xWcAZ!&u1%T=3ZW@ zX#7R0Xw+K0&6;R=Bi-s=ae0|jh-pvZSLk^<4$O&ARI|6S|5=n{VjAl;Do3TpV{UaC zm$qC^YHMg=4qmTTcf*w1*=oZUgc?KxN2|Bi5zRudh?AosG_T4PGPmWw;WvE59mp0B4PNWO0?i=ag#4&-F77QuFA^e&c z#Wwj75OwSEDoqqYpJ7TvS*Rf!YG8v1w zIzoGzM{tI|i%MQQtQ=PE(pHp=Y6^J_S^ORT6`HKI=58Q}b=h^S;rW>m6j@Y?%@~By zVO~TAAE%J*ni;-`{1|+6m)L4eV4=v!QVc7_T&87|5MND77tJzEg zYDc*YD^h$vdX}Xazl1mbDS%rJ;FbZ1NL1LAZ$sY5I#auf$+;fm9c~< ze-LUA{DQbxkct$atWbl7N|DHkvmlK6ATvPd@s;ZeHOL;oz!s@AQhA~B-1{SmbvVOu z%f-SOvxD?C^>IYWNTtD*g(7B{!Z>F!zsx5S!EBI(a)Hn$rLUOmSfJG)Lu-WdC~01tM|CiRGgpYtoC<|$bgtB90q1IX%G73&&Es~4un@Rhu&aGWy@bm(0*foEJU73L>X6s!$|?S3B>7*0 zK-3DRK0*aYK|=dsau(LR?cO85ORTD*@b<*codWkq+`RHI%sVJ7k03FuFEpQ>+48;R zCmIq0*pf3pU`qnc$I^I!^uyLu{w<+%W@$E zcEpGgYCDad1Qk*t_#P2~EH0%4y!;BK1hSBOlK(0Z(gO~{n6NZGj02>)yh|ZKXO6tS zB-TC0ihL+un1a3$N%Ij9#o**v!^c9_OK|!kjps{51SI`ArT&6QyxCHg+d{4j6-Z*) z*Qt_VfWM}c-~t)6A8q&Nh!q~hsxd&tNcU_zU8YEh?Yea8^1I7dGtQWXa$#-v?i`F(N(BEOgms4Oe0)h zQYE@cJR;W~A=04)7)*h3Q}*u&c!8j4?1Z$k2L3?VrggD8eA#VIgt$_@min>pmxG#>Ix^AP)zLZMnR!wMo^ef3D4aXCFoi^Vq^s;diFbs?g;uS8F*L4h;-b-x z{gH=OR@eK-?T_@~kaCIj;aZr*UBbo=3?y9DYc)1HZ9H%ah`RW=_%|vNrixR5TUD}d zwJlgzV$1$t;cZewIV_JfNp}7Skp&tECVxdZZ)3P|%tqVW>|P1gdfPGHhC6GX&`)rt z$&P{(2%#lmL5M3Curx59iym}-@jr$*ze6uDlo_3dzDOFpmZg#&1UykQ_@|IkQ7gi`iZ48IGN@#I zNBy>L$4EoH-8g(My5Lr>by)f7_-<19y|~~ufjWfe1oi@TNKd4TAV3{GMjdf2lUzjt zQIw|U>+=52HQcWAzyZU}jfPRruJ1ou375%wm@IU~6{=3*%fi38=uAfa3fH9OY2 z!~CefSKvHz?9!I#V8nL+M%@d+B#019IdMynuA5b<`~M=oA=hvb{IQ5HaShpyW4iOd z<@a0td6W(+LB zVYoAjo`xJ|3}9Ib4uf=yILzRE#PKuWu&keL<{-VY;4t}K$zgDHHuETbB@W~NjP)3? z7XKH@PBIq%SCl!{aA}>HIB+elS>ME61wP^=(l@2EhD@Ak!|otHD^2$P3cQw!<5_}> z96O42ft%vkR{TW7w(w*M44v8db)@*p%ZpB7vQTs=uD^Lv1v|=7hwq|^i9q~+P-=?9 zrxwdIup-_HDw3;&U{JVDz~~fPF9H||4i8}a|3s6DdUg%owtqrWB)%mR_k_u6mBB4j z4GRzz1fawXJqrMBYQ>14N!C(jw9!!iiAOhCfT>D@4em=HDPf5`z;+a9Xml?HHks^>Dk( zwp#8ZfSZK(5D2!ykdeKz+!T#A#+tKct$8bNx|c(zo|MPDoU~i*snJT2knV|l%8Ba>PxsBPckMNS zyXTJA-^|W@`_0TZ-~7IB=G(fu00rr<|9Z+dH&E0+$%~a-<-+q%p>Ut#s078)oGVM` zTnRUAmQey3b)JOR$bAW)kuwQDN)Q+ov4T02YCbJ4CIYm z1LudliEHG7kT-KpTpi>soa=om*1T{sFobzl;<>1hk6xZFMu+*SX#AJaWAW2R2TmV5 z5se>>p9lobu|;WCz^9{JdX~=zo{>3B#9Vqkn@$%oo4%D2(*ox8jsoV>88IW|Q&3>D z+0?YrkP?f9f`BC*Nqnv_oJM6Okjw*rH~gPJ0CJy_XvoTG&MwO?+5G@i_sCKowTPrD zDRV@EkxEsyRvksjpir4!fC|n<=u5JK72Y|oRhHp9CnqSJ0>!x}bbnS5#cW2DQZG<{ zTAcsRr~7AkDOF&lSy8XIT1BHmcL}17NRy7{giK!dNCHm7i^$INF^|qz>^d`@$?_?Q zoza<$n964I#hbbpO0$JDCSw$dx}qe#xSUMRU}2u`XSx1jTFB@7d9IkmyvVa4J9%MZ z&vMvl#RFBHC@c7`wO@9dgIL+kBE0c zf6b)hcY!QXTW$Naw!;r+pLo@_vs&BPrHQS!@VyW3CqGY?`qvJvGirQvV?d2xRNF3T zZI_@K`IpC5Zm#ZAQLl!273+Bp;;5g4Aq2tZxlCFDqZcs)lZyE;+5UQCRfAy#I0V(W zj*xmn02(OV2qfmgdq|n+8MhG9O2`{PD(j8g;8kxlmZwTGjzINp!Z`^97APFm!pGKn zpA4(vAuT+lpw_MMo_ja$%b&|i*NL^<`m8!Ip9r#cUf24&!A{@pd-B#NBRCHKF zhZXC=VG#OrSf#4PNENU!#TrY6{V+Yy{$9vxw6C{Slx%)sZf$%2nd{M3vmRyP9UyQY z{KfZyz@oHwYwai2K6)~*whwFV!wTB7)xPh`k*~+U7*}F%D(5aKSFWk&t|^~PtLLWG z_8F~xhVYPOQ5rgw{R1i*)X<<}J?6UDKn0zUvH_!g0Gdu11sJ29Ft+HGY`_F?gm#X8 z?0NuokL>y+0O_J{ky-RF24t`7lNs4B2RQeap2eW-!c!8_L^)O9S4Og@rmv7hc7?@h ze^sg;z12*~Ar8rI1E@Va0Ch+ro2H!RY`GHIG*+$Qd&!F&d;q$5RB<99=g4T_05JDX z=$;&##zKtNeE@}+oiQ-k@@={wEWz|@y4Q4Tx@-EF?z5blFULzblP>T6ASNorpgqdO zfl#^m^%$VL3Y_jg&r0ktW^;V3PWOJCl5mkHIWa{8DQb)qA0WMS5z=Y8yRN~I|Wr#(-Z z;uy!?^4Zeto$=W3+|@O*>=6J$IQB{npS`CZKo^g!ZYbcg$4yalBtS8eRGBKEi&L`m zZPDkL8!=WR0p9p(cdLVwtnmNGX(I*~aQ0m@N8*W%#MR&i@R5C1PBJw|{Irp{P-UV{ z+2^Ti*+u4+-=Wuh_uiu4F;g4c0Dl>-u0&#kml7Z};B&Q&arFzZHqxOAE8u($k8{jU zeZ$G2@p01wknzY~W16}%c_-%>9f=Q(RBIXFBQ8outX8lN5~LVu2OklT;#MnglEBAE zB@*1cl)wu$mv?f@4n88_7+<^fx%|@VlTh9W=UXRU(7i9{nC_X*3as>u>~?Slh`pH+ zzjHk!#{Z1$DB>WdGc#glf!BR`2(x)nf>TgkCNJ^0AY@q*1nMZAV+#v`%Yfa{EIY+# z7aAtb5D|`9Mg=CPn1#UY)BW-3#qPlE#b+E;=-xC>1nHisEStU#f?MFsm01w-5}ReS znVEbJ+I3G_$O>3*goxk7zf{h^KRAXI@-wl$x|bEDTLoTsi^ZJo$`{B`Z;+|z?m|}5 zy)c6!oYlOUoH#4o&|T@A?s+>~(B7rE(=4U`ume1?nn^GoyUDUm| ztlXFk)SJ1-e0sJJ6r$6T>y5PxIzMsU}1Q9}5l{1{@3O zLGnp?u6P{uwUa|M)l%k4!FeaA2gqol2E)+lj84zt1bi$Ktg<{JB45X^T%-Phj8rSK{XuL!tu4Me@Xp0wXuIKrJfwu zPL6L}B)@-5UeYEL8`rdxSJm)UBh2 zT17n?>QPY7t7>EwwQHzdLG1>I(hybAehuwc(EgviRDH|cUui8}rHfk2LA9<|tLxpS zkf+NaEA&>=uDiFjaPPw=EnMyzJiQ;9fvmi>6>7LMdiUZgz544PBS(IS98r#*TW?x_ zYrR!FI=n8bN5}p?q)c8_lGl{fCra`YZIV^Tr?l}YHImjM>CM=&%}D3A$5q$$ZwknN z2-&U>H+1q-fbuZQqo0i}k1dTo^#w_wad@lqkk&c0KDsfjc3#msuP98%R%h3r4u0GB z$3ErAuriuZuJP(9ugvDv(Y)FzXq^J#VS;ypO6wt&>C>1##d@}VAkvU)h8yjdF`LO} z@-tC3&qY6g?Z_FsJvt880BPItg4qzDFhUOr%m+db7iZi%$cR9ddHb;F+WD4)b3aB- zNCwwa6?)9sw;wVkRrOg7c&~T2oaeFkrGQ}P9_Uy0P!lRRN6h*DM-R3$sdX5RnO1@v zms5U?Mr`44>yrTqb8; ze2I{c2{A82D$;gDdVCdFV@&1Z49-5V;q(ys&bVfSfxZ&<@pbrGBx`Izx_LUUH-x0% z(-e_|^T0tyC%yr?+Y#CgIJ^soZ@918r=fjNAXTF)zy0dUH^)n#ss{$N0|V;bL2d8g zlLZBhsc1|?V^A>Jl3$(trlpir_s6yUaW!&WiyVJ)MnR)08r9G!6pRi&8($t@8YjhC zfQPFx746p0ZUyZ&SXbs%v`a&~6tv6e!2xr$C+jp|}Qk;-ZrSv9n`ou)CnJKtbBqiNz=B5!5GuA!%Vy1~8)1da_zfTE)c)lvOyQlTZ|w@EcC*0V|ZmyF*g)uLF> zQ|eu1r{^ivs94Xo*Gcm_Kby3!##23EMIynU9gW z#ct1Ovr85;+ZjZ*=iu&>b6QM08Tam-Gw#_*hS}N8FrQ?qMs~}m#*;9co+Rv^Y-s4& z&5}(v_g1M?B~d_6&+lZig>~PnSNFYoRrTKezsL8iRuch7;HS$59=$>k{~1rDL!Zh# zbZQ9VIzbbC1g)X9!MBZu=xY<)H%uOBWLvG>_i?G^SFL4Ct;;Ye{`F_al!C;A*T1+aw90a!|# z0G82afaSCWUV&ia(&doaA+3hA5YietkJb(nbkRGS z0UccoCAC;WJLr-@EtHf(Ngd#pLAru2S74KL1(em(PP!7(26{QY4ARC6+Q#aM^``BN z*B$cG&cLYi(9pQE(|gL<`nzdYeCev2;cxWiVv^#0fkayH{58rZ6 zqk+$LdpzTe+jGGc^aL0$Up2;fJ-(nXFzSMgdwAG2AY5<-$H&G3Oo+EK0guZ)?n&Y2 z;3>3l*30XllrQoOyMsaBfY0L&iH%^bMh%}k>^|ikcAfH#x*6Yu7fPI}s4m~gDfh5@ z)Z?WAN9PHi^=PHB0jUvwI`};t6^2Jx{bV*4LHMz?tRt+YHP9-R zRSc&S@T0W|-%GU`(S`|HR|+MUy26Ct44JG$>d6k`#Cn40hk6lRgbeFw-RTVk5!76< zgvmio$Rbtq=Sn#_h3huyOuFBDsi&sN37AR~Fln$~VIr*C1nHFxBoVSpb^L`=PENyw z)ItPm`5p2VDF^RCKVy)dR6|@K8jU@C{?LUnFEi$5+#}wQmkGKakM z!J$y_9`>&DmWJ2+MrrT)Mhjnlzpz^)!!N;0uMbtPV4^&?oF@@ZSQ65{u<^$?|rtyggCAi6u*x$}4YfxUu2p zrW>2shE43|UiRRLiFbT_YHDv-#lyW6m8`rKElzq;c66|*H@KX|)}-G7qna3@|o#d-(0 z7YDeOL9TvqDsNi%!~EL&IoQA_7Q#__efZk&^p-^N+UUMT(w-#CII@f_Up+g{l4T3z zix1%m2oKIt>A)8*%-Mes=dAXrb5{2m=By6ptoBpRSzRbSPyOk6Cv_F)kTf&=d2*?g zqjl-4={YG~BZ!c|*6`e4AeTxxspd28uaWS6Rrep%Mo5~Z^^odAY1-f~mij5Dek5&K zN0yO_x=$Mo(eH;->7eN zkT=03_0aE_v{+hxg)Lmd3I9%7H4;95^4ie8EKd{ROb6kad6RuXGY z*cy|z7S7f(Tb;12PuezdwoP9@@QuC?`VzLT?^pkz;X4iNL06*tWU_mR>mFi#rxV@7 z$?hQ69h?uHO@?0OLa!#eU%O8bwhnD{*P>(j^d`1=O?2N003Sm!iJ|30OMyM07f7HhEGWVjL(>W zzx4#*8PBi%RTH3R++QZKzZn8d{)V4l|Es?Lpdmtm9zwpOiR63+20(>#B%&TMJ%a(5 z!bS;KU;z3arXGlaK=~OYcc_941cvDrk=P;!$QRFLJ|JSWW?{4rOW_`vFlD)^c-ec^`rxU%y$=(px8)C=LC3?>%dtc*vU;Bh0I<(uhe}~C# z?Z;vQ;@y7D9_^CNj;!X_agSdfJJV|gcJ{zPI9}5JJj^VuMZ%Xcz)xUi1EMth8`nT( zaw^NNAuIG!IOMVk7>1xrrZr4|zJw#Ea9yP)ty5~5H2Ib`q6?)ZTs8etRijK3_6QkD zOP2mJ`R-ByfAdV#8@;E?FJ`T-1$){$vV^xE3yhXsG-65Gb5{0 zx@SxaUDmV!b;>B1)l#~Yz)^XFScK*mK^bYxWmZGaL_LZ0jfOVz=2WdGO$2$+Gi$MG zGXmsd;)o}iLzs6WI0gVzBsA^?!E}SV4hcmvpEHY!j)1`Q!6i5Rf)N0~-mLbkCoZ3O zs};=~32RNXbFrWzS!t~pb3<8BWAns(nA;F=C4 z3Jynif09eU*xYz+W71yF+3VxQUtV@=8SuNt7r<6>``GPn)?S~m?~Cq{w3Mc4^8(rQ zPoovSfb#ftj8+R+myr;HWjJHVnib{DVd#}|%rZxzfX`Q;xIXi^JpoOL;}-l51aDI| zsy2A@g~6-9VopN60Q`cJ0AK_cR3-~*xq{k%dEninZy!w*tcvaibvfy1^>Ogm*2kXkaBZH1g9bDh$3V`h8&tBSi7RP}pH4P+ zaLpaJs}s#T*;So)B6k98NmHWa6_zYY4P~~VhOJw_K(;+)Fc19}kY2EQ!@tjIY2APp z1&8!GnRTug&j)oNpZxsfS_evz;T_FPT4ez_1uaa%;V+SLa{8E@LiH$zlxvI$oN|k1 zE(ZCcAg|oRT%(THtl>2+yrxwc@3*Wnjb_#W0$(H#x>P%#cF?Ek^ew$|)`+c$gMSF` zVDbcpQ&up(__4bk5=eQLgi@3>9a3$MKyygYgYN=>IpL^JI+{60^9=LliCYs1M>`Df z#flZlicMU_rbNYNmMmSWSa$QJ8!z45b7K$NxS8F0kUerTdBn%T-&P+x%y3(oL`4t~ z?^hG{ns-{J_fN0o3hLR$=fAdmu4Ar-!%DqB+<$;Qc!=G9h}(LYwH=9eFI86EJa*$) zva*G%Y?)p8XZzU7mPF+tuuP|?Oar@e%L2LeDO2Xy{|8eBjK800iqLx6FsS{7rU-Ki zMu}(vNKXoe#xgGgB>&7H0v~{2)(EUn9f6`-@dAT{`3qRJj`avRMfS(b5V6=(kz|R!cgFo?eFz3u6 z7^>m{zCVqmGes;RY)LqJH1xAi4MxFx0q>5paF|NEpf>Aka41;MZY@+pV=_3VU z9dkBJ`g5|n52tCB!LIPvsCa_Z{4ZfUZGcwOyhUzZP#KH}(_g{<8Fb_V>8T=x&rk*m z!*=OTXj%jREY)*M$LmLY9wq>Z8xunh?6FN2Y896UKM1DEmAyb)92r#y)yK;UE0UGb5EyjAMBU?1Rf z)4dnZ&8c7pRI0%z)FPHr7{?&!+4wG)iwMF9keM;B0^l_+@sW+GM`3;kOTpYIz5r2X zMBNV+p2rSBuV=1eYY4s!zu+>MvQdK1TNUkEEG&-hT`Y1$dlm}{L2k^=kG`~+XNz{- zwU<92a`csx#wGjm=@s!6346;VwN$h1cH5mc_V}s!8V^S~Cv&FSVy&?=OS$E-&gqwC zkKU=9&+VPm-KC0>R0T&>#E#8VD;BBZBvr{#l`-Eu)v!p}lT;Z;mBrfTsp>^4?`rPl z+$mHkx7@}FhUN%B`pC-qOElZM|Y zoiz3^XK+f6BS2ElV0W0;5TG%Li6Z!20KuovY(7__143BS1?KZmX94Tg{;DSVmpbC^ z`1ZH=%+=55Zu@PS>S@azyt?#M?*^uFoyZVy(njG3l_B6j$X1qslM^vT%peA(i3kWd zB*q9T1RSfPZH*kL5Xn`v4dsuu4T4j|meIB*jPZ!dwhQF8LunyO%|DIe*+539)idz5 zVNimVp;?j;5V6bsenuO=@I4=UCxwxsNU`#r{E~YBK_{6{o%OyQAo}Ho?SncH0Q16y zkeb5z;UY-QVOzKuQfs&%>=@M1Mj(16VFL(<#-|1BMM|F@l;^j>Sj{@Zr8EVtrX3^l zWkER1$sl^ApTYB5k`@ljp5b>b3ztgcA|)J}dYBM&NF;jaFn1mSYG5E6lJ4`qU=|r{ z%*&GNL<_Rn6BvcyPVXrE1ZhY$D5nJ*N3RK!DcCiZ^A=ICOF3GVXRUY*g+2LN`mB|= zrUby$#f;Nd<_)};6M!P5Rdny>$+DnWV38tQ(eDgFhb>}@%pYLw9Fz$~JUKQN&s+Jd zhM>2KhFa4NwaRU@s>zgsKq(4^C`>X_0D|ardQ5zj3q%J|TI>{k#qUC$|Amzw3EXC> zs0M_urbN*e2pDSVW%r+8U8mV`bmO*mXgRVux^Jp~dSiV5QhsG@IKKCGJA3@(e7-xn z>!Z(JGq3WkQ#VI%jK+uOHnFR^?wpxlwtxQM;p9O#chJpx0_<3rJr%i65IePp*jaNT-^-h@WA7Xoa?AV$4Eeu2;fy;SZvmIT|mexTdo%Y&Zrn24&(?3dW1BD8 z{rv+!c;!2GN#=1*qGh-G+60#GCEa9>oT|T0a(=Piv#wA+WNzcKVgsV{GB( z#p1f@*XDBAqHT-zs_BaOS+<}Jq~lo0bSs;?Vllro_WZPm&0G2F6_- z*~1kuFtT{4MoGIPB$yd=Xl4j&nMhdY$68rOT3t4T()aMA6FlosQ?Hq@;>9=Qf6HcL5IWwj#`e^Ps1a#0G z7!Qq&hg|5SXe6_hL8BA|U#uRwt}tq$x(C|0ZaCoK4Pa;;9~ot;p{$cZ#?feEQ0C#w z)Z2NYx{NWFOeNMd_(macIm9f(){M;Sc;RyhWI70HyAZK&Vglz}qJxf^1(e5Q5z7Rl znrdH|uRx7*7)HTskckpY*6QiDc-vfP-nt{&$=jZfcK^g^UaVW4tn23Lx)XJKSgLxd zt{$Bs@Akjl&$jMidmu(RnDm7?`0EM7%-RaH0Os~SO()WV6)$PA4g{1`T&su;By4q) zhPzZQYK5`o)4OK-Cav?-)L`KWEHRTopLis82II?qeiiaL+Sp&Nx}Rp0O?tP zo_fCSnQc6KLNM=R+fSJdHgPs6ddB<-R`~$H<1?b}$umOP!kom#;;o5x3DjsLQbRULWZ+j zdl-YQb_0`x*^x@T8iD1xf!9HyS2KuHrez4G%qsMu2>z4yM8yj%ReF~yO;S!U3{CHk zU$|{zDd#-3dr>N6mv`RrLfHa!NS-?2htl;NJiu!^Iwy?19otv;9@(B@mBtSwu#y;% zd?hRKqvV)%$gxJiP{?SJtOPv+o6Jzu^fR(0+n{Jeh3UZZDIHBp1V7VC4wiCrC~g0R zsCD>Ic!2Hv&D2^POT?;(o-&A5p0Fk5Rn%mjOAMnQ-iK&E51ZA(N7DeNkqqpH$FXTMZJDu3?;`_yr3!ZL6R|POHS3`BK-9xXp{zp?_3{O1n<-G-yG24& z8wb+ch1m*t5esbxZ@UG(|9?c#g`n08dAjJ2NIL(EdhnLW-%I7)f>}Vo2eW`!uQ*&d zmrJ}+ilXH*ko$leF z*V!9@Wq`OU@BkP+?jikz`qV&RG$b@oa0nb#nMf0s1N*i+x8uUtptIqTrjaL4fe`_! znXsDZmYXmk(jE+rb52mshK|GAoi92c>6+IL+(V{&OUvgB`$oOzd^D_<)C)2^uQ@%D zzZZlsC)Sm!7@f8KV@X9G=W21lVQNXP|J5CsDp+ zR$)1k5+N8D!g52j=!Vmc4TpFGe7JE~tE}@355W3SSYpZibD@p%;u|tt5L11~7aH~o zi^O<6;~oXqrv+A+!XjiZd;nNbY8(Nx*U0z?WMJ_bW(>pLLG)baDV|bA_~eZ!A3`@8 zi+RHZSI7%qMM%*F^Iw<3HaLVgo81Nb& zpQ@gE7Odjq^@#5iEmcBYcr$0w9}BA=aQU?n&V@{lP|XFg$jsluorA>yWOI@-Cz_T( zNldNd4l+5w*j|KRa2{w%l=#S!dv(|4T~h-IOJ%hFu0EgDm*6@u-!^YglLDz2K=zvGVErs=oM@gslSF?(iq$F2TEar>ox(Vdg)rU!&MfJ&|)m+i)ujpo9`V-6CsYFr79nHM8JKC^h z&AWQ!@{y^o>6UqG9iHb;mBcnCthJDbH`aN%b87AH?~c~PW9kfVth}^x>ga;Da!IRy zqyAF;>y4OcxYRJ&F?nXPZK`AH4ES=_&uf>b%71UuytV}MP?5D(#}6!M*Q8IE$G0tL zSL12prN+sgSYNVo4F`XgHM46uOWW`?!p+Uu1j5$2d8ZCWCvrmYe%Nf%;tPZw%^x7 zeQ@BylVGPeP92**$gbUfyX(%u+nd?_u!Q6!>!#UbURXNJt{G&5p#|+YbTFwc;Isu( z8{ax`v+qVRXR)j{S+ojV7?jS?`#VUB@~2 z+jyLPh2}QWi83!BfrHt6**vu`ZvT*4b>B!BOoD@%r+U1Du+9{AE}$g(o}v1ng+@?) zv;bk%VoDvL!C!-7NJJyE<}e}qUeq=kkQF2FQ>7R+EWX9og+tAr28s>Dhf8AM<3T`5 z)kY5e5tI`uk^`!bS!zA3m#@Q$EvrkZX8xjX#v%)1iaV zAbOW%2w4`zdsa~@psmuGGWqF*@S^e_(OLctxS=Q_eG_wkhTty{d>cUmz=SO={74cC z^FzG&9}vtVxP#z32)>KpdkFqLfQg)RRptj+$RhYLg8zu%KOwk_;3o+F5hC5d0$om_mfWtVJe_qgBazQ!4>&GVhfF8Ld$9Jp6*lhlO<# zpYND*Pi?<^Jlgq@1%h{GTEG1Kt>rjN^Io!`sc;4TWF2VtBo_T zHXJJor#10Y3uFtP?z*&V^7Qn;0=X)6aq7hdayc56e+jf+Etm*(0D-RmJM!5kEug_b z;UwAyG#f{haH!I35?1rT7sCVSjck0vAk1_K<_=;?bD|AQP|%BrA!5uhZh-QaeS{5C zo3LItP4OV$F-Kt_F`>B&eiryy6(*=0sV*qeqbai&1b-^Flrg2uT)8Q;mof8Pq5?v1 zG4i1xY*q8Bgo>no17#P@Ue9Pz>KUk=X0;Z>`3ek+Duyx$bPQB@`Hh2tKHy?At+Rgm@?@N z$tm2WFie3-gHo8RpiJsJodZ15q_|_6FDTkb2iywS<}Vrqvki=HJTx%ct~?L#Mjp8; z{lIY!yj+C2(o@42mTNhM<}v{AGQ0j3V#@5=mTph5buDl!U|F#{LLGLY8Qt0baqtcO)3t+72S9 z5&Qul^Cm&h7fd3eOaa^0JzI$vw>^;GGUI?+F|4^Bh7M@*dA2MO;S)+?_h%Q#Wx)LF z8sf&;lDX#-wL2dYCHh`1#u*!?3^D2>Sc}8SXODcXWA4n{j`#Z$%eGzK zH@S0aeQf!nCGTqY%M@zz@u7GhyP^C0Yww(AUv?*U!{X=xwtJ8r9GwrG zNe0Hbz&LyMLLxB1UVN1u#l3R2YCF)4*R%~`k9HqA^KyE%_gnBie$s|-8nx>u2veV4 zTw-&V$3b3NuauWQhMm7Gi&c%us?A*0=0w$2ma15)T7L7~jdM4{H^OYwR(2a~7UWJI z^>gsI&CiYmxoyEjRR|G96`6&V-JTC22-^s4{r$r#qWHKbh|}#`ZOeC6QWN>G2im|m z?hWYN>xTIE^BL+g3NC_JMZvrj)MX8lI8aFdW*-6x6!y}KXkg(f*2_BNWzujxO&nz; zS|MtQ;zkI&DjaHomZETjdmw-i2QY<+!eG`x8(?`8Od33Zk;u5YVwn;`;3pMIi)?D^ zLe#Qh0Ggby7<({#QtOIi>T_2qirm2=^l}vW?HIMP}9Ng<2E{q2S#TX9#V> z-6TQLXq4^+A9>R~;NTq6Gmw&xG9qXF_zZLAv zI_w)99f1qH4c1=4ewd0P(Su}z0AZ0$xJCpv;Sz+sdx8x4mvGC*3Q@emZ^`UIfP)er z{+tko{8w1?*H~omjRc1R=XkAWgx7Dw%{v8i4Tx6g-hvn;SZXFXc0@ms;XL%`bP!xI zLaH$GKy|lN4Z}kGvzR%N}n~b&mnj5ConXc*4}fiS{E|2z-Qh z%2yz`U{}x)5a^(1P=z@iWKinkHGbHo)XU&2#x;51u!F5(01dk35}kK#D7c}z|G3GB z*#OrB@LfbQ3DVu7#WuC$@}5|Gvb>ptKTGqhowKZq?p$B4Q+}K5F0un4XRq+d74}Rn92WP(#{vgaA zIL!9D7RZy51;;yWSRm{1Y|o`VulHayU+J~dx7NlkCd!{n*qWlS>n;TA=Q2H=kHchq1F3wZWDb5zC=RSHG`oui7QbxzKRL9vo=RCPQB_2(N z=gA6?>~if>FC^`?oV|8>KkO5dur}ZlyZTAZlC5~^V$#vTIU3?N&VjvsKu|gLlO2nt zjq&C2{qb6^boFcvSGs<(d#TV7Gf#I+pPAmll{6*_S5EG{TiZNa@Xo#)jyVTh2MM<8 za@W-Um?rjOvV1iM8>5=$b|%WVEjG3+)~|-0UH{gov(|rXhCAHXK?goAM#-(ZDOtUP ztKN~Q?qsQ|rRv&u>))<_x9ROBc6BG)1^$eo2Rv0&(vl*6h>lIAU+wN(58y;Coz+Zd#ee4TAG?k?}L|toiDx-C8SMK zZ!$1dJREd71SxHoC{ng0=A~YOD4P}m)r35M`Qvp#WQ9$qpZMji87xofqf!&(^mOh_ zUKY#C?_o-jJfS%OmKlEw5~czRki{`5RzNJlhnQc9AZ_L64LflgFW3@#4{Z7hdlUf_ zw-y%}h{EEZCo+&UqY!m}B=X|!T|dLzeFT4l0PXyIzTDnZh`T|tpP$fxOxbrJsYM+_ zq{pJ)JXhEw7PlgFVf)NZK>7p`lYj_@`zO(4Nq19N*{>XabM4ft3HyqKwLZRVZeoG# zMl4Vng)8BNy*6R3n-0%yS|Gbp^^{N4U;7F6O2}#%n&LniT}b2oES7ATf%% zA41%_K#atc&6YB805dqe9z3KJWX)6w?4X<`BCsGY8_yKhf}C3g(y3WCmw|L@mIh$N z@`N1;6nhq2Sq`$KPNf~#B|*`S`o0C(?ZA>kfsj69>si=G#wcKE;P}_xC>Ls{3%A~q#J5c*!v}7M)D-ggZf3ZvgsE7~tJF{B?>xT6i1d!PRqgbYV z%vLOW%O^U_-x2>-9bW5mzK$b=53ix@@az+4tVR3n3hnXNyC6y~N^ z>WTYcn^NocfYzng>7}m->-6ACmN?AY)3?KY7Q}5=d0S?o(4eq|BJ3cC`+)PN)EPE- z0w3%mf?e>Qf;c5a0xUt15NEx$AcK1j>OtNKc0vXyaajdRIu=XHaQiHK$y@eVf1-3{ z!u}j=3}-8TGZ<_C!`BkFdXQ5cH4rVMUo~mQy%(Bh%?bOaNotWQPg2z!RXx3Nfog(q z82u_RdX`ip>T@FM%J}Joy&X}dF#M7QY9*pRcb_7x1?s}N3L}{8XUm&s{R?DAivCab zv85|#n-<6|KL^>L0Pic3{ioXm75gr26ZT!&#w4nOPp}CJl4YZ|v!_uvQE@1>W4qHm zN;{9Fx8ijo^A<3iPr~R(jdZ|T$s1iR8bZ%pybTmFjO82lor3=-0du@TOod?$n;DLe zj9n1j1V~}vD%{x}WKb^Q^Ma>`L3bW)miFloW(EqaqMHEuwz%*TwQasgLWgk+QG1~8 z1!3!9F_eMk-Cj74r!p7}PC!X#{(1+k80x-xB!j44Z zTF=kp>AMKNg5XaPV5A*`zJ9?{FKTrtK?sBQ79qTEVR%2HHjM=6@nP>~<^)`aKO*=| z0QYqojpk#L)Q~?j5gO-@h=LyxIX@!of2Soh+tB_D@W;fO9}_P=G!UAi9}|uTWUglO zgOxVT+6OD_0Gh0t^$(n8&8i0#X3d5N6~&qk&4X5i4_j+AYai~>h_Dj!TQw%lmWS0Q z&0!5}%T>2>mi}Sa$%n*7P4oS;8ltIX?hxDC$Mz4f`vxBpm6~R#uv)YBKCK~Y+vaxO cp?`St{6nHn)AsO?#;RHUNp-1at?-Ed7h(%}qW}N^ literal 0 HcmV?d00001 diff --git a/utils/__pycache__/plot_save_func.cpython-39.pyc b/utils/__pycache__/plot_save_func.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35f1877ca03919df7f433e475300e49ff6a28de6 GIT binary patch literal 13081 zcmd5?YmgkrU7z>P&OUDUZto?@_Q;lHtz#?F*>P-Habn9>LSbKwqZl&CY*yQIce~n` zdgdhEWflxdDi2CvUasK8u;=1+p$Lzvg!h}IfPz#}1y$61-~$vN4n7bdq(A|}@83N; zyLUR-4p0TVHUH`E>F(+2{`c?y>UyEzDERgM_V$$<-=QdfLz&5+iOjoj1!q-7VTz|P zm1%9Yqj|b2(}rh?GRw2XJ>%K9>+NjE@tjW1%XRWzzEkiDouXIlOnFnHozX6J%3fKd z&GvL>#+ym{t9Y|$Z?!9(Id2YS8D@W2@#a~UIk+#d9LwXr$O^28`vEq^O1K|nWj2lb zAvVJ*xZlQR*&Ob-vj^D%Tl}!<-ND|*4zPnrx$Gf!h~0+NVfJ=*JG%p^JK4j`WrvZv zi_J4_Rbh9&rmpDhE_MWEN2rY5&5o{WD7%LpL)qQz5msgQB6XC#WulG2?qm0(^d5GK zy_KCn>KJ>JEwKlXsxs|;O7-;S+nh7puZKQ!dtLY3+PZtzzu+!C;+|YOb>g8@Cm(c| zPAom>IPa>jhig54%4Ll;zw7MK!Yo&LM7`av zt%wJ;V7=e(@h~p%UZYlDZzT1zltmAh{a8n7Jlkm3gP^(6Y}7;93AL)Kaj{*$;J0fR zn%z2YZu%&3_pMrMb}rQ0^=`vwsG~Q6%MFbNoBW80EL>-B1s?{9)SQ`qV_sp}yt1VYv`At4BG_$RiogTSXD!-l>xP7 z{c9>S7L)cm9yJwY&N^hzgo@5mA&oy^>8iLdO>W6l+o%nyRp5} zZ2PsazKVvS-w9%^+mH2by&LOozZ+|P7H6NVhxN1c`q&D4HN3lO##4KGj~(i?Qx7mi zUcy*n{i44So0seDb$?soE{bkE^4zKOPc6Uux!RevUibWybBO_rh9GzwFmm*1L`4{f)RHa(iAngH8i_nWEb2f}Yn(nynqt9F^aRvW8Ai zl3xaQT4J%hmyw7xvAlY+ygHUwn^;~wG=~-~%P_-qP&+hR*)3I+s(VXUwSmD5X5wy! zPAfOeM@D3VNvyCiEJjwOZ;ckzT8^D|F9_PrAPj0ehmq`j9lzKPdA-?fc2{EubJk>n z%eSY%-f3-`VO~ZHK23zao>z#>fy5@Jv%V2$xF4+5`&cWi1vn?pth9Ue@PQ?)Wj!{} z)i1RDIJd%kotkJ6a01sjdxc~D#o}hh@2rVI#re?~Y8V>7jb3?7G#Zr$@l=|ub-^jS z3PO6P-~dSuQ`@3g(6klx7vw?2Qsz%!L5nJX2>Avr-RMX2W#bad_$4GFODtn1S;h>O z(VAGs%r~-(qOP`gaYZSyz|D+c>|1K?EnT$+Ho)3=O&w$t_OPQY*uxR*;Y68~JctReUM=%qnziH7kg z>IB61l0oiM@&8Xl7i{u&l->m9>26=c(>Dh5(2SI80BfQL6##aoWrK=5(~{^lo8Ts6I^iTBg+C!xp3}mN5yH`j+ zb*KxOYqSV{rOeHYWo~GEFw86 zV5(Z=Z^0NOf*Q?k_;8#{THR{u$J5N;N=@%2LKrIN*x>J=wBYr}k@E6MfBQ~kPNE=b zFhL!)1L{ao^CZtQ2UPw9a{ouu=jEZX5ida7+BJu{iE{oJ1~@s1{9P!BOA}RU|Lut+ zke3trBvM{^veSKMvm|D7qFN8__D<|&%0y=_jertC#7%F}p*=QMilM3M03ZhG6f{UrbF zo>h{44$xbo00{0cA6F|Xe=kaJy;yxb{oh(F=x*uqkP5!4-Ew*OS-e+@=yXB&d#LQ2 zh|rec-%R8gB5q0!BNZRo_v@wkvhgx%?3x6ZCgE9>o^23|j()R9kXj_vpua={45dQ7 zFjtjg#hWsoX}l#NtTq_Rj&iIjs>nNpWr=(?E*%1z`j^V8;FI@0-{N5;?q zsLw!ywuYH#hFOu(vH|JAq{fqm_fB^qbJpwb47Eop?t+R8~zW?yW>y5h}7 zf^^$OPCJb2L`Y;oo$mMjE{pS{dT(MUouby`oirO;AAn-D4&D3%V0xc<8MQMgtd`gjY;QFY~#?jwbQzWmZngQkzU@A=+s!__q@|Pviw6aGxl#T9cog zNj`;Npi-!aasu{)R!&OP&*KV6Ya?k4TLO z**5%xL>ffglnC4?j+RF_IyU|SPj9&v2i|BcT4>#{K}NrWNTh&r4yY!;LIYUX0xY13 zC18;qID;G@guuceCz=3@0_v-gj`~H^M=GrkU>TG}eKjH{oUA_`Dxuyo2!>V?kB7)> z5#*@!fS|9LP<^9X>3b@J*})vWXWteu$4C99W0(g}6{DF|9dI=jRd9ErQZ$QuE-FWJ zxEG@7Xns{^8La(6WW59^1{A{6Ggu_A&d`n)nH^ge=DWFjR*-5V3|?B+6?yaLv?Bo4oOWl*XVWO2lKnQ#O~eKEs)v+d^ZqP z4|;hCM2RssnOmY~1Ww~;X>Q3YB!HDX+?8G8EA%y_`FdmTpEMz-#@Hnb@*^i8*|wey z^b+NKm0D*B!(_eP9W8l#I>KltI$BCQS{nDUv>yfv02GiUr>{vPcT*yETndQ7{`?vq z2FE~@6z&N0j;c%QeXv54KLF1f0bUn%8zehMKiXiaqDgZlC9-7UfdTl_Nrp!nzYys| z6>MgOT1y`qk;ULrY?)v)T*9wJlGGpzl)#5sEQ8HuUx&XN`7G+g6P-E{bIa$zUb)_S z*k2E8q|#T7y)q$_xM97y{hg;1>J9-pgtFf5HDU|u-+HIZ??BmEPWE5b;q;yHf&Kfr zrPN$RBYuEdTFq|Y^NZ3f@pY==rbH~^0ikT|JCcLg zb&2n=sM?wkQ>vqtRK9^e{=cjnCkqCXkV^147xbaSzzyCEH-tFfv{(I}OAZ}RZ(gYL zOK9O8-uIg)T;P)i1uZD&An6Kbk%{kq!#>`4G5KZc`_?PzmE}rKjErBUCLbc=rbMjf z-M3!NiT3y1a$zHM2b)P>e&2Jk_W0S&%(IW5dF!*!pV_6A%%Ku}zEG_{2qiF7hESKW z)gb5L?N|7rw8Rin5K;|xoP|;y0T{@)Eq%8F4kbJnLIPb=p@_eT2tj^OfV;_(T1u1` zwxT+jyQZDjrBWW{hE|w){kn-dZY7Gj1J8+6%x$z)Af+;pb5mRJm_S8e4wY6GrA}nS zPniqzG%f~bHO7v(1LVEZDnxdYD~Cl==pl%^+wzObrxaF!oJNYxe#&5_S4|iXQ>|&pa+E=m)04TV!sf3Vs`8S0=@a59zJ)?< z(X?oF0LDntnjK_^uA5?1i%|yOdH~u%c>&R$Ee*MYl*6dt$Y-}vIF7m3Em8XrJIwB+ znTWKPG=3hyeH(y#HoU!c$Iy*t*)1gXFlk3XewB2Km zzyFN;4)=zB{DGAnGMl!wINNS^{VPp|xRWVRI#w@kmYxArxzu*lB;y`A-|rl8cSty5 zgPIU3Jlv#wJU4A+q|^yb5f9F8{ZK0>kU87 z`rY*otWBs~G*DhALh_v?77F{ZK`6|tRDLg!1$wAsl+xbP`|U8cuyE^$hw6>?3LvcaT8bh!R%YUnLWD zYT_&Rf#^6xIErCZ=VEK47Wx53wIK}A2@#tU3@L*-{6#T}4f0YTV%X<3`cQBS3~z)cpz(Pb4XH>R9*tG6#;B{ z09a{Qj`D0Cz%~s?ToAyP$2ezDy2uWsz;pvqZ=P!heCt=ZFaAEK`aCQxgH%Uq-fC;J-qp zze*%kUHPw3_ScF029e(+@>@iHo5=4FStYU`)`W*OM z@_JECIsA*Ly^EV?Q9#ZILha!5=n(=>C94GDPXD+C_DwB{*06%krVqsI1Zxx z5FyjouPf9aX9y!+q;g1=gsS6xR9S(O9cOcF>@O$EUDKXKbdOXS9JH9m7iKZe8GK*< zV@fm=WuR0bBunXQ8p~lHqv0>5mqhAZ!L#a68BG*co-x+;;>Q6w4FWY%tcdl^W}gccFQ`bbIxx%5Of|0(+2ES>3fSJvqunfpu~9_y>Su&jV84z4q}f>-c1 zNM;p>t@IX-T+z|30gO4I8;l_wx6&l8NhBi2Az{%$1=1R^3i7H5%rjfUyMi)=@5r;l za>7(F5sFq*1I8TYgdwC4^I?Hb5daSn=@19RJ{!tXLzaf7I2B}&V-V>nL`_7xdo-z+ zv`*?-6ZJ;v5))Av)PMvR#?>^wXNHv)(H7X;0xHWABPn@)U;~?h!}N`;TtRdwtiS=A zlmHhAXQwa1SmLMHOjlzS(nDIa&@D1-b~r~-9hg6JFspJft6=`XtgUxt1# zFZ6?j)*_6jJX=WSd@<1v%+^7a;&fw*SBI8wXznqwlVekyQj+?@MXX&_+o5IL5nJga zNX411U&q;|kffvdC5bP;vueZzse$*};@DJdG&*&tSJrSiYSa19spCs4gS(v%4dD3liS>bkGdTU&+=N+# z!-ehUYPW+2aT(E^5JwfqMZ!Sw%5dCD=B;%E^Ga;l5j#fcCWqvfehXgDA0aYF1Hs2% z5$@qJ+*r*{u-3a0YmH89K7mtqfzV81TUbwU1R-Q4>;`FuSXVKgivcu!ai7p zJT#_b9%Cc)uacqC;}~N+zh}q|KiD^NH7=zHYxv)T{4wFnX(SYNT0NrW zRbUA;{vvDx;0W$S^AsM@r%|IsS`+RfT=gG$Gr<|^>BgaR$$>n#-evCd=~*iGX&lJg zg*k-f1oQ%PNKd4LE5IBh#T;>nlw3msPt=wk8uGl+HK35t?#Wp)Dp#-J4`j(ej}qlW zOCXC19MVY37a9Z3Br-?re07**aLkbAz&``M!NEQ>!(5Aw#Fz|c5ydzPVjsc|X%$gQ zdd%odVriLw38NMp5iwLDH2;hWXy5U>K_J|Pj6F&Psg@mEPtj>694MtJ@nCQVfjqK% zlf;~oT%B^S6oSNg@(`0yr-fpj>e+E=+~4bPoHcR4Omr}2x35t5Zc2zr;-WawNJryp z)cse8U&u@+ej#pIAZ{T`QIj5F{x5iT!@U(RJf0kK-gQW^h$bmN+&~5zu!zVb=mnLE zRu|YyX zNOp~Qjbv9A_)Uk3iI*r3*I&w8=Fk?rh10F*X~bLR5O$>CEl9P5w@m(3oCwH+w+dmQ zRfN|9jUtVh^1HF zT_`#fhxCG^f|KN^LqZrC&iYqMxhWC6y=Srl6EhOrhdPRIi4gEnte#vTpa8x+0pMPS z|2NdHYG;@Cy>$K$c#MZS|Ef&f8)s@Y28T>7E<;?9L#y4qfFs+KiY<8;i-99|>z)3F z)Sb&{ji-RtdcYScGZkEHL&A5Dk4`0s_0;i5>dv&IWY{i4)n`Z5G21vp0jU(=aI7%w zy%LW4@9vAdrg5gffg{hoEAj-NL;c-Jn7NxpGHi%F- zfq#L>7m1K^DrAs=3{KNHu|O_m#VXmSJ3Y4E_8+6~3myhBa6CfB4_%T@h$dp8X6+e! d&Mw(S{1@#-=P3TJQ+CRgM=P$gGSkks%YQd-k1tyUyw*A8hMCK*~0rlPk8)x*KC z(EzI(q=>Dy!aG_k!ePcT9Dc4=Vc@WK$VcDyKltK<5AMT;BYbes7w^?Ni2kw~z&u3V z+=$A`uBxu8%*x9As?Zu6vlKl4`rrTI{rm4K%74vdN zt@s0ljj__F3M+Y;y|KO0-uS*=E30^^V(#c3b?q&FZ^e zV7-8t%2riPjOY<-zF>HJp%WoBq0no#H|s_4!{<#fRCfa@d>iVDl%5(XdseUzEAv&$cqKB zD3qWjh2)a^O^g858uS1^ndWawC^>vgU==Cb}`uc%h z#bv3r*}fvKrW%9nm&GNik4rTEkKnot{PuNK3G$a=S>eAufez?*k93LF(9Sin0_|KQ z?L-@?WkaHC;)EFA&%uu8*~#8Tl;~`1P7xQ;(%7fA^URj{$ezVX`fgu)u5k+^s*ipp za^RQQO8iE2sdrh}Vnw_mOp$s3a+S&*nX0Q>iBaz`H__|uPIs=^VxI57HPs)sTa6%c zb~?Vl(T^=g?tR}yhLQ2NEzh6 ze(`to{Q7&Ch34}_Zv;E=q-NWV8eUNMy0dq=+Xx!L1KSOl{ekN@7^*OljMr|}cQ>ng zVlXePCq^e|VE(@6KK8n0>sAnXoIbln&8~M#qkd#TlG%2n`tFxTl8IW8>+?mL(Grnm z5cKIr?u(ar(51WW`Q7rt*lC~Lx2(71^Y&m2`zP_mpTrmJ(Ma#L!tgpuFC3b;JuK{g zH26I2py2o=Qb+HjT?qjzBiG450Kky<43$Ys} z1!+^Ed`rp)W;7_#T47>P%f#4&#U*<9G)%H?yX^%mF&PcCod{7$&bHJVq3yTJob zR(YI5J!E%D-U~WSPi_MpR-fds4|d(Qmt@jRceg``+i4~_Iq>UO62or=50ZS><1Oa_ zccB0p`iaX~VuX>`PK*X1L1K81T|dbMgGQAQcI#|=VRTrU>`sd}Ax^37w<1S2+35uJ zT1n2%K@W}(^;O?Yrqf-&`?T%xL#ivL4M|JTnw5Sac{6b24hne%;`4dZfw|N0JtuM> zU^Ouy=8H)sGmsfdI5e-+4op~!-v~M%Cs}a2?K(e2om2?$9B<6e(jZkLWChYSyviRW z`8tPsIPLJf?>4uY>-d+GLKwL`N+&#gksCP4NV$`|w4DZ&p5()km0&4gTVqnI#FE02 zS&vM^2?w^})Q84Vq9#7f%1PJ0IC{rtd_l?vMWnNkdLJ~LkW+E$tqz=R`fVXCY6Q&t zI4PuykWM)F;h;(G-hj#atp^R*asf`3HR|Nu!lZog;>W>;`u=RZJONGZ!!B^LAU%!8A>j$@; z8@sLG{!N#2FS!51R?y<1b9dJZ8{z#<)bPXmsnBGu8||mbZ2F4+$boy#Fyc(2&+wl? z<}!v})Qjqjq^54FgP(5DPqp-{YP`&<+RKVzsug`f&8ntm>4ssbS=~~XR0}^-Gc>eS zSCPu9roL;G)Dpfgsuk*4MZfY_k`(RHGyUjW`lbT@U7fz6d1${q@wGXP((D}OW2r^7 z)?Q{Y;(}^?%|F4k>+&oFhMsdsc_az!e+#$rEx_KApxy)|*5U%7w+`rCL~bA-L!Loi zLJlYyk0Z|^FGtzET+hTo0PumCEbr%=Ry+Y9GQliS6q*Q7Zb`3zt` zK);0RlW`@Uj%VW8cn*+!9%%t-5&cd}SZKye%#4?jmLT~IVBsf#I@!J1-W=6O^U;C? z+KUoseZN*781yH=4=6fgT_%h;gbQ$rLqkP{O$Zn&>G(g?o22fu$E#pQY{*y#|=br5uU-8v?f;P4;~)_13{+DqOEhv2rNY|uv64aD$_+ZIp5wP^ zfC52lL@PoeDfak)igUma5wx9*AFQs#8T>~yfe(q)i2M-|samN^I%K>GNu&yOsrDlx z+eGR_93np^LRU4la>P5HBSAjlnJ}syioJu9=>s`asbs@s5angyubC8MDJ=7c=r*yW zm}z`fKTOnnyhTH{i9DilfeJ?Brb_-9)d?bhN`%5~O8yw!0LBdC=C4r{a*#QrNU>}X zEh&Oph>U5ZG>r;smh^=|3_Tw0DAG+L)=jE2h>n)7{SV^Z%mUgkP`n-Vh2FFZVr&@^ zwW1pu|8opWnmqF08cRMQ*}A9Twok!LU-mO)2bWn)|1@9InZYu+dow=6vAULlx5%X? zP204tr=LqoJoQI;NIoG_B2~cG_B5dic;fW%`=z>t<6kE10Q1V864P;TSd1Kpj)kG; z@5p6OEF3@w=n?14ypx_t9p^y1{O1_ORxuoHo*A{f@QpLa#<1@?nvEXfDcwK{vmip* z!?~P414{?lZ8wB*@vy4$=O{Y9s98E=`Fw3Sa#rikeRJfr5xO8vRkI{!fWwjFB$nd< z0OLpmzUVlQImuBvT8jB?dw$(?r<{VjfLGdiARLDuI*YPC>4sSJWd1|t|<^@|ieY?AA-Qv8( zSL{Qbu$C9v0icY$zgU4G%6KlZfDG#Eml$9LZQ=8=mCM9A z_zMeEm2zn;hs}#vwud12CIm-0cqIK{hM$iN1~&C39Qm15-y z4AY&r z7$F!9qYY_BG)k%+(JE=dNP&{-ru8Fw4aO-56;2n96e@XbsA#0fKo|(d${6lwMo2Aj znqZ7~2*$(~Oc^qRO(}ff(v&Gv_fY{sd<=K$8lf2K4lz>9lrqIY$IO_aRH9HT0cw_y zw15qF8r#BXhY3&Fe3!A8WBokK(2+2G9`1g2jP5)@ckDmh^}^xKy>#c^&izK?asMno z8Q~7oOkk1?8$ZKTydI6L<6mQ@KVzWu8Rq0O%qCQ_(=+{nETd3|>oLOTH!WCLyaw{= zYbu-PGgXjIWm%19r6M#dDfPM5~YHH1JgRAshP%m(G+ys{n2=DczR zluPo;4k%ml%9VUmmKZTrp2@4%Xlxf_j8(zygu7Zx2<41bun9WBE>s8(q4HkoN3fDU z&aS4oEkh?i6-?@R#at?{Vb9>kD>AwKRB*-f>^#?^=9DSBSH8`%YxsRRY@ucw541pj zv-=4;-j(Z@%L$~K0_Wrp=C=wip+=xT(t$kSPFX zc7kARYO1kXzB|_{=BJQYZ9?@8g&e0@Dihk7 zn%t_0*XPE`<=(M?jTs&1r)n#X=lRbZmKp~?Z)5wHT0x4tgpQqMuJVPuGnCYl5%$JCnU?%iJ(^k$ zu>-;Zp-R|44{Kr`b_Y&g9ipKnn-rEt$6b^fA1ecx!%pCjS^-D`QnSXcbz3 zc3TE7KUMcpuv?ZjOMejPXB?wNXq|`MQtY6xduJOG4yt9Uc&qk&9x{0~FZd}Odjal4 zI8|Bpo}GJi5!*a&Uj~mLgl+p>m1o=5#?(F0vmHr$w!=^HY=@u0v#D^N;MtyplkeG{ zgp=>to&o1;dA4rh@YnEcM-`Exy~4(bPNN~N!YY6@2K|+F z*&}#%mVnTs)&Z;?e{L43T@_e)k5DU=sdt;OSIXP-F!k|{T)$in`WCYw9C4sKT~PHGc$%z&lvADsQc{f>mS<*AAJJFhObTW*b^u= zer<}!pFpwcYg0UtuQ#=wg2fyajtj?x6XQAvb~JirU0{~+%LRd%*?GrwgkeKI2xkOt z%tV4=-ZwrQ3boJhvLO_SMnl0U?~|z+j%9)YJ{So{eX=!I^38COF~3hHr&*p02B6Ur zh=j*yqk!p~_Cwt**_flv;r$|nOUP4fILdliUe+`03_mHGCb-BPKk4Hj1S6XQk!ctr ziiya^u?WwH*f6js8yGG!qu_jogBzYAnF>x%0Pp^Am<>g_QmB*-VP7ERk4D)jR|aJc zAHz6&hT`y{XyX0%@b!yhz=bx%$pyjiIA&Ev->t@42;6KFrZ?f|iV_%qOIoz!eqaB_%XXz*vvVP3Z2PQo_ zSbkTY5PV0_X*d<~E+XtmPxJJ~G1x zr)Av<#y>3^XZ)NW_+h!IY#jg+e2~9A3?nnMA(oR1*;`=#QMn`(@iTc#MAnZ(4KLgJ z{9NevFdvzj0nPXNL!mK0n5ljS41woxyiV40{_q3~a>iCA8`$vdG^-eS0ob%$1Xe!j zpMmyNQs5tp!n|gu#bqpV|Z z`a^P2IE_>u1*`Opv9Ne^a+C|lBOEMB7>x%So|CdIwfHA*&#+vcC@6(nAD%Tx!3dNy zc{1!%mO+%m#~uzpBxQ0u7-D_Ae*);js8QJtvw~^njhr$W5)3O84A0(@3n4c-6X5XH zTrS69((T9`-Ys)@E6m}&Er&Oi9Nrsp5Pv2({8Umjt*rT-7P@Q*a3C2DFI_{(KRw3y zeW4E71XJhu)Fd(mWo62GMHaF_v5z3gTQ)?qmH=7+`zclginBf}?FtsF-xp2m-xtUj zfIVmVc*dZFl=`c3_30In*I1rwOn1U#ROV{)yWpnm3QSX{GAj}@$5n%bQ9NK=;bs@(baUOFcg^x!m2RkSc}Se%Rj{@ zc-IbDGb3A4l=FtSC>E5@P)R;Rcxt8F_{Mbx0KODY`j-{?>#r(_IgcJjbmZ~nbUjbTd914z$nQmMc<4;1WNkSYPJ3%Dn+>;V?qbu4I9;v(POVO@zaX~vie;yyvQwBV`tHOBq4z_m z?F1zq)vr~IO8OmdAW4Caj(y8p6x;JjGjl`6b2B5jo6aY3@~UpSkzG{kR6md=F%plvOaEOnx# zL9#R~3;+#lsgjhH1vWy)nKDZx)$lGp~dc4@FvS_3xik>#&4d!b9&+Qrq%Xc zcIDivL$r5F_D*!@61w^-nw&<~uxJfS*6-xwoOmwcsRM+W1Kc9&4M!efAX4><=T@qcCFLuoL`fYgsarLz-AL>{fp(urS{z#> zQBfOjlrXD|-~T1D^?%^)s$wO4U6qJ0&kc(wjiGJz{6wNt6m4a{{#2p0hUH^7MYzZ3{q zGuoR<^PJK=r(Haqw7bEk?adqZwuHTHbzHP}N%pSA)1aBI!)tx(O+WK~>J_Pu#s1}U z$lZ~oYBwlPg7SRQaIbCMbpMLjJSa5}iqv_DI*+LHNvdgsYEMw@pUmEip`P>T!X-33 z`rsSlp6k+{>mucoC?BGHK)tkV@nq7rYs1!%ur-LbCdt;c1$Mq$0n3`CtzyOfLBsnE z?={_R`k>|gmiOB3wgK9*c3BI(Y0t{-Rr^mXe^UALu1|YJ&k4zM;{L(k9QoCeUmpGS zQL*NdRC5VY&LriG8&*&Lz${YjP=t=f_6@2gLDh(qTcX^Ea^t${4XPnQHHZ|f?j}Sv zDR^&r--L-fCCZuZ>55xLsuoe$YKz_AyEf_%CF&1}_1zMc?OJGNqo5SRqa@+6u|ab$>0J&FyG1vY`y>iYf>9lnj=bcM7yj{IeOir<4WUby(Tms7MW5L5s?_nSXz@!HKeG6N zytOVKj0e{W|DO6=>SyLp&G+@cG5*TlG zh-~?iA}ZTfUtAklzx|*ACW8FnJc-x9*@>TRo05Poi z3X3M@pvS!C4DL}ffJF}w@`kfG4-lF$EF`EpWU7sKt+tD%Lz3wbEM-G!lCm!LJt}}4 zJgn%3yRV~`6Zc<3H3MSJfK)TEQ8SpR8GLX>thp@JTt+(QX1(Xbb7=25bnz0Z9~SF} zrTXEG`YVb0E9j+bV*Pcg{yIFEH>+#!zJfeG_d8H^pIF@|RrhUFpGj1od0-c-FG$s} zmDsge+w@@zYCrwpBB~t{Ylo!Tp^e&0iCXYTSH;>9sdfaOMtM)2ho(Bz(2aUV(C8d8 z%^}@f6vrU`wW6 zg?L_D3PZutKy)2ia-YmV?4B4Veys6ABt##bg_EPgiA?%@5I5m{K8SJXqHJiK8wUci z5n?f!^OnJv3eEU@QOqXnnisf5$Z&rFV1CzQ2k`a`$p_FL#qU|ZS|H$2{X26_yj&z3 zC9-jY+>;>ptQpo1i)6n<_M^=IbHFA7I82nea2`wQa|?iXVg6H@7MKsW1O*%j8uLe4 zUxS_3QSX|PbMuihe_29y~nDh_17Nq~WHs7R3}EZrx5$s`avTa`LK zDQKPvOUsoD+GoKG(|jJM+uwUX3k9J7zQ@orn!E5#%}&mub2jO8 zZ#Y{M&Q{UcE;-wiv}c3fpP=`P^Z|)JkUZRrXfAHZuS4{K-wlkQR~T`Cl?K?2ftkd> zj5xqa1678E?0H7`(D3%u>ft}2iHM%jVfh7z| zx}mgjy9HfLpC42gU@tCcL6JoQd|st~$pNoJ@?0jf?ZXHNmxF`FH>i2O*~`w+IWf_5 zL4J}RV|fTI(Kx6<&-tS?mL=nckS!J>*9*Wy?xZ1^%IF(+l?~Z=688h#5$i9C=bP44 z%I|QOaT^9QZa)hE{7X^UaxhU;gNkaB<@BOqldM?L!-m$g_98kmgf7Ap=!Qs!Br=4^ zkaD^K2iYx=-H7adSc3PIDVUP64rpesLdYl!`%`dz@c78oLz??Uapy$@j}+*W`|yza zasEP_%8s%fFQx+*0I$>^zXTnl$$emyH&*Ba9UXZpNy zA(RXA%0+zk<3ac!sc1Xrs#a3ae~-W)yM2%NzHU2a+KR!3g`)AoVM5i@xE&imnmr~j z_dO6LcNc^2W1#j9^QFDnw>@CP5aeat^VOGd&=$YIP(n&LGtNb(X(hNu&y%gR5__ZP z-F+Mv;oNi{4JN`y>2QRnm3X-uyi+G0bsq-!E2*?A!$YNG>3RMvy;_t7u+6FA2Q?SK zfWZKJwjGwoaomQ%f5096PXG(VABzlk=Dz)nZ+&AqA{I4DMU7iTiGJUvZPz;^@ed5Rh-9ZkcB0HbwR~a9*$eh+%Jn&p&FAeT^{|W7 zWOk7Zeh*m4odgsPulGQv7w(D|zv2L1h734ajX?Nty>dOxaROeJ4x@zp=(K-Sc8{(d-^FBi zkpBbt(cJ(Rh@=D5u~fWS=~|h6&w1Ag1#_~(0f};I)-9DR!Y?J+> 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_train = ['011', '014', '030', '037', '044', '050', '055', '058', '074', '083', '091', '098', '101', '106', '109', '119'] + uid_nsr_val = ['041', '056', '325'] + uid_nsr_test = ['003', '012', '020', '024', '027', '035', '036', '047'] + + uid_af_train = ['017', '301', '302', '305', '306', '318', '319', '320', '321', '322', '324', '329', '402', '405', '406', '407', '416', '420', '421'] + uid_af_val = ['400', '409', '422'] + uid_af_test = ['307', '310', '311', '312', '410', '413', '414', '415', '423'] + + uid_pacpvc_train = ['005', '007', '013', '021', '022', '026', '028', '029', '042', '064', '068', '073', '080', '086', '087', '089', '093', '104', '110', '113', '120', '327', '408'] + uid_pacpvc_val = ['045', '054', '112'] + uid_pacpvc_test = ['002', '038', '039', '052', '053', '069', '070', '075', '078', '090', '100', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_train + uid_nsr_val + uid_nsr_test + total_uid_af = uid_af_train + uid_af_val + uid_af_test + total_uid_pacpvc = uid_pacpvc_train + uid_pacpvc_val + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + train_set = uid_nsr_train + uid_af_train + uid_pacpvc_train + val_set = uid_nsr_val + uid_af_val + uid_pacpvc_val + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + # Limit data set size to reduce computational load for optimization + test_set = test_set + + return train_set, val_set, test_set + + +def split_uids_60_10_30_smote(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_train = ['003', '020', '024', '041', '044', '047', '049', '050', '058', '063', '077', '084', '088', '091', '098', '099', '106', '109', '111', '118', '325'] + uid_nsr_val = ['014', '030', '036', '074'] + uid_nsr_test = ['011', '012', '027', '035', '037', '055', '056', '057', '083', '094', '101', '119'] + + uid_af_train = ['017', '302', '306', '307', '310', '311', '319', '321', '324', '400', '402', '405', '406', '407', '409', '410', '415', '420', '421'] + uid_af_val = ['416', '422', '423'] + uid_af_test = ['301', '305', '312', '318', '320', '322', '329', '413', '414'] + + uid_pacpvc_train = ['005', '007', '013', '021', '022', '026', '028', '029', '042', '064', '068', '073', '080', '086', '087', '089', '093', '104', '110', '113', '120', '327', '408'] + uid_pacpvc_val = ['045', '054', '112'] + uid_pacpvc_test = ['002', '038', '039', '052', '053', '069', '070', '075', '078', '090', '100', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_train + uid_nsr_val + uid_nsr_test + total_uid_af = uid_af_train + uid_af_val + uid_af_test + total_uid_pacpvc = uid_pacpvc_train + uid_pacpvc_val + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + train_set = uid_nsr_train + uid_af_train + uid_pacpvc_train + val_set = uid_nsr_val + uid_af_val + uid_pacpvc_val + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + return train_set, val_set, test_set + + +def split_uids_60_10_30_noPACPVC(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_train = ['003', '020', '024', '041', '044', '047', '049', '050', '058', '063', '077', '084', '088', '091', '098', '099', '106', '109', '111', '118', '325'] + uid_nsr_val = ['014', '030', '036', '074'] + uid_nsr_test = ['011', '012', '027', '035', '037', '055', '056', '057', '083', '094', '101', '119'] + + uid_af_train = ['017', '302', '306', '307', '310', '311', '319', '321', '324', '400', '402', '405', '406', '407', '409', '410', '415', '420', '421'] + uid_af_val = ['416', '422', '423'] + uid_af_test = ['301', '305', '312', '318', '320', '322', '329', '413', '414'] + + uid_pacpvc_train = [] # ['005', '007', '013', '021', '022', '026', '028', '029', '042', '064', '068', '073', '080', '086', '087', '089', '093', '104', '110', '113', '120', '327', '408'] + uid_pacpvc_val = [] # ['045', '054', '112'] + uid_pacpvc_test = [] # ['002', '038', '039', '052', '053', '069', '070', '075', '078', '090', '100', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_train + uid_nsr_val + uid_nsr_test + total_uid_af = uid_af_train + uid_af_val + uid_af_test + total_uid_pacpvc = uid_pacpvc_train + uid_pacpvc_val + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + train_set = uid_nsr_train + uid_af_train + uid_pacpvc_train + val_set = uid_nsr_val + uid_af_val + uid_pacpvc_val + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + return train_set, val_set, test_set + + +def split_uids_60_10_30_noNSR(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_train = [] # ['003', '020', '024', '041', '044', '047', '049', '050', '058', '063', '077', '084', '088', '091', '098', '099', '106', '109', '111', '118', '325'] + uid_nsr_val = [] # ['014', '030', '036', '074'] + uid_nsr_test = [] # ['011', '012', '027', '035', '037', '055', '056', '057', '083', '094', '101', '119'] + + uid_af_train = ['017', '302', '306', '307', '310', '311', '319', '321', '324', '400', '402', '405', '406', '407', '409', '410', '415', '420', '421'] + uid_af_val = ['416', '422', '423'] + uid_af_test = ['301', '305', '312', '318', '320', '322', '329', '413', '414'] + + uid_pacpvc_train = ['005', '007', '013', '021', '022', '026', '028', '029', '042', '064', '068', '073', '080', '086', '087', '089', '093', '104', '110', '113', '120', '327', '408'] + uid_pacpvc_val = ['045', '054', '112'] + uid_pacpvc_test = ['002', '038', '039', '052', '053', '069', '070', '075', '078', '090', '100', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_train + uid_nsr_val + uid_nsr_test + total_uid_af = uid_af_train + uid_af_val + uid_af_test + total_uid_pacpvc = uid_pacpvc_train + uid_pacpvc_val + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + train_set = uid_nsr_train + uid_af_train + uid_pacpvc_train + val_set = uid_nsr_val + uid_af_val + uid_pacpvc_val + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + return train_set, val_set, test_set + + +def split_uids_60_10_30_balanced(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_train = ['041', '044', '047', '050', '058', '063', '091', '098', '106', '111', '325'] + uid_nsr_val = ['014', '030', '036', '074'] + uid_nsr_test = ['011', '012', '027', '035', '037', '055', '056', '057', '083', '094', '101', '119'] + + uid_af_train = ['017', '302', '306', '307', '310', '311', '319', '321', '324', '400', '402', '407', '409', '415', '420', '421'] + uid_af_val = ['416', '422', '423'] + uid_af_test = ['301', '305', '312', '318', '320', '322', '329', '413', '414'] + + uid_pacpvc_train = ['005', '007', '013', '021', '022', '026', '028', '029', '042', '064', '068', '073', '080', '086', '087', '089', '093', '104', '110', '113', '120', '327', '408'] + uid_pacpvc_val = ['045', '054', '112'] + uid_pacpvc_test = ['002', '038', '039', '052', '053', '069', '070', '075', '078', '090', '100', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_train + uid_nsr_val + uid_nsr_test + total_uid_af = uid_af_train + uid_af_val + uid_af_test + total_uid_pacpvc = uid_pacpvc_train + uid_pacpvc_val + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + train_set = uid_nsr_train + uid_af_train + uid_pacpvc_train + val_set = uid_nsr_val + uid_af_val + uid_pacpvc_val + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + return train_set, val_set, test_set + + +def split_uids_2fold_60_40(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_pacpvc_fold1 = ['007', '022', '028', '038', '054', '068', '075', '086', '087', '093', '120', '327'] + uid_pacpvc_fold2 = ['002', '005', '013', '021', '026', '029', '045', '073', '089', '100', '112', '408'] + uid_pacpvc_test = ['039', '042', '052', '053', '064', '069', '070', '078', '080', '090', '104', '110', '113', '419'] + + uid_af_fold1 = ['305', '307', '311', '318', '320', '322', '405', '415', '423'] + uid_af_fold2 = ['301', '319', '321', '324', '329', '400', '406', '409', '416'] + uid_af_test = ['017', '302', '306', '310', '312', '402', '407', '410', '413', '414', '420', '421', '422'] + + uid_nsr_fold1 = ['011', '014', '041', '050', '056', '058', '083', '106', '109'] + uid_nsr_fold2 = ['037', '047', '055', '074', '091', '098', '101', '119', '325'] + uid_nsr_test = ['003', '012', '020', '024', '027', '030', '035', '036', '044', '049', '057', '063', '077', '084', '088', '094', '099', '111', '118'] + + # Total UID counts + total_uid_pacpvc = uid_pacpvc_fold1 + uid_pacpvc_fold2 + uid_pacpvc_test + total_uid_af = uid_af_fold1 + uid_af_fold2 + uid_af_test + total_uid_nsr = uid_nsr_fold1 + uid_nsr_fold2 + uid_nsr_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + cross_val_fold1 = uid_nsr_fold1 + uid_af_fold1 + uid_pacpvc_fold1 + cross_val_fold2 = uid_nsr_fold2 + uid_af_fold2 + uid_pacpvc_fold2 + test = uid_nsr_test + uid_af_test + uid_pacpvc_test + + # # Limit data set size to reduce computational load for optimization + # cross_val_fold1 = uid_nsr_fold1[:2] + uid_af_fold1[:2] + uid_pacpvc_fold1[:2] + # cross_val_fold2 = uid_nsr_fold2[:2] + uid_af_fold2[:2] + uid_pacpvc_fold2[:2] + # test = uid_nsr_test[:2] + uid_af_test[:2] + uid_pacpvc_test[:2] + + return cross_val_fold1, cross_val_fold2, test + + +def split_uids_2fold_60_40_smote(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + # Filter out 0-segment UIDs and UIDs without NSR, AF, and/or PAC/PVC + remaining_UIDs = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + if row['TOTAL'] == 0: + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + elif (row['NSR'] > 0 or row['AF'] > 0 or row['PACPVC'] > 0): # Append UID only if it contains NSR, AF, or PAC/PVC + remaining_UIDs.append(UID) + else: + print(f'---------UID {UID} has no AF, NSR, or PAC/PVC segments.------------') + + # Split UIDs + uid_nsr_fold1 = ['020', '030', '037', '041', '058', '077', '084', '106', '109', '118', '325'] + uid_nsr_fold2 = ['003', '014', '036', '044', '047', '049', '063', '083', '088', '091', '099'] + uid_nsr_test = ['011', '012', '024', '027', '035', '050', '055', '056', '057', '074', '094', '098', '101', '111', '119'] + + uid_af_fold1 = ['302', '306', '307', '402', '405', '415', '420', '421', '422'] + uid_af_fold2 = ['310', '321', '324', '406', '407', '409', '414', '416', '423'] + uid_af_test = ['017', '301', '305', '311', '312', '318', '319', '320', '322', '329', '400', '410', '413'] + + uid_pacpvc_fold1 = ['007', '022', '028', '038', '054', '068', '075', '086', '087', '093', '120', '327'] + uid_pacpvc_fold2 = ['002', '005', '013', '021', '026', '029', '045', '073', '089', '100', '112', '408'] + uid_pacpvc_test = ['039', '042', '052', '053', '064', '069', '070', '078', '080', '090', '104', '110', '113', '419'] + + # Total UID counts + total_uid_nsr = uid_nsr_fold1 + uid_nsr_fold2 + uid_nsr_test + total_uid_af = uid_af_fold1 + uid_af_fold2 + uid_af_test + total_uid_pacpvc = uid_pacpvc_fold1 + uid_pacpvc_fold2 + uid_pacpvc_test + total_uid = total_uid_pacpvc + total_uid_af + total_uid_nsr + + print('Number of total and unique UIDs:', len(total_uid),'|', len(np.unique(total_uid))) + print('Number of total and unique NSR UIDs:', len(total_uid_nsr),'|', len(np.unique(total_uid_nsr))) + print('Number of total and unique AF UIDs:', len(total_uid_af),'|', len(np.unique(total_uid_af))) + print('Number of total and unique PAC/PVC UIDs:', len(total_uid_pacpvc),'|', len(np.unique(total_uid_pacpvc))) + + cross_val_fold1 = uid_nsr_fold1 + uid_af_fold1 + uid_pacpvc_fold1 + cross_val_fold2 = uid_nsr_fold2 + uid_af_fold2 + uid_pacpvc_fold2 + test_set = uid_nsr_test + uid_af_test + uid_pacpvc_test + + return cross_val_fold1, cross_val_fold2, test_set + + +def split_uids(pathmaster): + # ====== Load the per subject arrythmia summary ====== + file_path = pathmaster.summary_path() + # df_summary = pd.read_csv(file_path) + + # Read the CSV file using pyarrow.csv.read_csv + table_summary = csv.read_csv(file_path) + df_summary = table_summary.to_pandas() + + df_summary['UID'] = df_summary['UID'].astype(str).str.zfill(3) # Pads each UIDs with enough zeroes to be 3 characters + + df_summary['sample_nonAF'] = df_summary['NSR'] + df_summary['PACPVC'] + df_summary['SVT'] + df_summary['sample_AF'] = df_summary['AF'] + + df_summary['sample_nonAF_ratio'] = df_summary['sample_nonAF'] / (df_summary['sample_AF'] + df_summary['sample_nonAF']) + + all_UIDs = df_summary['UID'].unique() + + # ==================================================== + # ====== AF trial separation ====== + # R:\ENGR_Chon\Dong\Numbers\Pulsewatch_numbers\Fahimeh_CNNED_general_ExpertSystemwApplication\tbl_file_name\TrainingSet_final_segments + AF_trial_Fahimeh_train = ['402','410'] + AF_trial_Fahimeh_test = ['301', '302', '305', '306', '307', '310', '311', + '312', '318', '319', '320', '321', '322', '324', + '325', '327', '329', '400', '406', '407', '409', + '414'] + AF_trial_Fahimeh_did_not_use = ['405', '413', '415', '416', '420', '421', '422', '423'] + AF_trial_paroxysmal_AF = ['408','419'] + + AF_trial_train = AF_trial_Fahimeh_train + AF_trial_test = AF_trial_Fahimeh_test + AF_trial_unlabeled = AF_trial_Fahimeh_did_not_use + AF_trial_paroxysmal_AF + print(f'AF trial: {len(AF_trial_train)} training subjects {AF_trial_train}') + print(f'AF trial: {len(AF_trial_test)} testing subjects {AF_trial_test}') + print(f'AF trial: {len(AF_trial_unlabeled)} unlabeled subjects {AF_trial_unlabeled}') + + # ================================= + # === Clinical trial AF subjects separation === + clinical_trial_AF_subjects = ['005', '017', '026', '051', '075', '082'] + + # Filter out AF trial and 0-segment UIDs + remaining_UIDs = [] + count_NSR = [] + + for index, row in df_summary.iterrows(): + UID = row['UID'] + this_NSR = row['sample_nonAF'] + if math.isnan(row['sample_nonAF_ratio']): # sample_nonAF is never NaN, sample_nonAF_ratio may be NaN + # There is no segment in this subject, skip this UID. + print(f'---------UID {UID} has no segments.------------') + continue # If a UID has no segments, skip the rest of the for loop for this index, row + if UID not in AF_trial_train and UID not in AF_trial_test and UID not in clinical_trial_AF_subjects \ + and UID[0] != '3' and UID[0] != '4': + remaining_UIDs.append(UID) + count_NSR.append(this_NSR) + + # From the candidate UIDs, select a subset to be used for training, validation, and testing + random.seed(seed=42) + + list_of_candidates = remaining_UIDs + number_of_items_to_pick = round(len(list_of_candidates) * 0.25) # 15% labeled for training, 10% for testing. + sum_NSR = sum(count_NSR) + + # probability_distribution = [x/sum_NSR for x in count_NSR] # Proportion of total NSR segments for each UID + probability_distribution = [(1-x/sum_NSR)/ (len(count_NSR)-1) for x in count_NSR] # Subjects with fewer segments have higher chance to be selected. + draw = choice(list_of_candidates, number_of_items_to_pick, + p=probability_distribution, replace=False) + + # Ensures that training set contains both AF and non-AF + clinical_trial_train_nonAF = list(draw[:round(len(list_of_candidates) * 0.12)]) # Draws the first X number of candidates equal to 7% of the total list of candidates + clinical_trial_train_temp = clinical_trial_train_nonAF + clinical_trial_AF_subjects[:round(len(clinical_trial_AF_subjects)/2)] + clinical_trial_train = [] + + for UID in clinical_trial_train_temp: + # UID 051 and 108 and maybe other UIDs had no segments (unknown reason). + if UID in all_UIDs: + clinical_trial_train.append(UID) # Only use the UIDs that are in the summary to test + + # Ensures that the testing set contains both AF and non-AF + clinical_trial_test_nonAF = list(draw[round(len(list_of_candidates) * 0.12):]) # Draws the remaining candidates + clinical_trial_test_temp = clinical_trial_test_nonAF + clinical_trial_AF_subjects[round(len(clinical_trial_AF_subjects)/2):] + clinical_trial_test = [] + for UID in clinical_trial_test_temp: + # UID 051 and 108 and maybe other UIDs had no segments (unknown reason). + if UID in all_UIDs: + clinical_trial_test.append(UID) # Only use the UIDs that are in the summary to test + + # Uses all remaining subset of UIDs from original list not used in training or validating for testing + clinical_trial_unlabeled = [] + for UID in remaining_UIDs: # Changed from all_UIDs to remove UIDs with 0 segments (i.e. UID 108) + if UID not in clinical_trial_train and UID not in clinical_trial_test and UID[0] != '3' and UID[0] != '4': + clinical_trial_unlabeled.append(UID) + + # Sum up to 74 UIDs, all of the ones that do not start with '3' or '4' and dropping UID 108 which has 0 segments + print(f'Clinical trial: selected {len(clinical_trial_train)} UIDs for training {clinical_trial_train}') # Contains both non-AF and AF clinical trial subjects + print(f'Clinical trial: selected {len(clinical_trial_test)} UIDs for testing {clinical_trial_test}') # Contains both non-AF and AF clinical trial subjects + print(f'Clinical trial: selected {len(clinical_trial_unlabeled)} UIDs for unlabeled {clinical_trial_unlabeled}') # All remaining clinical trial subjects...probably contains both AF and non-AF + + # Used to make sure the model runs correctly + clinical_trial_train = ['063','416','005'] # Training + clinical_trial_test = ['058','409','054'] # Evaluation + clinical_trial_unlabeled = ['029','036','421'] # Testing + + return clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled + + +class CustomDataset(Dataset): + def __init__(self, data_path, labels_path, UIDs, standardize=True, data_format='csv', read_all_labels=False, + start_idx=0, img_channels=1, img_size=128, downsample=None, data_type=torch.float32, is_tfs=True, binary=False): + self.data_path = data_path + self.labels_path = labels_path + self.UIDs = UIDs + self.standardize = standardize + self.data_format = data_format + self.read_all_labels = read_all_labels + self.transforms = ToTensor() + self.start_idx = start_idx # Initial batch index to start from, useful for resuming training + self.img_channels = img_channels + self.img_size = img_size + self.downsample = downsample + self.is_tfs = is_tfs + self.binary = binary + + # Must be manually set so that the image resolution chosen is the one that is returned + self.dtype = data_type + + self.refresh_dataset() + + def refresh_dataset(self): + self.segment_names, self.labels = self.extract_segment_names_and_labels() + + def add_uids(self, new_uids): + unique_new_uids = [uid for uid in new_uids if uid not in self.UIDs] # Appends any unqiue new UID in self.UIDs to unique_new_uids + self.UIDs.extend(unique_new_uids) # Appends unique_new_uids to UIDs + self.refresh_dataset() + + def __len__(self): # Method is implicitly called when len() is used on an instance of CustomDataset + return len(self.segment_names) + + def save_checkpoint(self, checkpoint_path): # Likely not worth using, simply use the save_checkpoint() function in train_func.py + # Enhanced to automatically include 'start_idx' in the checkpoint + checkpoint = { + 'segment_names': self.segment_names, + 'labels': self.labels, + 'UIDs': self.UIDs, + 'start_idx': self.start_idx # Now also saving start_idx + } + torch.save(checkpoint, checkpoint_path) # Using standard Python methods like pickle or json is generally recommended for dictionaries, there are no benefits for using torch.save, no real harm either + + def load_checkpoint(self, checkpoint_path): # Reloads where you started off last time (not where you ended), just use analogous function in train_func.py + checkpoint = torch.load(checkpoint_path) + self.segment_names = checkpoint['segment_names'] # Seems redundant since it is overwritten by refresh_dataset() + self.labels = checkpoint['labels'] # Seems redundant since it is overwritten by refresh_dataset() + self.UIDs = checkpoint['UIDs'] + # Now also loading and setting start_idx from checkpoint + self.start_idx = checkpoint.get('start_idx', 0) # Returns 0 if no start_idx found + self.refresh_dataset() + + def __getitem__(self, idx): # Method is implicitly called when getitem() is used on an instance of CustomDataset. It is called batch_size number of times per iteration of dataloader | Loads segments as needed (lazy loading) + actual_idx = (idx + self.start_idx) % len(self.segment_names) # Adjust index based on start_idx and wrap around if needed (i.e. index falls out of bounds) + segment_name = self.segment_names[actual_idx] + label = self.labels[segment_name] + + if hasattr(self, 'all_data') and actual_idx < len(self.all_data): # When Luis uses adds data to train_loader in main_checkpoints.py, + # new data is added (creating all_data) only after train_loader is created with its original training data. This means that if self.all_data + # exists, then __getitem__ is only be called in order to retrieve data newly added to train_loader in all_data + time_freq_tensor = self.all_data[actual_idx] + else: + time_freq_tensor = self.load_data(segment_name) + + return {'data': time_freq_tensor, 'label': label, 'segment_name': segment_name} + + # When iterating over the dataloader, which returns batches of data, each batch will contain a dictionary with keys corresponding to the data and labels. + + # Since the dataloader's dataset's __getitem__ method returns a dictionary with keys 'data', 'label', and 'segment_name', the returned batch will be a dictionary where: + + # The 'data' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the data. + # The 'label' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the labels. + # The 'segment_name' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the segment_name. + + def set_start_idx(self, index): + self.start_idx = index + + def add_data_label_pair(self, data, label): + # Assign a unique ID or name for the new data + new_id = len(self.segment_names) + segment_name = f"new_data_{new_id}" + + # Append the new data and label + self.segment_names.append(segment_name) + self.labels[segment_name] = label + + # Append the new data tensor to an attribute that holds all of the newly added data + if hasattr(self, 'all_data'): + self.all_data.append(data) + else: + self.all_data = [data] + + # def extract_segment_names_and_labels(self): + # segment_names = [] + # labels = {} + + # for UID in self.UIDs: + # label_file = os.path.join(self.labels_path, UID + "_final_attemp_4_1_Dong.csv") + # if os.path.exists(label_file): + # # label_data = pd.read_csv(label_file, sep=',', header=0, names=['segment', 'label']) # Replaces the original headers with names + + # # Use PyArrow to read csv + # parse_options = csv.ParseOptions(delimiter=',') # Indicate delimiter + # read_options = csv.ReadOptions(column_names=['segment', 'label'], skip_rows=1) # Assign desired column names and skip the first row (headers) + # label_data = csv.read_csv(label_file, parse_options=parse_options, read_options=read_options) + # label_data = label_data.to_pandas() + + # label_segment_names = label_data['segment'].apply(lambda x: x.split('.')[0]) # Splits each segment name by '.' and retrieves the first part + # for idx, segment_name in enumerate(label_segment_names): # enumerate() returns the value and corresponding index of each element in an iterable + # label_val = label_data['label'].values[idx] + # # Will only use NSR (0), AF (1), and PAC/PVC(2) and not SVT (3) + # if self.read_all_labels: # If reading all labels, set all labels not 0, 1, or 2 to -1 and return all labels + # # Assign -1 if label is not in [0, 1, 2] + # labels[segment_name] = label_val if label_val in [0, 1, 2] else -1 + # if segment_name not in segment_names: + # segment_names.append(segment_name) + # else: + # # Only add segments with labels in [0, 1, 2] + # if label_val in [0, 1, 2] and segment_name not in segment_names: + # segment_names.append(segment_name) + # labels[segment_name] = label_val # Extracts the labels of the segments retrieved into a dictionary + + # # # Since shuffle=False for the dataloader in preprocess_data(), this is my work around for that while allowing for checkpointing + # # random.seed(seed=42) + # # random.shuffle(segment_names) # Will not affect the labels since the labels are in a dictionary + + # return segment_names, labels + + + def extract_segment_names_and_labels(self): # Only extract the segments and labels of a particular class, temporary solution + segment_names = [] + labels = {} + + # If a subject is not loading and there are no errors, just these lists + uid_nsr = ['011', '014', '041', '050', '056', '058', '083', '106', '109', + '037', '047', '055', '074', '091', '098', '101', '119', '325', + '003', '012', '020', '024', '027', '030', '035', '036', '044', '049', '057', '063', '077', '084', '088', '094', '099', '111', '118'] + uid_af = ['305', '307', '311', '318', '320', '322', '405', '415', '423', + '301', '319', '321', '324', '329', '400', '406', '409', '416', + '017', '302', '306', '310', '312', '402', '407', '410', '413', '414', '420', '421', '422'] + uid_pacpvc = ['007', '022', '028', '038', '054', '068', '075', '086', '087', '093', '120', '327', + '002', '005', '013', '021', '026', '029', '045', '073', '089', '100', '112', '408', + '039', '042', '052', '053', '064', '069', '070', '078', '080', '090', '104', '110', '113', '419'] + + for UID in self.UIDs: + label_file = os.path.join(self.labels_path, UID + "_final_attemp_4_1_Dong.csv") + if os.path.exists(label_file): + # label_data = pd.read_csv(label_file, sep=',', header=0, names=['segment', 'label']) # Replaces the original headers with names + + # Use PyArrow to read csv + parse_options = csv.ParseOptions(delimiter=',') # Indicate delimiter + read_options = csv.ReadOptions(column_names=['segment', 'label'], skip_rows=1) # Assign desired column names and skip the first row (headers) + label_data = csv.read_csv(label_file, parse_options=parse_options, read_options=read_options) + label_data = label_data.to_pandas() + + label_segment_names = label_data['segment'].apply(lambda x: x.split('.')[0]) # Splits each segment name by '.' and retrieves the first part + for idx, segment_name in enumerate(label_segment_names): # enumerate() returns the value and corresponding index of each element in an iterable + label_val = label_data['label'].values[idx] + # Will only use NSR (0), AF (1), and PAC/PVC(2) and not SVT (3) + if self.read_all_labels: # If reading all labels, set all labels not 0, 1, or 2 to -1 and return all labels + # Assign -1 if label is not in [0, 1, 2] + labels[segment_name] = label_val if label_val in [0, 1, 2] else -1 + if segment_name not in segment_names: + segment_names.append(segment_name) + else: + # Only add segments with labels in [0, 1, 2] + if label_val in [0, 1, 2] and segment_name not in segment_names: + # Temporary solution to ensure only segments of a particular class are loaded for each UID + if UID in uid_nsr and label_val == 0: + segment_names.append(segment_name) + labels[segment_name] = label_val + elif UID in uid_af and label_val == 1: + segment_names.append(segment_name) + labels[segment_name] = label_val + elif UID in uid_pacpvc and label_val == 2: + segment_names.append(segment_name) + if self.binary: + labels[segment_name] = 0 + else: + labels[segment_name] = label_val + + return segment_names, labels + + + def load_data(self, segment_name): + data_path_UID = os.path.join(self.data_path, segment_name.split('_')[0]) + if self.is_tfs: + seg_path = os.path.join(data_path_UID, segment_name + '_filt_STFT.' + self.data_format) + else: + seg_path = os.path.join(data_path_UID, segment_name + '_density_poincare.' + self.data_format) + + + try: # Allows to define a block of code to be executed and specify how to handle any errors that might occur during its execution + if self.data_format == 'csv' and seg_path.endswith('.csv'): + # time_freq_plot = np.array(pd.read_csv(seg_path, header=None)) + + # Use PyArrow to read csv + read_options = csv.ReadOptions(autogenerate_column_names=True) + seg_data = csv.read_csv(seg_path, read_options=read_options) + time_freq_plot = seg_data.to_pandas().to_numpy() + + time_freq_tensor = torch.tensor(time_freq_plot).reshape(self.img_channels, self.img_size, self.img_size) + elif self.data_format == 'png' and seg_path.endswith('.png'): + img = Image.open(seg_path) + img_data = np.array(img) + time_freq_tensor = torch.tensor(img_data).unsqueeze(0) + elif self.data_format == 'pt' and seg_path.endswith('.pt'): + time_freq_tensor = torch.load(seg_path) + else: + raise ValueError("Unsupported file format") + + if self.downsample is not None: + # Downsample the image + # Use OpenCV to resize the array to downsample x downsample using INTER_AREA interpolation + time_freq_array = cv2.resize(np.array(time_freq_tensor.reshape(self.img_size, self.img_size).to('cpu')), (self.downsample, self.downsample), interpolation=cv2.INTER_AREA) + time_freq_tensor = torch.tensor(time_freq_array, dtype=self.dtype).reshape(self.img_channels, self.downsample, self.downsample) + else: + time_freq_tensor = time_freq_tensor.reshape(self.img_channels, self.img_size, self.img_size).to(self.dtype) + + if self.standardize: + time_freq_tensor = self.standard_scaling(time_freq_tensor) # Standardize the data + + return time_freq_tensor + + except Exception as e: + print(f"Error processing segment: {segment_name}. Exception: {str(e)}") + if self.downsample is not None: + return torch.zeros((self.img_channels, self.downsample, self.downsample)) # Return zeros in case of an error + else: + return torch.zeros((self.img_channels, self.img_size, self.img_size)) # Return zeros in case of an error + + def standard_scaling(self, data): + scaler = StandardScaler() + data = scaler.fit_transform(data.reshape(-1, data.shape[-1])).reshape(data.shape) # Converts data into 2D array, standardizes it, reshapes it back into 3D (1,X,X) + return torch.tensor(data, dtype=self.dtype) + +def load_data_split_batched(data_path, labels_path, UIDs, batch_size, standardize=False, data_format='csv', + read_all_labels=False, drop_last=False, num_workers=4, start_idx=0, + img_channels=1, img_size=128, downsample=None, data_type=torch.float32, is_tfs=True, binary=False): + torch.manual_seed(42) + g = torch.Generator() + g.manual_seed(42) + + pin_memory = False + if torch.cuda.is_available(): + pin_memory = True + + dataset = CustomDataset(data_path, labels_path, UIDs, standardize, data_format, read_all_labels, start_idx=start_idx, + img_channels=img_channels, img_size=img_size, downsample=downsample, data_type=data_type, is_tfs=is_tfs, binary=binary) + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=drop_last, num_workers=num_workers, prefetch_factor=2, persistent_workers=True, pin_memory=pin_memory, worker_init_fn=seed_worker, generator=g) # Prefetches 2 batches ahead of current training iteration (allows loading of data simultaneously with training). Shuffle is set to False to resume training at a specific batch. + return dataloader + +def seed_worker(worker_id): + worker_seed = torch.initial_seed() % 2**32 + np.random.seed(worker_seed) + random.seed(worker_seed) + +# Function to extract and preprocess data +def preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, batch_size, standardize=False, + read_all_labels=False, img_channels=1, img_size=128, downsample=None, data_type=torch.float32, pathmaster=None, binary=False): + start_idx = 0 + data_path, labels_path = pathmaster.data_paths(data_format) + + if data_format == 'csv': + num_workers = 6 + elif data_format == 'pt': + num_workers = 8 + + train_loader = load_data_split_batched(data_path, labels_path, clinical_trial_train, batch_size, standardize=standardize, + data_format=data_format, read_all_labels=read_all_labels, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs, binary=binary) + val_loader = load_data_split_batched(data_path, labels_path, clinical_trial_test, batch_size, standardize=standardize, + data_format=data_format, read_all_labels=read_all_labels, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs, binary=binary) + test_loader = load_data_split_batched(data_path, labels_path, clinical_trial_unlabeled, batch_size, standardize=standardize, + data_format=data_format, read_all_labels=read_all_labels, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs, binary=binary) + return train_loader, val_loader, test_loader + +def map_samples_to_uids(uncertain_sample_indices, dataset): + """ + Maps indices of uncertain samples back to their corresponding segment names or UIDs. + + Args: + - uncertain_sample_indices: Indices of the uncertain samples in the dataset. + - dataset: The dataset object which contains the mapping of segment names and UIDs. + + Returns: + - List of UIDs or segment names corresponding to the uncertain samples. + """ + return [dataset.segment_names[i] for i in uncertain_sample_indices] + +def update_train_loader_with_labeled_samples(current_train_loader, labeled_samples, batch_size): # Luis' doesn't seem to use this + """ + Updates the training DataLoader with newly labeled samples. + + Args: + - current_train_loader: The current DataLoader for the training set. + - labeled_samples: A list of tuples, each containing a data tensor and its new label. + - batch_size: Batch size for the DataLoader. + + Returns: + - DataLoader: The updated DataLoader with the new labeled samples. + """ + + # Extract the current dataset from the DataLoader + current_dataset = current_train_loader.dataset + + # Update the dataset with new samples and labels + for data_tensor, label in labeled_samples: + # Assuming the CustomDataset class has a method to add new data and labels + current_dataset.add_data_label_pair(data_tensor, label) + + # Create a new DataLoader with the updated dataset + updated_train_loader = DataLoader(current_dataset, batch_size=batch_size, shuffle=True, drop_last=False, num_workers=4, prefetch_factor=2) + + return updated_train_loader + +def update_train_loader_with_uncertain_samples(current_train_loader, new_sample_indices, batch_size): # Luis' uses this method for active learning + # Extract current UIDs from the current_train_loader + current_dataset = current_train_loader.dataset + # Map new_samples back to their corresponding segment names or UIDs + new_uids = map_samples_to_uids(new_sample_indices, current_dataset) + # Add new UIDs to the current dataset and refresh it + current_dataset.add_uids(new_uids) + # Create new DataLoader with the updated dataset + updated_train_loader = DataLoader(current_dataset, batch_size=batch_size, shuffle=False) + return updated_train_loader + + \ No newline at end of file diff --git a/utils/dataloader_database.py b/utils/dataloader_database.py new file mode 100644 index 0000000..c3ab6b1 --- /dev/null +++ b/utils/dataloader_database.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Feb 26 18:29:59 2024 + +@author: dchen +""" +import os +import numpy as np +import pandas as pd +from PIL import Image +import torch +from torch.utils.data import Dataset, DataLoader +from sklearn.preprocessing import StandardScaler +from torchvision.transforms import ToTensor +import math +from numpy import random +from numpy.random import choice +import cv2 +from pyarrow import csv + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# Seeds +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +class CustomDataset(Dataset): + def __init__(self, data_path, labels_path, standardize=True, data_format='pt', start_idx=0, + img_channels=1, img_size=128, downsample=None, data_type=torch.float32, is_tfs=True, binary=False): + self.data_path = data_path + self.labels_path = labels_path + self.standardize = standardize + self.data_format = data_format + self.transforms = ToTensor() + self.start_idx = start_idx # Initial batch index to start from, useful for resuming training + self.img_channels = img_channels + self.img_size = img_size + self.downsample = downsample + self.is_tfs = is_tfs + self.dtype = data_type + self.binary = binary + + self.refresh_dataset() + + + def refresh_dataset(self): + self.segment_names, self.labels = self.extract_segment_names_and_labels() + + + def __len__(self): # Method is implicitly called when len() is used on an instance of CustomDataset + return len(self.segment_names) + + + def __getitem__(self, idx): # Method is implicitly called when getitem() is used on an instance of CustomDataset. It is called batch_size number of times per iteration of dataloader | Loads segments as needed (lazy loading) + actual_idx = (idx + self.start_idx) % len(self.segment_names) # Adjust index based on start_idx and wrap around if needed (i.e. index falls out of bounds) + segment_name = self.segment_names[actual_idx] + label = self.labels[segment_name] + + data_tensor = self.load_data(segment_name) + + return {'data': data_tensor, 'label': label, 'segment_name': segment_name} + + # When iterating over the dataloader, which returns batches of data, each batch will contain a dictionary with keys corresponding to the data and labels. + + # Since the dataloader's dataset's __getitem__ method returns a dictionary with keys 'data', 'label', and 'segment_name', the returned batch will be a dictionary where: + + # The 'data' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the data. + # The 'label' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the labels. + # The 'segment_name' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the segment_name. + + def set_start_idx(self, index): + self.start_idx = index + + + def extract_segment_names_and_labels(self): # Only extract the segments and labels of a particular class, temporary solution + segment_names = [] + labels = {} + label_file = self.labels_path + if os.path.exists(label_file): + # Use PyArrow to read csv + parse_options = csv.ParseOptions(delimiter=',') # Indicate delimiter + read_options = csv.ReadOptions(column_names=['segment_names', 'labels'], skip_rows=1) # Assign desired column names and skip the first row (headers) + label_data = csv.read_csv(label_file, parse_options=parse_options, read_options=read_options) + label_data = label_data.to_pandas() + + label_segment_names = label_data['segment_names'] + for idx, segment_name in enumerate(label_segment_names): # enumerate() returns the value and corresponding index of each element in an iterable + label_val = label_data['labels'].values[idx] + + if self.binary and label_val == 2: # If binary is true, set all PAC/PVC to 0 (non-AF) + label_val = 0 + + segment_names.append(segment_name) + labels[segment_name] = label_val + + return segment_names, labels + + + def second_to_last_directory_name(self, path): + # Normalize path separator to '/' + path = path.replace('\\', '/') + + # Split the path into its components + components = path.split('/') + + # Remove empty components + components = [c for c in components if c] + + # Check if the path ends with a separator (indicating it's a directory) + if path.endswith('/'): + # Remove the last empty component + components.pop() + + # If there's only one or zero directories in the path, return None + if len(components) <= 1: + return None + + # Return the name of the second-to-last directory + return components[-2] + + + def load_data(self, segment_name): + seg_path = os.path.join(self.data_path, segment_name + '.' + self.data_format) + + try: # Allows to define a block of code to be executed and specify how to handle any errors that might occur during its execution + if self.data_format == 'csv' and seg_path.endswith('.csv'): + # data_plot = np.array(pd.read_csv(seg_path, header=None)) + + # Use PyArrow to read csv + read_options = csv.ReadOptions(autogenerate_column_names=True) + seg_data = csv.read_csv(seg_path, read_options=read_options) + data_plot = seg_data.to_pandas().to_numpy() + + data_tensor = torch.tensor(data_plot).reshape(self.img_channels, self.img_size, self.img_size) + elif self.data_format == 'png' and seg_path.endswith('.png'): + img = Image.open(seg_path) + img_data = np.array(img) + data_tensor = torch.tensor(img_data).unsqueeze(0) + elif self.data_format == 'pt' and seg_path.endswith('.pt'): + data_tensor = torch.load(seg_path) + else: + raise ValueError("Unsupported file format") + + if self.downsample is not None: + # Downsample the image + # Use OpenCV to resize the array to downsample x downsample using INTER_AREA interpolation + data_array = cv2.resize(np.array(data_tensor.reshape(self.img_size, self.img_size).to('cpu')), (self.downsample, self.downsample), interpolation=cv2.INTER_AREA) + data_tensor = torch.tensor(data_array, dtype=self.dtype).reshape(self.img_channels, self.downsample, self.downsample) + else: + data_tensor = data_tensor.reshape(self.img_channels, self.img_size, self.img_size).to(self.dtype) + + if self.standardize: + data_tensor = self.standard_scaling(data_tensor) # Standardize the data + + return data_tensor + + except Exception as e: + print(f"Error processing segment: {segment_name}. Exception: {str(e)}") + if self.downsample is not None: + return torch.zeros((self.img_channels, self.downsample, self.downsample)) # Return zeros in case of an error + else: + return torch.zeros((self.img_channels, self.img_size, self.img_size)) # Return zeros in case of an error + + def standard_scaling(self, data): + scaler = StandardScaler() + data = scaler.fit_transform(data.reshape(-1, data.shape[-1])).reshape(data.shape) # Converts data into 2D array, standardizes it, reshapes it back into 3D (1,X,X) + return torch.tensor(data, dtype=self.dtype) + +def load_data_split_batched(data_path, labels_path, batch_size, standardize=False, data_format='csv', + drop_last=False, num_workers=4, start_idx=0, + img_channels=1, img_size=128, downsample=None, data_type=torch.float16, is_tfs=True, binary=False): + torch.manual_seed(42) + g = torch.Generator() + g.manual_seed(42) + + pin_memory = False + if torch.cuda.is_available(): + pin_memory = True + + dataset = CustomDataset(data_path, labels_path, standardize, data_format, start_idx=start_idx, + img_channels=img_channels, img_size=img_size, downsample=downsample, data_type=data_type, is_tfs=is_tfs, binary=binary) + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=drop_last, num_workers=num_workers, prefetch_factor=2, persistent_workers=True, pin_memory=pin_memory, worker_init_fn=seed_worker, generator=g) # Prefetches 2 batches ahead of current training iteration (allows loading of data simultaneously with training). Shuffle is set to False to resume training at a specific batch. + return dataloader + +def seed_worker(worker_id): + worker_seed = torch.initial_seed() % 2**32 + np.random.seed(worker_seed) + random.seed(worker_seed) + +# Function to extract and preprocess data +def preprocess_data(database, batch_size, standardize=False, img_channels=1, img_size=128, + downsample=None, data_type=torch.float32, pathmaster=None, binary=False): + start_idx = 0 + + if database == 'DeepBeat' or database == 'deepbeat' or database == 'Deepbeat': + data_path, labels_path = pathmaster.deepbeat_paths() + elif database == 'MIMICIII' or database == 'mimiciii' or database == 'mimicIII' or database == 'mimic3': + data_path, labels_path = pathmaster.mimic3_paths() + elif database == 'Simband' or database == 'simband': + data_path, labels_path = pathmaster.simband_paths() + else: + print('Invalid Database') + + data_format = 'pt' + + num_workers = 1 + + test_loader = load_data_split_batched(data_path, labels_path, batch_size, standardize=standardize, + data_format=data_format, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs, binary=binary) + # loader2 = load_data_split_batched(data_path, labels_path, batch_size, standardize=standardize, + # data_format=data_format, num_workers=num_workers, + # start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + # data_type=data_type, is_tfs=pathmaster.is_tfs, binary=False) + # loader3 = load_data_split_batched(data_path, labels_path, batch_size, standardize=standardize, + # data_format=data_format, num_workers=num_workers, + # start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + # data_type=data_type, is_tfs=pathmaster.is_tfs, binary=False) + return test_loader # loader1, loader2, loader3 + + \ No newline at end of file diff --git a/utils/dataloader_smote.py b/utils/dataloader_smote.py new file mode 100644 index 0000000..9266028 --- /dev/null +++ b/utils/dataloader_smote.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Feb 26 18:29:59 2024 + +@author: dchen +""" +import os +import numpy as np +import pandas as pd +from PIL import Image +import torch +from torch.utils.data import Dataset, DataLoader +from sklearn.preprocessing import StandardScaler +from torchvision.transforms import ToTensor +import math +from numpy import random +from numpy.random import choice +import cv2 +from pyarrow import csv + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# Seeds +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + +class CustomDataset(Dataset): + def __init__(self, smote_path, groups, standardize=True, data_format='pt', start_idx=0, + img_channels=1, img_size=128, downsample=None, data_type=torch.float32, is_tfs=True): + self.smote_path = smote_path + self.standardize = standardize + self.data_format = data_format + self.transforms = ToTensor() + self.start_idx = start_idx # Initial batch index to start from, useful for resuming training + self.img_channels = img_channels + self.img_size = img_size + self.downsample = downsample + self.is_tfs = is_tfs + self.groups = groups + self.dtype = data_type + + self.refresh_dataset() + + + def refresh_dataset(self): + self.segment_names, self.labels = self.extract_segment_names_and_labels() + + + def __len__(self): # Method is implicitly called when len() is used on an instance of CustomDataset + return len(self.segment_names) + + + def __getitem__(self, idx): # Method is implicitly called when getitem() is used on an instance of CustomDataset. It is called batch_size number of times per iteration of dataloader | Loads segments as needed (lazy loading) + actual_idx = (idx + self.start_idx) % len(self.segment_names) # Adjust index based on start_idx and wrap around if needed (i.e. index falls out of bounds) + segment_name = self.segment_names[actual_idx] + label = self.labels[segment_name] + + data_tensor = self.load_data(segment_name) + + return {'data': data_tensor, 'label': label, 'segment_name': segment_name} + + # When iterating over the dataloader, which returns batches of data, each batch will contain a dictionary with keys corresponding to the data and labels. + + # Since the dataloader's dataset's __getitem__ method returns a dictionary with keys 'data', 'label', and 'segment_name', the returned batch will be a dictionary where: + + # The 'data' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the data. + # The 'label' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the labels. + # The 'segment_name' key will correspond to a tensor of shape (batch_size, ...), representing the shape of the segment_name. + + def set_start_idx(self, index): + self.start_idx = index + + + def extract_segment_names_and_labels(self): # Only extract the segments and labels of a particular class, temporary solution + segment_names = [] + labels = {} + + group_directories = [entry for entry in os.listdir(self.smote_path) if os.path.isdir(os.path.join(self.smote_path, entry))] + group = list(set(self.groups).intersection(set(group_directories)))[0] + + smote_type = self.second_to_last_directory_name(self.smote_path) + label_file = os.path.join(self.smote_path, smote_type + '_' + group + '_names_labels.csv') + if os.path.exists(label_file): + # Use PyArrow to read csv + parse_options = csv.ParseOptions(delimiter=',') # Indicate delimiter + read_options = csv.ReadOptions(column_names=['segment_name', 'label'], skip_rows=1) # Assign desired column names and skip the first row (headers) + label_data = csv.read_csv(label_file, parse_options=parse_options, read_options=read_options) + label_data = label_data.to_pandas() + + label_segment_names = label_data['segment_name'] + for idx, segment_name in enumerate(label_segment_names): # enumerate() returns the value and corresponding index of each element in an iterable + label_val = label_data['label'].values[idx] + segment_names.append(segment_name) + labels[segment_name] = label_val + + return segment_names, labels + + + def second_to_last_directory_name(self, path): + # Normalize path separator to '/' + path = path.replace('\\', '/') + + # Split the path into its components + components = path.split('/') + + # Remove empty components + components = [c for c in components if c] + + # Check if the path ends with a separator (indicating it's a directory) + if path.endswith('/'): + # Remove the last empty component + components.pop() + + # If there's only one or zero directories in the path, return None + if len(components) <= 1: + return None + + # Return the name of the second-to-last directory + return components[-2] + + + def load_data(self, segment_name): + data_path_group = os.path.join(self.smote_path, segment_name.split('_')[1]) + seg_path = os.path.join(data_path_group, segment_name + '.' + self.data_format) + + try: # Allows to define a block of code to be executed and specify how to handle any errors that might occur during its execution + if self.data_format == 'csv' and seg_path.endswith('.csv'): + # data_plot = np.array(pd.read_csv(seg_path, header=None)) + + # Use PyArrow to read csv + read_options = csv.ReadOptions(autogenerate_column_names=True) + seg_data = csv.read_csv(seg_path, read_options=read_options) + data_plot = seg_data.to_pandas().to_numpy() + + data_tensor = torch.tensor(data_plot).reshape(self.img_channels, self.img_size, self.img_size) + elif self.data_format == 'png' and seg_path.endswith('.png'): + img = Image.open(seg_path) + img_data = np.array(img) + data_tensor = torch.tensor(img_data).unsqueeze(0) + elif self.data_format == 'pt' and seg_path.endswith('.pt'): + data_tensor = torch.load(seg_path) + else: + raise ValueError("Unsupported file format") + + if self.downsample is not None: + # Downsample the image + # Use OpenCV to resize the array to downsample x downsample using INTER_AREA interpolation + data_array = cv2.resize(np.array(data_tensor.reshape(self.img_size, self.img_size).to('cpu')), (self.downsample, self.downsample), interpolation=cv2.INTER_AREA) + data_tensor = torch.tensor(data_array, dtype=self.dtype).reshape(self.img_channels, self.downsample, self.downsample) + else: + data_tensor = data_tensor.reshape(self.img_channels, self.img_size, self.img_size).to(self.dtype) + + if self.standardize: + data_tensor = self.standard_scaling(data_tensor) # Standardize the data + + return data_tensor + + except Exception as e: + print(f"Error processing segment: {segment_name}. Exception: {str(e)}") + if self.downsample is not None: + return torch.zeros((self.img_channels, self.downsample, self.downsample)) # Return zeros in case of an error + else: + return torch.zeros((self.img_channels, self.img_size, self.img_size)) # Return zeros in case of an error + + def standard_scaling(self, data): + scaler = StandardScaler() + data = scaler.fit_transform(data.reshape(-1, data.shape[-1])).reshape(data.shape) # Converts data into 2D array, standardizes it, reshapes it back into 3D (1,X,X) + return torch.tensor(data, dtype=self.dtype) + +def load_data_split_batched(smote_path, groups, batch_size, standardize=False, data_format='csv', + drop_last=False, num_workers=4, start_idx=0, + img_channels=1, img_size=128, downsample=None, data_type=torch.float32, is_tfs=True): + torch.manual_seed(42) + g = torch.Generator() + g.manual_seed(42) + + pin_memory = False + if torch.cuda.is_available(): + pin_memory = True + + dataset = CustomDataset(smote_path, groups, standardize, data_format, start_idx=start_idx, + img_channels=img_channels, img_size=img_size, downsample=downsample, data_type=data_type, is_tfs=is_tfs) + dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=drop_last, num_workers=num_workers, prefetch_factor=2, persistent_workers=True, pin_memory=pin_memory, worker_init_fn=seed_worker, generator=g) # Prefetches 2 batches ahead of current training iteration (allows loading of data simultaneously with training). Shuffle is set to False to resume training at a specific batch. + return dataloader + +def seed_worker(worker_id): + worker_seed = torch.initial_seed() % 2**32 + np.random.seed(worker_seed) + random.seed(worker_seed) + +# Function to extract and preprocess data +def preprocess_data(smote_type, split, batch_size, standardize=False, img_channels=1, img_size=128, + downsample=None, data_type=torch.float32, pathmaster=None): + start_idx = 0 + smote_path = pathmaster.smote_path(smote_type, split) + data_format = 'pt' + + num_workers = 8 + + loader1 = load_data_split_batched(smote_path, ['fold1', 'train'], batch_size, standardize=standardize, + data_format=data_format, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs) + loader2 = load_data_split_batched(smote_path, ['fold2', 'validate'], batch_size, standardize=standardize, + data_format=data_format, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs) + loader3 = load_data_split_batched(smote_path, ['test', 'test'], batch_size, standardize=standardize, + data_format=data_format, num_workers=num_workers, + start_idx=start_idx, img_channels=img_channels, img_size=img_size, downsample=downsample, + data_type=data_type, is_tfs=pathmaster.is_tfs) + return loader1, loader2, loader3 + + \ No newline at end of file diff --git a/utils/get_paths.py b/utils/get_paths.py new file mode 100644 index 0000000..b22752e --- /dev/null +++ b/utils/get_paths.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Feb 27 14:55:43 2024 + +@author: dchen +""" +import os + +def data_paths(data_format, is_linux=False, is_hpc=False): + if is_linux: + base_path = "/mnt/r/ENGR_Chon/Dong/MATLAB_generate_results/NIH_PulseWatch" + labels_base_path = "/mnt/r/ENGR_Chon/NIH_Pulsewatch_Database/Adjudication_UConn" + saving_base_path = "/mnt/r/ENGR_Chon/Darren/Honors_Thesis/saves/analysis" + elif is_hpc: + base_path = "/gpfs/scratchfs1/kic14002/doh16101" + labels_base_path = "/gpfs/scratchfs1/hfp14002/lrm22005" + saving_base_path = "/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/saves/analysis" + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = r"R:\ENGR_Chon\Dong\MATLAB_generate_results\\NIH_PulseWatch" # Why double \\ before NIH_Pulsewatch_Database? + labels_base_path = r"R:\ENGR_Chon\\NIH_Pulsewatch_Database\Adjudication_UConn" # Why double \\ before NIH_Pulsewatch_Database? + saving_base_path = r"\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\saves" # Only when writing to file in the R drive do we need the entire address for the R drive + if data_format == 'csv': + data_path = os.path.join(base_path, "TFS_csv") + labels_path = os.path.join(labels_base_path, "final_attemp_4_1_Dong_Ohm") + saving_path = os.path.join(saving_base_path, "analysis") + elif data_format == 'png': + data_path = os.path.join(base_path, "TFS_plots") + labels_path = os.path.join(labels_base_path, "final_attemp_4_1_Dong_Ohm") + saving_path = os.path.join(saving_base_path, "analysis") + elif data_format == 'pt': + data_path = os.path.join(base_path, "PT_format") + labels_path = os.path.join(labels_base_path, "final_attemp_4_1_Dong_Ohm") + saving_path = os.path.join(saving_base_path, "analysis") + else: + raise ValueError("Invalid data format. Choose 'csv', 'png, or 'pt'.") + + return data_path, labels_path, saving_path + + +def models_path(is_linux=False, is_hpc=False): + if is_linux: + models_path = "/mnt/r/ENGR_Chon/Darren/Honors_Thesis/models" + elif is_hpc: + models_path = "/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/models" + else: + models_path = r"\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\models" + + return models_path + +# Base saving paths +focus = 'misc' +# focus = '2_layers_per_block' +# focus = '2_layers_per_block' +linux_saves_path = '/mnt/r/ENGR_Chon/Darren/Honors_Thesis/saves/' + focus + '/' +hpc_saves_path = '/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/saves/' + focus + '/' +saves_path = r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\saves' + '\\' + focus + '\\' + +def losslists_path(is_linux=False, is_hpc=False): + if is_linux: + losslists_path = linux_saves_path + 'losslists' + elif is_hpc: + losslists_path = hpc_saves_path + 'losslists' + else: + losslists_path = saves_path + 'losslists' + + return losslists_path + + +def runtime_lists_path(is_linux=False, is_hpc=False): + if is_linux: + runtime_lists_path = linux_saves_path + 'runtime_lists' + elif is_hpc: + runtime_lists_path = hpc_saves_path + 'runtime_lists' + else: + runtime_lists_path = saves_path + 'runtime_lists' + + return runtime_lists_path + + +def predictions_path(is_linux=False, is_hpc=False): + if is_linux: + predictions_path = linux_saves_path + 'predictions' + elif is_hpc: + predictions_path = hpc_saves_path + 'predictions' + else: + predictions_path = saves_path + 'predictions' + + return predictions_path + +def prediction_proba_path(is_linux=False, is_hpc=False): + if is_linux: + prediction_proba_path = linux_saves_path + 'prediction_proba' + elif is_hpc: + prediction_proba_path = hpc_saves_path + 'prediction_proba' + else: + prediction_proba_path = saves_path + 'prediction_proba' + + return prediction_proba_path + + +def metrics_path(is_linux=False, is_hpc=False): + if is_linux: + metrics_path = linux_saves_path + 'metrics' + elif is_hpc: + metrics_path = hpc_saves_path + 'metrics' + else: + metrics_path = saves_path + 'metrics' + + return metrics_path + + +def confusion_matrices_path(is_linux=False, is_hpc=False): + if is_linux: + confusion_matrices_path = linux_saves_path + 'confusion_matrices' + elif is_hpc: + confusion_matrices_path = hpc_saves_path + 'confusion_matrices' + else: + confusion_matrices_path = saves_path + 'confusion_matrices' + + return confusion_matrices_path + + +def checkpoints_path(is_linux=False, is_hpc=False): + if is_linux: + checkpoints_path = linux_saves_path + 'checkpoints' + elif is_hpc: + checkpoints_path = hpc_saves_path + 'checkpoints' + else: + checkpoints_path = saves_path + 'checkpoints' + + return checkpoints_path + +def hyperparameters_path(is_linux=False, is_hpc=False): + if is_linux: + hyperparameters_path = linux_saves_path + 'hyperparameters' + elif is_hpc: + hyperparameters_path = hpc_saves_path + 'hyperparameters' + else: + hyperparameters_path = saves_path + 'hyperparameters' + + return hyperparameters_path + +def loss_curves_path(is_linux=False, is_hpc=False): + if is_linux: + loss_curves_path = linux_saves_path + 'loss_curves' + elif is_hpc: + loss_curves_path = hpc_saves_path + 'loss_curves' + else: + loss_curves_path = saves_path + 'loss_curves' + + return loss_curves_path + + diff --git a/utils/misc_func.py b/utils/misc_func.py new file mode 100644 index 0000000..6893a71 --- /dev/null +++ b/utils/misc_func.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Mar 3 03:56:36 2024 + +@author: dchen +""" + +def substring_between_strings(main_string, start_string, end_string): + start_index = main_string.find(start_string) + if start_index == -1: + return None + + end_index = main_string.find(end_string, start_index + len(start_string)) + if end_index == -1: + return None + + return main_string[start_index + len(start_string):end_index] + + +def string_to_boolean(input_string): + if input_string.lower() in ['true', 't', 'yes', 'y', '1']: + return True + elif input_string.lower() in ['false', 'f', 'no', 'n', '0']: + return False + else: + raise ValueError("String does not represent a boolean value") diff --git a/utils/model_func.py b/utils/model_func.py new file mode 100644 index 0000000..95f19de --- /dev/null +++ b/utils/model_func.py @@ -0,0 +1,2145 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Feb 26 14:58:20 2024 + +@author: dchen +""" + +import os +import sys +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from tqdm import tqdm +import random +import time +import torch.autograd as autograd +from torch.cuda.amp import autocast, GradScaler + +# Import my own functions and classes +# from utils import get_paths +from utils import plot_save_func +from models.densenet import DenseNet3 as DenseNet +from models.densenet_configurable import DenseNet as DenseNet_config + +# If GPU is available, use GPU, else use CPU +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +# Seeds +torch.manual_seed(42) +np.random.seed(42) +random.seed(42) + + +def cross_val_2fold_DenseNet(model_hyperparameters, fold1_loader, fold2_loader, model_type=torch.float32, + n_epochs=100, n_classes=3, patience=10, save=False, + resume_checkpoint_path=None, pathmaster=None): + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Optimizer and scheduler hyperparameters + lr = 0.0001 + + # Define img_channels + img_channels = 1 + + # Resume checkpoint if specified + if resume_checkpoint_path is not None and os.path.exists(resume_checkpoint_path): + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + model_fold1, optimizer_fold1, scheduler_fold1, model_fold2, optimizer_fold2, scheduler_fold2, epoch, loss = load_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, pathmaster) + start_epoch = epoch + 1 + best_loss_cross_val = loss + else: + # Extract model hyperparameters + depth = model_hyperparameters['depth'] + growth_rate = model_hyperparameters['growth_rate'] + compression = model_hyperparameters['compression'] + bottleneck = model_hyperparameters['bottleneck'] + drop_rate = model_hyperparameters['drop_rate'] + class_weights = model_hyperparameters['class_weights'] + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + best_loss_cross_val = float('inf') # If no checkpoint is loaded, set to infinity + start_epoch = 0 + + if save: + # Save hyperparameters + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Create criterion for loss + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + + # Regularization + lambda_l1 = 0.01 + + # Initialize losslists + losslist_train = [] + losslist_cross_val = [] + + losslist_train_fold1 = [] + losslist_val_fold1 = [] + + losslist_train_fold2 = [] + losslist_val_fold2 = [] + + # Initialize runtime list + runtime_list = [] + + # Cross-validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in tqdm(range(start_epoch, n_epochs), desc='Cross-Validation', unit='epoch', leave=False): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + sys.stdout.flush() + + # Fold 1 training =============================================================================================================================================================== + model_fold1.train() + train_cum_loss_fold1 = 0 + for data_batch in tqdm(fold1_loader, total=len(fold1_loader), desc='Training', unit='batch', leave=False): + # Extract input and labels + X_train = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_train = data_batch['label'].to(device=device) + + # Forward pass + logits, _, _ = model_fold1(X_train) + + # Regularization (if applicable) + l1 = 0 + for p in model_fold1.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate total loss with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) + lambda_l1 * l1 + train_cum_loss_fold1 += batch_loss_train.item() + + # Clear gradients + optimizer_fold1.zero_grad() + + # Backwards pass + batch_loss_train.backward() + + # Optimizer step + optimizer_fold1.step() + + # Update scheduler + scheduler_fold1.step() + + loss_train_fold1 = train_cum_loss_fold1 / len(fold1_loader) + + sys.stderr.flush() + print('\nTraining for Fold #1 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 1 validation ============================================================================================================================================================= + model_fold1.eval() + with torch.no_grad(): + val_cum_loss_fold1 = 0 + for data_batch in tqdm(fold2_loader, total=len(fold2_loader), desc='Validation', unit='batch', leave=False): + X_val = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_val = data_batch['label'].to(device=device) + + logits, _, _ = model_fold1(X_val) + val_cum_loss_fold1 += criterion_val(logits.float(), Y_val.long()).item() + + loss_val_fold1 = val_cum_loss_fold1 / len(fold2_loader) + + sys.stderr.flush() + print('\nValidation for Fold #1 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 2 training =============================================================================================================================================================== + model_fold2.train() + train_cum_loss_fold2 = 0 + for data_batch in tqdm(fold2_loader, total=len(fold2_loader), desc='Training', unit='batch', leave=False): + # Extract input and labels + X_train = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_train = data_batch['label'].to(device=device) + + # Forward pass + logits, _, _ = model_fold2(X_train) + + # Regularization (if applicable) + l1 = 0 + for p in model_fold2.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate total loss with regularization + batch_loss_train = criterion(logits.to(torch.float32), Y_train.long()) + lambda_l1 * l1 + train_cum_loss_fold2 += batch_loss_train.item() + + # Clear gradients + optimizer_fold2.zero_grad() + + # Backwards pass + batch_loss_train.backward() + + # Optimizer step + optimizer_fold2.step() + + # Update scheduler + scheduler_fold2.step() + + loss_train_fold2 = train_cum_loss_fold2 / len(fold2_loader) + + sys.stderr.flush() + print('\nTraining for Fold #2 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 2 validation ============================================================================================================================================================= + model_fold2.eval() + with torch.no_grad(): + val_cum_loss_fold2 = 0 + for data_batch in tqdm(fold1_loader, total=len(fold1_loader), desc='Validation', unit='batch', leave=False): + X_val = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_val = data_batch['label'].to(device=device) + + logits, _, _ = model_fold2(X_val) + val_cum_loss_fold2 += criterion(logits.float(), Y_val.long()).item() + + loss_val_fold2 = val_cum_loss_fold2 / len(fold1_loader) + + sys.stderr.flush() + print('\nValidation for Fold #2 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + # =============================================================================================================================================================================== + + # Caluclate epoch losses + epoch_loss_train = (loss_train_fold1 + loss_train_fold2) / 2 + epoch_loss_cross_val = (loss_val_fold1 + loss_val_fold2) / 2 + + # Append to losslists + losslist_train.append(epoch_loss_train) + losslist_cross_val.append(epoch_loss_cross_val) + + losslist_train_fold1.append(loss_train_fold1) + losslist_val_fold1.append(loss_val_fold1) + + losslist_train_fold2.append(loss_train_fold2) + losslist_val_fold2.append(loss_val_fold2) + + # Return the best cross-validation loss and save best checkpoint (epoch) + best_loss_cross_val = save_best_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, epoch_loss_cross_val, best_loss_cross_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Cross-Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_cross_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_cross_val): + break + + # Saving + if save: + title = 'Training and Cross-Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_cross_val, title, save, pathmaster) + + plot_save_func.save_losslists_2fold(losslist_train_fold1, losslist_val_fold1, losslist_train_fold2, losslist_val_fold2, losslist_train, losslist_cross_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +def cross_val_2fold_DenseNet_mixed(model_hyperparameters, fold1_loader, fold2_loader, + n_epochs=100, n_classes=3, patience=10, save=False, + resume_checkpoint_path=None, pathmaster=None): + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Optimizer and scheduler hyperparameters + lr = 0.0005 + + # Define img_channels + img_channels = 1 + + # Resume checkpoint if specified + if resume_checkpoint_path is not None and os.path.exists(resume_checkpoint_path): + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + model_fold2 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + model_fold1, optimizer_fold1, scheduler_fold1, model_fold2, optimizer_fold2, scheduler_fold2, epoch, loss = load_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, pathmaster) + start_epoch = epoch + 1 + best_loss_cross_val = loss + else: + # Extract model hyperparameters + depth = model_hyperparameters['depth'] + growth_rate = model_hyperparameters['growth_rate'] + compression = model_hyperparameters['compression'] + bottleneck = model_hyperparameters['bottleneck'] + drop_rate = model_hyperparameters['drop_rate'] + class_weights = model_hyperparameters['class_weights'] + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + model_fold2 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + best_loss_cross_val = float('inf') # If no checkpoint is loaded, set to infinity + start_epoch = 0 + + if save: + # Save hyperparameters + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Create criterion for loss + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + + # Regularization + lambda_l1 = 0.01 + + # Initialize losslists + losslist_train = [] + losslist_cross_val = [] + + losslist_train_fold1 = [] + losslist_val_fold1 = [] + + losslist_train_fold2 = [] + losslist_val_fold2 = [] + + # Initialize runtime list + runtime_list = [] + + # Initialize predictions lists + predictions_list_train = [] + predictions_list_val = [] + + # Initialize true labels lists + true_labels_list_train = [] + true_labels_list_val = [] + + # Scalers + scaler_fold1 = GradScaler() + scaler_fold2 = GradScaler() + + # Cross-validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in tqdm(range(start_epoch, n_epochs), desc='Cross-Validation', unit='epoch', leave=False): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + sys.stdout.flush() + + # Epoch predictions + predictions_epoch_train = [] + predictions_epoch_val = [] + + # Fold 1 training =============================================================================================================================================================== + model_fold1.train() + train_cum_loss_fold1 = 0 + for data_batch in tqdm(fold1_loader, total=len(fold1_loader), desc='Training', unit='batch', leave=False): + # Extract input and labels + X_train = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_train = data_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_train.append(torch.reshape(Y_train, (-1,1))) + + with autocast(): + # Forward pass + logits, predictions, _ = model_fold1(X_train) + + predictions_epoch_train.append(torch.reshape(predictions, (-1,1))) + + # Regularization (if applicable) + l1 = 0 + for p in model_fold1.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate total loss with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) + lambda_l1 * l1 + train_cum_loss_fold1 += batch_loss_train.item() + + # Clear gradients + optimizer_fold1.zero_grad() + + # Backwards pass + scaler_fold1.scale(batch_loss_train).backward() + + # Optimizer step + scaler_fold1.step(optimizer_fold1) + + # Scaler update + scaler_fold1.update() + + # Update scheduler + scheduler_fold1.step() + + loss_train_fold1 = train_cum_loss_fold1 / len(fold1_loader) + + sys.stderr.flush() + print('\nTraining for Fold #1 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 1 validation ============================================================================================================================================================= + model_fold1.eval() + with torch.no_grad(): + val_cum_loss_fold1 = 0 + for data_batch in tqdm(fold2_loader, total=len(fold2_loader), desc='Validation', unit='batch', leave=False): + X_val = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_val = data_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_val.append(torch.reshape(Y_val, (-1,1))) + + logits, predictions, _ = model_fold1(X_val) + + predictions_epoch_val.append(torch.reshape(predictions, (-1,1))) + + val_cum_loss_fold1 += criterion_loss(logits.float(), Y_val.long()).item() + + loss_val_fold1 = val_cum_loss_fold1 / len(fold2_loader) + + sys.stderr.flush() + print('\nValidation for Fold #1 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 2 training =============================================================================================================================================================== + model_fold2.train() + train_cum_loss_fold2 = 0 + for data_batch in tqdm(fold2_loader, total=len(fold2_loader), desc='Training', unit='batch', leave=False): + # Extract input and labels + X_train = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_train = data_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_train.append(torch.reshape(Y_train, (-1,1))) + + with autocast(): + # Forward pass + logits, predictions, _ = model_fold2(X_train) + + predictions_epoch_train.append(torch.reshape(predictions, (-1,1))) + + # Regularization (if applicable) + l1 = 0 + for p in model_fold2.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate total loss with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) + lambda_l1 * l1 + train_cum_loss_fold2 += batch_loss_train.item() + + # Clear gradients + optimizer_fold2.zero_grad() + + # Backwards pass + scaler_fold2.scale(batch_loss_train).backward() + + # Optimizer step + scaler_fold2.step(optimizer_fold2) + + # Scaler update + scaler_fold2.update() + + # Update scheduler + scheduler_fold2.step() + + loss_train_fold2 = train_cum_loss_fold2 / len(fold2_loader) + + sys.stderr.flush() + print('\nTraining for Fold #2 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Fold 2 validation ============================================================================================================================================================= + model_fold2.eval() + with torch.no_grad(): + val_cum_loss_fold2 = 0 + for data_batch in tqdm(fold1_loader, total=len(fold1_loader), desc='Validation', unit='batch', leave=False): + X_val = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_val = data_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_val.append(torch.reshape(Y_val, (-1,1))) + + logits, predictions, _ = model_fold2(X_val) + + predictions_epoch_val.append(torch.reshape(predictions, (-1,1))) + + val_cum_loss_fold2 += criterion_val(logits.float(), Y_val.long()).item() + + loss_val_fold2 = val_cum_loss_fold2 / len(fold1_loader) + + sys.stderr.flush() + print('\nValidation for Fold #2 in Epoch', epoch, 'has been completed!') + sys.stdout.flush() + # =============================================================================================================================================================================== + + # Caluclate epoch losses + epoch_loss_train = (loss_train_fold1 + loss_train_fold2) / 2 + epoch_loss_cross_val = (loss_val_fold1 + loss_val_fold2) / 2 + + # Append to losslists + losslist_train.append(epoch_loss_train) + losslist_cross_val.append(epoch_loss_cross_val) + + losslist_train_fold1.append(loss_train_fold1) + losslist_val_fold1.append(loss_val_fold1) + + losslist_train_fold2.append(loss_train_fold2) + losslist_val_fold2.append(loss_val_fold2) + + # Return the best cross-validation loss and save best checkpoint (epoch) + best_loss_cross_val = save_best_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, epoch_loss_cross_val, best_loss_cross_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Cross-Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_cross_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch predictions + predictions_epoch_train = np.array(torch.cat(predictions_epoch_train, dim=0).to('cpu')) + predictions_epoch_val = np.array(torch.cat(predictions_epoch_val, dim=0).to('cpu')) + + predictions_list_train.append(predictions_epoch_train) + predictions_list_val.append(predictions_epoch_val) + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_cross_val): + break + + # Convert true label list into array + true_labels_train = np.array(torch.cat(true_labels_list_train, dim=0).to('cpu')) + true_labels_val = np.array(torch.cat(true_labels_list_val, dim=0).to('cpu')) + + # Saving + if save: + title = 'Training and Cross-Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_cross_val, title, save, pathmaster) + + title = 'Training and Cross-Validation Accuracy' + plot_save_func.accuracy_curves(true_labels_train, true_labels_val, predictions_list_train, predictions_list_val, title, save, pathmaster) + + plot_save_func.save_losslists_2fold(losslist_train_fold1, losslist_val_fold1, losslist_train_fold2, losslist_val_fold2, losslist_train, losslist_cross_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +# Utilizes train() and validate() functions +def cross_val_2fold_DenseNet_func(model_hyperparameters, fold1_loader, fold2_loader, model_type=torch.float32, + n_epochs=100, n_classes=3, patience=10, save=False, + resume_checkpoint_path=None, pathmaster=None): + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Optimizer and scheduler hyperparameters + lr = 0.0005 + + # Define img_channels + img_channels = 1 + + # Resume checkpoint if specified + if resume_checkpoint_path is not None and os.path.exists(resume_checkpoint_path): + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + model_fold1, optimizer_fold1, scheduler_fold1, model_fold2, optimizer_fold2, scheduler_fold2, epoch, loss = load_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, pathmaster) + start_epoch = epoch + 1 + best_loss_cross_val = loss + else: + # Extract model hyperparameters + depth = model_hyperparameters['depth'] + growth_rate = model_hyperparameters['growth_rate'] + compression = model_hyperparameters['compression'] + bottleneck = model_hyperparameters['bottleneck'] + drop_rate = model_hyperparameters['drop_rate'] + class_weights = model_hyperparameters['class_weights'] + + # Define DenseNet model based on loaded hyperparameters + model_fold1 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer_fold1 = torch.optim.Adam(model_fold1.parameters(), lr=lr) + optimizer_fold2 = torch.optim.Adam(model_fold2.parameters(), lr=lr) + + scheduler_fold1 = IdentityScheduler(optimizer_fold1) + scheduler_fold2 = IdentityScheduler(optimizer_fold2) + + best_loss_cross_val = float('inf') # If no checkpoint is loaded, set to infinity + start_epoch = 0 + + if save: + # Save hyperparameters + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Create criterion for loss + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + + # Regularization + lambda_l1 = 0.01 + + # Initialize losslists + losslist_train = [] + losslist_cross_val = [] + + losslist_train_fold1 = [] + losslist_val_fold1 = [] + + losslist_train_fold2 = [] + losslist_val_fold2 = [] + + # Initialize runtime list + runtime_list = [] + + # Cross-validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in tqdm(range(start_epoch, n_epochs), desc='Cross-Validation', unit='epoch', leave=False): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + sys.stdout.flush() + + # Fold 1 (train on fold1, validate on fold2) + model_fold1, optimizer_fold1, scheduler_fold1, loss_train_fold1 = train(model_fold1, fold1_loader, optimizer_fold1, scheduler_fold1, criterion_train, lambda_l1) + loss_val_fold1 = validate(model_fold1, fold2_loader, criterion_val) + + # Fold 2 (train on fold2, validate on fold1) + model_fold2, optimizer_fold2, scheduler_fold2, loss_train_fold2 = train(model_fold2, fold2_loader, optimizer_fold2, scheduler_fold2, criterion_train, lambda_l1) + loss_val_fold2 = validate(model_fold2, fold1_loader, criterion_val) + + # Caluclate epoch losses + epoch_loss_train = (loss_train_fold1 + loss_train_fold2) / 2 + epoch_loss_cross_val = (loss_val_fold1 + loss_val_fold2) / 2 + + # Append to losslists + losslist_train.append(epoch_loss_train) + losslist_cross_val.append(epoch_loss_cross_val) + + losslist_train_fold1.append(loss_train_fold1) + losslist_val_fold1.append(loss_val_fold1) + + losslist_train_fold2.append(loss_train_fold2) + losslist_val_fold2.append(loss_val_fold2) + + # Return the best cross-validation loss and save best checkpoint (epoch) + best_loss_cross_val = save_best_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, epoch_loss_cross_val, best_loss_cross_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Cross-Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_cross_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_cross_val): + break + + # Saving + if save: + title = 'Training and Cross-Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_cross_val, title, save, pathmaster) + + plot_save_func.save_losslists_2fold(losslist_train_fold1, losslist_val_fold1, losslist_train_fold2, losslist_val_fold2, losslist_train, losslist_cross_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +def train_validate_DenseNet(model_hyperparameters, train_loader, val_loader, model_type=torch.float32, + n_epochs=100, n_classes=3, patience=10, save=False, + resume_checkpoint_path=None, pathmaster=None): + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Optimizer and scheduler hyperparameters + lr = 0.0005 + + # Resume checkpoint if specified + if resume_checkpoint_path is not None and os.path.exists(resume_checkpoint_path): + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + + # Define DenseNet model based on loaded hyperparameters + model = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + scheduler = IdentityScheduler(optimizer) + + model, optimizer, scheduler, epoch, loss = load_checkpoint(model, optimizer, scheduler, pathmaster) + start_epoch = epoch + 1 + best_loss_val = loss + else: + # Extract model hyperparameters + depth = model_hyperparameters['depth'] + growth_rate = model_hyperparameters['growth_rate'] + compression = model_hyperparameters['compression'] + bottleneck = model_hyperparameters['bottleneck'] + drop_rate = model_hyperparameters['drop_rate'] + class_weights = model_hyperparameters['class_weights'] + + # Define DenseNet model based on input hyperparameters + model = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create optimizer and scheduler + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + scheduler = IdentityScheduler(optimizer) + + best_loss_val = float('inf') # If no checkpoint is loaded, set to infinity + start_epoch = 0 + + if save: + # Save hyperparameters + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Create criterion for loss + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + + # Regularization + lambda_l1 = 0.01 + + # Initialize losslists + losslist_train = [] + losslist_val = [] + + # Initialize runtime list + runtime_list = [] + + # Training and validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in tqdm(range(start_epoch, n_epochs), desc='Training and Validation', unit='epoch', leave=False): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + # Training + model.train() + # Reset training sum of epoch loss and batch_count + sum_epoch_loss_train = 0 + sys.stdout.flush() + for train_batch in tqdm(train_loader, total=len(train_loader), desc='Training Epoch', unit='batch', leave=False): + # Extract input and labels + # train_batch['data'].shape = [batch_size, img_channels, img_size, img_size] + X_train = train_batch['data'].reshape(train_batch['data'].shape[0], train_batch['data'].shape[1], train_batch['data'].shape[-1], train_batch['data'].shape[-1]).to(device=device) + Y_train = train_batch['label'].to(device=device) + + # Forward pass + logits, _, _ = model(X_train) + + # Regularization + l1 = 0 + for p in model.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate sum of total loss for epoch with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) # Criterion returns a scalar tensor + batch_loss_train += lambda_l1 * l1 + + # Clear gradients + optimizer.zero_grad(set_to_none=True) + + # Backwards pass + batch_loss_train.backward() + + # Optimizer step + optimizer.step() + + # Generate epoch loss + sum_epoch_loss_train += batch_loss_train.item() + + # Update scheduler + scheduler.step() + + # Calculate epoch loss for training + epoch_loss_train = sum_epoch_loss_train / len(train_loader) + losslist_train.append(epoch_loss_train) + + sys.stderr.flush() + print('\nTraining for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Validation + model.eval() + sum_epoch_loss_val = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for val_batch in tqdm(val_loader, total=len(val_loader), desc='Validation Epoch', unit='batch', leave=False): + # Extract input and labels + X_val = val_batch['data'].reshape(val_batch['data'].shape[0], val_batch['data'].shape[1], val_batch['data'].shape[-1], val_batch['data'].shape[-1]).to(device=device) + Y_val = val_batch['label'].to(device=device) + + # Forward pass + logits, _, _ = model(X_val) + + # Calculate sum of total loss for epoch + sum_epoch_loss_val += criterion_val(logits.float(), Y_val.long()).item() # Criterion returns a scalar tensor + + # Calculate epoch loss for validation + epoch_loss_val = sum_epoch_loss_val / len(val_loader) + losslist_val.append(epoch_loss_val) + + sys.stderr.flush() + print('\nValidation for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # # Temporarily save checkpoint after each epoch + # save_checkpoint(model, optimizer, scheduler, epoch, loss=epoch_loss_val, checkpoint_path=temp_checkpoint_path) + + # Return the best validation loss and save best checkpoint (epoch) + best_loss_val = save_best_checkpoint(model, optimizer, scheduler, epoch, epoch_loss_val, best_loss_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_val): + break + + # Saving + if save: + title = 'Training and Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_val, title, save, pathmaster) + + plot_save_func.save_losslists(losslist_train, losslist_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +def train_validate_DenseNet_mixed(model_hyperparameters, train_loader, val_loader, + n_epochs=100, n_classes=3, patience=10, save=False, + resume_checkpoint_path=None, pathmaster=None): + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Optimizer and scheduler hyperparameters + lr = 0.0005 + + # Resume checkpoint if specified + if resume_checkpoint_path is not None and os.path.exists(resume_checkpoint_path): + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + + # Define DenseNet model based on loaded hyperparameters + model = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + + # Create optimizer and scheduler + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + scheduler = IdentityScheduler(optimizer) + + model, optimizer, scheduler, epoch, loss = load_checkpoint(model, optimizer, scheduler, pathmaster) + start_epoch = epoch + 1 + best_loss_val = loss + else: + # Extract model hyperparameters + depth = model_hyperparameters['depth'] + growth_rate = model_hyperparameters['growth_rate'] + compression = model_hyperparameters['compression'] + bottleneck = model_hyperparameters['bottleneck'] + drop_rate = model_hyperparameters['drop_rate'] + class_weights = model_hyperparameters['class_weights'] + + # Define DenseNet model based on input hyperparameters + model = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device) + + # Create optimizer and scheduler + optimizer = torch.optim.Adam(model.parameters(), lr=lr) + scheduler = IdentityScheduler(optimizer) + + best_loss_val = float('inf') # If no checkpoint is loaded, set to infinity + start_epoch = 0 + + if save: + # Save hyperparameters + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Create criterion for loss + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + + # Regularization + lambda_l1 = 0.01 + + # Initialize losslists + losslist_train = [] + losslist_val = [] + + # Initialize runtime list + runtime_list = [] + + # Initialize predictions lists + predictions_list_train = [] + predictions_list_val = [] + + # Initialize true labels lists + true_labels_list_train = [] + true_labels_list_val = [] + + # Scalers + scaler = GradScaler() + + # Training and validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in tqdm(range(start_epoch, n_epochs), desc='Training and Validation', unit='epoch', leave=False): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + # Training + model.train() + # Reset training sum of epoch loss and batch_count + sum_epoch_loss_train = 0 + sys.stdout.flush() + + # Epoch predictions + predictions_epoch_train = [] + predictions_epoch_val = [] + + for train_batch in tqdm(train_loader, total=len(train_loader), desc='Training Epoch', unit='batch', leave=False): + # Extract input and labels + # train_batch['data'].shape = [batch_size, img_channels, img_size, img_size] + X_train = train_batch['data'].reshape(train_batch['data'].shape[0], train_batch['data'].shape[1], train_batch['data'].shape[-1], train_batch['data'].shape[-1]).to(device=device) + Y_train = train_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_train.append(torch.reshape(Y_train, (-1,1))) + + with autocast(): + # Forward pass + logits, predictions, _ = model(X_train) + + predictions_epoch_train.append(torch.reshape(predictions, (-1,1))) + + # Regularization + l1 = 0 + for p in model.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate sum of total loss for epoch with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) # Criterion returns a scalar tensor + batch_loss_train += lambda_l1 * l1 + + # Clear gradients + optimizer.zero_grad(set_to_none=True) + + # Backwards pass + scaler.scale(batch_loss_train).backward() + + # Optimizer step + scaler.step(optimizer) + + # Scaler update + scaler.update() + + # Generate epoch loss + sum_epoch_loss_train += batch_loss_train.item() + + # Update scheduler + scheduler.step() + + # Calculate epoch loss for training + epoch_loss_train = sum_epoch_loss_train / len(train_batch) + losslist_train.append(epoch_loss_train) + + sys.stderr.flush() + print('\nTraining for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Validation + model.eval() + sum_epoch_loss_val = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for val_batch in tqdm(val_loader, total=len(val_loader), desc='Validation Epoch', unit='batch', leave=False): + # Extract input and labels + X_val = val_batch['data'].reshape(val_batch['data'].shape[0], val_batch['data'].shape[1], val_batch['data'].shape[-1], val_batch['data'].shape[-1]).to(device=device) + Y_val = val_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_val.append(torch.reshape(Y_val, (-1,1))) + + # Forward pass + logits, predictions, _ = model(X_val) + predictions_epoch_val.append(torch.reshape(predictions, (-1,1))) + + # Calculate sum of total loss for epoch + sum_epoch_loss_val += criterion_val(logits.float(), Y_val.long()).item() # Criterion returns a scalar tensor + + # Calculate epoch loss for validation + epoch_loss_val = sum_epoch_loss_val / len(val_loader) + losslist_val.append(epoch_loss_val) + + sys.stderr.flush() + print('\nValidation for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # # Temporarily save checkpoint after each epoch + # save_checkpoint(model, optimizer, scheduler, epoch, loss=epoch_loss_val, checkpoint_path=temp_checkpoint_path) + + # Return the best validation loss and save best checkpoint (epoch) + best_loss_val = save_best_checkpoint(model, optimizer, scheduler, epoch, epoch_loss_val, best_loss_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch predictions + predictions_epoch_train = np.array(torch.cat(predictions_epoch_train, dim=0).to('cpu')) + predictions_epoch_val = np.array(torch.cat(predictions_epoch_val, dim=0).to('cpu')) + + predictions_list_train.append(predictions_epoch_train) + predictions_list_val.append(predictions_epoch_val) + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_val): + break + + # Convert true label list into array + true_labels_train = np.array(torch.cat(true_labels_list_train, dim=0).to('cpu')) + true_labels_val = np.array(torch.cat(true_labels_list_val, dim=0).to('cpu')) + + # Saving + if save: + title = 'Training and Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_val, title, save, pathmaster) + + title = 'Training and Validation Accuracy' + plot_save_func.accuracy_curves(true_labels_train, true_labels_val, predictions_list_train, predictions_list_val, title, save, pathmaster) + + plot_save_func.save_losslists(losslist_train, losslist_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +def train_validate_DenseNet_config(config, train_loader, val_loader, + n_epochs=100, n_classes=3, patience=10, save=False, + pathmaster=None): + # # Set filetag + # file_tag = str(dt.datetime.now()) + # # Define characters to replace with underscores + # chars_to_replace = [' ', ':', '.', '-'] + + # # Replace characters with underscores + # for char in chars_to_replace: + # file_tag = file_tag.replace(char, '_') + # pathmaster.set_file_tag(file_tag) + + # Save hyperparameters + model_hyperparameters = { # Default, no bottleneck or compression + 'depth': config['depth'], + 'growth_rate': config['growth_rate'], + 'compression': config['compression'], + 'bottleneck': config['bottleneck'], + 'drop_rate': config['drop_rate'], + 'class_weights': config['class_weights'], + 'learning_rate': config['learning_rate'], + 'num_dense_tran': config['num_dense_tran'], + 'lambda_l1': config['lambda_l1'], + 'activation': activation_to_string(config['activation']), + } + + if save: + plot_save_func.save_hyperparameters(model_hyperparameters, pathmaster) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + img_channels = 1 + + model = DenseNet_config(img_channels, config['depth'], n_classes, config['growth_rate'], config['compression'], + config['bottleneck'], config['drop_rate'], config['activation'], config['num_dense_tran']).to(device=device) + + # Loss function and optimizer + criterion_train = nn.CrossEntropyLoss(weight=torch.tensor(config['class_weights']).to(device=device)) + criterion_val = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=config['learning_rate']) + scheduler = IdentityScheduler(optimizer) + + + # Scalers + scaler = GradScaler() + + # Initialize losslists + losslist_train = [] + losslist_val = [] + + # Initialize predictions lists + predictions_list_train = [] + predictions_list_val = [] + + # Initialize true labels lists + true_labels_list_train = [] + true_labels_list_val = [] + + # Initialize runtime list + runtime_list = [] + + # Create EarlyStoppingCallback object + early_stopping_callback = EarlyStoppingCallback(patience) + + # Initialize best validation loss + best_loss_val = float('inf') # If no checkpoint is loaded, set to infinity + + start_epoch = 0 + # Training and validation + print('\n===========================================================================================') + sys.stdout.flush() + for epoch in range(start_epoch, n_epochs): # Creates a training progress bar with units of epoch + start_time = time.time() + sys.stderr.flush() + print("\nEntering Epoch:", epoch) + # Training + model.train() + # Reset training sum of epoch loss and batch_count + sum_epoch_loss_train = 0 + sys.stdout.flush() + + # Epoch predictions + predictions_epoch_train = [] + predictions_epoch_val = [] + + for train_batch in tqdm(train_loader, total=len(train_loader), desc='Training Epoch', unit='batch', leave=False): + # Extract input and labels + # train_batch['data'].shape = [batch_size, img_channels, img_size, img_size] + X_train = train_batch['data'].reshape(train_batch['data'].shape[0], train_batch['data'].shape[1], train_batch['data'].shape[-1], train_batch['data'].shape[-1]).to(device=device) + Y_train = train_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_train.append(torch.reshape(Y_train, (-1,1))) + + with autocast(): + # Forward pass + logits, predictions, _ = model(X_train) + + predictions_epoch_train.append(torch.reshape(predictions, (-1,1))) + + # Regularization + l1 = 0 + for p in model.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate sum of total loss for epoch with regularization + batch_loss_train = criterion_train(logits.to(torch.float32), Y_train.long()) # Criterion returns a scalar tensor + batch_loss_train += config['lambda_l1'] * l1 + + # Clear gradients + optimizer.zero_grad(set_to_none=True) + + # Backwards pass + scaler.scale(batch_loss_train).backward() + + # Optimizer step + scaler.step(optimizer) + + # Scaler update + scaler.update() + + # Generate epoch loss + sum_epoch_loss_train += batch_loss_train.item() + + # Update scheduler + scheduler.step() + + # Calculate epoch loss for training + epoch_loss_train = sum_epoch_loss_train / len(train_batch) + losslist_train.append(epoch_loss_train) + + sys.stderr.flush() + print('\nTraining for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Validation + model.eval() + sum_epoch_loss_val = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for val_batch in tqdm(val_loader, total=len(val_loader), desc='Validation Epoch', unit='batch', leave=False): + # Extract input and labels + X_val = val_batch['data'].reshape(val_batch['data'].shape[0], val_batch['data'].shape[1], val_batch['data'].shape[-1], val_batch['data'].shape[-1]).to(device=device) + Y_val = val_batch['label'].to(device=device) + + if epoch == start_epoch: + true_labels_list_val.append(torch.reshape(Y_val, (-1,1))) + + # Forward pass + logits, predictions, _ = model(X_val) + predictions_epoch_val.append(torch.reshape(predictions, (-1,1))) + + # Calculate sum of total loss for epoch + sum_epoch_loss_val += criterion_val(logits.float(), Y_val.long()).item() # Criterion returns a scalar tensor + + # Calculate epoch loss for validation + epoch_loss_val = sum_epoch_loss_val / len(val_loader) + losslist_val.append(epoch_loss_val) + + sys.stderr.flush() + print('\nValidation for Epoch', epoch, 'has been completed!') + sys.stdout.flush() + + # Return the best validation loss and save best checkpoint (epoch) + best_loss_val = save_best_checkpoint(model, optimizer, scheduler, epoch, epoch_loss_val, best_loss_val, pathmaster) + + # Update line + sys.stderr.flush() + print("\n======> Epoch: {}/{}, Training Loss: {:.4f}, Validation Loss: {:.4f}".format(epoch, n_epochs-1, epoch_loss_train, epoch_loss_val)) + print('\n===========================================================================================') + sys.stdout.flush() + + # Add epoch predictions + predictions_epoch_train = np.array(torch.cat(predictions_epoch_train, dim=0).to('cpu')) + predictions_epoch_val = np.array(torch.cat(predictions_epoch_val, dim=0).to('cpu')) + + predictions_list_train.append(predictions_epoch_train) + predictions_list_val.append(predictions_epoch_val) + + # Add epoch time to runtime_list + end_time = time.time() + time_passed = end_time-start_time # in seconds + runtime_list.append(time_passed) + + # Call the early stopping callback + if early_stopping_callback(epoch, epoch_loss_val): + break + + # Convert true label list into array + true_labels_train = np.array(torch.cat(true_labels_list_train, dim=0).to('cpu')) + true_labels_val = np.array(torch.cat(true_labels_list_val, dim=0).to('cpu')) + + if save: + title = 'Training and Validation Loss' + plot_save_func.train_val_loss(losslist_train, losslist_val, title, save, pathmaster) + + title = 'Training and Validation Accuracy' + plot_save_func.accuracy_curves(true_labels_train, true_labels_val, predictions_list_train, predictions_list_val, title, save, pathmaster) + + plot_save_func.save_losslists(losslist_train, losslist_val, pathmaster) + plot_save_func.save_runtime_list(runtime_list, pathmaster) + + +def best_DenseNet_2fold(fold1_loader, fold2_loader, model_type=torch.float32, n_classes=3, save=False, pathmaster=None): + print('\n===========================================================================================') + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Get paths + checkpoints_path = pathmaster.checkpoints_path() + + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate = load_hyperparameters(pathmaster) + # When testing on the test set, drop_rate should always be 0 + + # Define img_channels + img_channels = 1 + + # Initialize model + model_fold1 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create criterion for loss + criterion = nn.CrossEntropyLoss() + + # If checkpoint is not specified, terminate the function + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + assert os.path.exists(checkpoint_path), 'Function terminated. Not a valid checkpoint path.' + + # Load models + model_fold1, model_fold2 = load_model_2fold(model_fold1, model_fold2, pathmaster) + + # Fold 1 ======================================================================================================================================================================= + # Initialize true label lists + true_labels_list_fold1 = [] + + # Intialize output (prediction) lists + predictions_list_fold1 = [] + prediction_proba_list_fold1 = [] + + # Validation + model_fold1.eval() + cum_loss_fold1 = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(fold2_loader, total=len(fold2_loader), desc='Testing Fold #1', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + true_labels_list_fold1.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model_fold1(X) + predictions_list_fold1.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list_fold1.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss_fold1 += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss_fold1 = cum_loss_fold1 / len(fold2_loader) + + # Convert true label list into array + true_labels_fold1 = np.array(torch.cat(true_labels_list_fold1, dim=0).to('cpu')) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions_fold1 = np.array(torch.cat(predictions_list_fold1, dim=0).to('cpu')) + prediction_proba_fold1 = np.array(torch.cat(prediction_proba_list_fold1, dim=0).to('cpu')) + + # Fold 2 ======================================================================================================================================================================= + # Initialize true label lists + true_labels_list_fold2 = [] + + # Intialize output (prediction) lists + predictions_list_fold2 = [] + prediction_proba_list_fold2 = [] + + # Validation + model_fold2.eval() + cum_loss_fold2 = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(fold1_loader, total=len(fold1_loader), desc='Testing Fold #2', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + true_labels_list_fold2.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model_fold2(X) + predictions_list_fold2.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list_fold2.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss_fold2 += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss_fold2 = cum_loss_fold2 / len(fold1_loader) + + # Convert true label list into array + true_labels_fold2 = np.array(torch.cat(true_labels_list_fold2, dim=0).to('cpu')) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions_fold2 = np.array(torch.cat(predictions_list_fold2, dim=0).to('cpu')) + prediction_proba_fold2 = np.array(torch.cat(prediction_proba_list_fold2, dim=0).to('cpu')) + # ============================================================================================================================================================================== + + # Create overall lists + true_labels = np.concatenate((true_labels_fold1, true_labels_fold2), axis=0) + predictions = np.concatenate((predictions_fold1, predictions_fold2), axis=0) + prediction_proba = np.concatenate((prediction_proba_fold1, prediction_proba_fold2), axis=0) + + # Print mean validation loss + mean_loss = (loss_fold1 + loss_fold2) / 2 + print('\n=====> Fold #1 Loss: %.4f' % loss_fold1) + print('=====> Fold #2 Loss: %.4f' % loss_fold2) + print('=====> Mean Loss: %.4f' % mean_loss) + + # Saving + if save: + from sklearn.metrics import confusion_matrix + conf_matrix = confusion_matrix(true_labels, predictions) + title = 'Cross-Validation Confusion Matrix' + plot_save_func.conf_matrix(conf_matrix, title, save, pathmaster) + + plot_save_func.save_labels(true_labels, pathmaster) + plot_save_func.save_predictions(predictions, pathmaster) + plot_save_func.save_prediction_proba(prediction_proba, pathmaster) + plot_save_func.metrics_2fold(true_labels_fold1, true_labels_fold2, predictions_fold1, predictions_fold2, prediction_proba_fold1, prediction_proba_fold2, save, pathmaster) + + clf_names = ['Fold #1', 'Fold #2', 'Combined'] + plot_save_func.mean_roc_curves([true_labels_fold1, true_labels_fold2], [prediction_proba_fold1, prediction_proba_fold2], clf_names, save, pathmaster) + + +# Utilizes test() function +def best_DenseNet_2fold_func(fold1_loader, fold2_loader, model_type=torch.float32, n_classes=3, save=False, pathmaster=None): + print('\n===========================================================================================') + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Get paths + checkpoints_path = pathmaster.checkpoints_path() + + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, _ = load_hyperparameters(pathmaster) + # When testing on the test set, drop_rate should always be 0 + + # Define img_channels + img_channels = 1 + + # Initialize model + model_fold1 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + model_fold2 = DenseNet(img_channels=img_channels, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create criterion for loss + criterion = nn.CrossEntropyLoss() + + # If checkpoint is not specified, terminate the function + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + assert os.path.exists(checkpoint_path), 'Function terminated. Not a valid checkpoint path.' + + # Load models + model_fold1, model_fold2 = load_model_2fold(model_fold1, model_fold2, pathmaster) + + # Validation + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + + true_labels_fold1, predictions_fold1, prediction_proba_fold1, loss_fold1 = test(model_fold1, fold1_loader, criterion, n_classes) + true_labels_fold2, predictions_fold2, prediction_proba_fold2, loss_fold2 = test(model_fold2, fold2_loader, criterion, n_classes) + + # Create overall lists + true_labels = true_labels_fold1 + true_labels_fold2 + predictions = predictions_fold1 + predictions_fold2 + prediction_proba = prediction_proba_fold1 + prediction_proba_fold2 + + # Print mean validation loss + mean_loss = (loss_fold1 + loss_fold2) / 2 + print('\n======> Fold #1 Loss: %.4f' % loss_fold1) + print('=====> Fold #2 Loss: %.4f' % loss_fold2) + print('======> Mean Loss: %.4f' % mean_loss) + + # Saving + if save: + from sklearn.metrics import confusion_matrix + conf_matrix = confusion_matrix(true_labels, predictions) + title = 'Cross-Validation Confusion Matrix' + plot_save_func.conf_matrix(conf_matrix, title, save, pathmaster) + + plot_save_func.save_labels(true_labels, pathmaster) + plot_save_func.save_predictions(predictions, pathmaster) + plot_save_func.save_prediction_proba(prediction_proba, pathmaster) + plot_save_func.metrics_2fold(true_labels_fold1, true_labels_fold2, predictions_fold1, predictions_fold2, save, pathmaster) + + clf_names = ['Fold #1', 'Fold #2', 'Combined'] + plot_save_func.mean_roc_curves([true_labels_fold1, true_labels_fold2], [prediction_proba_fold1, prediction_proba_fold2], clf_names, save, pathmaster) + + +def best_DenseNet(data_loader, model_type=torch.float32, n_classes=3, save=False, pathmaster=None): + print('\n===========================================================================================') + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Get paths + checkpoints_path = pathmaster.checkpoints_path() + + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, class_weights = load_hyperparameters(pathmaster) + # When testing on the test set, drop_rate should always be 0 + + # Initialize model + model = DenseNet(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate).to(device=device, dtype=model_type) + + # Create criterion for loss + criterion = nn.CrossEntropyLoss() + + # If checkpoint is not specified, terminate the function + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + assert os.path.exists(checkpoint_path), 'Function terminated. Not a valid checkpoint path.' + + # Load model + model = load_model(model, pathmaster) + + # Initialize true label lists + true_labels_list = [] + + # Intialize output (prediction) lists + predictions_list = [] + prediction_proba_list = [] + + # Evaluation + model.eval() + cum_loss = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Testing', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + true_labels_list.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model(X) + predictions_list.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss = cum_loss / len(data_loader) + + # Convert true label list into array + true_labels = np.array(torch.cat(true_labels_list, dim=0).to('cpu')) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions = np.array(torch.cat(predictions_list, dim=0).to('cpu')) + prediction_proba = np.array(torch.cat(prediction_proba_list, dim=0).to('cpu')) + + # Print validation loss + print('\n======> Loss: %.4f' % loss) + + # Saving + if save: + from sklearn.metrics import confusion_matrix + conf_matrix = confusion_matrix(true_labels, predictions) + title = 'Evaluation Confusion Matrix' + plot_save_func.conf_matrix(conf_matrix, title, save, pathmaster) + + plot_save_func.save_labels(true_labels, pathmaster) + plot_save_func.save_predictions(predictions, pathmaster) + plot_save_func.save_prediction_proba(prediction_proba, pathmaster) + plot_save_func.metrics(true_labels, predictions, prediction_proba, save, pathmaster) + + plot_save_func.save_classification_report(true_labels, predictions, save, pathmaster) + plot_save_func.save_classification_report_imbalanced(true_labels, predictions, save, pathmaster) + + clf_names = ['Model'] + plot_save_func.mean_roc_curves([true_labels], [prediction_proba], clf_names, save, pathmaster) + plot_save_func.roc_curves(true_labels, prediction_proba, save, pathmaster) + + +def best_DenseNet_config(data_loader, model_type=torch.float32, n_classes=3, save=False, pathmaster=None): + print('\n===========================================================================================') + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Get paths + checkpoints_path = pathmaster.checkpoints_path() + + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, _, _, num_dense_tran, _, activation = load_hyperparameters_random_search(pathmaster) + # When testing on the test set, drop_rate, class_weights, learning_rate, and lambda_l1 are not needed + + # Initialize model + model = DenseNet_config(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate, + activation=activation, num_dense_tran=num_dense_tran).to(device=device, dtype=model_type) + + # Create criterion for loss + criterion = nn.CrossEntropyLoss() + + # If checkpoint is not specified, terminate the function + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + assert os.path.exists(checkpoint_path), 'Function terminated. Not a valid checkpoint path.' + + # Load model + model = load_model(model, pathmaster) + + # Initialize true label lists + true_labels_list = [] + + # Intialize output (prediction) lists + predictions_list = [] + prediction_proba_list = [] + + # # Initialize segment names list + # segment_names_list = [] + + # Evaluation + model.eval() + cum_loss = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Testing', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + # Z = data_batch['segment_name'] + # segment_names_list.append(Z) + true_labels_list.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model(X) + predictions_list.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss = cum_loss / len(data_loader) + + # Convert true label list into array + true_labels = np.array(torch.cat(true_labels_list, dim=0).to('cpu')) + + # # Convert segment names list into array + # segment_names = np.concatenate(segment_names_list, axis=0) + # segment_names = segment_names.reshape(-1,1) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions = np.array(torch.cat(predictions_list, dim=0).to('cpu')) + prediction_proba = np.array(torch.cat(prediction_proba_list, dim=0).to('cpu')) + + # Print validation loss + print('\n======> Loss: %.4f' % loss) + + # Saving + if save: + # pathmaster.set_file_tag(pathmaster.file_tag + '_test') + from sklearn.metrics import confusion_matrix + conf_matrix = confusion_matrix(true_labels, predictions) + title = 'Evaluation Confusion Matrix' + plot_save_func.conf_matrix(conf_matrix, title, save, pathmaster) + + plot_save_func.save_labels(true_labels, pathmaster) + # plot_save_func.save_labels(np.hstack([segment_names, true_labels]), pathmaster) + plot_save_func.save_predictions(predictions, pathmaster) + plot_save_func.save_prediction_proba(prediction_proba, pathmaster) + plot_save_func.metrics(true_labels, predictions, prediction_proba, save, pathmaster) + + plot_save_func.save_classification_report(true_labels, predictions, save, pathmaster) + plot_save_func.save_classification_report_imbalanced(true_labels, predictions, save, pathmaster) + + clf_names = ['Model'] + plot_save_func.mean_roc_curves([true_labels], [prediction_proba], clf_names, save, pathmaster) + plot_save_func.roc_curves(true_labels, prediction_proba, save, pathmaster) + + +def best_DenseNet_config_binary(data_loader, model_type=torch.float32, n_classes=3, save=False, pathmaster=None): + print('\n===========================================================================================') + + # If GPU is available, use GPU, else use CPU + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Get paths + checkpoints_path = pathmaster.checkpoints_path() + + # Load model hyperparameters + depth, growth_rate, compression, bottleneck, drop_rate, _, _, num_dense_tran, _, activation = load_hyperparameters_random_search(pathmaster) + # When testing on the test set, drop_rate, class_weights, learning_rate, and lambda_l1 are not needed + + # Initialize model + model = DenseNet_config(img_channels=1, depth=depth, n_classes=n_classes, growth_rate=growth_rate, + compression=compression, bottleneck=bottleneck, drop_rate=drop_rate, + activation=activation, num_dense_tran=num_dense_tran).to(device=device, dtype=model_type) + + # Create criterion for loss + criterion = nn.CrossEntropyLoss() + + # If checkpoint is not specified, terminate the function + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + assert os.path.exists(checkpoint_path), 'Function terminated. Not a valid checkpoint path.' + + # Load model + model = load_model(model, pathmaster) + + # Initialize true label lists + true_labels_list = [] + + # Intialize output (prediction) lists + predictions_list = [] + prediction_proba_list = [] + + # Evaluation + model.eval() + cum_loss = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Testing', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + true_labels_list.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model(X) + predictions_list.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss = cum_loss / len(data_loader) + + # Convert true label list into array + true_labels = np.array(torch.cat(true_labels_list, dim=0).to('cpu')) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions = np.array(torch.cat(predictions_list, dim=0).to('cpu')) + prediction_proba = np.array(torch.cat(prediction_proba_list, dim=0).to('cpu')) + + # Print validation loss + print('\n======> Loss: %.4f' % loss) + + # Saving + if save: + # pathmaster.set_file_tag(pathmaster.file_tag + '_test') + from sklearn.metrics import confusion_matrix + conf_matrix = confusion_matrix(true_labels, predictions) + title = 'Evaluation Confusion Matrix' + plot_save_func.conf_matrix(conf_matrix, title, save, pathmaster, class_names=['non-AF', 'AF']) + + plot_save_func.save_labels(true_labels, pathmaster) + plot_save_func.save_predictions(predictions, pathmaster) + plot_save_func.save_prediction_proba_binary(prediction_proba, pathmaster) + plot_save_func.metrics_binary(true_labels, predictions, prediction_proba, save, pathmaster) + + plot_save_func.save_classification_report(true_labels, predictions, save, pathmaster) + plot_save_func.save_classification_report_imbalanced(true_labels, predictions, save, pathmaster) + + plot_save_func.roc_curves_binary(true_labels, prediction_proba, save, pathmaster, class_names=['non-AF', 'AF']) + + +def train(model, dataloader, optimizer, scheduler, criterion, regularization): + model.train() + cum_loss = 0 + for data_batch in tqdm(dataloader, total=len(dataloader), desc='Training', unit='batch', leave=False): + # Extract input and labels + X_train = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_train = data_batch['label'].to(device=device) + + # Forward pass + logits, _, _ = model(X_train) + + # Regularization (if applicable) + l1 = 0 + for p in model.parameters(): + l1 = l1 + p.abs().sum() + + # Calculate total loss with regularization + batch_loss_train = criterion(logits.to(torch.float32), Y_train.long()) + regularization * l1 + cum_loss += batch_loss_train.item() + + # Clear gradients + optimizer.zero_grad(set_to_none=True) + + # Backwards pass + batch_loss_train.backward() + + # Optimizer step + optimizer.step() + + # Update scheduler + scheduler.step() + + epoch_loss = cum_loss / len(dataloader) + + return model, optimizer, scheduler, epoch_loss + + +def validate(model, dataloader, criterion): + model.eval() + with torch.no_grad(): + cum_loss = 0 + for data_batch in tqdm(dataloader, total=len(dataloader), desc='Validation', unit='batch', leave=False): + X_val = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y_val = data_batch['label'].to(device=device) + + logits, _, _ = model(X_val) + cum_loss += criterion(logits.float(), Y_val.long()).item() + + epoch_loss = cum_loss / len(dataloader) + + return epoch_loss + + +def test(model, dataloader, criterion, n_classes): + # Initialize true label lists + true_labels_list = [] + + # Intialize output (prediction) lists + predictions_list = [] + prediction_proba_list = [] + + # Validation + model.eval() + cum_loss = 0 + with torch.no_grad(): # Disable gradient computation during validation + sys.stdout.flush() + for data_batch in tqdm(dataloader, total=len(dataloader), desc='Testing', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[1], data_batch['data'].shape[-1], data_batch['data'].shape[-1]).to(device=device) + Y = data_batch['label'].to(device=device) + true_labels_list.append(torch.reshape(Y, (-1,1))) + + # Forward pass + logits, predictions, prediction_proba = model(X) + predictions_list.append(torch.reshape(predictions, (-1,1))) + prediction_proba_list.append(torch.reshape(prediction_proba, (-1,n_classes))) + + # Calculate sum of total loss for epoch + cum_loss += criterion(logits.float(), Y.long()).item() # Criterion returns a scalar tensor + + # Calculate loss for validation + loss = cum_loss / len(dataloader) + + # Convert true label list into array + true_labels = np.array(torch.cat(true_labels_list, dim=0).to('cpu')) + + # Convert the output lists into arrays and concatenate along dim=0 (rows) + predictions = np.array(torch.cat(predictions_list, dim=0).to('cpu')) + prediction_proba = np.array(torch.cat(prediction_proba_list, dim=0).to('cpu')) + + return true_labels, predictions, prediction_proba, loss + + +class IdentityScheduler(torch.optim.lr_scheduler._LRScheduler): + def __init__(self, optimizer, last_epoch=-1): + super(IdentityScheduler, self).__init__(optimizer, last_epoch) + + def get_lr(self): + # Returns the current learning rate without any modifications. + return self.base_lrs + + +def save_checkpoint(model, optimizer, scheduler, epoch, loss, checkpoint_path): # Will also be called to save the most recent checkpoint locally in the runtime so I always have the most recent checkpoint + torch.save({ + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'scheduler_state_dict': scheduler.state_dict() if scheduler else IdentityScheduler(optimizer).state_dict(), # Create identity scheduler if missing, actually doesn't work since the parameter is required + 'epoch': epoch, + 'loss': loss + }, checkpoint_path) + +def save_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, loss, checkpoint_path): # Will also be called to save the most recent checkpoint locally in the runtime so I always have the most recent checkpoint + torch.save({ + 'model_fold1_state_dict': model_fold1.state_dict(), + 'model_fold2_state_dict': model_fold2.state_dict(), + 'optimizer_fold1_state_dict': optimizer_fold1.state_dict(), + 'optimizer_fold2_state_dict': optimizer_fold2.state_dict(), + 'scheduler_fold1_state_dict': scheduler_fold1.state_dict(), + 'scheduler_fold2_state_dict': scheduler_fold2.state_dict(), + 'epoch': epoch, + 'loss': loss + }, checkpoint_path) + +def save_best_checkpoint(model, optimizer, scheduler, epoch, current_loss, best_loss, pathmaster): # When training the model, best_loss should be initialized to float.('inf') + # Might be good to have two different checkpoint paths, one for the best and one for the most recent checkpoint, maybe also have temp vs permanent checkpoint paths + if current_loss < best_loss: + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + best_loss = current_loss + save_checkpoint(model, optimizer, scheduler, epoch, best_loss, checkpoint_path) + print('\nNew checkpoint with better loss was saved!') + + return best_loss + else: + return best_loss + + +def save_best_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, current_loss, best_loss, pathmaster): # When training the model, best_loss should be initialized to float.('inf') + # Might be good to have two different checkpoint paths, one for the best and one for the most recent checkpoint, maybe also have temp vs permanent checkpoint paths + if current_loss < best_loss: + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + best_loss = current_loss + save_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, epoch, best_loss, checkpoint_path) + print('\nNew checkpoint with better loss was saved!') + + return best_loss + else: + return best_loss + + +def load_checkpoint(model, optimizer, scheduler, pathmaster): + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=device) + + model.load_state_dict(checkpoint['model_state_dict']) + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + scheduler.load_state_dict(checkpoint['scheduler_state_dict']) + start_epoch = checkpoint['epoch'] + loss = checkpoint['loss'] + + print('\nCheckpoint loaded!') + # print(f'Resuming training from epoch {start_epoch}, batch {start_batch}') + + return model, optimizer, scheduler, start_epoch, loss + else: + print('\nError! Checkpoint does not exist!') + + +def load_checkpoint_2fold(model_fold1, model_fold2, optimizer_fold1, optimizer_fold2, scheduler_fold1, scheduler_fold2, pathmaster): + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=device) + + model_fold1.load_state_dict(checkpoint['model_fold1_state_dict']) + optimizer_fold1.load_state_dict(checkpoint['optimizer_fold1_state_dict']) + scheduler_fold1.load_state_dict(checkpoint['scheduler_fold1_state_dict']) + + model_fold2.load_state_dict(checkpoint['model_fold2_state_dict']) + optimizer_fold2.load_state_dict(checkpoint['optimizer_fold2_state_dict']) + scheduler_fold2.load_state_dict(checkpoint['scheduler_fold2_state_dict']) + + start_epoch = checkpoint['epoch'] + loss = checkpoint['loss'] + + print('\nCheckpoint loaded!') + # print(f'Resuming training from epoch {start_epoch}, batch {start_batch}') + + return model_fold1, optimizer_fold1, scheduler_fold1, model_fold2, optimizer_fold2, scheduler_fold2, start_epoch, loss + else: + print('\nError! Checkpoint does not exist!') + + +def load_model_2fold(model_fold1, model_fold2, pathmaster): + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=device) + + model_fold1.load_state_dict(checkpoint['model_fold1_state_dict']) + model_fold2.load_state_dict(checkpoint['model_fold2_state_dict']) + + print('\nModels loaded!') + # print(f'Resuming training from epoch {start_epoch}, batch {start_batch}') + + return model_fold1, model_fold2 + else: + print('\nError! Models do not exist!') + + +def load_model(model, pathmaster): + checkpoints_path = pathmaster.checkpoints_path() + checkpoint_path = os.path.join(checkpoints_path, 'checkpoint_' + pathmaster.file_tag + '.pt') + if os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location=device) + + model.load_state_dict(checkpoint['model_state_dict']) + + print('\nModel loaded!') + # print(f'Resuming training from epoch {start_epoch}, batch {start_batch}') + + return model + else: + print('\nError! Model does not exist!') + + +def load_hyperparameters(pathmaster): + hyperparameters_path = pathmaster.hyperparameters_path() + + # Extract model hyperparameters + model_hyperparameters_file = os.path.join(hyperparameters_path, 'hyperparameters_' + pathmaster.file_tag + '.csv') + model_hyperparameters = pd.read_csv(model_hyperparameters_file) + depth = int(model_hyperparameters['depth'].iloc[0]) + growth_rate = int(model_hyperparameters['growth_rate'].iloc[0]) + compression = model_hyperparameters['compression'].iloc[0] + bottleneck = model_hyperparameters['bottleneck'].iloc[0] + drop_rate = model_hyperparameters['drop_rate'].iloc[0] + class_weights = model_hyperparameters['class_weights'] + + return depth, growth_rate, compression, bottleneck, drop_rate, class_weights + + +def load_hyperparameters_random_search(pathmaster): + hyperparameters_path = pathmaster.hyperparameters_path() + + # Extract model hyperparameters + model_hyperparameters_file = os.path.join(hyperparameters_path, 'hyperparameters_' + pathmaster.file_tag + '.csv') + model_hyperparameters = pd.read_csv(model_hyperparameters_file) + depth = int(model_hyperparameters['depth'].iloc[0]) + growth_rate = int(model_hyperparameters['growth_rate'].iloc[0]) + compression = model_hyperparameters['compression'].iloc[0] + bottleneck = model_hyperparameters['bottleneck'].iloc[0] + drop_rate = model_hyperparameters['drop_rate'].iloc[0] + class_weights = model_hyperparameters['class_weights'] + learning_rate = model_hyperparameters['learning_rate'].iloc[0] + num_dense_tran = int(model_hyperparameters['num_dense_tran'].iloc[0]) + lambda_l1 = model_hyperparameters['lambda_l1'].iloc[0] + activation = string_to_activation((model_hyperparameters['activation'].iloc[0])) + + return depth, growth_rate, compression, bottleneck, drop_rate, class_weights, learning_rate, num_dense_tran, lambda_l1, activation + + +def string_to_activation(activation_string): + activation_map = { + 'relu': nn.ReLU(), + 'leaky_relu': nn.LeakyReLU(), + 'sigmoid': nn.Sigmoid(), + 'tanh': nn.Tanh(), + 'softmax': nn.Softmax(), + 'softplus': nn.Softplus(), + 'softshrink': nn.Softshrink(), + 'softmin': nn.Softmin(), + 'log_softmax': nn.LogSoftmax(), + 'elu': nn.ELU(), + 'prelu': nn.PReLU(), + 'relu6': nn.ReLU6(), + 'rrelu': nn.RReLU(), + 'celu': nn.CELU(), + 'selu': nn.SELU(), + 'gelu': nn.GELU(), + 'silu': nn.SiLU(), + # Add more activation functions if needed + } + + return activation_map.get(activation_string, None) + + +def activation_to_string(activation_func): + activation_map = { + nn.ReLU: 'relu', + nn.LeakyReLU: 'leaky_relu', + nn.Sigmoid: 'sigmoid', + nn.Tanh: 'tanh', + nn.Softmax: 'softmax', + nn.Softplus: 'softplus', + nn.Softshrink: 'softshrink', + nn.Softmin: 'softmin', + nn.LogSoftmax: 'log_softmax', + nn.ELU: 'elu', + nn.PReLU: 'prelu', + nn.ReLU6: 'relu6', + nn.RReLU: 'rrelu', + nn.CELU: 'celu', + nn.SELU: 'selu', + nn.GELU: 'gelu', + nn.SiLU: 'silu', + # Add more activation functions if needed + } + + return activation_map.get(activation_func.__class__, 'unknown') + + +class EarlyStoppingCallback: + def __init__(self, patience=10): + self.patience = patience + self.best_loss = float('inf') + self.counter = 0 + self.best_epoch = 0 + + def __call__(self, epoch, current_loss): + if current_loss < self.best_loss: + self.best_loss = current_loss + self.counter = 0 + self.best_epoch = epoch + else: + self.counter += 1 + if self.counter >= self.patience: + print(f"\nEarly stopping at epoch {epoch}. No improvement for {self.patience} epochs.") + + return True + + return False \ No newline at end of file diff --git a/utils/pathmaster.py b/utils/pathmaster.py new file mode 100644 index 0000000..38c5718 --- /dev/null +++ b/utils/pathmaster.py @@ -0,0 +1,321 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Mar 4 13:04:27 2024 + +@author: dchen +""" +import os + +class PathMaster(): + def __init__(self, is_linux=False, is_hpc=False, is_tfs=True, is_internal=False, is_external=False, focus='misc', file_tag='temp', img_res='not_an_img_res'): + self.focus = focus + self.file_tag = file_tag + self.is_linux = is_linux + self.is_hpc = is_hpc + self.is_tfs = is_tfs + self.is_internal = is_internal + self.is_external = is_external + self.img_res = img_res + + # Select correct root saves path + if self.is_linux: + if self.is_tfs: + self.saves_path = '/mnt/R/ENGR_Chon/Darren/Honors_Thesis/saves_tfs/' + self.focus + '/' + else: + self.saves_path = '/mnt/R/ENGR_Chon/Darren/Honors_Thesis/saves_poincare/' + self.focus + '/' + elif self.is_hpc: + if self.is_tfs: + self.saves_path = '/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/saves_tfs/' + self.focus + '/' + else: + self.saves_path = '/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/saves_poincare/' + self.focus + '/' + else: # Using your own computer + if self.is_tfs: + self.saves_path = r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\saves_tfs' + '\\' + self.focus + '\\' + else: + self.saves_path = r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\saves_poincare' + '\\' + self.focus + '\\' + + + def set_saves_path(self, saves_path): + self.saves_path = saves_path + + + def set_file_tag(self, file_tag): + self.file_tag = file_tag + + + def set_focus(self, focus): + self.focus = focus + + + def data_paths(self, data_format): + if data_format == 'pt': + # Base path + if self.is_linux: + base_path = "/mnt/R/ENGR_Chon/Darren/NIH_PulseWatch" + labels_base_path = "/mnt/R/ENGR_Chon/Darren/NIH_Pulsewatch" + # labels_base_path = "/mnt/R/ENGR_Chon/NIH_Pulsewatch_Database/Adjudication_UConn" + elif self.is_hpc: + base_path = "/gpfs/scratchfs1/kic14002/doh16101" + labels_base_path = "/gpfs/scratchfs1/hfp14002/lrm22005" + else: + if self.is_internal: + base_path = r'C:\\Chon_Lab\\NIH_Pulsewatch' + labels_base_path = r'C:\\Chon_Lab\\NIH_Pulsewatch' + elif self.is_external: + base_path = r'D:\\Chon_Lab\\NIH_Pulsewatch' + labels_base_path = r'D:\\Chon_Lab\\NIH_Pulsewatch' + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = "R:\\ENGR_Chon\\Darren\\NIH_Pulsewatch" # Why double \\ before NIH_Pulsewatch_Database? + labels_base_path = "R:\\ENGR_Chon\\Darren\\NIH_Pulsewatch" # Why double \\ before NIH_Pulsewatch_Database? + # labels_base_path = "R:\ENGR_Chon\\NIH_Pulsewatch_Database\Adjudication_UConn" + + # Type path + if self.is_tfs: + format_path = 'TFS_pt' + else: + format_path = 'Poincare_pt' + + # Join paths + data_path = os.path.join(base_path, format_path, self.img_res) + + else: + if self.is_linux: + base_path = "/mnt/R/ENGR_Chon/Dong/MATLAB_generate_results/NIH_PulseWatch" + labels_base_path = "/mnt/R/ENGR_Chon/Darren/NIH_Pulsewatch" + # labels_base_path = "/mnt/R/ENGR_Chon/NIH_Pulsewatch_Database/Adjudication_UConn" + elif self.is_hpc: + base_path = "/gpfs/scratchfs1/kic14002/doh16101" + labels_base_path = "/gpfs/scratchfs1/hfp14002/lrm22005" + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = "R:\ENGR_Chon\Dong\MATLAB_generate_results\\NIH_PulseWatch" # Why double \\ before NIH_Pulsewatch_Database? + labels_base_path = "R:\ENGR_Chon\Darren\\NIH_Pulsewatch" # Why double \\ before NIH_Pulsewatch_Database? + # labels_base_path = "R:\ENGR_Chon\\NIH_Pulsewatch_Database\Adjudication_UConn" + + if data_format == 'csv': + if self.is_tfs: + data_path = os.path.join(base_path, "TFS_csv") + else: + data_path = os.path.join(base_path, "Poincare_Density_csv") + elif data_format == 'png': + if not self.is_tfs: + print('No png image available for Density Poincare plot') + return + data_path = os.path.join(base_path, "TFS_plots") + else: + raise ValueError("Invalid data format. Choose 'csv', 'png, or 'pt'.") + + # Complete labels path + # labels_path = os.path.join(labels_base_path, "final_attemp_4_1_Dong_Ohm_2024_02_18_copy") + labels_path = os.path.join(labels_base_path, "Ground_Truths") + + # Check if directories exist + if not os.path.exists(data_path): + print("Data path does not exist") + return + if not os.path.exists(labels_path): + print("Labels path does not exist") + return + + return data_path, labels_path + + + def smote_path(self, smote_type, split): + if self.is_internal: + base_path = r'C:\Chon_Lab\NIH_Pulsewatch' + elif self.is_external: + base_path = r'D:\Chon_Lab\NIH_Pulsewatch' + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = "R:\ENGR_Chon\Darren\\NIH_Pulsewatch" # Why double \\ before NIH_Pulsewatch_Database? + + # Type path + if self.is_tfs: + format_path = 'TFS_pt' + else: + format_path = 'Poincare_pt' + + smote_path = os.path.join(base_path, format_path, smote_type, split) + + return smote_path + + + def deepbeat_paths(self): + if self.is_internal: + base_path = r'C:\Chon_Lab\Public_Database\DeepBeat\Concatenated_DeepBeat\test\Darren_conversion' + elif self.is_external: + base_path = r'D:\Chon_Lab\Public_Database\DeepBeat\Concatenated_DeepBeat\test\Darren_conversion' + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = r'R:\ENGR_Chon\Darren\Public_Database\DeepBeat\Concatenated_DeepBeat\test\Darren_conversion' + + # Type path + if self.is_tfs: + format_path = 'tfs_float16_pt' + else: + format_path = 'poincare_float16_pt' + + data_path = os.path.join(base_path, format_path) + labels_path = os.path.join(base_path, 'DeepBeat_segment_names_labels_STFT.csv') + + return data_path, labels_path + + + def mimic3_paths(self): + if self.is_internal: + base_path = r'C:\Chon_Lab\Public_Database\PPG_PeakDet_MIMICIII\Darren_conversion' + elif self.is_external: + base_path = r'D:\Chon_Lab\Public_Database\PPG_PeakDet_MIMICIII\Darren_conversion' + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = r'R:\ENGR_Chon\Darren\Public_Database\PPG_PeakDet_MIMICIII\Darren_conversion' + + # Type path + if self.is_tfs: + format_path = 'test_tfs_float16_pt' + else: + format_path = 'test_poincare_float16_pt' + + data_path = os.path.join(base_path, format_path) + labels_path = os.path.join(base_path, '2020_Han_Sensors_MIMICIII_Ground_Truth_STFT.csv') + + return data_path, labels_path + + + def simband_paths(self): + if self.is_internal: + base_path = r'C:\Chon_Lab\Public_Database\PPG_PeakDet_Simband\Darren_conversion' + elif self.is_external: + base_path = r'D:\Chon_Lab\Public_Database\PPG_PeakDet_Simband\Darren_conversion' + else: + # R:\ENGR_Chon\Dong\MATLAB_generate_results\NIH_PulseWatch + base_path = r'R:\ENGR_Chon\Darren\Public_Database\PPG_PeakDet_Simband\Darren_conversion' + + # Type path + if self.is_tfs: + format_path = 'tfs_float16_pt' + else: + format_path = 'poincare_float16_pt' + + data_path = os.path.join(base_path, format_path) + labels_path = os.path.join(base_path, 'simband_segments_labels_STFT.csv') + + return data_path, labels_path + + + def summary_path(self): + if self.is_linux: + summary_path = "/mnt/R/ENGR_Chon/Darren/NIH_Pulsewatch/labels_summary_2_18_Darren.csv" + elif self.is_hpc: + summary_path = "/gpfs/scratchfs1/hfp14002/dac20022/NIH_Pulsewatch/labels_summary_2_18_Darren.csv" + else: + if self.is_internal: + summary_path = r'C:\Chon_Lab\NIH_Pulsewatch\labels_summary_2_18_Darren.csv' + elif self.is_external: + summary_path = r'D:\Chon_Lab\NIH_Pulsewatch\labels_summary_2_18_Darren.csv' + else: + summary_path = r"\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch\labels_summary_2_18_Darren.csv" + + return summary_path + + + def models_path(self): + if self.is_linux: + models_path = "/mnt/R/ENGR_Chon/Darren/Honors_Thesis/models" + elif self.is_hpc: + models_path = "/gpfs/scratchfs1/hfp14002/dac20022/Honors_Thesis/models" + else: + models_path = r"\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\Honors_Thesis\models" + + return models_path + + + def losslists_path(self): + losslists_path = self.saves_path + 'losslists' + + return losslists_path + + + def runtime_lists_path(self): + runtime_lists_path = self.saves_path + 'runtime_lists' + + return runtime_lists_path + + + def labels_path(self): + labels_path = self.saves_path + 'labels' + + return labels_path + + + def predictions_path(self): + predictions_path = self.saves_path + 'predictions' + + return predictions_path + + + def prediction_proba_path(self): + prediction_proba_path = self.saves_path + 'prediction_proba' + + return prediction_proba_path + + + def metrics_path(self): + metrics_path = self.saves_path + 'metrics' + + return metrics_path + + + def classification_report_path(self): + classification_report_path = self.saves_path + 'classification_reports' + + return classification_report_path + + + def classification_report_imbalanced_path(self): + classification_report_imbalanced_path = self.saves_path + 'classification_reports_imbalanced' + + return classification_report_imbalanced_path + + + def confusion_matrices_path(self): + confusion_matrices_path = self.saves_path + 'confusion_matrices' + + return confusion_matrices_path + + + def checkpoints_path(self): + checkpoints_path = self.saves_path + 'checkpoints' + + return checkpoints_path + + + def hyperparameters_path(self): + hyperparameters_path = self.saves_path + 'hyperparameters' + + return hyperparameters_path + + + def loss_curves_path(self): + loss_curves_path = self.saves_path + 'loss_curves' + + return loss_curves_path + + + def roc_curves_path(self): + roc_curves_path = self.saves_path + 'roc_curves' + + return roc_curves_path + + + def mean_roc_curves_path(self): + mean_roc_curves_path = self.saves_path + 'mean_roc_curves' + + return mean_roc_curves_path + + + def accuracy_curves_path(self): + accuracy_curves_path = self.saves_path + 'accuracy_curves' + + return accuracy_curves_path \ No newline at end of file diff --git a/utils/plot_save_func.py b/utils/plot_save_func.py new file mode 100644 index 0000000..abe201e --- /dev/null +++ b/utils/plot_save_func.py @@ -0,0 +1,542 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Feb 29 12:06:14 2024 + +@author: dchen +""" +import matplotlib.pyplot as plt +import numpy as np +import os +import pandas as pd +from sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score, roc_curve, auc, classification_report +from sklearn.preprocessing import label_binarize +from imblearn.metrics import classification_report_imbalanced + +# For increased csv speed +import pyarrow as pa +from pyarrow import csv + +def save_hyperparameters(hyperparameters, pathmaster): + hyperparameters_path = pathmaster.hyperparameters_path() + hyperparameters_path = os.path.join(hyperparameters_path, 'hyperparameters_' + pathmaster.file_tag + '.csv') + + # If there are class weights, make sure all other columns have same length + if hyperparameters['class_weights'] is not None: + # Update the dictionary + for key, value in hyperparameters.items(): + # If the length of the value is less than max_length + if key != 'class_weights': + # Fill missing values with np.nan + hyperparameters[key] = [value] + [np.nan] * (len(hyperparameters['class_weights']) - 1) + + hyperparameters = pd.DataFrame(hyperparameters) + hyperparameters.to_csv(hyperparameters_path, index=False) + + # # Using PyArrow (need each hyperparameter to be a list) + # hyperparameters_table = pa.Table.from_pydict(hyperparameters) + # csv.write_csv(hyperparameters_table, hyperparameters_path) + + +def save_losslists(losslist_train, losslist_val, pathmaster): # For holdout training and validation + losslists_path = pathmaster.losslists_path() + losslists_path = os.path.join(losslists_path, 'losslists_' + pathmaster.file_tag + '.csv') + # losslists = pd.DataFrame(dtype='float32') + # losslists['training'] = losslist_train + # losslists['validation'] = losslist_val + # losslists.to_csv(losslists_path, index=False, chunksize=500) + + # Using PyArrow + # losslists = { + # 'training': losslist_train, + # 'validation': losslist_val + # } + # losslists_table = pa.Table.from_pydict(losslists) + losslists = [np.array(losslist_train).reshape(-1).astype(np.float32), np.array(losslist_val).reshape(-1).astype(np.float32)] + losslists_names = ['training', 'validation'] + losslists_table = pa.Table.from_arrays(losslists, losslists_names) + csv.write_csv(losslists_table, losslists_path) + +def save_losslists_2fold(losslist_train_fold1, losslist_val_fold1, losslist_train_fold2, losslist_val_fold2, losslist_train, losslist_val, pathmaster): # For holdout training and validation + losslists_path = pathmaster.losslists_path() + losslists_path = os.path.join(losslists_path, 'losslists_' + pathmaster.file_tag + '.csv') + # losslists = pd.DataFrame(dtype='float32') + # losslists['training'] = losslist_train + # losslists['validation'] = losslist_val + # losslists.to_csv(losslists_path, index=False, chunksize=500) + + # Using PyArrow + # losslists = { + # 'training': losslist_train, + # 'validation': losslist_val + # } + # losslists_table = pa.Table.from_pydict(losslists) + losslists = [np.array(losslist_train_fold1).reshape(-1).astype(np.float32), np.array(losslist_val_fold1).reshape(-1).astype(np.float32), + np.array(losslist_train_fold2).reshape(-1).astype(np.float32), np.array(losslist_val_fold2).reshape(-1).astype(np.float32), + np.array(losslist_train).reshape(-1).astype(np.float32), np.array(losslist_val).reshape(-1).astype(np.float32)] + losslists_names = ['fold1_training', 'fold1_validation', 'fold2_training', 'fold2_validation', 'mean_training', 'mean_validation'] + losslists_table = pa.Table.from_arrays(losslists, losslists_names) + csv.write_csv(losslists_table, losslists_path) + + +def save_runtime_list(epoch_time_list, pathmaster): + # epoch_time_array = np.array(epoch_time_list).reshape(-1).astype(np.float32) + runtime_lists_path = pathmaster.runtime_lists_path() + runtime_lists_path = os.path.join(runtime_lists_path, 'runtime_lists_' + pathmaster.file_tag + '.csv') + # runtime_list = pd.DataFrame(dtype='float32') + # runtime_list['time_sec'] = epoch_time_list + # runtime_list.to_csv(runtime_lists_path, index=False, chunksize=500) + + # Using PyArrow + runtime_dict = {'epoch_time_sec': epoch_time_list, + 'mean_time_sec': [sum(epoch_time_list)/len(epoch_time_list)] + [np.nan] * (len(epoch_time_list) - 1)} + runtime_table = pa.Table.from_pydict(runtime_dict) + # runtime_table = pa.Table.from_arrays([epoch_time_array, np.array([np.mean(epoch_time_array)])], names=['epoch_time_sec', 'mean_time_sec']) + csv.write_csv(runtime_table, runtime_lists_path) + + +def save_labels(labels, pathmaster): + labels = labels.astype(np.int8) + labels_path = pathmaster.labels_path() + labels_path = os.path.join(labels_path, 'labels_' + pathmaster.file_tag + '.csv') + # labels = pd.DataFrame(np.array(labels), dtype='int') + # labels.to_csv(labels_path, index=False, chunksize=500) + + # Using PyArrow + # labels_dict = {'labels': labels.reshape(-1)} # Convert to 1D array + # labels_table = pa.Table.from_pydict(labels_dict) + labels_table = pa.Table.from_arrays([labels.reshape(-1)], names=['labels']) + csv.write_csv(labels_table, labels_path) + + +def save_predictions(predictions, pathmaster): + predictions = predictions.astype(np.int8) + predictions_path = pathmaster.predictions_path() + predictions_path = os.path.join(predictions_path, 'predictions_' + pathmaster.file_tag + '.csv') + # predictions = pd.DataFrame(np.array(predictions), dtype='int') + # predictions.to_csv(predictions_path, index=False, chunksize=500) + + # Using PyArrow + # predictions_dict = {'predictions': predictions.reshape(-1)} # Convert to 1D array + # predictions_table = pa.Table.from_pydict(predictions_dict) + predictions_table = pa.Table.from_arrays([predictions.reshape(-1)], names=['predictions']) + csv.write_csv(predictions_table, predictions_path) + + +def save_prediction_proba(prediction_proba, pathmaster): + prediction_proba = prediction_proba.astype(np.float32) + prediction_proba_path = pathmaster.prediction_proba_path() + prediction_proba_path = os.path.join(prediction_proba_path, 'prediction_proba_' + pathmaster.file_tag + '.csv') + # prediction_proba = pd.DataFrame(np.array(prediction_proba), dtype='float32') + # prediction_proba.to_csv(prediction_proba_path, index=False, chunksize=500) + + # Using PyArrow + # # Create PyArrow arrays with specific data type (float64) + # prediction_proba_dict = { + # '0': prediction_proba[:,0], + # '1': prediction_proba[:,1], + # '2': prediction_proba[:,2] + # } + + # Create a PyArrow table + # prediction_proba_Table = pa.Table.from_pydict(prediction_proba_dict) + # col_arrays = [prediction_proba[:,0], prediction_proba[:,1]] + # prediction_proba_Table = pa.Table.from_arrays(col_arrays, names=['0', '1']) + # csv.write_csv(prediction_proba_Table, prediction_proba_path) + col_arrays = [prediction_proba[:,0], prediction_proba[:,1], prediction_proba[:,2]] + prediction_proba_Table = pa.Table.from_arrays(col_arrays, names=['0', '1', '2']) + csv.write_csv(prediction_proba_Table, prediction_proba_path) + + +def save_prediction_proba_binary(prediction_proba, pathmaster): + prediction_proba = prediction_proba.astype(np.float32) + prediction_proba_path = pathmaster.prediction_proba_path() + prediction_proba_path = os.path.join(prediction_proba_path, 'prediction_proba_' + pathmaster.file_tag + '.csv') + # prediction_proba = pd.DataFrame(np.array(prediction_proba), dtype='float32') + # prediction_proba.to_csv(prediction_proba_path, index=False, chunksize=500) + + # Using PyArrow + # # Create PyArrow arrays with specific data type (float64) + # prediction_proba_dict = { + # '0': prediction_proba[:,0], + # '1': prediction_proba[:,1], + # '2': prediction_proba[:,2] + # } + + # Create a PyArrow table + # prediction_proba_Table = pa.Table.from_pydict(prediction_proba_dict) + # col_arrays = [prediction_proba[:,0], prediction_proba[:,1]] + # prediction_proba_Table = pa.Table.from_arrays(col_arrays, names=['0', '1']) + # csv.write_csv(prediction_proba_Table, prediction_proba_path) + col_arrays = [prediction_proba[:,0], prediction_proba[:,1]] + prediction_proba_Table = pa.Table.from_arrays(col_arrays, names=['0', '1']) + csv.write_csv(prediction_proba_Table, prediction_proba_path) + + +def metrics(Y_true, Y_pred, Y_proba, save=False, pathmaster=None): + averages = ['micro', 'macro', 'weighted'] + accuracy_list = [] + precision_list = [] + recall_list = [] + f1_list = [] + auc_list = [] + + for average in averages: + accuracy = accuracy_score(Y_true, Y_pred) + precision, recall, f1, _ = precision_recall_fscore_support(Y_true, Y_pred, average=average) + auc = roc_auc_score(Y_true, Y_proba, average=average, multi_class='ovr') + + accuracy_list.append(accuracy) + precision_list.append(precision) + recall_list.append(recall) + f1_list.append(f1) + auc_list.append(auc) + + metrics = { + 'accuracy': accuracy_list, + 'precision': precision_list, + 'recall': recall_list, + 'f1': f1_list, + 'auc': auc_list + } + + if save: + metrics_path = pathmaster.metrics_path() + metrics_path = os.path.join(metrics_path, 'metrics_' + pathmaster.file_tag + '.csv') + # metrics = pd.DataFrame(metrics, index=[0], dtype='float32') + # metrics.to_csv(metrics_path, index=False) + + # Using PyArrow + metrics_table = pa.Table.from_pydict(metrics) + csv.write_csv(metrics_table, metrics_path) + + +def metrics_binary(Y_true, Y_pred, Y_proba, save=False, pathmaster=None): + averages = ['micro', 'macro', 'weighted'] + accuracy_list = [] + precision_list = [] + recall_list = [] + f1_list = [] + auc_list = [] + + for average in averages: + accuracy = accuracy_score(Y_true, Y_pred) + precision, recall, f1, _ = precision_recall_fscore_support(Y_true, Y_pred, average=average) + auc = roc_auc_score(Y_true, Y_proba[:,1], average=average) + + accuracy_list.append(accuracy) + precision_list.append(precision) + recall_list.append(recall) + f1_list.append(f1) + auc_list.append(auc) + + metrics = { + 'accuracy': accuracy_list, + 'precision': precision_list, + 'recall': recall_list, + 'f1': f1_list, + 'auc': auc_list + } + + if save: + metrics_path = pathmaster.metrics_path() + metrics_path = os.path.join(metrics_path, 'metrics_' + pathmaster.file_tag + '.csv') + # metrics = pd.DataFrame(metrics, index=[0], dtype='float32') + # metrics.to_csv(metrics_path, index=False) + + # Using PyArrow + metrics_table = pa.Table.from_pydict(metrics) + csv.write_csv(metrics_table, metrics_path) + + +def metrics_2fold(Y_true_fold1, Y_true_fold2, Y_pred_fold1, Y_pred_fold2, Y_proba_fold1, Y_proba_fold2, save=False, pathmaster=None): + accuracy_fold1 = accuracy_score(Y_true_fold1, Y_pred_fold1) + precision_fold1, recall_fold1, f1_fold1, _ = precision_recall_fscore_support(Y_true_fold1, Y_pred_fold1, average='weighted') + auc_fold1 = roc_auc_score(Y_true_fold1, Y_proba_fold1, average='weighted', multi_class='ovr') + + accuracy_fold2 = accuracy_score(Y_true_fold2, Y_pred_fold2) + precision_fold2, recall_fold2, f1_fold2, _ = precision_recall_fscore_support(Y_true_fold2, Y_pred_fold2, average='weighted') + auc_fold2 = roc_auc_score(Y_true_fold2, Y_proba_fold2, average='weighted', multi_class='ovr') + + accuracy = accuracy_score(np.concatenate((Y_true_fold1,Y_true_fold2), axis=0), np.concatenate((Y_pred_fold1,Y_pred_fold2), axis=0)) + precision, recall, f1, _ = precision_recall_fscore_support(np.concatenate((Y_true_fold1,Y_true_fold2), axis=0), np.concatenate((Y_pred_fold1,Y_pred_fold2), axis=0), average='weighted') + auc = roc_auc_score(np.concatenate((Y_true_fold1,Y_true_fold2), axis=0), np.concatenate((Y_proba_fold1,Y_proba_fold2), axis=0), average='weighted', multi_class='ovr') + + metrics = { + 'accuracy': [accuracy_fold1, accuracy_fold2, accuracy], + 'precision': [precision_fold1, precision_fold2, precision], + 'recall': [recall_fold1, recall_fold2, recall], + 'f1': [f1_fold1, f1_fold2, f1], + 'auc': [auc_fold1, auc_fold2, auc] + } + + if save: + metrics_path = pathmaster.metrics_path() + metrics_path = os.path.join(metrics_path, 'metrics_' + pathmaster.file_tag + '.csv') + # metrics = pd.DataFrame(metrics, index=[0], dtype='float32') + # metrics.to_csv(metrics_path, index=False) + + # Using PyArrow + metrics_table = pa.Table.from_pydict(metrics) + csv.write_csv(metrics_table, metrics_path) + + +def save_classification_report(Y_true, Y_pred, save=False, pathmaster=None): + report = classification_report(Y_true, Y_pred, output_dict=True) + row_labels = ['precision', 'recall', 'f1', 'support'] + + if save: + classification_report_path = pathmaster.classification_report_path() + classification_report_path = os.path.join(classification_report_path, 'classification_report_' + pathmaster.file_tag + '.csv') + report = pd.DataFrame(report) + # report.reset_index(inplace=True) + report.insert(loc=0, column='metrics', value=row_labels) + report.to_csv(classification_report_path, index=False) + + # # Using PyArrow + # report_table = pa.Table.from_pydict(report) + # csv.write_csv(report_table, classification_report_path) + + +def save_classification_report_imbalanced(Y_true, Y_pred, save=False, pathmaster=None): + report_imbalanced = classification_report_imbalanced(Y_true, Y_pred, output_dict=True) + row_labels = ['precision', 'recall', 'specificity', 'f1', 'geo mean', 'iba', 'support'] + + if save: + classification_report_imbalanced_path = pathmaster.classification_report_imbalanced_path() + classification_report_imbalanced_path = os.path.join(classification_report_imbalanced_path, 'classification_report_imbalanced_' + pathmaster.file_tag + '.csv') + report_imbalanced = pd.DataFrame(report_imbalanced) + # report_imbalanced.reset_index(inplace=True) + report_imbalanced.insert(loc=0, column='metrics', value=row_labels) + report_imbalanced.to_csv(classification_report_imbalanced_path, index=False) + + # # Using PyArrow + # report_imbalanced_table = pa.Table.from_pydict(report_imbalanced) + # csv.write_csv(report_imbalanced_table, classification_report_imbalanced_path) + + +def roc_curves(y_test, y_prob, save=False, pathmaster=None, class_names=['NSR', 'AF', 'PAC/PVC']): + # Get the unique class labels + classes = np.unique(y_test) + + if class_names is None: + class_names = np.unique(y_test) + + # Convert labels to binary matrix + y_bin = label_binarize(y_test, classes=classes) + + # Pre-allocate arrays for ROC curves + fpr_mean = np.linspace(0, 1, 100) + tpr_mean = [] + fpr = [] + tpr = [] + AUC = [] + + # Calculate ROC curves for each class + for i, class_label in enumerate(classes): + fpr_i, tpr_i, _ = roc_curve(y_bin[:, i], y_prob[:, i]) + AUC.append(roc_auc_score(y_bin[:, i], y_prob[:, i])) + fpr.append(fpr_i) + tpr.append(tpr_i) + + # Interpolate TPR for mean ROC curve + tpr_mean.append(np.interp(fpr_mean, fpr_i, tpr_i)) + + # Calculate mean and standard deviation for TPR and AUC + tpr_mean = np.mean(np.array(tpr_mean).reshape(len(classes), -1), axis=0) + tpr_stdv = np.std(tpr_mean, axis=0) + mean_auc = auc(fpr_mean, tpr_mean) + std_auc = np.std(AUC) + + # Create the plot + plt.figure(figsize=(12, 9)) + plt.clf() + plt.plot([0, 1], [0, 1], 'k--') + plt.axis([0, 1, 0, 1]) + plt.xlabel('False Positive Rate', fontsize=16) + plt.ylabel('True Positive Rate', fontsize=16) + plt.title('ROC Curves (' + pathmaster.file_tag + ')', fontweight='bold') + + # Plot individual ROC curves + for i in range(len(classes)): + label_str = f"ROC Label {class_names[i]} (AUC = {AUC[i]:.3f})" + plt.plot(fpr[i], tpr[i], linewidth=3, label=label_str) + + # Plot mean ROC curve with standard deviation + plt.plot(fpr_mean, tpr_mean, color='k', label=rf"Mean ROC (AUC = {mean_auc:.3f} $\pm$ {std_auc:.3f})", linewidth=5) + plt.fill_between(fpr_mean, np.maximum(tpr_mean - tpr_stdv, 0), np.minimum(tpr_mean + tpr_stdv, 1), color='grey', alpha=0.2, label=r"$\pm$ 1 std. dev.") + + plt.legend(loc="lower right") + + if save: + roc_curves_path = pathmaster.roc_curves_path() + roc_curves_path = os.path.join(roc_curves_path, 'roc_curves_' + pathmaster.file_tag + '.jpg') + plt.savefig(roc_curves_path, dpi=150) + + +def roc_curves_binary(y_test, y_prob, save=False, pathmaster=None, class_names=['Negative', 'Positive']): + y_prob = y_prob[:,1] + # Convert labels to binary matrix + y_bin = label_binarize(y_test, classes=np.unique(y_test)) + + # Pre-allocate arrays for ROC curves + fpr_mean = np.linspace(0, 1, 100) + tpr_mean = [] + fpr = [] + tpr = [] + AUC = [] + + # Calculate ROC curve for the positive class + fpr, tpr, _ = roc_curve(y_bin, y_prob) + AUC = roc_auc_score(y_bin, y_prob) + + # Create the plot + plt.figure(figsize=(12, 9)) + plt.plot([0, 1], [0, 1], 'k--') + plt.plot(fpr, tpr, linewidth=3, label=f'ROC Curve (AUC = {AUC:.3f})') + plt.axis([0, 1, 0, 1]) + plt.xlabel('False Positive Rate', fontsize=16) + plt.ylabel('True Positive Rate', fontsize=16) + plt.title('ROC Curve', fontweight='bold') + plt.legend(loc="lower right") + + if save: + roc_curves_path = pathmaster.roc_curves_path() + roc_curves_path = os.path.join(roc_curves_path, 'roc_curves_' + pathmaster.file_tag + '.jpg') + plt.savefig(roc_curves_path, dpi=150) + + +def mean_roc_curves(Y_tests, Y_probas, clf_names, save=False, pathmaster=None): + # Pre-allocate arrays for ROC curves + fpr_mean = np.linspace(0, 1, 100) + # tpr_mean = np.zeros_like(fpr_mean) + + # Set figure size + plt.figure(figsize=(12,9)) + + # Plot individual mean ROC curves for each classifier + for y_test, y_prob, clf_name in zip(Y_tests, Y_probas, clf_names): + # Get the unique class labels + classes = np.unique(y_test) + + # Convert labels to binary matrix + y_bin = label_binarize(y_test, classes=classes) + + # Pre-allocate arrays for ROC curves + fpr = [] + tpr = [] + AUC = [] + + # Calculate ROC curves for each class + for i, class_label in enumerate(classes): + fpr_i, tpr_i, _ = roc_curve(y_bin[:, i], y_prob[:, i]) + AUC.append(roc_auc_score(y_bin[:, i], y_prob[:, i])) + fpr.append(fpr_i) + tpr.append(tpr_i) + + # Interpolate TPR for mean ROC curve + tpr_interp = [np.interp(fpr_mean, fpr_i, tpr_i) for fpr_i, tpr_i in zip(fpr, tpr)] + tpr_mean = np.mean(tpr_interp, axis=0) + + # Plot mean ROC curve + plt.plot(fpr_mean, tpr_mean, label=f"{clf_name} - Mean ROC (AUC = {auc(fpr_mean, tpr_mean):.3f} $\pm$ {np.std(AUC):.3f})", linewidth=2) + + # Additional plot configurations + plt.plot([0, 1], [0, 1], 'k--') + plt.axis([0, 1, 0, 1]) + plt.xlabel('False Positive Rate', fontsize=12) + plt.ylabel('True Positive Rate', fontsize=12) + plt.title('Mean ROC Curve(s)', fontweight='bold') + plt.legend(loc="lower right") + # plt.show() + + if save: + mean_roc_curves_path = pathmaster.mean_roc_curves_path() + mean_roc_curves_path = os.path.join(mean_roc_curves_path, 'mean_roc_curves_' + pathmaster.file_tag + '.jpg') + plt.savefig(mean_roc_curves_path, dpi=150) + + +def conf_matrix(conf_matrix, title='Confusion Matrix', save=False, pathmaster=None, class_names=['NSR', 'AF', 'PAC/PVC']): + title = title + ' (' + pathmaster.file_tag + ')' + conf_matrix_norm = conf_matrix.astype('float') / conf_matrix.sum(axis=1)[:, np.newaxis] # Normalize + + plt.figure(figsize=(10, 8)) # Adjust the figure size as per your preference + plt.imshow(conf_matrix_norm, interpolation='nearest', cmap=plt.cm.Blues, vmin=0.0, vmax=1.0) + plt.title(title, fontweight='bold') + plt.colorbar() + tick_marks = np.arange(len(conf_matrix)) + + if class_names is not None: + tick_marks = np.arange(len(class_names)) + plt.xticks(tick_marks, class_names) + plt.yticks(tick_marks, class_names) + else: + tick_marks = np.arange(len(conf_matrix)) + plt.xticks(tick_marks, tick_marks) + plt.yticks(tick_marks, tick_marks) + + plt.xlabel('Predicted label') + plt.ylabel('True label') + + # Add counts and percentages in each box + for i in range(conf_matrix.shape[0]): + for j in range(conf_matrix.shape[1]): + percentage = conf_matrix_norm[i, j] * 100 + count = int(conf_matrix[i, j]) + # text_color = 'black' if conf_matrix[i, j] < np.max(conf_matrix) / 1.5 else 'white' + text_color = 'black' if percentage < 80 else 'white' + plt.text(j, i, "{:.2f}%\n{}".format(percentage, count), + horizontalalignment="center", + verticalalignment="center", + color=text_color) + + if save: + confusion_matrices_path = pathmaster.confusion_matrices_path() + confusion_matrices_path = os.path.join(confusion_matrices_path, 'confusion_matrix_' + pathmaster.file_tag + '.jpg') + plt.savefig(confusion_matrices_path, dpi=200) + + # plt.show() + + +def train_val_loss(losslist_train, losslist_val, title='Training and Validation Loss', save=False, pathmaster=None): + title = title + ' (' + pathmaster.file_tag + ')' + plt.figure(figsize=(12, 8)) + plt.plot(range(len(losslist_train)), losslist_train, label='training') + plt.plot(range(len(losslist_val)), losslist_val, label='validation') + plt.legend() + plt.title(title, fontweight='bold') + plt.xlabel('Epochs') + plt.ylabel('Loss') + + if save: + loss_curves_path = pathmaster.loss_curves_path() + loss_curves_path = os.path.join(loss_curves_path, 'loss_curve_' + pathmaster.file_tag + '.jpg') + plt.savefig(loss_curves_path, dpi=150) + + # plt.show() + +def accuracy_curves(Y_true_train, Y_true_val, Y_pred_train, Y_pred_val, title='Training and Validation Accuracy', save=False, pathmaster=None): + accuracy_list_train = [] + accuracy_list_val = [] + epochs_train = range(len(Y_pred_train)) + epochs_val = range(len(Y_pred_val)) + + for predictions in Y_pred_train: + accuracy = accuracy_score(Y_true_train, predictions) + accuracy_list_train.append(accuracy) + for predictions in Y_pred_val: + accuracy = accuracy_score(Y_true_val, predictions) + accuracy_list_val.append(accuracy) + + title = title + ' (' + pathmaster.file_tag + ')' + plt.figure(figsize=(12, 8)) + plt.plot(epochs_train, accuracy_list_train, label='training') + plt.plot(epochs_val, accuracy_list_val, label='validation') + plt.legend() + plt.title(title, fontweight='bold') + plt.xlabel('Epochs') + plt.ylabel('Accuracy') + + if save: + accuracy_curves_path = pathmaster.accuracy_curves_path() + accuracy_curves_path = os.path.join(accuracy_curves_path, 'accuracy_curve_' + pathmaster.file_tag + '.jpg') + plt.savefig(accuracy_curves_path, dpi=150) \ No newline at end of file diff --git a/utils/smote.py b/utils/smote.py new file mode 100644 index 0000000..890d2fb --- /dev/null +++ b/utils/smote.py @@ -0,0 +1,158 @@ +import torch +import torch.nn as nn +import torchvision.transforms as transforms +import os +import csv +from imblearn.over_sampling import SMOTE +import numpy as np +from tqdm import tqdm +import pandas as pd +from concurrent.futures import ProcessPoolExecutor + +import sys +sys.path.append('R:\ENGR_Chon\Darren\Honors_Thesis') + +# Import my own functions and classes +from utils.pathmaster import PathMaster +from utils import dataloader + +def apply_cassey_smote(data, labels): + cassey_smote = SMOTE(random_state=42,sampling_strategy='not majority',k_neighbors=5) + data_resampled, labels_resampled = cassey_smote.fit_resample(data, labels) + return data_resampled, labels_resampled + +def save_image(i, image, group, save_dir): + # Generate a unique file name with zero-padding + file_name = f'{i+1:06d}' + '_' + group + '_tfs' + + # Convert the image to a PyTorch tensor + tensor_image = torch.tensor(image).to(dtype=torch.float16) + + # Save the tensor to a .pt file + torch.save(tensor_image, os.path.join(save_dir, file_name + '.pt')) + + return file_name + +def save_images_parallel(data_resampled, group, save_dir): + file_names = [] + with ProcessPoolExecutor() as executor: + results = [executor.submit(save_image, i, image, group, save_dir) for i, image in enumerate(data_resampled)] + for future in results: + file_names.append(future.result()) + return file_names + +def main(): + # Initialize save location specifics + smote_type = 'Cassey_SMOTE' + split = '2foldCV_60_40' + groups = ['fold1', 'fold2', 'test'] + + # Device and drives + is_linux = False + is_hpc = False + is_internal = True + is_external = False + + # Input + is_tfs = True + + # Intialize the focus + focus = 'misc' + + # Initialize the file tag + file_tag = 'temp' + + # Image resolution + img_res = '128x128_float16' + + # Data type: the type to convert the data into when it is loaded in + data_type = torch.float32 + + # Create a PathMaster object + pathmaster = PathMaster(is_linux, is_hpc, is_tfs, is_internal, is_external, focus, file_tag, img_res) + + # Image dimensions + img_channels = 1 + img_size = 128 + downsample = None + standardize = None + + # Split UIDs + # train_set, val_set, test_set = dataloader.split_uids(pathmaster) + cross_val_fold1, cross_val_fold2, test_set = dataloader.split_uids_2fold_60_40_smote(pathmaster) + # train_set, val_set, test_set = dataloader.split_uids_60_10_30(pathmaster) + + # Preprocess data + data_format = 'pt' + batch_size = 256 + + # train_loader, val_loader, _ = dataloader.preprocess_data(data_format, clinical_trial_train, clinical_trial_test, clinical_trial_unlabeled, + # batch_size, standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + fold1_loader, fold2_loader, test_loader = dataloader.preprocess_data(data_format, cross_val_fold1, cross_val_fold2, test_set, batch_size, + standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + # train_loader, val_loader, test_loader = dataloader.preprocess_data(data_format, train_set, val_set, test_set, + # batch_size, standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + data_loaders = [fold1_loader, fold2_loader, test_loader] + print() + sys.stdout.flush() + for data_loader, group in tqdm(zip(data_loaders,groups), total=len(data_loaders), desc='SMOTE', unit='Data Loader', leave=False): + sys.stderr.flush() + + # Define your original data and labels + data = np.empty((0,img_size*img_size)) + labels = np.empty((0,1)) + + sys.stdout.flush() + + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Loading', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[-1] * data_batch['data'].shape[-1]).numpy() + Y = data_batch['label'].numpy().reshape(-1,1) + + data = np.concatenate((data, X), axis=0) + labels = np.concatenate((labels, Y), axis=0) + + sys.stderr.flush() + print('\nData shape:', data.shape) + print('Labels shape:', labels.shape) + sys.stdout.flush() + + if group != 'test': + # SMOTE + data_resampled, labels_resampled = apply_cassey_smote(data, labels) + data_resampled = data_resampled.reshape(len(data_resampled), img_channels, img_size, img_size) + sys.stderr.flush() + print('\nResampled Data shape:', data_resampled.shape) + print('Resampled Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + else: + data_resampled = data + data_resampled = data_resampled.reshape(len(data_resampled), img_channels, img_size, img_size) + labels_resampled = labels + sys.stderr.flush() + print('\nResampled Data shape:', data_resampled.shape) + print('Resampled Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + + # Define a directory to save the images + # save_dir = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch', smote_type, split, group) + save_dir = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch', smote_type, split, group) + os.makedirs(save_dir, exist_ok=True) + + file_names = save_images_parallel(data_resampled, group, save_dir) + + # Ground truths + data_labels = pd.DataFrame({ + 'segment_name': file_names, + 'label': labels_resampled + }) + + csv_file_name = os.path.join(r'C:\Chon_Lab\NIH_Pulsewatch', smote_type, split, smote_type + '_' + group + '_names_labels.csv') + data_labels.to_csv(csv_file_name, index=False) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/smote_accelerated.py b/utils/smote_accelerated.py new file mode 100644 index 0000000..555f8cc --- /dev/null +++ b/utils/smote_accelerated.py @@ -0,0 +1,178 @@ +import torch +import torch.nn as nn +import torchvision.transforms as transforms +import os +import csv +from imblearn.over_sampling import SMOTE, BorderlineSMOTE, ADASYN +import numpy as np +from tqdm import tqdm +import pandas as pd +from concurrent.futures import ProcessPoolExecutor + +import sys +sys.path.append('R:\ENGR_Chon\Darren\Honors_Thesis') + +# Import my own functions and classes +from utils.pathmaster import PathMaster +from utils import dataloader + +def apply_cassey_smote(data, labels): + cassey_smote = SMOTE(random_state=42,sampling_strategy='not majority',k_neighbors=5) + data_resampled, labels_resampled = cassey_smote.fit_resample(data, labels) + return data_resampled, labels_resampled + +def apply_borderline_smote(data, labels): + borderline_smote = BorderlineSMOTE(random_state=42,sampling_strategy='not majority',k_neighbors=5) + data_resampled, labels_resampled = borderline_smote.fit_resample(data, labels) + return data_resampled, labels_resampled + +def apply_adasyn(data, labels): + adasyn = ADASYN(random_state=42,sampling_strategy='not majority',n_neighbors=5) + data_resampled, labels_resampled = adasyn.fit_resample(data, labels) + return data_resampled, labels_resampled + +def save_image(i, image, group, save_dir): + + # Generate a unique file name with zero-padding + file_name = f'{i+1:06d}' + '_' + group + '_tfs' + + # Convert the image to a PyTorch tensor + tensor_image = torch.tensor(image).to(dtype=torch.float16) + tensor_image = tensor_image.reshape(tensor_image.size()[-2], tensor_image.size()[-2]) + + + # Save the tensor to a .pt file + torch.save(tensor_image, os.path.join(save_dir, file_name + '.pt')) + + return file_name + +def save_images_parallel(data_resampled, group, save_dir): + file_names = [] + with ProcessPoolExecutor() as executor: + results = [executor.submit(save_image, i, image, group, save_dir) for i, image in enumerate(data_resampled)] + for future in results: + file_names.append(future.result()) + return file_names + +def main(): + # Initialize save location specifics + # smote_type = 'Cassey_SMOTE' + smote_type = 'Borderline_SMOTE' + # smote_type = 'ADASYN' + + # split = '2foldCV_60_40' + split = 'holdout_60_10_30' + + # groups = ['fold1', 'fold2', 'test'] + groups = ['train', 'validate', 'test'] + + # Device and drives + is_linux = False + is_hpc = False + is_internal = True + is_external = False + + # Input + is_tfs = True + + # Intialize the focus + focus = 'misc' + + # Initialize the file tag + file_tag = 'temp' + + # Image resolution + img_res = '128x128_float16' + + # Data type: the type to convert the data into when it is loaded in + data_type = torch.float32 + + # Create a PathMaster object + pathmaster = PathMaster(is_linux, is_hpc, is_tfs, is_internal, is_external, focus, file_tag, img_res) + + # Image dimensions + img_channels = 1 + img_size = 128 + downsample = None + standardize = None + + # Split UIDs + # cross_val_fold1, cross_val_fold2, test_set = dataloader.split_uids_2fold_60_40_smote(pathmaster) + train_set, val_set, test_set = dataloader.split_uids_60_10_30_smote(pathmaster) + + # Preprocess data + data_format = 'pt' + batch_size = 256 + + # fold1_loader, fold2_loader, test_loader = dataloader.preprocess_data(data_format, cross_val_fold1, cross_val_fold2, test_set, batch_size, + # standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + # data_loaders = [fold1_loader, fold2_loader, test_loader] + + train_loader, val_loader, test_loader = dataloader.preprocess_data(data_format, train_set, val_set, test_set, + batch_size, standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + data_loaders = [train_loader, val_loader, test_loader] + print() + sys.stdout.flush() + for data_loader, group in tqdm(zip(data_loaders,groups), total=len(data_loaders), desc='SMOTE', unit='Data Loader', leave=False): + sys.stderr.flush() + + # Define your original data and labels + data = np.empty((0,img_size*img_size)) + labels = np.empty((0,1)) + + sys.stdout.flush() + + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Loading', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[-1] * data_batch['data'].shape[-1]).numpy() + Y = data_batch['label'].numpy().reshape(-1,1) + + data = np.concatenate((data, X), axis=0) + labels = np.concatenate((labels, Y), axis=0) + + sys.stderr.flush() + print('\nData shape:', data.shape) + print('Labels shape:', labels.shape) + sys.stdout.flush() + + if group != 'test': + # SMOTE + # data_resampled, labels_resampled = apply_cassey_smote(data, labels) + data_resampled, labels_resampled = apply_borderline_smote(data, labels) + # data_resampled, labels_resampled = apply_adasyn(data, labels) + data_resampled = data_resampled.reshape(len(data_resampled), img_channels, img_size, img_size) + sys.stderr.flush() + print('\nResampled Data shape:', data_resampled.shape) + print('Resampled Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + else: + data_resampled = data + data_resampled = data_resampled.reshape(len(data_resampled), img_channels, img_size, img_size) + labels_resampled = labels + sys.stderr.flush() + print('\nResampled Data shape:', data_resampled.shape) + print('Resampled Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + + # Define a directory to save the images + # save_dir = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch', smote_type, split, group) + save_dir = os.path.join(r'C:\Chon_Lab\NIH_Pulsewatch\TFS_pt', smote_type, split, group) + os.makedirs(save_dir, exist_ok=True) + + file_names = save_images_parallel(data_resampled, group, save_dir) + + # Ground truths + data_labels = pd.DataFrame({ + 'segment_name': file_names, + 'label': labels_resampled.reshape(-1) + }) + + csv_file_name = os.path.join(r'C:\Chon_Lab\NIH_Pulsewatch\TFS_pt', smote_type, split, smote_type + '_' + group + '_names_labels.csv') + data_labels.to_csv(csv_file_name, index=False) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/smote_accelerated_lab.py b/utils/smote_accelerated_lab.py new file mode 100644 index 0000000..90201ba --- /dev/null +++ b/utils/smote_accelerated_lab.py @@ -0,0 +1,177 @@ +import torch +import torch.nn as nn +import torchvision.transforms as transforms +import os +import csv +from imblearn.over_sampling import SMOTE, BorderlineSMOTE, ADASYN +import numpy as np +from tqdm import tqdm +import pandas as pd +from concurrent.futures import ProcessPoolExecutor + +import sys +sys.path.append('R:\ENGR_Chon\Darren\Honors_Thesis') + +# Import my own functions and classes +from utils.pathmaster import PathMaster +from utils import dataloader + +def apply_cassey_smote(data, labels): + cassey_smote = SMOTE(random_state=42,sampling_strategy='not majority',k_neighbors=5) + data_resampled, labels_resampled = cassey_smote.fit_resample(data, labels) + return data_resampled, labels_resampled + +def apply_borderline_smote(data, labels): + borderline_smote = BorderlineSMOTE(random_state=42,sampling_strategy='not majority',k_neighbors=5) + data_resampled, labels_resampled = borderline_smote.fit_resample(data, labels) + return data_resampled, labels_resampled + +def apply_adasyn(data, labels): + adasyn = ADASYN(random_state=42,sampling_strategy='not majority',n_neighbors=4) + data_resampled, labels_resampled = adasyn.fit_resample(data, labels) + return data_resampled, labels_resampled + +def save_image(i, image, group, save_dir): + + # Generate a unique file name with zero-padding + file_name = f'{i+1:06d}' + '_' + group + '_tfs' + + # Convert the image to a PyTorch tensor + tensor_image = torch.tensor(image).to(dtype=torch.float16) + tensor_image = tensor_image.reshape(tensor_image.size()[-2], tensor_image.size()[-2]) + + # Save the tensor to a .pt file + torch.save(tensor_image, os.path.join(save_dir, file_name + '.pt')) + + return file_name + +def save_images_parallel(data_resampled, group, save_dir): + file_names = [] + with ProcessPoolExecutor() as executor: + results = [executor.submit(save_image, i, image, group, save_dir) for i, image in enumerate(data_resampled)] + for future in results: + file_names.append(future.result()) + return file_names + +def main(): + # Initialize save location specifics + # smote_type = 'Cassey4k_SMOTE' + # smote_type = 'Borderline5k_SMOTE' + smote_type = 'ADASYN6k' + + # split = '2foldCV_60_40' + split = 'holdout_60_10_30' + + # groups = ['fold1', 'fold2', 'test'] + groups = ['train', 'validate', 'test'] + + # Device and drives + is_linux = False + is_hpc = False + is_internal = False + is_external = False + + # Input + is_tfs = True + + # Intialize the focus + focus = 'misc' + + # Initialize the file tag + file_tag = 'temp' + + # Image resolution + img_res = '128x128_float16' + + # Data type: the type to convert the data into when it is loaded in + data_type = torch.float32 + + # Create a PathMaster object + pathmaster = PathMaster(is_linux, is_hpc, is_tfs, is_internal, is_external, focus, file_tag, img_res) + + # Image dimensions + img_channels = 1 + img_size = 128 + downsample = None + standardize = None + + # Split UIDs + # cross_val_fold1, cross_val_fold2, test_set = dataloader.split_uids_2fold_60_40_smote(pathmaster) + train_set, val_set, test_set = dataloader.split_uids_60_10_30_smote(pathmaster) + + # Preprocess data + data_format = 'pt' + batch_size = 256 + + # fold1_loader, fold2_loader, test_loader = dataloader.preprocess_data(data_format, cross_val_fold1, cross_val_fold2, test_set, batch_size, + # standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + # data_loaders = [fold1_loader, fold2_loader, test_loader] + + train_loader, val_loader, test_loader = dataloader.preprocess_data(data_format, train_set, val_set, test_set, + batch_size, standardize, False, img_channels, img_size, downsample, data_type, pathmaster) + data_loaders = [train_loader, val_loader, test_loader] + print() + sys.stdout.flush() + for data_loader, group in tqdm(zip(data_loaders,groups), total=len(data_loaders), desc='SMOTE', unit='Data Loader', leave=False): + sys.stderr.flush() + + # Define your original data and labels + data = np.empty((0,img_size*img_size)) + labels = np.empty((0,1)) + + sys.stdout.flush() + + for data_batch in tqdm(data_loader, total=len(data_loader), desc='Loading', unit='batch', leave=False): + sys.stderr.flush() + + # Extract input and labels + X = data_batch['data'].reshape(data_batch['data'].shape[0], data_batch['data'].shape[-1] * data_batch['data'].shape[-1]).numpy() + Y = data_batch['label'].numpy().reshape(-1,1) + + data = np.concatenate((data, X), axis=0) + labels = np.concatenate((labels, Y), axis=0) + + sys.stderr.flush() + print('\nData shape:', data.shape) + print('Labels shape:', labels.shape) + sys.stdout.flush() + + if group != 'test': + # SMOTE + # data_resampled, labels_resampled = apply_cassey_smote(data, labels) + # data_resampled, labels_resampled = apply_borderline_smote(data, labels) + data_resampled, labels_resampled = apply_adasyn(data, labels) + data_resampled = data_resampled.reshape(len(data_resampled), img_size, img_size) + sys.stderr.flush() + print('\nResampled Data shape:', data_resampled.shape) + print('Resampled Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + else: + data_resampled = data + data_resampled = data_resampled.reshape(len(data_resampled), img_size, img_size) + labels_resampled = labels + sys.stderr.flush() + print('\nData shape:', data_resampled.shape) + print('Labels shape:', labels_resampled.shape) + print() + sys.stdout.flush() + + # Define a directory to save the images + # save_dir = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch', smote_type, split, group) + save_dir = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch\TFS_pt', smote_type, split, group) + os.makedirs(save_dir, exist_ok=True) + + file_names = save_images_parallel(data_resampled, group, save_dir) + + # Ground truths + data_labels = pd.DataFrame({ + 'segment_name': file_names, + 'label': labels_resampled.reshape(-1) + }) + + csv_file_name = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_Pulsewatch\TFS_pt', smote_type, split, smote_type + '_' + group + '_names_labels.csv') + data_labels.to_csv(csv_file_name, index=False) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/smote_transfer_location.py b/utils/smote_transfer_location.py new file mode 100644 index 0000000..93ffd22 --- /dev/null +++ b/utils/smote_transfer_location.py @@ -0,0 +1,93 @@ +import os +import pandas as pd +import numpy as np +from PIL import Image +import torch +from concurrent.futures import ProcessPoolExecutor +from pyarrow import csv +import cv2 +from tqdm import tqdm +import sys + + +def preprocess_and_save_data(data_path, output_path): + if not os.path.exists(output_path): + os.makedirs(output_path) + group_directories = [entry for entry in os.listdir(data_path) if os.path.isdir(os.path.join(data_path, entry))] + for group in tqdm(group_directories, total=len(group_directories), desc='Data Transfer', unit='Group', leave=False): + sys.stderr.flush() + group_path = os.path.join(data_path, group) + group_output_path = os.path.join(output_path, group) + if not os.path.exists(group_output_path): + os.makedirs(group_output_path) + # else: # Only use for resuming converting + # print('Skipping', group) + # continue + files_to_process = [file for file in os.listdir(group_path) if file.endswith(('.csv', '.png', '.pt'))] + with ProcessPoolExecutor() as executor: + executor.map(preprocess_file, [group_path]*len(files_to_process), files_to_process, [group_output_path]*len(files_to_process)) + print() + print(group, 'data transfer done!') + sys.stdout.flush() + +def preprocess_file(group_path, file, group_output_path): + is_tfs = True + if is_tfs: + dtype = torch.float16 + input_size = 128 + else: + dtype = torch.uint8 + input_size = 500 + + downsample = None + + file_path = os.path.join(group_path, file) + if file.endswith('.csv'): + # data = pd.read_csv(file_path, header=None).to_numpy() + + # Use PyArrow + read_options = csv.ReadOptions(autogenerate_column_names=True) + data = csv.read_csv(file_path, read_options=read_options).to_pandas().to_numpy() + + if data.shape != (input_size, input_size): + print(f"Warning: File {file_path} has shape {data.shape} instead of", input_size + 'x', input_size + '.') + elif file.endswith('.png'): + data = np.array(Image.open(file_path)) + if data.shape != (input_size, input_size): + print(f"Warning: Image {file_path} has shape {data.shape} instead of", input_size + 'x', input_size + '.') + elif file.endswith('.pt'): + data = torch.load(file_path) + if data.shape != (input_size, input_size): + print(f"Warning: Image {file_path} has shape {data.shape} instead of", input_size + 'x', input_size + '.') + else: + print('Incorrect data type') + return + + if downsample is not None: + # Downsample the image + # Use OpenCV to resize the array to downsample x downsample using INTER_AREA interpolation + data_array = cv2.resize(np.array(data), (downsample, downsample), interpolation=cv2.INTER_AREA) + data_tensor = torch.tensor(data_array, dtype=dtype).view(downsample, downsample) + elif file.endswith('.pt'): + data_tensor = data.to(dtype).view(input_size, input_size) + else: + data_tensor = torch.tensor(data, dtype=dtype).view(input_size, input_size) + + # base_name, extension = os.path.splitext(file) + output_file_path = os.path.join(group_output_path, file) + torch.save(data_tensor, output_file_path) + +def main(): + smote_type = 'ADASYN6k' + split = 'holdout_60_10_30' + input_path = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_PulseWatch\TFS_pt', smote_type, split) + # input_path = os.path.join(r'\\grove.ad.uconn.edu\research\ENGR_Chon\Darren\NIH_PulseWatch\Poincare_pt', smote_type, split) + + output_path = os.path.join(r'C:\Chon_Lab\NIH_Pulsewatch\TFS_pt', smote_type, split) + # output_path = os.path.join(r'C:\Chon_Lab\NIH_Pulsewatch\Poincare_pt', smote_type, split) + + preprocess_and_save_data(input_path, output_path) + print('Data transfer complete!') + +if __name__ == '__main__': + main() From 5de15262aa1d74421fc23b97d7c2e2410808c3d7 Mon Sep 17 00:00:00 2001 From: Luis Roberto Mercado Diaz Date: Sun, 21 Apr 2024 16:55:05 -0400 Subject: [PATCH 6/6] UPDATES --- main_darren_v1.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/main_darren_v1.py b/main_darren_v1.py index 29ec642..6178aff 100644 --- a/main_darren_v1.py +++ b/main_darren_v1.py @@ -95,7 +95,8 @@ def load_data(data_path, labels_path, batch_size, binary=False): return dataloader def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, patience=10, - checkpoint_path='model_checkpoint.pt', resume_training=False): + checkpoint_path='model_checkpoint.pt', data_checkpoint_path='data_checkpoint.pt', + resume_training=False, plot_path='training_plot.png'): model = MultitaskGPModel().to(device) likelihood = gpytorch.likelihoods.SoftmaxLikelihood(num_features=4, num_classes=4).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=0.1) @@ -136,9 +137,9 @@ def train_gp_model(train_loader, val_loader, num_iterations=50, n_classes=4, pat # Stochastic validation model.eval() likelihood.eval() + val_loss = 0.0 # Initialize val_loss here with torch.no_grad(): val_indices = torch.randperm(len(val_loader.dataset))[:int(0.1 * len(val_loader.dataset))] - val_loss = 0.0 val_labels = [] val_predictions = [] for idx in val_indices: @@ -212,6 +213,7 @@ def evaluate_gp_model(test_loader, model, likelihood, n_classes=4): return metrics def main(): + print("Step 1: Loading paths and parameters") # Paths base_path = r"\\grove.ad.uconn.edu\\research\\ENGR_Chon\Darren\\NIH_Pulsewatch" smote_type = 'Cassey5k_SMOTE' @@ -231,23 +233,37 @@ def main(): else: n_classes = 3 patience = round(n_epochs / 10) if n_epochs > 50 else 5 - save = True resume_checkpoint_path = None batch_size = 256 + print("Step 2: Loading data") # Data loading train_loader = load_data(data_path_train, labels_path_train, batch_size, binary) val_loader = load_data(data_path_val, labels_path_val, batch_size, binary) test_loader = load_data(data_path_test, labels_path_test, batch_size, binary) - # Training and validation + print("Step 3: Loading data checkpoints") + # Data loading with checkpointing + data_checkpoint_path = 'data_checkpoint.pt' + if os.path.exists(data_checkpoint_path): + train_loader.dataset.load_checkpoint(data_checkpoint_path) + val_loader.dataset.load_checkpoint(data_checkpoint_path) + test_loader.dataset.load_checkpoint(data_checkpoint_path) + + print("Step 4: Training and validation") + # Training and validation with checkpointing and plotting + model_checkpoint_path = 'model_checkpoint.pt' + plot_path = 'training_plot.png' start_time = time.time() model, likelihood, metrics = train_gp_model(train_loader, val_loader, n_epochs, - n_classes, patience, save) + n_classes, patience, + model_checkpoint_path, data_checkpoint_path, + resume_checkpoint_path is not None, plot_path) end_time = time.time() time_passed = end_time - start_time print('\nTraining and validation took %.2f minutes' % (time_passed / 60)) + print("Step 5: Evaluation") # Evaluation start_time = time.time() test_metrics = evaluate_gp_model(test_loader, model, likelihood, n_classes) @@ -255,6 +271,7 @@ def main(): time_passed = end_time - start_time print('\nTesting took %.2f seconds' % time_passed) + print("Step 6: Printing test metrics") print('Test Metrics:') print('Precision: %.4f' % test_metrics['precision']) print('Recall: %.4f' % test_metrics['recall'])