# SeeAlso
# ~/code/geowatch/docs/source/manual/baselines/baseline-2023-06-22-sc_truth.rst
DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)


MLOPS_DPATH=$DVC_EXPT_DPATH/_drop8_ara_sc_v1
MODEL_SHORTLIST="
- $DVC_EXPT_DPATH/models/fusion/Drop7-Cropped2GSD/packages/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84_epoch17_step1548.pt
- $DVC_EXPT_DPATH/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_100pctphase.pt
- $DVC_EXPT_DPATH/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_85pctphase.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch0_step169.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch11_step2028.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch12_step2197.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch1_step338.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch2_step507.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch3_step676.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch43_step7436.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch5_step1014.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch10_step1859.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch18_step3211.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch24_step4225.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch25_step4394.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch26_step4563.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch27_step4732.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch28_step4901.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch32_step5577.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch33_step5746.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch34_step5915.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch35_step6084.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch36_step6253.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch37_step6422.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch38_step6591.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch3_step676.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch9_step1690.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch3_step1368.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch6_step2394.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch4_step1710.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch78_step27018.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch359_step122881.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch2_step1026.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch70_step24282.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch142_step48906.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch104_step35910.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch5_step2052.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch113_step38988.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch0_step21021.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch1_step42042.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch3_step84084.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch2_step63063.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch4_step105105.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch5_step122881.pt
"


MODEL_SHORTLIST="
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop7-Cropped2GSD/packages/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84_epoch17_step1548.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch0_step169.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch11_step2028.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch12_step2197.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch1_step338.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch2_step507.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch3_step676.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_100pctphase.pt
- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_85pctphase.pt

- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch38_step6591.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch3_step676.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch9_step1690.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch5_step2052.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch113_step38988.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch2_step1026.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch78_step27018.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch359_step122881.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch0_step21021.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch1_step42042.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch3_step84084.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch2_step63063.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch4_step105105.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch5_step122881.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch18_step3211.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch24_step4225.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch25_step4394.pt
- $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v2_V003_epoch26_step4563.pt
"


#MODEL_SHORTLIST="
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop7-Cropped2GSD/packages/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84_epoch17_step1548.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch0_step21021.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_100pctphase.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_85pctphase.pt
#"


#MODEL_SHORTLIST="
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop7-Cropped2GSD/packages/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84_epoch17_step1548.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch0_step21021.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_V001/Drop8-ARA-Cropped2GSD-V1_allsensors_V001_epoch5_step122881.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_V002_epoch2_step1026.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Cropped2GSD-V1/packages/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_from_v1_V002_epoch12_step2197.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_100pctphase.pt
#- /home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_85pctphase.pt
#"


mkdir -p "$MLOPS_DPATH"
echo "$MODEL_SHORTLIST" > "$MLOPS_DPATH/ac_model_shortlist.yaml"
cat "$MLOPS_DPATH/ac_model_shortlist.yaml"

DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
MLOPS_DNAME=_drop8_ara_sc_v1
MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_DNAME
python -m geowatch.mlops.schedule_evaluation --params="
    pipeline: sc
    matrix:
        #####################
        ## AC PIXEL PARAMS ##
        #####################
        sc_pxl.test_dataset:
            - $DVC_DATA_DPATH/Drop8-ARA-Cropped2GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip

        sc_pxl.package_fpath:
            #- $DVC_EXPT_DPATH/models/fusion/Drop7-Cropped2GSD/packages/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84/Drop7-Cropped2GSD_SC_bgrn_gnt_split6_V84_epoch17_step1548.pt
            #- $DVC_EXPT_DPATH/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_100pctphase.pt
            #- $DVC_EXPT_DPATH/models/fusion/dzyne/Drop8-ARA-Cropped2GSD-V1_allsensors_rebalance_85pctphase.pt
            - $MLOPS_DPATH/ac_model_shortlist.yaml

        sc_pxl.tta_fliprot: 0.0
        sc_pxl.tta_time: 0.0
        sc_pxl.chip_overlap:
            - 0.3
            #- 0.0
        sc_pxl.num_workers: 4
        sc_pxl.batch_size: 1
        sc_pxl.write_workers: 0
        sc_pxl.observable_threshold: 0.0
        sc_pxl.drop_unused_frames: true

        sc_pxl.fixed_resolution:
            #- 8GSD
            #- 6GSD
            - 4GSD
            - 2GSD
        #####################
        ## SC POLY PARAMS  ##
        #####################

        sc_poly.boundaries_as:
            - bounds
            #- poly
        sc_poly.new_algo: crall
        sc_poly.polygon_simplify_tolerance: 2
        sc_poly.site_score_thresh:
            - 0.275
            - 0.3
            - 0.325
        sc_poly.smoothing: 0.0
        sc_poly.thresh:
            #- 0.07
            #- 0.1
            - 0.2
            - 0.25
            - 0.275
            - 0.3
            - 0.325
        sc_poly.resolution:
            - 8GSD
            - 4GSD
        sc_poly.min_area_square_meters: 7200

        ##########################
        ## SC POLY EVAL PARAMS  ##
        ##########################

        sc_poly_eval.true_site_dpath: $DVC_DATA_DPATH/annotations/drop8-v1/site_models
        sc_poly_eval.true_region_dpath: $DVC_DATA_DPATH/annotations/drop8-v1/region_models

        ##################################
        ## HIGH LEVEL PIPELINE CONTROLS ##
        ##################################
        sc_pxl.enabled: 1
        sc_pxl_eval.enabled: 0
        sc_poly.enabled: 1
        sc_poly_eval.enabled: 1
        sc_poly_viz.enabled: 0

    submatrices:
        - sc_pxl.input_space_scale: 8GSD
          sc_pxl.window_space_scale: 8GSD
          sc_pxl.output_space_scale: 8GSD
          sc_pxl.fixed_resolution: 8GSD
        - sc_pxl.input_space_scale: 2GSD
          sc_pxl.window_space_scale: 2GSD
          sc_pxl.output_space_scale: 2GSD
          sc_pxl.fixed_resolution: 2GSD
        - sc_pxl.input_space_scale: 4GSD
          sc_pxl.window_space_scale: 4GSD
          sc_pxl.output_space_scale: 4GSD
          sc_pxl.fixed_resolution: 4GSD
        - sc_pxl.input_space_scale: 6GSD
          sc_pxl.window_space_scale: 6GSD
          sc_pxl.output_space_scale: 6GSD
          sc_pxl.fixed_resolution: 6GSD

    submatrices1:
        # Because there is no BAS component, we need to provide site summaries.
        # We should use BAS system outputs, but if we dont have those use truth.
        #- sc_pxl.test_dataset: $DVC_DATA_DPATH/Drop8-ARA-Cropped2GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip
        #  sc_poly.site_summary: $DVC_DATA_DPATH/annotations/drop8-v1/region_models/KR_R002.geojson

        - sc_pxl.test_dataset: $DVC_DATA_DPATH/Drop8-ARA-Cropped2GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip
          sc_poly.site_summary: $HOME/data/dvc-repos/smart_phase3_expt/static-results/bas-predictions/eval21/KR_R002/KR_R002.geojson

        #- sc_pxl.test_dataset: $BUNDLE_DPATH/KW_C001/imgonly-KW_C001-rawbands-small.kwcoco.zip
        #  sc_poly.site_summary: $BUNDLE_DPATH/bas_small_output/region_models/KW_C501.geojson
    " \
    --root_dpath="$MLOPS_DPATH" \
    --queue_name "$MLOPS_DNAME" \
    --devices="0,1,2,3" \
    --backend=tmux --tmux_workers=8 \
    --cache=1 --skip_existing=1 --run=1


DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
MLOPS_DNAME=_drop8_ara_sc_v1
MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_DNAME
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
echo "DVC_EXPT_DPATH = $DVC_EXPT_DPATH"
python -m geowatch.mlops.aggregate \
    --pipeline=sc \
    --target "
        - $MLOPS_DPATH
    " \
    --primary_metric_cols="
        - metrics.sc_poly_eval.bas_faa_f1
        - metrics.sc_poly_eval.sc_macro_f1
    " \
    --output_dpath="$MLOPS_DPATH/aggregate" \
    --resource_report=0 \
    --eval_nodes="
        - sc_poly_eval
    " \
    --plot_params="
        enabled: 1
        stats_ranking: 0
        min_variations: 1
        params_of_interest:
            - resolved_params.sc_poly.thresh
            - resolved_params.sc_poly.boundaries_as
            - resolved_params.sc_poly.resolution
            - resolved_params.sc_poly.resolution
            - resolved_params.sc_pxl.fixed_resolution
            - resolved_params.sc_pxl.chip_overlap
            - resolved_params.sc_pxl.package_fpath
    " \
    --stdout_report="
        top_k: 100
        per_group: 1
        macro_analysis: 0
        analyze: 0
        print_models: True
        reference_region: final
        concise: 1
        show_csv: 0
    " \
    --rois="KR_R002" \
    --query "df['params.sc_poly.site_summary'] == '$HOME/data/dvc-repos/smart_phase3_expt/static-results/bas-predictions/eval21/KR_R002/KR_R002.geojson'"



#sc_pxl.tta_fliprot: 0.0
#sc_pxl.tta_time: 0.0
#sc_pxl.chip_overlap: 0.3
#sc_pxl.input_space_scale: 8GSD
#sc_pxl.window_space_scale: 8GSD
#sc_pxl.output_space_scale: 8GSD
#sc_pxl.time_span: 6m
#sc_pxl.time_sampling: auto
#sc_pxl.time_steps: 12
#sc_pxl.chip_dims: auto
#sc_pxl.set_cover_algo: null
#sc_pxl.resample_invalid_frames: 3
#sc_pxl.observable_threshold: 0.0
#sc_pxl.mask_low_quality: true
#sc_pxl.drop_unused_frames: true
#sc_poly.thresh: 0.07
#sc_poly.boundaries_as: polys
#sc_poly.resolution: 8GSD
#sc_poly.min_area_square_meters: 7200
#

DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
MLOPS_DNAME=_drop8_ara_sc_v1
MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_DNAME
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
echo "DVC_EXPT_DPATH = $DVC_EXPT_DPATH"
python -m geowatch.mlops.aggregate \
    --pipeline=sc \
    --export_tables=True \
    --target "
        - $MLOPS_DPATH
    " \
    --primary_metric_cols="
        - metrics.sc_poly_eval.bas_faa_f1
        - metrics.sc_poly_eval.sc_macro_f1
        - metrics.sc_poly_eval.macro_f1_siteprep
    " \
    --output_dpath="$MLOPS_DPATH/aggregate" \
    --resource_report=0 \
    --eval_nodes="
        - sc_poly_eval
    " \
    --plot_params="
        enabled: 0
    " \
    --stdout_report="
        top_k: 10
        per_group: 1
        macro_analysis: 0
        analyze: 0
        print_models: True
        reference_region: final
        concise: 0
        show_csv: 0
    " \
    --rois="KR_R002" \
    --query "df['params.sc_poly.site_summary'] == '$HOME/data/dvc-repos/smart_phase3_expt/static-results/bas-predictions/eval21/KR_R002/KR_R002.geojson'"


DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
MLOPS_DNAME=_drop8_ara_sc_v1
MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_DNAME
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto)
echo "DVC_EXPT_DPATH = $DVC_EXPT_DPATH"
geowatch mlops.aggregate \
    --pipeline=sc \
    --target "
        - $MLOPS_DPATH/sc_poly_eval_horologic_01488_2024-05-21T121410-5.csv.zip
    " \
    --primary_metric_cols="
        - metrics.sc_poly_eval.macro_f1_siteprep
        - metrics.sc_poly_eval.bas_faa_f1
        - metrics.sc_poly_eval.sc_macro_f1
    " \
    --output_dpath="$MLOPS_DPATH/aggregate" \
    --resource_report=0 \
    --eval_nodes="
        - sc_poly_eval
    " \
    --plot_params="
        enabled: 0
        stats_ranking: 0
        min_variations: 1
        params_of_interest:
            - resolved_params.sc_poly.thresh
            - resolved_params.sc_poly.boundaries_as
            - resolved_params.sc_poly.resolution
            - resolved_params.sc_poly.resolution
            - resolved_params.sc_pxl.fixed_resolution
            - resolved_params.sc_pxl.chip_overlap
            - resolved_params.sc_pxl.package_fpath
        vantage_points:
            - metric1: metrics.sc_poly_eval.sc_macro_f1
              metric2: metrics.sc_poly_eval.bas_faa_f1

            - metric1: metrics.sc_poly_eval.bas_f1
              metric2: metrics.sc_poly_eval.bas_ffpa

            - metric1: metrics.sc_poly_eval.macro_f1_siteprep
              metric2: metrics.sc_poly_eval.sc_macro_f1

            - metric1: metrics.sc_poly_eval.macro_f1_active
              metric2: metrics.sc_poly_eval.sc_macro_f1

            - metric1: metrics.sc_poly_eval.bas_ffpa
              metric2: metrics.sc_poly_eval.sc_macro_f1
    " \
    --stdout_report="
        top_k: 1000
        per_group: 1
        macro_analysis: 0
        analyze: 0
        print_models: True
        reference_region: final
        concise: 0
        show_csv: 0
    " \
    --rois="KR_R002" \
    --query "
    (df['params.sc_poly.site_summary'].str.contains('data/dvc-repos/smart_phase3_expt/static-results/bas-predictions/eval21/KR_R002/KR_R002.geojson')) &
    (df['params.sc_pxl.package_fpath'].str.contains('V1_allsensors_V001_epoch0_step21021'))
    "
    #&
    #(df['params.sc_pxl.fixed_resolution'].str.contains('4GSD'))
    'enkgdgoaaktv': {
        'params.sc_poly.boundaries_as': 'bounds',
        'params.sc_poly.site_score_thresh': 0.3,
        'params.sc_poly.thresh': 0.25,
        'params.sc_poly.resolution': '4GSD',
        'params.sc_pxl.fixed_resolution': '2GSD',
        'params.sc_pxl.input_space_scale': '2GSD',
        'params.sc_pxl.window_space_scale': '2GSD',
        'params.sc_pxl.output_space_scale': '2GSD',
    },
