WORKDIR=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp
mkdir -p $WORKDIR
cd $WORKDIR
rsync -avpr horologic:/data/david.joy/Ph2Oct5/./KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco "$WORKDIR"



WORKDIR=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp
BUNDLE_DPATH=$WORKDIR/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco
BAS_TEST_DATASET=$BUNDLE_DPATH/cropped_kwcoco_for_bas.json
SC_TEST_DATASET=$BUNDLE_DPATH/cropped_kwcoco_for_sc.json
DVC_EXPT_DPATH=$(geowatch_dvc --tags="phase2_expt")
BAS_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt.pt
SC_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt
echo "

DVC_EXPT_DPATH = $DVC_EXPT_DPATH
SC_MODEL_FPATH = $SC_MODEL_FPATH
BAS_MODEL_FPATH = $BAS_MODEL_FPATH

"
mkdir -p $BUNDLE_DPATH/testing
python -m geowatch.mlops.schedule_evaluation \
    --trk_model_globstr="$BAS_MODEL_FPATH" \
    --act_model_globstr="$SC_MODEL_FPATH" \
    --model_globstr="$BAS_MODEL_FPATH" \
    --trk_test_dataset="$BAS_TEST_DATASET" \
    --act_test_dataset="$SC_TEST_DATASET" \
    --grid_pred_pxl='
    include:
    - {
      "tta_fliprot": 0,
      "tta_time": 0,
      "chip_overlap": 0.3,
      "input_space_scale": "15GSD",
      "window_space_scale": "10GSD",
      "output_space_scale": "15GSD",
      "time_span": "auto",
      "time_sampling": "auto",
      "time_steps": "auto",
      "chip_dims": "auto",
      "set_cover_algo": "None",
      "resample_invalid_frames": 1,
      "use_cloudmask": 1
    }
    ' \
    --grid_pred_trk='
    include: 
    - {
        "thresh": 0.10,
        "morph_kernel": 3,
        "norm_ord": 1,
        "agg_fn": "probs",
        "thresh_hysteresis": None,
        "moving_window_size": None,
        "polygon_fn": "heatmaps_to_polys",
    }
    ' \
    --expt_dvc_dpath="$BUNDLE_DPATH/testing"  \
    --enable_pred_trk=1 \
    --enable_pred_pxl=1 \
    --enable_eval_pxl=1 \
    --enable_eval_trk=1 \
    --enable_pred_trk_viz=1 \
    --backend=tmux --run=0  --skip_existing=1 --check_other_sessions=0




WORKDIR=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp
BUNDLE_DPATH=$WORKDIR/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco
BAS_TEST_DATASET=$BUNDLE_DPATH/cropped_kwcoco_for_bas.json
SC_TEST_DATASET=$BUNDLE_DPATH/cropped_kwcoco_for_sc.json
DVC_EXPT_DPATH=$(geowatch_dvc --tags="phase2_expt")
BAS_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt.pt
SC_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt
python -m geowatch.mlops.schedule_evaluation \
    --params="
    - matrix:
        ###
        ### BAS Pixel Prediction
        ###
        trk.pxl.model: $BAS_MODEL_FPATH
        trk.pxl.data.test_dataset: $BAS_TEST_DATASET
        trk.pxl.data.tta_time: 0
        trk.pxl.data.chip_overlap: 0.3
        trk.pxl.data.window_scale_space: 10GSD
        trk.pxl.data.input_scale_space: 15GSD
        trk.pxl.data.output_scale_space: 15GSD
        trk.pxl.data.time_span: auto
        trk.pxl.data.time_sampling: auto
        trk.pxl.data.time_steps: auto
        trk.pxl.data.chip_dims: auto
        trk.pxl.data.set_cover_algo: None
        trk.pxl.data.resample_invalid_frames: 1
        trk.pxl.data.use_cloudmask: 1
        ###
        ### BAS Polygon Prediction
        ###
        trk.poly.thresh: 0.10
        trk.poly.morph_kernel: 3
        trk.poly.norm_ord: 1
        trk.poly.agg_fn: probs
        trk.poly.thresh_hysteresis: None
        trk.poly.moving_window_size: None
        trk.poly.polygon_fn: heatmaps_to_polys
        ###
        ### SC Pixel Prediction
        ###
        act.pxl.model: $SC_MODEL_FPATH
        act.pxl.data.test_dataset: $SC_TEST_DATASET
    " \
    --expt_dvc_dpath="$BUNDLE_DPATH/testing"  \
    --enable_pred_trk=1 \
    --enable_pred_pxl=1 \
    --enable_eval_pxl=1 \
    --enable_eval_trk=1 \
    --enable_pred_trk_viz=1 \
    --backend=tmux --run=0  --skip_existing=1 --check_other_sessions=0



python -m geowatch.tasks.fusion.predict \
    --write_probs=True \
    --write_preds=False \
    --with_class=auto \
    --with_saliency=auto \
    --with_change=False \
    --package_fpath=/home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt.pt \
    --pred_dataset=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/pred.kwcoco.json \
    --test_dataset=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/cropped_kwcoco_for_bas.json \
    --num_workers=4 \
    --tta_fliprot=0 \
    --tta_time=0 \
    --chip_overlap=0.3 \
    --input_space_scale=15GSD \
    --window_space_scale=10GSD \
    --output_space_scale=15GSD \
    --time_span=auto \
    --time_sampling=auto \
    --time_steps=auto \
    --chip_dims=auto \
    --set_cover_algo=None \
    --resample_invalid_frames=1 \
    --use_cloudmask=1 \
    --devices=0, \
    --accelerator=gpu \
    --batch_size=1
    


python -m geowatch.cli.run_tracker \
    /home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/pred.kwcoco.json \
    --default_track_fn saliency_heatmaps \
    --track_kwargs '{"thresh": 0.1, "morph_kernel": 3, "norm_ord": 1, "agg_fn": "probs", "thresh_hysteresis": null, "moving_window_size": null, "polygon_fn": "heatmaps_to_polys"}' \
    --clear_annots \
    --bas_mode \
    --out_dir /home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/tracking/trackcfg_6db2a013/tracked_sites_bas \
    --out_fpath /home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/tracking/trackcfg_6db2a013/tracks_bas.json \
    --out_kwcoco /home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/tracking/trackcfg_6db2a013/tracks_bas.kwcoco.json


source "$HOME/code/watch/secrets/secrets"
export GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR"
export GDAL_HTTP_MERGE_CONSECUTIVE_RANGES="YES"
export GDAL_HTTP_MULTIPLEX="YES"
export GDAL_HTTP_VERSION="2"
export VSI_CACHE="TRUE"
export VSI_CACHE_SIZE=25000000
export AWS_PROFILE=iarpa
WORKDIR=/home/joncrall/data/dvc-repos/smart_data_dvc/tmp
BUNDLE_DPATH=$WORKDIR/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco
TEST_DATASET=$BUNDLE_DPATH/cropped_kwcoco_for_bas.json
DVC_EXPT_DPATH=$(geowatch_dvc --tags="phase2_expt")
BAS_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt.pt
SC_MODEL_FPATH=$DVC_EXPT_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt
mkdir -p $BUNDLE_DPATH/testing
XDEV_PROFILE=1 python -m geowatch.cli.coco_align_geotiffs \
    --visualize False \
    --src $BUNDLE_DPATH/kwcoco_for_sc.json \
    --dst $BUNDLE_DPATH/cropped_kwcoco_for_sc.json \
    --regions /home/joncrall/data/dvc-repos/smart_data_dvc/tmp/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco/testing/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/pred/Drop4_BAS_Retrain_V002/Drop4_BAS_Retrain_V002_epoch=31-step=16384.pt/KR_R001_0.1BASThresh_40cloudcover_debug10_kwcoco_cropped_kwcoco_for_bas/predcfg_41fd3894/tracking/trackcfg_6db2a013/tracked_sites_bas/KR_R001.geojson \
    --force_nodata -9999 \
    --include_channels 'red|green|blue|cloudmask' \
    --site_summary True \
    --geo_preprop auto \
    --keep none \
    --target_gsd 4  \
    --context_factor 1.5   \
    --workers 24  \
    --aux_workers 4 \
    --rpc_align_method affine_warp  
