IMAGE=registry.smartgitlab.com/kitware/watch:0.7.5-6ae5ccba-strict-pyenv3.11.2-20230627T174359-0400-from-a19930d6

DVC_LORES_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=ssd)
DVC_HIRES_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=hdd)
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto)

IMAGE_LORES_DATA_DPATH=/root/data/dvc-repos/smart_data_dvc-ssd
IMAGE_HIRES_DATA_DPATH=/root/data/dvc-repos/smart_data_dvc-hdd
IMAGE_EXPT_DPATH=/root/data/dvc-repos/smart_expt_dvc

# Note: need to be careful to mount directories that symlinks reference
# **exactly**.
EXPT_CACHE_DIR=$(python -m geowatch.utils.simple_dvc cache_dir "$DVC_EXPT_DPATH")
LORES_CACHE_DIR=$(python -m geowatch.utils.simple_dvc cache_dir "$DVC_LORES_DATA_DPATH")
HIRES_CACHE_DIR=$(python -m geowatch.utils.simple_dvc cache_dir "$DVC_HIRES_DATA_DPATH")


# Ensure models exist locally
python -m geowatch.utils.simple_dvc request \
    "$DVC_EXPT_DPATH"/models/fusion/Drop6-MeanYear10GSD-V2/packages/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47_epoch47_step3026.pt \
    "$DVC_EXPT_DPATH"/models/fusion/Drop7-MedianNoWinter10GSD/packages/Drop7-MedianNoWinter10GSD_bgrn_split6_V68/Drop7-MedianNoWinter10GSD_bgrn_split6_V68_epoch34_stepNone.pt \
    "$DVC_EXPT_DPATH"/models/depth_pcd/basicModel2.h5 \
    "$DVC_EXPT_DPATH"/models/depth_pcd/model3.h5


_note_='
Theoretically this would let us write new files such that the host filesytem
could read them without chown, but doing this doesnt let us login as root which
is where all the python stuff is installed.

    --user "$(id -u):$(id -g)" \
    -v /etc/passwd:/etc/passwd:ro \
    -v /etc/group:/etc/group:ro \
    --mount "type=bind,source=$DVC_LORES_DATA_DPATH,target=$IMAGE_LORES_DATA_DPATH" \
    --mount "type=bind,source=$DVC_HIRES_DATA_DPATH,target=$IMAGE_HIRES_DATA_DPATH" \
    --mount "type=bind,source=$DVC_EXPT_DPATH,target=$IMAGE_EXPT_DPATH" \
    --mount "type=bind,source=$EXPT_CACHE_DIR,target=$EXPT_CACHE_DIR,readonly" \
    --mount "type=bind,source=$LORES_CACHE_DIR,target=$LORES_CACHE_DIR,readonly" \
    --mount "type=bind,source=$HIRES_CACHE_DIR,target=$HIRES_CACHE_DIR,readonly" \

Other docker run options:
    #--memory=60g --cpus=8  --gpus='"device=1"' \
'

docker run \
    --gpus all \
    --shm-size=60g \
    --volume "$DVC_LORES_DATA_DPATH:$IMAGE_LORES_DATA_DPATH" \
    --volume "$DVC_HIRES_DATA_DPATH:$IMAGE_HIRES_DATA_DPATH" \
    --volume "$DVC_EXPT_DPATH:$IMAGE_EXPT_DPATH" \
    --volume "$EXPT_CACHE_DIR:$EXPT_CACHE_DIR:ro" \
    --volume "$LORES_CACHE_DIR:$LORES_CACHE_DIR:ro" \
    --volume "$HIRES_CACHE_DIR:$HIRES_CACHE_DIR:ro" \
    --env "DVC_LORES_DATA_DPATH=$IMAGE_LORES_DATA_DPATH" \
    --env "DVC_HIRES_DATA_DPATH=$IMAGE_HIRES_DATA_DPATH" \
    --env "DVC_EXPT_DPATH=$IMAGE_EXPT_DPATH" \
    -it $IMAGE \
    bash

geowatch schedule --params="
    matrix:
        bas_pxl.package_fpath:
            - $DVC_EXPT_DPATH/models/fusion/Drop6-MeanYear10GSD-V2/packages/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47_epoch47_step3026.pt
            - $DVC_EXPT_DPATH/models/fusion/Drop7-MedianNoWinter10GSD/packages/Drop7-MedianNoWinter10GSD_bgrn_split6_V68/Drop7-MedianNoWinter10GSD_bgrn_split6_V68_epoch34_stepNone.pt
        bas_pxl.test_dataset:
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-KR_R002_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-CH_R001_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-NZ_R001_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-BR_R002_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-KR_R001_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-AE_R001_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-PE_R001_EI2LMSC.kwcoco.zip
            - $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-BR_R004_EI2LMSC.kwcoco.zip
        bas_pxl.chip_overlap: 0.3
        bas_pxl.chip_dims: auto
        bas_pxl.time_span: auto
        bas_pxl.time_sampling: soft4
        bas_poly.thresh:
            - 0.35
            - 0.375
            - 0.4
            - 0.425
        bas_poly.inner_window_size: 1y
        bas_poly.inner_agg_fn: mean
        bas_poly.norm_ord: inf
        bas_poly.polygon_simplify_tolerance: 1
        bas_poly.agg_fn: probs
        bas_poly.time_thresh:
            - 0.8
        bas_poly.resolution: 10GSD
        bas_poly.moving_window_size: null
        bas_poly.poly_merge_method: 'v2'
        bas_poly.min_area_square_meters: 7200
        bas_poly.max_area_square_meters: 8000000
        bas_poly.boundary_region: $DVC_LORES_DATA_DPATH/annotations/drop6/region_models
        bas_poly_eval.true_site_dpath: $DVC_LORES_DATA_DPATH/annotations/drop6/site_models
        bas_poly_eval.true_region_dpath: $DVC_LORES_DATA_DPATH/annotations/drop6/region_models
        bas_pxl.enabled: 1
        bas_pxl_eval.enabled: 0
        bas_poly.enabled: 1
        bas_poly_eval.enabled: 1
        bas_poly_viz.enabled: 0
        sv_crop.enabled: 1
        sv_crop.minimum_size: '256x256@2GSD'
        sv_crop.num_start_frames: 3
        sv_crop.num_end_frames: 3
        sv_crop.context_factor: 1.5

        sv_dino_boxes.enabled: 1
        sv_dino_boxes.package_fpath: $DVC_EXPT_DPATH/models/kitware/xview_dino.pt
        sv_dino_boxes.window_dims: 256
        sv_dino_boxes.window_overlap: 0.5
        sv_dino_boxes.fixed_resolution: 3GSD

        sv_dino_filter.enabled: 1
        sv_dino_filter.end_min_score:
            - 0.1
            - 0.15
        sv_dino_filter.start_max_score: 1.0
        sv_dino_filter.box_score_threshold: 0.01
        sv_dino_filter.box_isect_threshold: 0.1

        sv_depth_score.enabled: 1
        sv_depth_score.model_fpath:
            - $DVC_EXPT_DPATH/models/depth_pcd/basicModel2.h5
            #- $DVC_EXPT_DPATH/models/depth_pcd/model3.h5
        sv_depth_filter.threshold:
            - 0.25
            - 0.30
            - 0.35
    submatrices:
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-KR_R001_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/KR_R001/imgonly-KR_R001.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-KR_R002_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/KR_R002/imgonly-KR_R002.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-AE_R001_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/AE_R001/imgonly-AE_R001.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-BR_R002_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/BR_R002/imgonly-BR_R002.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-CH_R001_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/CH_R001/imgonly-CH_R001.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-NZ_R001_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/NZ_R001/imgonly-NZ_R001.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-PE_R001_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/PE_R001/imgonly-PE_R001.kwcoco.zip
        - bas_pxl.test_dataset: $DVC_LORES_DATA_DPATH/Drop7-MedianNoWinter10GSD/combo_imganns-BR_R004_EI2LMSC.kwcoco.zip
          sv_crop.crop_src_fpath: $DVC_HIRES_DATA_DPATH/Aligned-Drop7/BR_R004/imgonly-BR_R004.kwcoco.zip
    " \
    --root_dpath="$DVC_EXPT_DPATH/_namek_sv_sweep" \
    --devices="0,1" --tmux_workers=8 \
    --backend=tmux --queue_name "_namek_sv_sweep" \
    --pipeline=bas_building_and_depth_vali \
    --skip_existing=1 \
    --run=1



#docker run --shm-size=60g -it registry.smartgitlab.com/kitware/watch:0.7.5-4246df07-strict-pyenv3.11.2-20230627T150436-0400-from-176d15dd-metrics-e3d2ed8
#xdoctest watch/cli/run_metrics_framework.py
#FAILED watch/cli/run_metrics_framework.py::main:0
#FAILED watch/tasks/fusion/methods/channelwise_transformer.py::MultimodalTransformer.forward_step:0
#FAILED tests/test_heterogeneous_model.py::test_heterogeneous_with_split_attention_backbone - RuntimeError: DataLoader worker (pid(s) 6466) exited unexpectedly

#pytest watch/tasks/fusion/methods/channelwise_transformer.py
#pytest tests/test_heterogeneous_model.py

# Pull out baseline tables
DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto)
#sudo chown -R "$USER":smart "$DVC_EXPT_DPATH"/_namek_sv_sweep

python -m geowatch.mlops.aggregate \
    --pipeline=bas_building_and_depth_vali \
    --target "
        - $DVC_EXPT_DPATH/_namek_sv_sweep
    " \
    --output_dpath="$DVC_EXPT_DPATH/_namek_sv_sweep/aggregate" \
    --resource_report=0 \
    --eval_nodes="
        - sv_poly_eval
        - bas_poly_eval
        #- bas_pxl_eval
    " \
    --plot_params="
        enabled: 0
        stats_ranking: 0
        min_variations: 1
        params_of_interest:
            - params.sv_depth_filter.threshold
            - params.sv_depth_score.model_fpath
            - params.bas_poly.thresh
    " \
    --stdout_report="
        top_k: 5
        per_group: 1
        macro_analysis: 0
        analyze: 0
        print_models: True
        reference_region: final
    " \
    --rois="KR_R002,PE_R001,NZ_R001,CH_R001,KR_R001,BR_R002,BR_R004"

    #--rois="PE_R001"
    --query='
        ((df["params.sv_depth_filter.threshold"] > 0) &
         (df["params.bas_poly.thresh"] >= 0.3) &
    '

         #(df["params.sv_depth_filter.threshold"] == 0.35))
        # (df["param_hashid"] == "hmownacdredt") &

    #
    #
    #--rois="KR_R002,PE_R001,NZ_R001,CH_R001,KR_R001,AE_R001,BR_R002,BR_R004"
    #--query='(df["param_hashid"] == "hmownacdredt")' \
    #--rois="PE_R001"

