From 98fd2b3a3d26a3da0d89a6183a7c1bab3a921a9e Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 19 Jan 2021 13:58:14 -0500 Subject: [PATCH 1/5] Start branch for 0.5.14 --- CHANGELOG.md | 5 +- netharn/__init__.py | 2 +- publish.sh | 259 +++++++++++++++++++++++++++++--------------- super_setup.py | 2 +- 4 files changed, 176 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e89c985..19a16be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ This changelog follows the specifications detailed in: [Keep a Changelog](https: This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), although we have not yet reached a `1.0.0` release. -## Version 0.5.13 - Unreleased +## Version 0.5.14 - Unreleased + + +## Version 0.5.13 - Released 2021-01-19 ### Added diff --git a/netharn/__init__.py b/netharn/__init__.py index b3bdcf6..427c4c2 100644 --- a/netharn/__init__.py +++ b/netharn/__init__.py @@ -4,7 +4,7 @@ mkinit netharn --noattrs --dry mkinit netharn --noattrs """ -__version__ = '0.5.13' +__version__ = '0.5.14' try: # PIL 7.0.0 removed PIL_VERSION, which breaks torchvision, monkey patch it diff --git a/publish.sh b/publish.sh index 917c631..1c9c912 100755 --- a/publish.sh +++ b/publish.sh @@ -9,13 +9,13 @@ setup.py script to create the wheels as well. Running this script with the default arguments will perform any builds and gpg signing, but nothing will be uploaded to pypi unless the user explicitly sets -TAG_AND_UPLOAD=True or answers yes to the prompts. +DO_UPLOAD=True or answers yes to the prompts. Args: # These environment variables must / should be set TWINE_USERNAME : username for pypi TWINE_PASSWORD : password for pypi - USE_GPG : defaults to True + DO_GPG : defaults to True Requirements: twine >= 1.13.0 @@ -45,45 +45,68 @@ Usage: echo "MB_PYTHON_TAG = $MB_PYTHON_TAG" MB_PYTHON_TAG=$MB_PYTHON_TAG ./run_multibuild.sh - DEPLOY_BRANCH=master DEPLOY_REMOTE=ibeis MB_PYTHON_TAG=$MB_PYTHON_TAG ./publish.sh yes + DEPLOY_REMOTE=ibeis MB_PYTHON_TAG=$MB_PYTHON_TAG ./publish.sh yes - MB_PYTHON_TAG=py2.py3-none-any ./publish.sh + MB_PYTHON_TAG=py3-none-any ./publish.sh ''' check_variable(){ KEY=$1 + HIDE=$2 VAL=${!KEY} - echo "[DEBUG] CHECK VARIABLE: $KEY=\"$VAL\"" + if [[ "$HIDE" == "" ]]; then + echo "[DEBUG] CHECK VARIABLE: $KEY=\"$VAL\"" + else + echo "[DEBUG] CHECK VARIABLE: $KEY=" + fi if [[ "$VAL" == "" ]]; then echo "[ERROR] UNSET VARIABLE: $KEY=\"$VAL\"" exit 1; fi } + +normalize_boolean(){ + ARG=$1 + ARG=$(echo "$ARG" | awk '{print tolower($0)}') + if [ "$ARG" = "true" ] || [ "$ARG" = "1" ] || [ "$ARG" = "yes" ] || [ "$ARG" = "on" ]; then + echo "True" + elif [ "$ARG" = "false" ] || [ "$ARG" = "0" ] || [ "$ARG" = "no" ] || [ "$ARG" = "off" ]; then + echo "False" + else + echo "$ARG" + fi +} + # Options -CURRENT_BRANCH=${CURRENT_BRANCH:=$(git branch | grep \* | cut -d ' ' -f2)} -DEPLOY_BRANCH=${DEPLOY_BRANCH:=release} DEPLOY_REMOTE=${DEPLOY_REMOTE:=origin} NAME=${NAME:=$(python -c "import setup; print(setup.NAME)")} VERSION=$(python -c "import setup; print(setup.VERSION)") -MB_PYTHON_TAG=${MB_PYTHON_TAG:=$(python -c "import setup; print(setup.native_mb_python_tag())")} +MB_PYTHON_TAG=${MB_PYTHON_TAG:py3-none-any} # The default should change depending on the application #DEFAULT_MODE_LIST=("sdist" "universal" "bdist") -DEFAULT_MODE_LIST=("sdist" "universal") +DEFAULT_MODE_LIST=("sdist" "native" "universal") #DEFAULT_MODE_LIST=("sdist" "bdist") -check_variable CURRENT_BRANCH -check_variable DEPLOY_BRANCH check_variable DEPLOY_REMOTE check_variable VERSION || exit 1 -TAG_AND_UPLOAD=${TAG_AND_UPLOAD:=$1} +ARG_1=$1 + +DO_UPLOAD=${DO_UPLOAD:=$ARG_1} +DO_TAG=${DO_TAG:=$ARG_1} +DO_GPG=${DO_GPG:="True"} +DO_BUILD=${DO_BUILD:="True"} + +DO_GPG=$(normalize_boolean "$DO_GPG") +DO_BUILD=$(normalize_boolean "$DO_BUILD") +DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD") +DO_TAG=$(normalize_boolean "$DO_TAG") + TWINE_USERNAME=${TWINE_USERNAME:=""} TWINE_PASSWORD=${TWINE_PASSWORD:=""} -USE_GPG=${USE_GPG:="True"} - if [[ "$(which gpg2)" != "" ]]; then GPG_EXECUTABLE=${GPG_EXECUTABLE:=gpg2} else @@ -96,23 +119,48 @@ GPG_KEYID=${GPG_KEYID:=$(git config --global user.signingkey)} echo " === PYPI BUILDING SCRIPT == -CURRENT_BRANCH='$CURRENT_BRANCH' -DEPLOY_BRANCH='$DEPLOY_BRANCH' VERSION='$VERSION' TWINE_USERNAME='$TWINE_USERNAME' GPG_KEYID = '$GPG_KEYID' MB_PYTHON_TAG = '$MB_PYTHON_TAG' + +DO_UPLOAD=${DO_UPLOAD} +DO_TAG=${DO_TAG} +DO_GPG=${DO_GPG} +DO_BUILD=${DO_BUILD} " -echo " -=== === -" +# Verify that we want to tag +if [[ "$DO_TAG" == "True" ]]; then + echo "About to tag VERSION='$VERSION'" +else + if [[ "$DO_TAG" == "False" ]]; then + echo "We are NOT about to tag VERSION='$VERSION'" + else + read -p "Do you want to git tag version='$VERSION'? (input 'yes' to confirm)" ANS + echo "ANS = $ANS" + DO_TAG="$ANS" + DO_TAG=$(normalize_boolean "$DO_TAG") + fi +fi + +# Verify that we want to publish +if [[ "$DO_UPLOAD" == "True" ]]; then + echo "About to publish VERSION='$VERSION'" +else + if [[ "$DO_UPLOAD" == "False" ]]; then + echo "We are NOT about to publish VERSION='$VERSION'" + else + read -p "Are you ready to publish version='$VERSION'? (input 'yes' to confirm)" ANS + echo "ANS = $ANS" + DO_UPLOAD="$ANS" + DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD") + fi +fi -echo "LIVE BUILDING" -# Build wheel and source distribution MODE=${MODE:=all} @@ -124,21 +172,69 @@ fi MODE_LIST_STR=$(printf '"%s" ' "${MODE_LIST[@]}") + + +if [ "$DO_BUILD" == "True" ]; then + + echo " + === === + " + + echo "LIVE BUILDING" + # Build wheel and source distribution + + #WHEEL_PATHS=() + for _MODE in "${MODE_LIST[@]}" + do + echo "_MODE = $_MODE" + if [[ "$_MODE" == "sdist" ]]; then + python setup.py sdist || { echo 'failed to build sdist wheel' ; exit 1; } + WHEEL_PATH=$(ls dist/$NAME-$VERSION*.tar.gz) + #WHEEL_PATHS+=($WHEEL_PATH) + elif [[ "$_MODE" == "native" ]]; then + python setup.py bdist_wheel || { echo 'failed to build native wheel' ; exit 1; } + WHEEL_PATH=$(ls dist/$NAME-$VERSION*.whl) + #WHEEL_PATHS+=($WHEEL_PATH) + elif [[ "$_MODE" == "universal" ]]; then + python setup.py bdist_wheel --universal || { echo 'failed to build universal wheel' ; exit 1; } + UNIVERSAL_TAG="py3-none-any" + WHEEL_PATH=$(ls dist/$NAME-$VERSION-$UNIVERSAL_TAG*.whl) + #WHEEL_PATHS+=($WHEEL_PATH) + elif [[ "$_MODE" == "bdist" ]]; then + echo "Assume wheel has already been built" + WHEEL_PATH=$(ls wheelhouse/$NAME-$VERSION-$MB_PYTHON_TAG*.whl) + #WHEEL_PATHS+=($WHEEL_PATH) + else + echo "bad mode" + exit 1 + fi + echo "WHEEL_PATH = $WHEEL_PATH" + done + + echo " + === === + " + +else + echo "DO_BUILD=False, Skipping build" +fi + + WHEEL_PATHS=() for _MODE in "${MODE_LIST[@]}" do echo "_MODE = $_MODE" if [[ "$_MODE" == "sdist" ]]; then - python setup.py sdist WHEEL_PATH=$(ls dist/$NAME-$VERSION*.tar.gz) WHEEL_PATHS+=($WHEEL_PATH) + elif [[ "$_MODE" == "native" ]]; then + WHEEL_PATH=$(ls dist/$NAME-$VERSION*.whl) + WHEEL_PATHS+=($WHEEL_PATH) elif [[ "$_MODE" == "universal" ]]; then - python setup.py bdist_wheel --universal - UNIVERSAL_TAG="py2.py3-none-any" + UNIVERSAL_TAG="py3-none-any" WHEEL_PATH=$(ls dist/$NAME-$VERSION-$UNIVERSAL_TAG*.whl) WHEEL_PATHS+=($WHEEL_PATH) elif [[ "$_MODE" == "bdist" ]]; then - echo "Assume wheel has already been built" WHEEL_PATH=$(ls wheelhouse/$NAME-$VERSION-$MB_PYTHON_TAG*.whl) WHEEL_PATHS+=($WHEEL_PATH) else @@ -156,99 +252,84 @@ VERSION='$VERSION' WHEEL_PATHS='$WHEEL_PATHS_STR' " -echo " -=== === -" -echo " -=== === -" +if [ "$DO_GPG" == "True" ]; then -for WHEEL_PATH in "${WHEEL_PATHS[@]}" -do - echo "WHEEL_PATH = $WHEEL_PATH" - check_variable WHEEL_PATH - if [ "$USE_GPG" == "True" ]; then - # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci - # secure gpg --export-secret-keys > all.gpg - - # REQUIRES GPG >= 2.2 - check_variable GPG_EXECUTABLE - check_variable GPG_KEYID - - echo "Signing wheels" - GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" - ls wheelhouse - echo "GPG_SIGN_CMD = $GPG_SIGN_CMD" - $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH - - echo "Checking wheels" - twine check $WHEEL_PATH.asc $WHEEL_PATH - - echo "Verifying wheels" - $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH - else - echo "USE_GPG=False, Skipping GPG sign" - fi -done -echo " -=== === -" + echo " + === === + " + + for WHEEL_PATH in "${WHEEL_PATHS[@]}" + do + echo "WHEEL_PATH = $WHEEL_PATH" + check_variable WHEEL_PATH + # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci + # secure gpg --export-secret-keys > all.gpg + + # REQUIRES GPG >= 2.2 + check_variable GPG_EXECUTABLE || { echo 'failed no gpg exe' ; exit 1; } + check_variable GPG_KEYID || { echo 'failed no gpg key' ; exit 1; } + + echo "Signing wheels" + GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" + echo "GPG_SIGN_CMD = $GPG_SIGN_CMD" + $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH + + echo "Checking wheels" + twine check $WHEEL_PATH.asc $WHEEL_PATH || { echo 'could not check wheels' ; exit 1; } -if [[ "$CURRENT_BRANCH" != "$DEPLOY_BRANCH" ]]; then - TAG_AND_UPLOAD="no" - echo "current branch is not the deploy branch. Forcing tag_and_upload=no" + echo "Verifying wheels" + $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH || { echo 'could not verify wheels' ; exit 1; } + done + echo " + === === + " +else + echo "DO_GPG=False, Skipping GPG sign" fi -# Verify that we want to publish -if [[ "$TAG_AND_UPLOAD" != "yes" ]]; then - if [[ "$TAG_AND_UPLOAD" != "no" ]]; then - read -p "Are you ready to publish version='$VERSION' on branch='$CURRENT_BRANCH'? (input 'yes' to confirm)" ANS - echo "ANS = $ANS" - TAG_AND_UPLOAD="$ANS" - else - echo "WRONG BRANCH: Not ready to publish VERSION='$VERSION' on branch='$CURRENT_BRANCH'" - fi +if [[ "$DO_TAG" == "True" ]]; then + git tag $VERSION -m "tarball tag $VERSION" + git push --tags $DEPLOY_REMOTE $DEPLOY_BRANCH else - echo "Do not want to publish VERSION='$VERSION' on branch='$CURRENT_BRANCH'" + echo "Not tagging" fi -if [[ "$TAG_AND_UPLOAD" == "yes" ]]; then +if [[ "$DO_UPLOAD" == "True" ]]; then check_variable TWINE_USERNAME - check_variable TWINE_PASSWORD - - #git tag $VERSION -m "tarball tag $VERSION" - #git push --tags $DEPLOY_REMOTE $DEPLOY_BRANCH + check_variable TWINE_PASSWORD "hide" for WHEEL_PATH in "${WHEEL_PATHS[@]}" do - if [ "$USE_GPG" == "True" ]; then - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH + if [ "$DO_GPG" == "True" ]; then + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } else - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } fi done echo """ !!! FINISH: LIVE RUN !!! """ else - ls wheelhouse echo """ - DRY RUN ... Skiping tag and upload + DRY RUN ... Skiping upload - VERSION = '$VERSION' DEPLOY_REMOTE = '$DEPLOY_REMOTE' - CURRENT_BRANCH = '$CURRENT_BRANCH' - DEPLOY_BRANCH = '$DEPLOY_BRANCH' - TAG_AND_UPLOAD = '$TAG_AND_UPLOAD' + DO_UPLOAD = '$DO_UPLOAD' WHEEL_PATH = '$WHEEL_PATH' WHEEL_PATHS_STR = '$WHEEL_PATHS_STR' MODE_LIST_STR = '$MODE_LIST_STR' - To do live run set TAG_AND_UPLOAD=yes and ensure deploy and curent branch are the same + VERSION='$VERSION' + NAME='$NAME' + TWINE_USERNAME='$TWINE_USERNAME' + GPG_KEYID = '$GPG_KEYID' + MB_PYTHON_TAG = '$MB_PYTHON_TAG' + + To do live run set DO_UPLOAD=1 and ensure deploy and current branch are the same !!! FINISH: DRY RUN !!! """ diff --git a/super_setup.py b/super_setup.py index 007cc4c..77e747a 100755 --- a/super_setup.py +++ b/super_setup.py @@ -810,7 +810,7 @@ DEVEL_REPOS = [ # netharn - training harness { - 'name': 'netharn', 'branch': 'dev/0.5.13', 'remote': 'public', + 'name': 'netharn', 'branch': 'dev/0.5.14', 'remote': 'public', 'remotes': {'public': 'git@gitlab.kitware.com:computer-vision/netharn.git'}, }, ] -- GitLab From 3ee21b468bd744c9887b4505f23a9040985dd366 Mon Sep 17 00:00:00 2001 From: "jon.crall" Date: Wed, 20 Jan 2021 12:23:09 -0500 Subject: [PATCH 2/5] Removed deprecated util attributes --- netharn/util/__init__.py | 186 +++++++-------------------- netharn/util/_deprecation_helpers.py | 86 +++++++++++++ netharn/util/util_cachestamp.py | 14 -- netharn/util/util_dataframe.py | 8 -- 4 files changed, 136 insertions(+), 158 deletions(-) create mode 100644 netharn/util/_deprecation_helpers.py delete mode 100644 netharn/util/util_cachestamp.py delete mode 100644 netharn/util/util_dataframe.py diff --git a/netharn/util/__init__.py b/netharn/util/__init__.py index e8b1151..b4c59ea 100644 --- a/netharn/util/__init__.py +++ b/netharn/util/__init__.py @@ -1,27 +1,40 @@ +# -*- coding: utf-8 -*- +# flake8: noqa """ - mkinit ~/code/kwarray/kwarray/__init__.py --relative --nomods -w mkinit ~/code/kwimage/kwimage/__init__.py --relative --nomods -w mkinit ~/code/kwplot/kwplot/__init__.py --relative --nomods -w -mkinit netharn.util --relative --nomods -w +mkinit netharn.util --relative --nomods """ from __future__ import absolute_import, division, print_function, unicode_literals -__extra_all__ = [ - 'profiler', - 'util_dataframe', - 'dataframe_light', -] -__private__ = ['util_dataframe'] +def __getattr__(key): + from netharn.util._deprecation_helpers import _lookup_deprecated_attribute + return _lookup_deprecated_attribute(key) - -__external__ = [ - 'kwarray', 'kwimage', 'kwplot' +__submodules__ = [ + 'imutil', + 'mplutil', + 'profiler', + 'util_averages', + 'util_filesys', + 'util_fname', + 'util_idstr', + 'util_inspect', + 'util_io', + 'util_iter', + 'util_json', + 'util_misc', + 'util_resources', + 'util_slider', + 'util_subextreme', + 'util_tensorboard', + 'util_torch', + 'util_zip', ] -# from .imutil import (adjust_gamma, ensure_grayscale, get_num_channels, image_slices, load_image_paths, overlay_colorized, wide_strides_1d,) @@ -35,7 +48,6 @@ from .profiler import (IS_PROFILING, profile, profile_now,) from .util_averages import (CumMovingAve, ExpMovingAve, InternalRunningStats, MovingAve, RunningStats, WindowedMovingAve, absdev,) -from .util_cachestamp import (CacheStamp,) from .util_filesys import (get_file_info,) from .util_fname import (align_paths, check_aligned, dumpsafe, shortest_unique_prefixes, shortest_unique_suffixes,) @@ -43,14 +55,13 @@ from .util_idstr import (compact_idstr, make_idstr, make_short_idstr,) from .util_inspect import (default_kwargs,) from .util_io import (read_arr, read_h5arr, write_arr, write_h5arr,) from .util_iter import (roundrobin,) -from .util_json import (LossyJSONEncoder, NumpyEncoder, +from .util_json import (IndexableWalker, LossyJSONEncoder, NumpyEncoder, ensure_json_serializable, read_json, walk_json, write_json,) from .util_misc import (FlatIndexer, SupressPrint, align, align_lines, strip_ansi,) -from .util_resources import (ensure_ulimit, resource_usage) +from .util_resources import (ensure_ulimit, resource_usage,) from .util_slider import (SlidingWindow, Stitcher,) -from .util_slider_dep import (SlidingIndexDataset, SlidingSlices,) from .util_subextreme import (argsubmax, argsubmaxima,) from .util_tensorboard import (read_tensorboard_scalars,) from .util_torch import (BatchNormContext, DisableBatchNorm, @@ -60,124 +71,27 @@ from .util_torch import (BatchNormContext, DisableBatchNorm, trainable_layers,) from .util_zip import (split_archive, zopen,) - -try: - # external modules in netharn.util are DEPRECATED - # Don't depend on these external utilities at import time - # In the future these will become optional, and these will no longer be - # exposed via netharn.util - from kwarray import (ArrayAPI, DataFrameArray, DataFrameLight, LocLight, - apply_grouping, arglexmax, argmaxima, argminima, - atleast_nd, boolmask, ensure_rng, group_consecutive, - group_consecutive_indices, group_indices, group_items, - isect_flags, iter_reduce_ufunc, maxvalue_assignment, - mincost_assignment, mindist_assignment, one_hot_embedding, - one_hot_lookup, random_combinations, random_product, - seed_global, shuffle, standard_normal, standard_normal32, - standard_normal64, stats_dict, uniform, uniform32,) - from kwimage import (BASE_COLORS, Boxes, CSS4_COLORS, Color, Coords, - Detections, Heatmap, Mask, MaskList, MultiPolygon, Points, - PointsList, Polygon, PolygonList, Segmentation, - SegmentationList, TABLEAU_COLORS, - TORCH_GRID_SAMPLE_HAS_ALIGN, XKCD_COLORS, - atleast_3channels, available_nms_impls, - convert_colorspace, daq_spatial_nms, decode_run_length, - draw_boxes_on_image, draw_clf_on_image, - draw_text_on_image, draw_vector_field, encode_run_length, - ensure_alpha_channel, ensure_float01, ensure_uint255, - fourier_mask, gaussian_patch, grab_test_image, - grab_test_image_fpath, imread, imresize, imscale, imwrite, - load_image_shape, make_channels_comparable, make_heatmask, - make_orimask, make_vector_field, non_max_supression, - num_channels, overlay_alpha_images, overlay_alpha_layers, - radial_fourier_mask, rle_translate, smooth_prob, - stack_images, stack_images_grid, subpixel_accum, - subpixel_align, subpixel_getvalue, subpixel_maximum, - subpixel_minimum, subpixel_set, subpixel_setvalue, - subpixel_slice, subpixel_translate, warp_points, - warp_tensor,) - from kwplot import (PlotNums, autompl, autoplt, distinct_colors, - distinct_markers, draw_boxes, draw_boxes_on_image, - draw_clf_on_image, draw_line_segments, draw_text_on_image, - ensure_fnum, figure, imshow, legend, make_conv_images, - make_heatmask, make_orimask, make_vector_field, multi_plot, - next_fnum, plot_convolutional_features, plot_matrix, - plot_surface3d, set_figtitle, set_mpl_backend, - show_if_requested,) -except Exception: - pass - -__all__ = ['ArrayAPI', 'BASE_COLORS', 'BatchNormContext', 'Boxes', - 'CSS4_COLORS', 'CacheStamp', 'Color', 'Coords', - 'CumMovingAve', 'DataFrameArray', 'DataFrameLight', 'Detections', - 'DisableBatchNorm', 'ExpMovingAve', 'FlatIndexer', 'Heatmap', - 'IS_PROFILING', 'IgnoreLayerContext', 'InternalRunningStats', - 'LocLight', 'LossyJSONEncoder', 'Mask', 'MaskList', - 'ModuleMixin', 'MovingAve', 'MultiPolygon', 'NumpyEncoder', - 'PlotNums', 'Points', 'PointsList', 'Polygon', 'PolygonList', - 'RunningStats', 'Segmentation', 'SegmentationList', - 'SlidingIndexDataset', 'SlidingSlices', 'SlidingWindow', 'Stitcher', - 'SupressPrint', 'TABLEAU_COLORS', 'TORCH_GRID_SAMPLE_HAS_ALIGN', - 'WindowedMovingAve', 'XKCD_COLORS', 'absdev', 'adjust_gamma', - 'adjust_subplots', 'aggensure', 'align', 'align_lines', - 'align_paths', 'apply_grouping', 'arglexmax', 'argmaxima', - 'argminima', 'argsubmax', 'argsubmaxima', 'atleast_3channels', - 'atleast_nd', 'autompl', 'autoplt', 'available_nms_impls', - 'axes_extent', 'boolmask', 'check_aligned', 'colorbar', - 'colorbar_image', 'compact_idstr', 'convert_colorspace', - 'copy_figure_to_clipboard', 'daq_spatial_nms', 'dataframe_light', - 'decode_run_length', 'default_kwargs', 'distinct_colors', - 'distinct_markers', 'draw_border', 'draw_boxes', - 'draw_boxes_on_image', 'draw_boxes_on_image', 'draw_clf_on_image', - 'draw_clf_on_image', 'draw_line_segments', 'draw_text_on_image', - 'draw_text_on_image', 'draw_vector_field', 'dumpsafe', - 'encode_run_length', 'ensure_alpha_channel', 'ensure_float01', - 'ensure_fnum', 'ensure_grayscale', 'ensure_json_serializable', - 'ensure_rng', 'ensure_uint255', 'ensure_ulimit', - 'extract_axes_extents', 'figure', 'fourier_mask', 'freeze_params', - 'gaussian_patch', 'get_file_info', 'get_num_channels', - 'grab_test_image', 'grab_test_image_fpath', 'grad_context', - 'group_consecutive', 'group_consecutive_indices', 'group_indices', - 'group_items', 'image_slices', 'imread', 'imresize', 'imscale', - 'imshow', 'imwrite', 'interpolated_colormap', 'isect_flags', - 'iter_reduce_ufunc', 'legend', 'load_image_paths', - 'load_image_shape', 'make_channels_comparable', 'make_conv_images', - 'make_heatmask', 'make_heatmask', 'make_idstr', 'make_legend_img', - 'make_orimask', 'make_orimask', 'make_short_idstr', - 'make_vector_field', 'make_vector_field', 'maxvalue_assignment', - 'mincost_assignment', 'mindist_assignment', 'multi_plot', - 'next_fnum', 'non_max_supression', 'num_channels', - 'number_of_parameters', 'one_hot_embedding', 'one_hot_embedding', - 'one_hot_lookup', 'one_hot_lookup', 'overlay_alpha_images', - 'overlay_alpha_layers', 'overlay_colorized', 'pandas_plot_matrix', - 'plot_convolutional_features', 'plot_matrix', 'plot_surface3d', - 'profile', 'profile_now', 'profiler', 'qtensure', - 'radial_fourier_mask', 'random_combinations', 'random_product', - 'read_arr', 'read_h5arr', 'read_json', 'read_tensorboard_scalars', - 'render_figure_to_image', 'reverse_colormap', 'rle_translate', - 'roundrobin', 'save_parts', 'savefig2', 'scores_to_cmap', - 'scores_to_color', 'seed_global', 'set_figtitle', 'set_mpl_backend', +__all__ = ['BatchNormContext', 'CumMovingAve', 'DisableBatchNorm', + 'ExpMovingAve', 'FlatIndexer', 'IS_PROFILING', 'IgnoreLayerContext', + 'IndexableWalker', 'InternalRunningStats', 'LossyJSONEncoder', + 'ModuleMixin', 'MovingAve', 'NumpyEncoder', 'RunningStats', + 'SlidingWindow', 'Stitcher', 'SupressPrint', 'WindowedMovingAve', + 'absdev', 'adjust_gamma', 'adjust_subplots', 'aggensure', 'align', + 'align_lines', 'align_paths', 'argsubmax', 'argsubmaxima', + 'axes_extent', 'check_aligned', 'colorbar', 'colorbar_image', + 'compact_idstr', 'copy_figure_to_clipboard', 'default_kwargs', + 'draw_border', 'dumpsafe', 'ensure_grayscale', + 'ensure_json_serializable', 'ensure_ulimit', 'extract_axes_extents', + 'freeze_params', 'get_file_info', 'get_num_channels', + 'grad_context', 'image_slices', 'interpolated_colormap', + 'load_image_paths', 'make_idstr', 'make_legend_img', + 'make_short_idstr', 'number_of_parameters', 'one_hot_embedding', + 'one_hot_lookup', 'overlay_colorized', 'pandas_plot_matrix', + 'profile', 'profile_now', 'qtensure', 'read_arr', 'read_h5arr', + 'read_json', 'read_tensorboard_scalars', 'render_figure_to_image', + 'resource_usage', 'reverse_colormap', 'roundrobin', 'save_parts', + 'savefig2', 'scores_to_cmap', 'scores_to_color', 'shortest_unique_prefixes', 'shortest_unique_suffixes', - 'show_if_requested', 'shuffle', 'smooth_prob', 'split_archive', - 'stack_images', 'stack_images_grid', 'standard_normal', - 'standard_normal32', 'standard_normal64', 'stats_dict', - 'strip_ansi', 'subpixel_accum', 'subpixel_align', - 'subpixel_getvalue', 'subpixel_maximum', 'subpixel_minimum', - 'subpixel_set', 'subpixel_setvalue', 'subpixel_slice', - 'subpixel_translate', 'torch_ravel_multi_index', 'trainable_layers', - 'uniform', 'uniform32', 'util_dataframe', 'walk_json', - 'warp_points', 'warp_tensor', 'wide_strides_1d', 'write_arr', - 'write_h5arr', 'write_json', 'zopen', 'resource_usage'] -# - - -# Backwards compatibility patch -try: - # external modules in netharn.util are DEPRECATED - # Don't depend on these external utilities at import time - # In the future these will become optional, and these will no longer be - # exposed via netharn.util - from . import util_dataframe - from kwarray import dataframe_light -except Exception: - pass + 'split_archive', 'strip_ansi', 'torch_ravel_multi_index', + 'trainable_layers', 'walk_json', 'wide_strides_1d', 'write_arr', + 'write_h5arr', 'write_json', 'zopen'] diff --git a/netharn/util/_deprecation_helpers.py b/netharn/util/_deprecation_helpers.py new file mode 100644 index 0000000..4c60d35 --- /dev/null +++ b/netharn/util/_deprecation_helpers.py @@ -0,0 +1,86 @@ + +def _lookup_deprecated_attribute(key): + """ + Provide a module level getattr to give better error messages on deprecated + and removed attributes + + Our deprecation errors will only work in Python 3.7 + """ + import ubelt as ub + # mapping from module name to the attributes that were moved there. + refactored = { + 'kwarray': [ + 'ArrayAPI', 'DataFrameArray', 'DataFrameLight', 'FlatIndexer', + 'LocLight', 'RunningStats', 'apply_grouping', 'arglexmax', + 'argmaxima', 'argminima', 'atleast_nd', 'boolmask', 'ensure_rng', + 'group_consecutive', 'group_consecutive_indices', 'group_indices', + 'group_items', 'isect_flags', 'iter_reduce_ufunc', + 'maxvalue_assignment', 'mincost_assignment', 'mindist_assignment', + 'one_hot_embedding', 'one_hot_lookup', 'random_combinations', + 'random_product', 'seed_global', 'setcover', 'shuffle', + 'standard_normal', 'standard_normal32', 'standard_normal64', + 'stats_dict', 'uniform', 'uniform32' + ], + + 'kwimage': [ + 'BASE_COLORS', 'Boxes', 'CSS4_COLORS', 'Color', 'Coords', + 'Detections', 'Heatmap', 'Mask', 'MaskList', 'MultiPolygon', + 'Points', 'PointsList', 'Polygon', 'PolygonList', 'Segmentation', + 'SegmentationList', 'TABLEAU_COLORS', + 'TORCH_GRID_SAMPLE_HAS_ALIGN', 'XKCD_COLORS', 'add_homog', + 'atleast_3channels', 'available_nms_impls', 'convert_colorspace', + 'daq_spatial_nms', 'decode_run_length', 'draw_boxes_on_image', + 'draw_clf_on_image', 'draw_line_segments_on_image', + 'draw_text_on_image', 'draw_vector_field', 'encode_run_length', + 'ensure_alpha_channel', 'ensure_float01', 'ensure_uint255', + 'fourier_mask', 'gaussian_patch', 'grab_test_image', + 'grab_test_image_fpath', 'imread', 'imresize', 'imscale', + 'imwrite', 'load_image_shape', 'make_channels_comparable', + 'make_heatmask', 'make_orimask', 'make_vector_field', + 'non_max_supression', 'normalize', 'num_channels', + 'overlay_alpha_images', 'overlay_alpha_layers', + 'radial_fourier_mask', 'remove_homog', 'rle_translate', + 'smooth_prob', 'stack_images', 'stack_images_grid', + 'subpixel_accum', 'subpixel_align', 'subpixel_getvalue', + 'subpixel_maximum', 'subpixel_minimum', 'subpixel_set', + 'subpixel_setvalue', 'subpixel_slice', 'subpixel_translate', + 'warp_image', 'warp_points', 'warp_tensor', + ], + + 'kwplot': [ + 'BackendContext', 'Color', 'PlotNums', 'autompl', 'autoplt', + 'distinct_colors', 'distinct_markers', 'draw_boxes', + 'draw_boxes_on_image', 'draw_clf_on_image', 'draw_line_segments', + 'draw_points', 'draw_text_on_image', 'ensure_fnum', 'figure', + 'imshow', 'legend', 'make_conv_images', 'make_heatmask', + 'make_legend_img', 'make_orimask', 'make_vector_field', + 'multi_plot', 'next_fnum', 'plot_convolutional_features', + 'plot_matrix', 'plot_surface3d', 'set_figtitle', 'set_mpl_backend', + 'show_if_requested', + ], + + 'ubelt': [ + 'CacheStamp', + ] + + } + ERROR_ON_ACCESS = True + for modname, attrs in refactored.items(): + if key in attrs: + text = ub.paragraph( + ''' + The attribute `netharn.util.{key}` is deprecated. + It was refactored and moved to `{modname}.{key}`. + ''').format(key=key, modname=modname) + if ERROR_ON_ACCESS: + raise AttributeError(text) + else: + module = ub.import_module_from_name(modname) + import warnings + warnings.warn(text) + return getattr(module, key) + + if key in ['SlidingIndexDataset', 'SlidingSlices']: + raise AttributeError( + 'Deprecated {}, but still available in ' + 'netharn.util.util_slider_dep'.format(key)) diff --git a/netharn/util/util_cachestamp.py b/netharn/util/util_cachestamp.py deleted file mode 100644 index 76df078..0000000 --- a/netharn/util/util_cachestamp.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, division, print_function, unicode_literals -import ubelt as ub - -CacheStamp = ub.CacheStamp - - -if __name__ == '__main__': - """ - CommandLine: - python -m netharn.util.util_cachestamp all - """ - import xdoctest - xdoctest.doctest_module(__file__) diff --git a/netharn/util/util_dataframe.py b/netharn/util/util_dataframe.py deleted file mode 100644 index 95a3bb2..0000000 --- a/netharn/util/util_dataframe.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -mkinit ~/code/kwarray/kwarray/__init__.py --relative --nomods -w -mkinit ~/code/netharn/netharn/util/util_dataframe.py --relative --nomods - -""" -# this module is a hack for backwards compatibility -from kwarray.dataframe_light import DataFrameLight, DataFrameArray, LocLight -__all__ = ['DataFrameLight', 'DataFrameArray', 'LocLight'] -- GitLab From ec7acef3a777af84251d4d7bd232b37853ca4ca6 Mon Sep 17 00:00:00 2001 From: "jon.crall" Date: Wed, 20 Jan 2021 14:23:54 -0500 Subject: [PATCH 3/5] Fix issue when psutil is not installed --- CHANGELOG.md | 7 +++++++ netharn/fit_harn.py | 5 +++-- netharn/util/util_slider.py | 4 +++- netharn/util/util_zip.py | 14 ++++++++++---- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a16be..9473db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.5.14 - Unreleased +### Deprecated + +* Removed deprecated kwimage, kwarray, and kwplot attributes. Accessing them + will now raise errors. + + + ## Version 0.5.13 - Released 2021-01-19 ### Added diff --git a/netharn/fit_harn.py b/netharn/fit_harn.py index 07fa35d..3f641a0 100644 --- a/netharn/fit_harn.py +++ b/netharn/fit_harn.py @@ -2040,8 +2040,9 @@ class CoreMixin(object): if harn.preferences['log_resources']: usage = util.resource_usage() key = 'ram' - value = usage['ram_percent'] - harn.log_value(tag + ' iter ' + key, value, iter_idx) + if 'ram_percent' in usage: + value = usage['ram_percent'] + harn.log_value(tag + ' iter ' + key, value, iter_idx) harn.debug(ub.repr2(usage, nl=1)) if harn._tlog is not None: diff --git a/netharn/util/util_slider.py b/netharn/util/util_slider.py index 755e924..ef594a4 100644 --- a/netharn/util/util_slider.py +++ b/netharn/util/util_slider.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import ubelt as ub import numpy as np -import netharn as nh import torch import itertools as it @@ -85,6 +84,7 @@ class SlidingWindow(ub.NiceRepr): """ def __init__(self, shape, window, overlap=None, stride=None, keepbound=False, allow_overshoot=False): + import netharn as nh if overlap is None and stride is None: overlap = 0 @@ -374,6 +374,7 @@ class Stitcher(ub.NiceRepr): Example: >>> import sys + >>> import netharn as nh >>> # Build a high resolution image and slice it into chips >>> frames = np.random.rand(1, 200, 100, 100).astype(np.float32) >>> window = (frames.shape[0], 15, 15, 15) @@ -416,6 +417,7 @@ class Stitcher(ub.NiceRepr): Benchmark: >>> import sys + >>> import netharn as nh >>> # setup benchmark >>> frames = np.random.rand(1, 50, 100, 100).astype(np.float32) >>> window = (frames.shape[0], 20, 20, 20) diff --git a/netharn/util/util_zip.py b/netharn/util/util_zip.py index 0fda05b..c8770fe 100644 --- a/netharn/util/util_zip.py +++ b/netharn/util/util_zip.py @@ -1,13 +1,19 @@ # -*- coding: utf-8 -*- +""" +Utilities for opening files within a zip archive without explicitly unpacking +it to disk. + +TODO: + - [ ] Move to ubelt? +""" from __future__ import absolute_import, division, print_function, unicode_literals -from os.path import exists +import os import io -from os.path import join -from os.path import os +import re import tempfile import zipfile import ubelt as ub -import re +from os.path import exists, join def split_archive(fpath): -- GitLab From cb4082ae4bae8cca61f3b7abecac454e0bb48cdc Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 21 Jan 2021 15:53:44 -0500 Subject: [PATCH 4/5] Fix bug in configure hacks --- dev/debug_memory.py | 6 ++++++ netharn/api.py | 23 ++++++++++++++++++++--- netharn/examples/classification.py | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/dev/debug_memory.py b/dev/debug_memory.py index 5124e16..cb17ed0 100644 --- a/dev/debug_memory.py +++ b/dev/debug_memory.py @@ -280,6 +280,12 @@ if __name__ == '__main__': python debug_memory.py --storage_mode=ndsampler --total=1000 --shuffle=False --workers=4 python debug_memory.py --storage_mode=ndsampler --total=1000 --shuffle=True --workers=4 + srun -c 5 -p community --gres=gpu:1 \ + python debug_memory.py --storage_mode=ndsampler --total=1000 --shuffle=True --workers=4 + + python debug_memory.py --storage_mode=python --total=24e5 --shuffle=False --workers=4 + + python debug_memory.py numpy dict 24e5 python debug_memory.py python list 24e7 diff --git a/netharn/api.py b/netharn/api.py index 814c07e..ac2f40c 100644 --- a/netharn/api.py +++ b/netharn/api.py @@ -807,18 +807,35 @@ def configure_hacks(config={}, **kw): Configures hacks to fix global settings in external modules Args: - config (dict): exected to contain they key "workers" with an - integer value equal to the number of dataloader processes. + config (dict): exected to contain certain special keys. + + * "workers" with an integer value equal to the number of dataloader + processes. + + * "sharing_strategy" to specify the torch multiprocessing backend + **kw: can also be used to specify config items Modules we currently hack: * cv2 - fix thread count + * torch sharing strategy """ config = _update_defaults(config, kw) - if config['workers'] > 0: + + if config.get('workers', 0) > 0: import cv2 cv2.setNumThreads(0) + strat = config.get('sharing_strategy', None) + if strat is not None and strat != 'default': + if strat == 'auto': + # TODO: can we add a better auto test? + strat = torch.multiprocessing.get_sharing_strategy() + valid_strats = torch.multiprocessing.get_all_sharing_strategies() + if strat not in valid_strats: + raise KeyError('start={} is not in valid_strats={}'.format(strat, valid_strats)) + torch.multiprocessing.set_sharing_strategy(strat) + def configure_workdir(config={}, **kw): config = _update_defaults(config, kw) diff --git a/netharn/examples/classification.py b/netharn/examples/classification.py index ff6a1c8..ab40beb 100644 --- a/netharn/examples/classification.py +++ b/netharn/examples/classification.py @@ -302,7 +302,7 @@ class ClfDataset(torch.utils.data.Dataset): self.augmenter = None self.conditional_augmentors = None self.input_dims = input_dims - self.classes = self.sampler.catgraph + self.classes = self.sampler.classes self.augmenter = self._coerce_augmenter(augmenter) -- GitLab From 5727817125ec489a98ea1b98de61174d740c1ef9 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 21 Jan 2021 16:12:57 -0500 Subject: [PATCH 5/5] Fix pytest issues with __getattr__ --- netharn/util/__init__.py | 7 ++++++ netharn/util/_deprecation_helpers.py | 8 ++----- netharn/util/imutil.py | 32 ++++++++++++---------------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/netharn/util/__init__.py b/netharn/util/__init__.py index b4c59ea..6128445 100644 --- a/netharn/util/__init__.py +++ b/netharn/util/__init__.py @@ -10,9 +10,16 @@ from __future__ import absolute_import, division, print_function, unicode_litera def __getattr__(key): + """ + Provide a module level getattr to give better error messages on deprecated + and removed attributes + + Our deprecation errors will only work in Python 3.7+ + """ from netharn.util._deprecation_helpers import _lookup_deprecated_attribute return _lookup_deprecated_attribute(key) + __submodules__ = [ 'imutil', 'mplutil', diff --git a/netharn/util/_deprecation_helpers.py b/netharn/util/_deprecation_helpers.py index 4c60d35..dbc1fe2 100644 --- a/netharn/util/_deprecation_helpers.py +++ b/netharn/util/_deprecation_helpers.py @@ -1,11 +1,5 @@ def _lookup_deprecated_attribute(key): - """ - Provide a module level getattr to give better error messages on deprecated - and removed attributes - - Our deprecation errors will only work in Python 3.7 - """ import ubelt as ub # mapping from module name to the attributes that were moved there. refactored = { @@ -84,3 +78,5 @@ def _lookup_deprecated_attribute(key): raise AttributeError( 'Deprecated {}, but still available in ' 'netharn.util.util_slider_dep'.format(key)) + + raise AttributeError(key) diff --git a/netharn/util/imutil.py b/netharn/util/imutil.py index e27df33..4e183d5 100644 --- a/netharn/util/imutil.py +++ b/netharn/util/imutil.py @@ -3,15 +3,10 @@ from __future__ import absolute_import, division, print_function, unicode_litera import glob from os.path import expanduser, exists, join, basename import numpy as np -# try: -# # Don't import skimage.io immediately because it imports pyplot -# # See GH Issue https://github.com/scikit-image/scikit-image/issues/3347 -# import skimage.io -# except ImportError: -# pass def load_image_paths(dpath, ext=('.png', '.tiff', 'tif')): + # DEPRECATE dpath = expanduser(dpath) if not exists(dpath): raise ValueError('dpath = {} does not exist'.format(dpath)) @@ -208,22 +203,23 @@ def adjust_gamma(img, gamma=1.0): http://www.pyimagesearch.com/2015/10/05/opencv-gamma-correction/ Ignore: - >>> from netharn import util - >>> fpath = util.grab_test_image() - >>> img = util.imread(fpath) + >>> # xdoctest: +REQUIRES(module:kwplot, module:kwimage) + >>> import kwimage + >>> fpath = kwimage.grab_test_image() + >>> img = kwimage.imread(fpath) >>> gamma = .5 >>> imgf = ensure_float01(img) >>> img2 = adjust_gamma(img, gamma) >>> img3 = adjust_gamma(imgf, gamma) - >>> import plottool as pt - >>> pt.qtensure() - >>> pt.imshow(img, pnum=(3, 3, 1), fnum=1) - >>> pt.imshow(img2, pnum=(3, 3, 2), fnum=1) - >>> pt.imshow(img3, pnum=(3, 3, 3), fnum=1) - >>> pt.imshow(adjust_gamma(img, 1), pnum=(3, 3, 5), fnum=1) - >>> pt.imshow(adjust_gamma(imgf, 1), pnum=(3, 3, 6), fnum=1) - >>> pt.imshow(adjust_gamma(img, 2), pnum=(3, 3, 8), fnum=1) - >>> pt.imshow(adjust_gamma(imgf, 2), pnum=(3, 3, 9), fnum=1) + >>> import kwplot + >>> kwplot.autompl() + >>> kwplot.imshow(img, pnum=(3, 3, 1), fnum=1) + >>> kwplot.imshow(img2, pnum=(3, 3, 2), fnum=1) + >>> kwplot.imshow(img3, pnum=(3, 3, 3), fnum=1) + >>> kwplot.imshow(adjust_gamma(img, 1), pnum=(3, 3, 5), fnum=1) + >>> kwplot.imshow(adjust_gamma(imgf, 1), pnum=(3, 3, 6), fnum=1) + >>> kwplot.imshow(adjust_gamma(img, 2), pnum=(3, 3, 8), fnum=1) + >>> kwplot.imshow(adjust_gamma(imgf, 2), pnum=(3, 3, 9), fnum=1) """ import cv2 if img.dtype.kind in ('i', 'u'): -- GitLab