From 74dc135b4166fa4ec0542ba99c1283e1e99c8808 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 20 May 2021 13:33:08 -0400 Subject: [PATCH 1/8] Start branch for 0.6.4 --- CHANGELOG.md | 5 ++++- ndsampler/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 844c684..c61702b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,10 @@ This changelog follows the specifications detailed in: [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), although we have not yet reached a `1.0.0` release. -## Version 0.6.3 - Unreleased +## Version 0.6.4 - Unreleased + + +## Version 0.6.3 - Released 2021-05-20 ## Version 0.6.2 - Released 2021-05-19 diff --git a/ndsampler/__init__.py b/ndsampler/__init__.py index 8bac9a8..fdbd4c6 100644 --- a/ndsampler/__init__.py +++ b/ndsampler/__init__.py @@ -2,7 +2,7 @@ mkinit ~/code/ndsampler/ndsampler/__init__.py --diff mkinit ~/code/ndsampler/ndsampler/__init__.py -w """ -__version__ = '0.6.3' +__version__ = '0.6.4' from ndsampler.utils.util_misc import (HashIdentifiable,) -- GitLab From e0305708ab6ee540949e8a09839a23cc7fe1a7de Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 20 May 2021 21:57:48 -0400 Subject: [PATCH 2/8] Better downsampling --- ndsampler/coco_sampler.py | 4 ++- ndsampler/delayed.py | 70 ++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index 5ff82ad..de23afd 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -885,6 +885,7 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, # MECHANISM IS for time_idx, gid in enumerate(time_gids): img = self.dset.imgs[gid] + frame_index = img.get('frame_index', gid) tf_img_to_vid = Affine.coerce(img['warp_img_to_vid']) alignable = self.frames._load_alignable(gid) @@ -924,7 +925,8 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, vid_chan_frame[None, ...], dims=('t', 'y', 'x', 'c'), coords={ - 't': np.array([time_idx]), + # TODO: this should be a timestamp if we have it + 't': np.array([frame_index]), # 'y': np.arange(vid_chan_frame.shape[0]), # 'x': np.arange(vid_chan_frame.shape[1]), 'c': list(matching_coords), diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index dfe16dd..fccf02f 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -710,29 +710,6 @@ class DelayedWarp(DelayedImageOperation): >>> tf = np.array([[5.2, 0, 1.1], [0, 3.1, 2.2], [0, 0, 1]]) >>> self = DelayedWarp(np.random.rand(3, 5, 13), tf, dsize=dsize) >>> self.finalize().shape - - Example: - >>> # Test aliasing - >>> from ndsampler.delayed import * # NOQA - >>> s = DelayedIdentity.demo('checkerboard') - >>> s = DelayedIdentity.demo() - >>> a = s.delayed_warp(Affine.scale(0.05), dsize='auto') - >>> b = s.delayed_warp(Affine.scale(3), dsize='auto') - - >>> # xdoctest: +REQUIRES(--show) - >>> import kwplot - >>> kwplot.autompl() - >>> # It looks like downsampling linear and area is the same - >>> # Does warpAffine have no alias handling? - >>> pnum_ = kwplot.PlotNums(nRows=2, nCols=4) - >>> kwplot.imshow(a.finalize(interpolation='area'), pnum=pnum_(), title='warpAffine area') - >>> kwplot.imshow(a.finalize(interpolation='linear'), pnum=pnum_(), title='warpAffine linear') - >>> kwplot.imshow(a.finalize(interpolation='nearest'), pnum=pnum_(), title='warpAffine nearest') - >>> kwplot.imshow(a.finalize(interpolation='cubic'), pnum=pnum_(), title='warpAffine cubic') - >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='area'), pnum=pnum_(), title='resize area') - >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='linear'), pnum=pnum_(), title='resize linear') - >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='nearest'), pnum=pnum_(), title='resize nearest') - >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='cubic'), pnum=pnum_(), title='resize cubic') """ def __init__(self, sub_data, transform=None, dsize=None): self.sub_data = sub_data @@ -910,6 +887,29 @@ class DelayedWarp(DelayedImageOperation): >>> kwplot.imshow(final1, pnum=(1, 3, 2), fnum=1) >>> kwplot.imshow(final2, pnum=(1, 3, 3), fnum=1) >>> kwplot.show_if_requested() + + Example: + >>> # Test aliasing + >>> from ndsampler.delayed import * # NOQA + >>> s = DelayedIdentity.demo('checkerboard') + >>> s = DelayedIdentity.demo() + >>> a = s.delayed_warp(Affine.scale(0.05), dsize='auto') + >>> b = s.delayed_warp(Affine.scale(3), dsize='auto') + + >>> # xdoctest: +REQUIRES(--show) + >>> import kwplot + >>> kwplot.autompl() + >>> # It looks like downsampling linear and area is the same + >>> # Does warpAffine have no alias handling? + >>> pnum_ = kwplot.PlotNums(nRows=2, nCols=4) + >>> kwplot.imshow(a.finalize(interpolation='area'), pnum=pnum_(), title='warpAffine area') + >>> kwplot.imshow(a.finalize(interpolation='linear'), pnum=pnum_(), title='warpAffine linear') + >>> kwplot.imshow(a.finalize(interpolation='nearest'), pnum=pnum_(), title='warpAffine nearest') + >>> kwplot.imshow(a.finalize(interpolation='cubic'), pnum=pnum_(), title='warpAffine cubic') + >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='area'), pnum=pnum_(), title='resize area') + >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='linear'), pnum=pnum_(), title='resize linear') + >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='nearest'), pnum=pnum_(), title='resize nearest') + >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='cubic'), pnum=pnum_(), title='resize cubic') """ # todo: needs to be extended for the case where the sub_data is a # nested chain of transforms. @@ -943,22 +943,32 @@ class DelayedWarp(DelayedImageOperation): # TODO: should we blur the source if the determanent of M is less # than 1? If so by how much if kwargs.get('antialias', True) and interpolation != 'nearest': - factor = transform.det() - # [0:2, 0:2]) + """ + transform = Affine.scale(0.2) + """ # Hacked in heuristic for antialiasing before a downsample + factor = np.sqrt(transform.det()) if factor < 0.99: - k = int(1 / np.sqrt(factor) * 1.2) + # compute the number of 2x downsamples + num_downscales = np.log2(1 / factor) + + # Define b0 = kernel size for one downsample operation + b0 = 5 + # Define s0 = sigma for one downsample operation + s0 = 1 + + # The kernel size and sigma doubles for each 2x downsample + k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) + sigma = s0 * (2 ** (num_downscales - 1)) + if k % 2 == 0: k += 1 - sigma = 0.3 * ((k - 1) * 0.5 - 1) + 0.8 - sigma = sigma ** 1.2 - sub_data_ = sub_data_.copy() + sub_data_ = cv2.GaussianBlur(sub_data_, (k, k), sigma, sigma) M = np.asarray(transform) final = cv2.warpAffine(sub_data_, M[0:2], dsize=dsize, flags=flags) # final = cv2.warpPerspective(sub_data_, M, dsize=dsize, flags=flags) - print(final.mean()) # Ensure that the last dimension is channels final = kwarray.atleast_nd(final, 3, front=False) if as_xarray: -- GitLab From fc512b79babae9b5106655fcc49f24db5f819a6d Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 20 May 2021 22:57:19 -0400 Subject: [PATCH 3/8] wip --- dev/antialias_warp.py | 175 ++++++++++++++++++++++++++++++++++++++++++ ndsampler/delayed.py | 6 +- 2 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 dev/antialias_warp.py diff --git a/dev/antialias_warp.py b/dev/antialias_warp.py new file mode 100644 index 0000000..4ddaf88 --- /dev/null +++ b/dev/antialias_warp.py @@ -0,0 +1,175 @@ + + +def warp_image_test(image, transform, dsize=None): + """ + + from kwimage.transform import Affine + import kwimage + image = kwimage.grab_test_image('checkerboard', dsize=(2048, 2048)).astype(np.float32) + image = kwimage.grab_test_image('astro', dsize=(2048, 2048)) + transform = Affine.random() @ Affine.scale(0.01) + + """ + from kwimage.transform import Affine + import kwimage + import numpy as np + import ubelt as ub + + # Choose a random affine transform that probably has a small scale + # transform = Affine.random(rng=432) @ Affine.scale(0.01) + transform = Affine.scale(0.01) + + image = kwimage.grab_test_image('checkerboard', dsize=(2048, 2048)) + # image = kwimage.grab_test_image('astro', dsize=(2048, 2048)) + + image = kwimage.ensure_float01(image) + + from kwimage import im_cv2 + import kwarray + import cv2 + transform = Affine.coerce(transform) + + if 1 or dsize is None: + h, w = image.shape[0:2] + + boxes = kwimage.Boxes(np.array([[0, 0, w, h]]), 'xywh') + poly = boxes.to_polygons()[0] + warped_poly = poly.warp(transform.matrix) + warped_box = warped_poly.to_boxes().to_ltrb().quantize() + dsize = tuple(map(int, warped_box.data[0, 2:4])) + + import timerit + ti = timerit.Timerit(10, bestof=3, verbose=2) + + def _full_gauss_kernel(k0, sigma0, scale): + num_downscales = np.log2(1 / scale) + + # Define b0 = kernel size for one downsample operation + b0 = 5 + # Define s0 = sigma for one downsample operation + s0 = 1 + + # The kernel size and sigma doubles for each 2x downsample + k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) + sigma = s0 * (2 ** (num_downscales - 1)) + + if k % 2 == 0: + k += 1 + + return k, sigma + + # -------------------- + # METHOD 1 + # + for timer in ti.reset('resize+warp'): + with timer: + params = transform.decompose() + + sx, sy = params['scale'] + noscale_params = ub.dict_diff(params, {'scale'}) + noscale_warp = Affine.affine(**noscale_params) + + h, w = image.shape[0:2] + resize_dsize = (int(np.ceil(sx * w)), int(np.ceil(sy * h))) + + downsampled = cv2.resize(image, dsize=resize_dsize, fx=sx, fy=sy, + interpolation=cv2.INTER_AREA) + + interpolation = 'linear' + flags = im_cv2._coerce_interpolation(interpolation) + final_v1 = cv2.warpAffine(downsampled, noscale_warp.matrix[0:2], dsize=dsize, flags=flags) + + # -------------------- + # METHOD 2 + for timer in ti.reset('fullblur+warp'): + with timer: + k_x, sigma_x = _full_gauss_kernel(k0=5, sigma0=1, scale=sx) + k_y, sigma_y = _full_gauss_kernel(k0=5, sigma0=1, scale=sy) + image_ = image.copy() + image_ = cv2.GaussianBlur(image_, (k_x, k_y), sigma_x, sigma_y) + image_ = kwarray.atleast_nd(image_, 3) + # image_ = image_.clip(0, 1) + + interpolation = 'linear' + flags = im_cv2._coerce_interpolation(interpolation) + final_v2 = cv2.warpAffine(image_, transform.matrix[0:2], dsize=dsize, flags=flags) + + # -------------------- + # METHOD 3 + + for timer in ti.reset('pyrDown+blur+warp'): + with timer: + temp = image + + smallest_scale = max(sx, sy) + num_downscales = int(np.log2(1 / smallest_scale)) + pyr_scale = 1 / (2 ** num_downscales) + + # Does the gaussian downsampling + temp = image + for _ in range(num_downscales): + temp = cv2.pyrDown(temp) + + rest_sx = sx / pyr_scale + rest_sy = sy / pyr_scale + + partial_scale = Affine.scale((rest_sx, rest_sy)) + rest_warp = noscale_warp @ partial_scale + + k_x, sigma_x = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sx) + k_y, sigma_y = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sy) + temp = temp.copy() + temp = cv2.GaussianBlur(temp, (k_x, k_y), sigma_x, sigma_y) + temp = kwarray.atleast_nd(temp, 3) + + interpolation = 'linear' + flags = im_cv2._coerce_interpolation(interpolation) + final_v3 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, flags=flags) + + # -------------------- + # METHOD 4 - dont do the final blur + + for timer in ti.reset('pyrDown+warp'): + with timer: + temp = image + + smallest_scale = max(sx, sy) + num_downscales = int(np.log2(1 / smallest_scale)) + pyr_scale = 1 / (2 ** num_downscales) + + # Does the gaussian downsampling + temp = image + for _ in range(num_downscales): + temp = cv2.pyrDown(temp) + + rest_sx = sx / pyr_scale + rest_sy = sy / pyr_scale + + partial_scale = Affine.scale((rest_sx, rest_sy)) + rest_warp = noscale_warp @ partial_scale + + interpolation = 'linear' + flags = im_cv2._coerce_interpolation(interpolation) + final_v4 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, flags=flags) + + if 1: + + def get_title(key): + from ubelt.timerit import _choose_unit + value = ti.measures['mean'][key] + suffix, mag = _choose_unit(value) + unit_val = value / mag + + return key + ' ' + ub.repr2(unit_val, precision=2) + ' ' + suffix + + final_v2 = final_v2.clip(0, 1) + final_v1 = final_v1.clip(0, 1) + final_v3 = final_v3.clip(0, 1) + final_v4 = final_v4.clip(0, 1) + import kwplot + kwplot.autompl() + kwplot.imshow(final_v2, pnum=(1, 4, 1), title=get_title('fullblur+warp')) + kwplot.imshow(final_v1, pnum=(1, 4, 2), title=get_title('resize+warp')) + kwplot.imshow(final_v3, pnum=(1, 4, 3), title=get_title('pyrDown+blur+warp')) + kwplot.imshow(final_v4, pnum=(1, 4, 4), title=get_title('pyrDown+warp')) + # kwplot.imshow(np.abs(final_v2 - final_v1), pnum=(1, 4, 4)) diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index fccf02f..914a77e 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -942,10 +942,13 @@ class DelayedWarp(DelayedImageOperation): # TODO: should we blur the source if the determanent of M is less # than 1? If so by how much - if kwargs.get('antialias', True) and interpolation != 'nearest': + if kwargs.get('antialias', 0) and interpolation != 'nearest': """ transform = Affine.scale(0.2) + See: ~/code/ndsampler/dev/antialias_warp.py """ + # FIXME! This is too slow for large images. + # Hacked in heuristic for antialiasing before a downsample factor = np.sqrt(transform.det()) if factor < 0.99: @@ -964,6 +967,7 @@ class DelayedWarp(DelayedImageOperation): if k % 2 == 0: k += 1 + sub_data_ = sub_data_.copy() sub_data_ = cv2.GaussianBlur(sub_data_, (k, k), sigma, sigma) M = np.asarray(transform) -- GitLab From 45221e77479f824462cf8534d6243609f03802dd Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 21 May 2021 00:03:17 -0400 Subject: [PATCH 4/8] wip --- dev/antialias_warp.py | 67 ++++++++++++++++++++++++++++--------------- ndsampler/delayed.py | 2 +- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/dev/antialias_warp.py b/dev/antialias_warp.py index 4ddaf88..f96b778 100644 --- a/dev/antialias_warp.py +++ b/dev/antialias_warp.py @@ -16,11 +16,14 @@ def warp_image_test(image, transform, dsize=None): import ubelt as ub # Choose a random affine transform that probably has a small scale - # transform = Affine.random(rng=432) @ Affine.scale(0.01) - transform = Affine.scale(0.01) + # transform = Affine.random() @ Affine.scale((0.3, 2)) + # transform = Affine.scale((0.1, 1.2)) + # transform = Affine.scale(0.05) + transform = Affine.random() @ Affine.scale(0.01) + # transform = Affine.random() - image = kwimage.grab_test_image('checkerboard', dsize=(2048, 2048)) - # image = kwimage.grab_test_image('astro', dsize=(2048, 2048)) + image = kwimage.grab_test_image('astro') + image = kwimage.grab_test_image('checkerboard') image = kwimage.ensure_float01(image) @@ -43,6 +46,8 @@ def warp_image_test(image, transform, dsize=None): def _full_gauss_kernel(k0, sigma0, scale): num_downscales = np.log2(1 / scale) + if num_downscales < 0: + return 1, 0 # Define b0 = kernel size for one downsample operation b0 = 5 @@ -58,6 +63,18 @@ def warp_image_test(image, transform, dsize=None): return k, sigma + def pyrDownK(a, k=1): + assert k >= 0 + for _ in range(k): + a = cv2.pyrDown(a) + return a + + for timer in ti.reset('naive'): + with timer: + interpolation = 'nearest' + flags = im_cv2._coerce_interpolation(interpolation) + final_v5 = cv2.warpAffine(image, transform.matrix[0:2], dsize=dsize, flags=flags) + # -------------------- # METHOD 1 # @@ -99,16 +116,18 @@ def warp_image_test(image, transform, dsize=None): for timer in ti.reset('pyrDown+blur+warp'): with timer: - temp = image + temp = image.copy() + params = transform.decompose() + sx, sy = params['scale'] - smallest_scale = max(sx, sy) - num_downscales = int(np.log2(1 / smallest_scale)) + biggest_scale = max(sx, sy) + # The -2 allows the gaussian to be a little bigger. This + # seems to help with border effects at only a small runtime cost + num_downscales = max(int(np.log2(1 / biggest_scale)) - 2, 0) pyr_scale = 1 / (2 ** num_downscales) # Does the gaussian downsampling - temp = image - for _ in range(num_downscales): - temp = cv2.pyrDown(temp) + temp = pyrDownK(image, num_downscales) rest_sx = sx / pyr_scale rest_sy = sy / pyr_scale @@ -118,29 +137,29 @@ def warp_image_test(image, transform, dsize=None): k_x, sigma_x = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sx) k_y, sigma_y = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sy) - temp = temp.copy() temp = cv2.GaussianBlur(temp, (k_x, k_y), sigma_x, sigma_y) temp = kwarray.atleast_nd(temp, 3) - interpolation = 'linear' + interpolation = 'cubic' flags = im_cv2._coerce_interpolation(interpolation) - final_v3 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, flags=flags) + final_v3 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, + flags=flags) # -------------------- # METHOD 4 - dont do the final blur for timer in ti.reset('pyrDown+warp'): with timer: - temp = image + temp = image.copy() + params = transform.decompose() + sx, sy = params['scale'] - smallest_scale = max(sx, sy) - num_downscales = int(np.log2(1 / smallest_scale)) + biggest_scale = max(sx, sy) + num_downscales = max(int(np.log2(1 / biggest_scale)), 0) pyr_scale = 1 / (2 ** num_downscales) # Does the gaussian downsampling - temp = image - for _ in range(num_downscales): - temp = cv2.pyrDown(temp) + temp = pyrDownK(image, num_downscales) rest_sx = sx / pyr_scale rest_sy = sy / pyr_scale @@ -166,10 +185,12 @@ def warp_image_test(image, transform, dsize=None): final_v1 = final_v1.clip(0, 1) final_v3 = final_v3.clip(0, 1) final_v4 = final_v4.clip(0, 1) + final_v5 = final_v5.clip(0, 1) import kwplot kwplot.autompl() - kwplot.imshow(final_v2, pnum=(1, 4, 1), title=get_title('fullblur+warp')) - kwplot.imshow(final_v1, pnum=(1, 4, 2), title=get_title('resize+warp')) - kwplot.imshow(final_v3, pnum=(1, 4, 3), title=get_title('pyrDown+blur+warp')) - kwplot.imshow(final_v4, pnum=(1, 4, 4), title=get_title('pyrDown+warp')) + kwplot.imshow(final_v5, pnum=(1, 5, 1), title=get_title('naive')) + kwplot.imshow(final_v2, pnum=(1, 5, 2), title=get_title('fullblur+warp')) + kwplot.imshow(final_v1, pnum=(1, 5, 3), title=get_title('resize+warp')) + kwplot.imshow(final_v3, pnum=(1, 5, 4), title=get_title('pyrDown+blur+warp')) + kwplot.imshow(final_v4, pnum=(1, 5, 5), title=get_title('pyrDown+warp')) # kwplot.imshow(np.abs(final_v2 - final_v1), pnum=(1, 4, 4)) diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index 914a77e..f3c8c9c 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -891,8 +891,8 @@ class DelayedWarp(DelayedImageOperation): Example: >>> # Test aliasing >>> from ndsampler.delayed import * # NOQA - >>> s = DelayedIdentity.demo('checkerboard') >>> s = DelayedIdentity.demo() + >>> s = DelayedIdentity.demo('checkerboard') >>> a = s.delayed_warp(Affine.scale(0.05), dsize='auto') >>> b = s.delayed_warp(Affine.scale(3), dsize='auto') -- GitLab From 0187551209d28139b51a300d1291b3d2d78f8d56 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 21 May 2021 18:58:32 -0400 Subject: [PATCH 5/8] Bump required kwimage --- dev/antialias_warp.py | 287 +++++++++++++++++++++++++++++++++++++- ndsampler/coco_regions.py | 3 +- ndsampler/delayed.py | 70 +++++----- requirements/runtime.txt | 2 +- 4 files changed, 323 insertions(+), 39 deletions(-) diff --git a/dev/antialias_warp.py b/dev/antialias_warp.py index f96b778..35cdf25 100644 --- a/dev/antialias_warp.py +++ b/dev/antialias_warp.py @@ -51,16 +51,15 @@ def warp_image_test(image, transform, dsize=None): # Define b0 = kernel size for one downsample operation b0 = 5 - # Define s0 = sigma for one downsample operation - s0 = 1 + # Define sigma0 = sigma for one downsample operation + sigma0 = 1 # The kernel size and sigma doubles for each 2x downsample k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) - sigma = s0 * (2 ** (num_downscales - 1)) + sigma = sigma0 * (2 ** (num_downscales - 1)) if k % 2 == 0: k += 1 - return k, sigma def pyrDownK(a, k=1): @@ -194,3 +193,283 @@ def warp_image_test(image, transform, dsize=None): kwplot.imshow(final_v3, pnum=(1, 5, 4), title=get_title('pyrDown+blur+warp')) kwplot.imshow(final_v4, pnum=(1, 5, 5), title=get_title('pyrDown+warp')) # kwplot.imshow(np.abs(final_v2 - final_v1), pnum=(1, 4, 4)) + + +def warp_affine(image, transform, dsize=None, antialias=True, + interpolation='linear'): + """ + Applies an affine transformation to an image with optional antialiasing. + + Args: + image (ndarray): the input image + + transform (ndarray | Affine): a coercable affine matrix + + dsize (Tuple[int, int] | None | str): + width and height of the resulting image. If "auto", it is computed + such that the positive coordinates of the warped image will fit in + the new canvas. If None, then the image size will not change. + + antialias (bool, default=True): + if True determines if the transform is downsampling and applies + antialiasing via gaussian a blur. + + TODO: + - [ ] This will be moved to kwimage.im_cv2 + + Example: + >>> import kwimage + >>> image = kwimage.grab_test_image('astro') + >>> image = kwimage.grab_test_image('checkerboard') + >>> transform = Affine.random() @ Affine.scale(0.05) + >>> transform = Affine.scale(0.02) + >>> warped1 = warp_affine(image, transform, dsize='auto', antialias=1, interpolation='nearest') + >>> warped2 = warp_affine(image, transform, dsize='auto', antialias=0) + >>> # xdoctest: +REQUIRES(--show) + >>> import kwplot + >>> kwplot.autompl() + >>> pnum_ = kwplot.PlotNums(nRows=1, nCols=2) + >>> kwplot.imshow(warped1, pnum=pnum_(), title='antialias=True') + >>> kwplot.imshow(warped2, pnum=pnum_(), title='antialias=False') + >>> kwplot.show_if_requested() + + Example: + >>> import kwimage + >>> image = kwimage.grab_test_image('astro') + >>> image = kwimage.grab_test_image('checkerboard') + >>> transform = Affine.random() @ Affine.scale((.1, 1.2)) + >>> warped1 = warp_affine(image, transform, dsize='auto', antialias=1) + >>> warped2 = warp_affine(image, transform, dsize='auto', antialias=0) + >>> # xdoctest: +REQUIRES(--show) + >>> import kwplot + >>> kwplot.autompl() + >>> pnum_ = kwplot.PlotNums(nRows=1, nCols=2) + >>> kwplot.imshow(warped1, pnum=pnum_(), title='antialias=True') + >>> kwplot.imshow(warped2, pnum=pnum_(), title='antialias=False') + >>> kwplot.show_if_requested() + """ + from kwimage import im_cv2 + from kwimage.transform import Affine + import kwimage + import numpy as np + import cv2 + import ubelt as ub + transform = Affine.coerce(transform) + flags = im_cv2._coerce_interpolation(interpolation) + + # TODO: expose these params + # borderMode = cv2.BORDER_DEFAULT + # borderMode = cv2.BORDER_CONSTANT + borderMode = None + borderValue = None + + """ + Variations that could change in the future: + + * In _gauss_params I'm not sure if we want to compute integer or + fractional "number of downsamples". + + * The fudge factor bothers me, but seems necessary + """ + + def _gauss_params(scale, k0=5, sigma0=1, fractional=True): + # Compute a gaussian to mitigate aliasing for a requested downsample + # Args: + # scale: requested downsample factor + # k0 (int): kernel size for one downsample operation + # sigma0 (float): sigma for one downsample operation + # fractional (bool): controls if we compute params for integer downsample + # ops + num_downs = np.log2(1 / scale) + if not fractional: + num_downs = max(int(num_downs), 0) + if num_downs <= 0: + k = 1 + sigma = 0 + else: + # The kernel size and sigma doubles for each 2x downsample + sigma = sigma0 * (2 ** (num_downs - 1)) + k = int(np.ceil(k0 * (2 ** (num_downs - 1)))) + k = k + int(k % 2 == 0) + return k, sigma + + def _pyrDownK(a, k=1): + # Downsamples by (2 ** k)x with antialiasing + if k == 0: + a = a.copy() + for _ in range(k): + a = cv2.pyrDown(a) + return a + + if dsize is None: + dsize = tuple(image.shape[0:2][::-1]) + elif dsize == 'auto': + h, w = image.shape[0:2] + boxes = kwimage.Boxes(np.array([[0, 0, w, h]]), 'xywh') + poly = boxes.to_polygons()[0] + warped_poly = poly.warp(transform.matrix) + warped_box = warped_poly.to_boxes().to_ltrb().quantize() + dsize = tuple(map(int, warped_box.data[0, 2:4])) + + if not antialias: + M = np.asarray(transform) + result = cv2.warpAffine(image, M[0:2], + dsize=dsize, flags=flags, + borderMode=borderMode, + borderValue=borderValue) + else: + # Decompose the affine matrix into its 6 core parameters + params = transform.decompose() + sx, sy = params['scale'] + + if sx >= 1 and sy > 1: + # No downsampling detected, no need to antialias + M = np.asarray(transform) + result = cv2.warpAffine(image, M[0:2], dsize=dsize, flags=flags, + borderMode=borderMode, + borderValue=borderValue) + else: + # At least one dimension is downsampled + + # Compute the transform with all scaling removed + noscale_warp = Affine.affine(**ub.dict_diff(params, {'scale'})) + + max_scale = max(sx, sy) + # The "fudge" factor limits the number of downsampled pyramid + # operations. A bigger fudge factor means means that the final + # gaussian kernel for the antialiasing operation will be bigger. + # It essentials say that at most "fudge" downsampling ops will + # be handled by the final blur rather than the pyramid downsample. + # It seems to help with border effects at only a small runtime cost + # I don't entirely understand why the border artifact is introduced + # when this is enabled though + + # TODO: should we allow for this fudge factor? + # TODO: what is the real name of this? num_down_prevent ? + # skip_final_downs? + fudge = 2 + # TODO: should final antialiasing be on? + # Note, if fudge is non-zero it is important to do this. + do_final_aa = 1 + # TODO: should fractional be True or False by default? + # If fudge is 0 and fractional=0, then I think is the same as + # do_final_aa=0. + fractional = 0 + + num_downs = max(int(np.log2(1 / max_scale)) - fudge, 0) + pyr_scale = 1 / (2 ** num_downs) + + # Downsample iteratively with antialiasing + downscaled = _pyrDownK(image, num_downs) + + rest_sx = sx / pyr_scale + rest_sy = sy / pyr_scale + + # Compute the transform from the downsampled image to the destination + rest_warp = noscale_warp @ Affine.scale((rest_sx, rest_sy)) + + # Do a final small blur to acount for the potential aliasing + # in any remaining scaling operations. + if do_final_aa: + # Computed as the closest sigma to the [1, 4, 6, 4, 1] approx + # used in cv2.pyrDown + aa_sigma0 = 1.0565137190917149 + aa_k0 = 5 + k_x, sigma_x = _gauss_params(scale=rest_sx, k0=aa_k0, + sigma0=aa_sigma0, + fractional=fractional) + k_y, sigma_y = _gauss_params(scale=rest_sy, k0=aa_k0, + sigma0=aa_sigma0, + fractional=fractional) + + # Note: when k=1, no blur occurs + # blurBorderType = cv2.BORDER_REPLICATE + # blurBorderType = cv2.BORDER_CONSTANT + blurBorderType = cv2.BORDER_DEFAULT + downscaled = cv2.GaussianBlur( + downscaled, (k_x, k_y), sigma_x, sigma_y, + borderType=blurBorderType + ) + + result = cv2.warpAffine(downscaled, rest_warp.matrix[0:2], + dsize=dsize, flags=flags, + borderMode=borderMode, + borderValue=borderValue) + + return result + + +def _check(): + # Find the sigma closest to the pyrDown op [1, 4, 6, 4, 1] / 16 + import cv2 + import numpy as np + import scipy + import ubelt as ub + def sigma_error(sigma): + sigma = np.asarray(sigma).ravel()[0] + got = (cv2.getGaussianKernel(5, sigma) * 16).ravel() + want = np.array([1, 4, 6, 4, 1]) + loss = ((got - want) ** 2).sum() + return loss + result = scipy.optimize.minimize(sigma_error, x0=1.0, method='Nelder-Mead') + print('result = {}'.format(ub.repr2(result, nl=1))) + + best_loss = float('inf') + best = None + methods = ['Nelder-Mead', 'Powell', 'CG', 'BFGS', + # 'Newton-CG', + 'L-BFGS-B', 'TNC', 'COBYLA', 'SLSQP', + # 'trust-constr', + # 'dogleg', + # 'trust-ncg', 'trust-exact', + # 'trust-krylov' + ] + # results = {} + x0 = 1.06 + for i in range(100): + for method in methods: + best_method_loss, best_method_sigma, best_method_x0 = results.get(method, (float('inf'), 1.06, x0)) + result = scipy.optimize.minimize(sigma_error, x0=x0, method=method) + sigma = np.asarray(result.x).ravel()[0] + loss = sigma_error(sigma) + if loss <= best_method_loss: + results[method] = (loss, sigma, x0) + + best_method = ub.argmin(results) + best_loss, best_sigma = results[best_method][0:2] + rng = np.random + if rng.rand() > 0.5: + x0 = best_sigma + else: + x0 = best_sigma + rng.rand() * 0.0001 + print('best_method = {!r}'.format(best_method)) + print('best_loss = {!r}'.format(best_loss)) + print('best_sigma = {!r}'.format(best_sigma)) + + print('results = {}'.format(ub.repr2(results, nl=1, align=':'))) + print('best_method = {}'.format(ub.repr2(best_method, nl=1))) + print('best_method = {!r}'.format(best_method)) + + sigma_error(1.0565139268118493) + sigma_error(1.0565137190917149) + # scipy.optimize.minimize_scalar(sigma_error, bounds=(1, 1.1)) + + import kwarray + import numpy as np + a = (kwarray.ensure_rng(0).rand(32, 32) * 256).astype(np.uint8) + a = kwimage.ensure_float01(a) + b = cv2.GaussianBlur(a.copy(), (1, 1), 3, 3) + assert np.all(a == b) + + import timerit + ti = timerit.Timerit(100, bestof=10, verbose=2) + for timer in ti.reset('time'): + with timer: + b = cv2.GaussianBlur(a.copy(), (9, 9), 3, 3) + + import timerit + ti = timerit.Timerit(100, bestof=10, verbose=2) + for timer in ti.reset('time'): + with timer: + c = cv2.GaussianBlur(a.copy(), (1, 9), 3, 3) + zR= cv2.GaussianBlur(c.copy(), (9, 1), 3, 3) diff --git a/ndsampler/coco_regions.py b/ndsampler/coco_regions.py index de962ff..c8f2418 100644 --- a/ndsampler/coco_regions.py +++ b/ndsampler/coco_regions.py @@ -849,7 +849,8 @@ def select_positive_regions(targets, window_dims=(300, 300), thresh=0.0, return selection -def new_video_sample_grid(dset, window_dims, window_overlap=0.0, +def new_video_sample_grid(dset, window_dims=None, window_overlap=0.0, + space_dims=None, time_dim=None, # TODO classes_of_interest=None, ignore_coverage_thresh=0.6, negative_classes={'ignore', 'background'}): """ diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index f3c8c9c..edd9b74 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -905,7 +905,7 @@ class DelayedWarp(DelayedImageOperation): >>> kwplot.imshow(a.finalize(interpolation='area'), pnum=pnum_(), title='warpAffine area') >>> kwplot.imshow(a.finalize(interpolation='linear'), pnum=pnum_(), title='warpAffine linear') >>> kwplot.imshow(a.finalize(interpolation='nearest'), pnum=pnum_(), title='warpAffine nearest') - >>> kwplot.imshow(a.finalize(interpolation='cubic'), pnum=pnum_(), title='warpAffine cubic') + >>> kwplot.imshow(a.finalize(interpolation='nearest', antialias=False), pnum=pnum_(), title='warpAffine nearest AA=0') >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='area'), pnum=pnum_(), title='resize area') >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='linear'), pnum=pnum_(), title='resize linear') >>> kwplot.imshow(kwimage.imresize(s.finalize(), dsize=a.dsize, interpolation='nearest'), pnum=pnum_(), title='resize nearest') @@ -913,8 +913,8 @@ class DelayedWarp(DelayedImageOperation): """ # todo: needs to be extended for the case where the sub_data is a # nested chain of transforms. - import cv2 - from kwimage import im_cv2 + # import cv2 + # from kwimage import im_cv2 if dsize is None: dsize = self.dsize transform = Affine.coerce(transform) @ self.transform @@ -932,7 +932,7 @@ class DelayedWarp(DelayedImageOperation): else: as_xarray = kwargs.get('as_xarray', False) # Leaf finalize - flags = im_cv2._coerce_interpolation(interpolation) + # flags = im_cv2._coerce_interpolation(interpolation) if dsize == (None, None): dsize = None sub_data_ = np.asarray(sub_data) @@ -942,36 +942,40 @@ class DelayedWarp(DelayedImageOperation): # TODO: should we blur the source if the determanent of M is less # than 1? If so by how much - if kwargs.get('antialias', 0) and interpolation != 'nearest': - """ - transform = Affine.scale(0.2) - See: ~/code/ndsampler/dev/antialias_warp.py - """ - # FIXME! This is too slow for large images. - - # Hacked in heuristic for antialiasing before a downsample - factor = np.sqrt(transform.det()) - if factor < 0.99: - # compute the number of 2x downsamples - num_downscales = np.log2(1 / factor) - - # Define b0 = kernel size for one downsample operation - b0 = 5 - # Define s0 = sigma for one downsample operation - s0 = 1 - - # The kernel size and sigma doubles for each 2x downsample - k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) - sigma = s0 * (2 ** (num_downscales - 1)) - - if k % 2 == 0: - k += 1 - - sub_data_ = sub_data_.copy() - sub_data_ = cv2.GaussianBlur(sub_data_, (k, k), sigma, sigma) - + # if kwargs.get('antialias', 0) and interpolation != 'nearest': + # """ + # transform = Affine.scale(0.2) + # See: ~/code/ndsampler/dev/antialias_warp.py + # """ + # # FIXME! This is too slow for large images. + + # # Hacked in heuristic for antialiasing before a downsample + # factor = np.sqrt(transform.det()) + # if factor < 0.99: + # # compute the number of 2x downsamples + # num_downscales = np.log2(1 / factor) + + # # Define b0 = kernel size for one downsample operation + # b0 = 5 + # # Define s0 = sigma for one downsample operation + # s0 = 1 + + # # The kernel size and sigma doubles for each 2x downsample + # k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) + # sigma = s0 * (2 ** (num_downscales - 1)) + + # if k % 2 == 0: + # k += 1 + + # sub_data_ = sub_data_.copy() + # sub_data_ = cv2.GaussianBlur(sub_data_, (k, k), sigma, sigma) M = np.asarray(transform) - final = cv2.warpAffine(sub_data_, M[0:2], dsize=dsize, flags=flags) + # final = cv2.warpAffine(sub_data_, M[0:2], dsize=dsize, flags=flags) + antialias = kwargs.get('antialias', 1) + # Requires update to kwimage version + final = kwimage.warp_affine(sub_data_, M, dsize=dsize, + interpolation=interpolation, + antialias=antialias) # final = cv2.warpPerspective(sub_data_, M, dsize=dsize, flags=flags) # Ensure that the last dimension is channels final = kwarray.atleast_nd(final, 3, front=False) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 99257c2..1202527 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -23,7 +23,7 @@ pyqtree >= 1.0.0 # There are likely several more undocumented deps, see what fails and submit a PR please! # TODO : fill me in! -kwimage >= 0.4.0 +kwimage >= 0.7.5 kwarray >= 0.5.16 kwcoco >= 0.2.1 -- GitLab From 711400dc2529361803cf110a26106abf1dc7c1a5 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 25 May 2021 14:28:20 -0400 Subject: [PATCH 6/8] modifications --- dev/antialias_warp.py | 475 --------------------------------------- ndsampler/delayed.py | 39 +--- requirements/runtime.txt | 2 +- 3 files changed, 3 insertions(+), 513 deletions(-) delete mode 100644 dev/antialias_warp.py diff --git a/dev/antialias_warp.py b/dev/antialias_warp.py deleted file mode 100644 index 35cdf25..0000000 --- a/dev/antialias_warp.py +++ /dev/null @@ -1,475 +0,0 @@ - - -def warp_image_test(image, transform, dsize=None): - """ - - from kwimage.transform import Affine - import kwimage - image = kwimage.grab_test_image('checkerboard', dsize=(2048, 2048)).astype(np.float32) - image = kwimage.grab_test_image('astro', dsize=(2048, 2048)) - transform = Affine.random() @ Affine.scale(0.01) - - """ - from kwimage.transform import Affine - import kwimage - import numpy as np - import ubelt as ub - - # Choose a random affine transform that probably has a small scale - # transform = Affine.random() @ Affine.scale((0.3, 2)) - # transform = Affine.scale((0.1, 1.2)) - # transform = Affine.scale(0.05) - transform = Affine.random() @ Affine.scale(0.01) - # transform = Affine.random() - - image = kwimage.grab_test_image('astro') - image = kwimage.grab_test_image('checkerboard') - - image = kwimage.ensure_float01(image) - - from kwimage import im_cv2 - import kwarray - import cv2 - transform = Affine.coerce(transform) - - if 1 or dsize is None: - h, w = image.shape[0:2] - - boxes = kwimage.Boxes(np.array([[0, 0, w, h]]), 'xywh') - poly = boxes.to_polygons()[0] - warped_poly = poly.warp(transform.matrix) - warped_box = warped_poly.to_boxes().to_ltrb().quantize() - dsize = tuple(map(int, warped_box.data[0, 2:4])) - - import timerit - ti = timerit.Timerit(10, bestof=3, verbose=2) - - def _full_gauss_kernel(k0, sigma0, scale): - num_downscales = np.log2(1 / scale) - if num_downscales < 0: - return 1, 0 - - # Define b0 = kernel size for one downsample operation - b0 = 5 - # Define sigma0 = sigma for one downsample operation - sigma0 = 1 - - # The kernel size and sigma doubles for each 2x downsample - k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) - sigma = sigma0 * (2 ** (num_downscales - 1)) - - if k % 2 == 0: - k += 1 - return k, sigma - - def pyrDownK(a, k=1): - assert k >= 0 - for _ in range(k): - a = cv2.pyrDown(a) - return a - - for timer in ti.reset('naive'): - with timer: - interpolation = 'nearest' - flags = im_cv2._coerce_interpolation(interpolation) - final_v5 = cv2.warpAffine(image, transform.matrix[0:2], dsize=dsize, flags=flags) - - # -------------------- - # METHOD 1 - # - for timer in ti.reset('resize+warp'): - with timer: - params = transform.decompose() - - sx, sy = params['scale'] - noscale_params = ub.dict_diff(params, {'scale'}) - noscale_warp = Affine.affine(**noscale_params) - - h, w = image.shape[0:2] - resize_dsize = (int(np.ceil(sx * w)), int(np.ceil(sy * h))) - - downsampled = cv2.resize(image, dsize=resize_dsize, fx=sx, fy=sy, - interpolation=cv2.INTER_AREA) - - interpolation = 'linear' - flags = im_cv2._coerce_interpolation(interpolation) - final_v1 = cv2.warpAffine(downsampled, noscale_warp.matrix[0:2], dsize=dsize, flags=flags) - - # -------------------- - # METHOD 2 - for timer in ti.reset('fullblur+warp'): - with timer: - k_x, sigma_x = _full_gauss_kernel(k0=5, sigma0=1, scale=sx) - k_y, sigma_y = _full_gauss_kernel(k0=5, sigma0=1, scale=sy) - image_ = image.copy() - image_ = cv2.GaussianBlur(image_, (k_x, k_y), sigma_x, sigma_y) - image_ = kwarray.atleast_nd(image_, 3) - # image_ = image_.clip(0, 1) - - interpolation = 'linear' - flags = im_cv2._coerce_interpolation(interpolation) - final_v2 = cv2.warpAffine(image_, transform.matrix[0:2], dsize=dsize, flags=flags) - - # -------------------- - # METHOD 3 - - for timer in ti.reset('pyrDown+blur+warp'): - with timer: - temp = image.copy() - params = transform.decompose() - sx, sy = params['scale'] - - biggest_scale = max(sx, sy) - # The -2 allows the gaussian to be a little bigger. This - # seems to help with border effects at only a small runtime cost - num_downscales = max(int(np.log2(1 / biggest_scale)) - 2, 0) - pyr_scale = 1 / (2 ** num_downscales) - - # Does the gaussian downsampling - temp = pyrDownK(image, num_downscales) - - rest_sx = sx / pyr_scale - rest_sy = sy / pyr_scale - - partial_scale = Affine.scale((rest_sx, rest_sy)) - rest_warp = noscale_warp @ partial_scale - - k_x, sigma_x = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sx) - k_y, sigma_y = _full_gauss_kernel(k0=5, sigma0=1, scale=rest_sy) - temp = cv2.GaussianBlur(temp, (k_x, k_y), sigma_x, sigma_y) - temp = kwarray.atleast_nd(temp, 3) - - interpolation = 'cubic' - flags = im_cv2._coerce_interpolation(interpolation) - final_v3 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, - flags=flags) - - # -------------------- - # METHOD 4 - dont do the final blur - - for timer in ti.reset('pyrDown+warp'): - with timer: - temp = image.copy() - params = transform.decompose() - sx, sy = params['scale'] - - biggest_scale = max(sx, sy) - num_downscales = max(int(np.log2(1 / biggest_scale)), 0) - pyr_scale = 1 / (2 ** num_downscales) - - # Does the gaussian downsampling - temp = pyrDownK(image, num_downscales) - - rest_sx = sx / pyr_scale - rest_sy = sy / pyr_scale - - partial_scale = Affine.scale((rest_sx, rest_sy)) - rest_warp = noscale_warp @ partial_scale - - interpolation = 'linear' - flags = im_cv2._coerce_interpolation(interpolation) - final_v4 = cv2.warpAffine(temp, rest_warp.matrix[0:2], dsize=dsize, flags=flags) - - if 1: - - def get_title(key): - from ubelt.timerit import _choose_unit - value = ti.measures['mean'][key] - suffix, mag = _choose_unit(value) - unit_val = value / mag - - return key + ' ' + ub.repr2(unit_val, precision=2) + ' ' + suffix - - final_v2 = final_v2.clip(0, 1) - final_v1 = final_v1.clip(0, 1) - final_v3 = final_v3.clip(0, 1) - final_v4 = final_v4.clip(0, 1) - final_v5 = final_v5.clip(0, 1) - import kwplot - kwplot.autompl() - kwplot.imshow(final_v5, pnum=(1, 5, 1), title=get_title('naive')) - kwplot.imshow(final_v2, pnum=(1, 5, 2), title=get_title('fullblur+warp')) - kwplot.imshow(final_v1, pnum=(1, 5, 3), title=get_title('resize+warp')) - kwplot.imshow(final_v3, pnum=(1, 5, 4), title=get_title('pyrDown+blur+warp')) - kwplot.imshow(final_v4, pnum=(1, 5, 5), title=get_title('pyrDown+warp')) - # kwplot.imshow(np.abs(final_v2 - final_v1), pnum=(1, 4, 4)) - - -def warp_affine(image, transform, dsize=None, antialias=True, - interpolation='linear'): - """ - Applies an affine transformation to an image with optional antialiasing. - - Args: - image (ndarray): the input image - - transform (ndarray | Affine): a coercable affine matrix - - dsize (Tuple[int, int] | None | str): - width and height of the resulting image. If "auto", it is computed - such that the positive coordinates of the warped image will fit in - the new canvas. If None, then the image size will not change. - - antialias (bool, default=True): - if True determines if the transform is downsampling and applies - antialiasing via gaussian a blur. - - TODO: - - [ ] This will be moved to kwimage.im_cv2 - - Example: - >>> import kwimage - >>> image = kwimage.grab_test_image('astro') - >>> image = kwimage.grab_test_image('checkerboard') - >>> transform = Affine.random() @ Affine.scale(0.05) - >>> transform = Affine.scale(0.02) - >>> warped1 = warp_affine(image, transform, dsize='auto', antialias=1, interpolation='nearest') - >>> warped2 = warp_affine(image, transform, dsize='auto', antialias=0) - >>> # xdoctest: +REQUIRES(--show) - >>> import kwplot - >>> kwplot.autompl() - >>> pnum_ = kwplot.PlotNums(nRows=1, nCols=2) - >>> kwplot.imshow(warped1, pnum=pnum_(), title='antialias=True') - >>> kwplot.imshow(warped2, pnum=pnum_(), title='antialias=False') - >>> kwplot.show_if_requested() - - Example: - >>> import kwimage - >>> image = kwimage.grab_test_image('astro') - >>> image = kwimage.grab_test_image('checkerboard') - >>> transform = Affine.random() @ Affine.scale((.1, 1.2)) - >>> warped1 = warp_affine(image, transform, dsize='auto', antialias=1) - >>> warped2 = warp_affine(image, transform, dsize='auto', antialias=0) - >>> # xdoctest: +REQUIRES(--show) - >>> import kwplot - >>> kwplot.autompl() - >>> pnum_ = kwplot.PlotNums(nRows=1, nCols=2) - >>> kwplot.imshow(warped1, pnum=pnum_(), title='antialias=True') - >>> kwplot.imshow(warped2, pnum=pnum_(), title='antialias=False') - >>> kwplot.show_if_requested() - """ - from kwimage import im_cv2 - from kwimage.transform import Affine - import kwimage - import numpy as np - import cv2 - import ubelt as ub - transform = Affine.coerce(transform) - flags = im_cv2._coerce_interpolation(interpolation) - - # TODO: expose these params - # borderMode = cv2.BORDER_DEFAULT - # borderMode = cv2.BORDER_CONSTANT - borderMode = None - borderValue = None - - """ - Variations that could change in the future: - - * In _gauss_params I'm not sure if we want to compute integer or - fractional "number of downsamples". - - * The fudge factor bothers me, but seems necessary - """ - - def _gauss_params(scale, k0=5, sigma0=1, fractional=True): - # Compute a gaussian to mitigate aliasing for a requested downsample - # Args: - # scale: requested downsample factor - # k0 (int): kernel size for one downsample operation - # sigma0 (float): sigma for one downsample operation - # fractional (bool): controls if we compute params for integer downsample - # ops - num_downs = np.log2(1 / scale) - if not fractional: - num_downs = max(int(num_downs), 0) - if num_downs <= 0: - k = 1 - sigma = 0 - else: - # The kernel size and sigma doubles for each 2x downsample - sigma = sigma0 * (2 ** (num_downs - 1)) - k = int(np.ceil(k0 * (2 ** (num_downs - 1)))) - k = k + int(k % 2 == 0) - return k, sigma - - def _pyrDownK(a, k=1): - # Downsamples by (2 ** k)x with antialiasing - if k == 0: - a = a.copy() - for _ in range(k): - a = cv2.pyrDown(a) - return a - - if dsize is None: - dsize = tuple(image.shape[0:2][::-1]) - elif dsize == 'auto': - h, w = image.shape[0:2] - boxes = kwimage.Boxes(np.array([[0, 0, w, h]]), 'xywh') - poly = boxes.to_polygons()[0] - warped_poly = poly.warp(transform.matrix) - warped_box = warped_poly.to_boxes().to_ltrb().quantize() - dsize = tuple(map(int, warped_box.data[0, 2:4])) - - if not antialias: - M = np.asarray(transform) - result = cv2.warpAffine(image, M[0:2], - dsize=dsize, flags=flags, - borderMode=borderMode, - borderValue=borderValue) - else: - # Decompose the affine matrix into its 6 core parameters - params = transform.decompose() - sx, sy = params['scale'] - - if sx >= 1 and sy > 1: - # No downsampling detected, no need to antialias - M = np.asarray(transform) - result = cv2.warpAffine(image, M[0:2], dsize=dsize, flags=flags, - borderMode=borderMode, - borderValue=borderValue) - else: - # At least one dimension is downsampled - - # Compute the transform with all scaling removed - noscale_warp = Affine.affine(**ub.dict_diff(params, {'scale'})) - - max_scale = max(sx, sy) - # The "fudge" factor limits the number of downsampled pyramid - # operations. A bigger fudge factor means means that the final - # gaussian kernel for the antialiasing operation will be bigger. - # It essentials say that at most "fudge" downsampling ops will - # be handled by the final blur rather than the pyramid downsample. - # It seems to help with border effects at only a small runtime cost - # I don't entirely understand why the border artifact is introduced - # when this is enabled though - - # TODO: should we allow for this fudge factor? - # TODO: what is the real name of this? num_down_prevent ? - # skip_final_downs? - fudge = 2 - # TODO: should final antialiasing be on? - # Note, if fudge is non-zero it is important to do this. - do_final_aa = 1 - # TODO: should fractional be True or False by default? - # If fudge is 0 and fractional=0, then I think is the same as - # do_final_aa=0. - fractional = 0 - - num_downs = max(int(np.log2(1 / max_scale)) - fudge, 0) - pyr_scale = 1 / (2 ** num_downs) - - # Downsample iteratively with antialiasing - downscaled = _pyrDownK(image, num_downs) - - rest_sx = sx / pyr_scale - rest_sy = sy / pyr_scale - - # Compute the transform from the downsampled image to the destination - rest_warp = noscale_warp @ Affine.scale((rest_sx, rest_sy)) - - # Do a final small blur to acount for the potential aliasing - # in any remaining scaling operations. - if do_final_aa: - # Computed as the closest sigma to the [1, 4, 6, 4, 1] approx - # used in cv2.pyrDown - aa_sigma0 = 1.0565137190917149 - aa_k0 = 5 - k_x, sigma_x = _gauss_params(scale=rest_sx, k0=aa_k0, - sigma0=aa_sigma0, - fractional=fractional) - k_y, sigma_y = _gauss_params(scale=rest_sy, k0=aa_k0, - sigma0=aa_sigma0, - fractional=fractional) - - # Note: when k=1, no blur occurs - # blurBorderType = cv2.BORDER_REPLICATE - # blurBorderType = cv2.BORDER_CONSTANT - blurBorderType = cv2.BORDER_DEFAULT - downscaled = cv2.GaussianBlur( - downscaled, (k_x, k_y), sigma_x, sigma_y, - borderType=blurBorderType - ) - - result = cv2.warpAffine(downscaled, rest_warp.matrix[0:2], - dsize=dsize, flags=flags, - borderMode=borderMode, - borderValue=borderValue) - - return result - - -def _check(): - # Find the sigma closest to the pyrDown op [1, 4, 6, 4, 1] / 16 - import cv2 - import numpy as np - import scipy - import ubelt as ub - def sigma_error(sigma): - sigma = np.asarray(sigma).ravel()[0] - got = (cv2.getGaussianKernel(5, sigma) * 16).ravel() - want = np.array([1, 4, 6, 4, 1]) - loss = ((got - want) ** 2).sum() - return loss - result = scipy.optimize.minimize(sigma_error, x0=1.0, method='Nelder-Mead') - print('result = {}'.format(ub.repr2(result, nl=1))) - - best_loss = float('inf') - best = None - methods = ['Nelder-Mead', 'Powell', 'CG', 'BFGS', - # 'Newton-CG', - 'L-BFGS-B', 'TNC', 'COBYLA', 'SLSQP', - # 'trust-constr', - # 'dogleg', - # 'trust-ncg', 'trust-exact', - # 'trust-krylov' - ] - # results = {} - x0 = 1.06 - for i in range(100): - for method in methods: - best_method_loss, best_method_sigma, best_method_x0 = results.get(method, (float('inf'), 1.06, x0)) - result = scipy.optimize.minimize(sigma_error, x0=x0, method=method) - sigma = np.asarray(result.x).ravel()[0] - loss = sigma_error(sigma) - if loss <= best_method_loss: - results[method] = (loss, sigma, x0) - - best_method = ub.argmin(results) - best_loss, best_sigma = results[best_method][0:2] - rng = np.random - if rng.rand() > 0.5: - x0 = best_sigma - else: - x0 = best_sigma + rng.rand() * 0.0001 - print('best_method = {!r}'.format(best_method)) - print('best_loss = {!r}'.format(best_loss)) - print('best_sigma = {!r}'.format(best_sigma)) - - print('results = {}'.format(ub.repr2(results, nl=1, align=':'))) - print('best_method = {}'.format(ub.repr2(best_method, nl=1))) - print('best_method = {!r}'.format(best_method)) - - sigma_error(1.0565139268118493) - sigma_error(1.0565137190917149) - # scipy.optimize.minimize_scalar(sigma_error, bounds=(1, 1.1)) - - import kwarray - import numpy as np - a = (kwarray.ensure_rng(0).rand(32, 32) * 256).astype(np.uint8) - a = kwimage.ensure_float01(a) - b = cv2.GaussianBlur(a.copy(), (1, 1), 3, 3) - assert np.all(a == b) - - import timerit - ti = timerit.Timerit(100, bestof=10, verbose=2) - for timer in ti.reset('time'): - with timer: - b = cv2.GaussianBlur(a.copy(), (9, 9), 3, 3) - - import timerit - ti = timerit.Timerit(100, bestof=10, verbose=2) - for timer in ti.reset('time'): - with timer: - c = cv2.GaussianBlur(a.copy(), (1, 9), 3, 3) - zR= cv2.GaussianBlur(c.copy(), (9, 1), 3, 3) diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index edd9b74..8220af2 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -388,7 +388,7 @@ class DelayedLoad(DelayedImageOperation): dsize = self.meta.get('dsize', None) if dsize is not None: final = np.asarray(final) - final = kwimage.imresize(final, dsize=dsize) + final = kwimage.imresize(final, dsize=dsize, antialias=True) self.cache['final'] = final as_xarray = kwargs.get('as_xarray', False) @@ -936,43 +936,8 @@ class DelayedWarp(DelayedImageOperation): if dsize == (None, None): dsize = None sub_data_ = np.asarray(sub_data) - # sub_data_ = sub_data_.astype(np.float32) / 255. - # print('flags = {!r}'.format(flags)) - # print('sub_data_.dtype = {!r}'.format(sub_data_.dtype)) - - # TODO: should we blur the source if the determanent of M is less - # than 1? If so by how much - # if kwargs.get('antialias', 0) and interpolation != 'nearest': - # """ - # transform = Affine.scale(0.2) - # See: ~/code/ndsampler/dev/antialias_warp.py - # """ - # # FIXME! This is too slow for large images. - - # # Hacked in heuristic for antialiasing before a downsample - # factor = np.sqrt(transform.det()) - # if factor < 0.99: - # # compute the number of 2x downsamples - # num_downscales = np.log2(1 / factor) - - # # Define b0 = kernel size for one downsample operation - # b0 = 5 - # # Define s0 = sigma for one downsample operation - # s0 = 1 - - # # The kernel size and sigma doubles for each 2x downsample - # k = int(np.ceil(b0 * (2 ** (num_downscales - 1)))) - # sigma = s0 * (2 ** (num_downscales - 1)) - - # if k % 2 == 0: - # k += 1 - - # sub_data_ = sub_data_.copy() - # sub_data_ = cv2.GaussianBlur(sub_data_, (k, k), sigma, sigma) M = np.asarray(transform) - # final = cv2.warpAffine(sub_data_, M[0:2], dsize=dsize, flags=flags) - antialias = kwargs.get('antialias', 1) - # Requires update to kwimage version + antialias = kwargs.get('antialias', True) final = kwimage.warp_affine(sub_data_, M, dsize=dsize, interpolation=interpolation, antialias=antialias) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 1202527..a9735f1 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -23,7 +23,7 @@ pyqtree >= 1.0.0 # There are likely several more undocumented deps, see what fails and submit a PR please! # TODO : fill me in! -kwimage >= 0.7.5 +kwimage >= 0.7.6 kwarray >= 0.5.16 kwcoco >= 0.2.1 -- GitLab From f55bafb36f03a2802f8cd99b15f0d20b10ecd6dc Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 4 Jun 2021 18:03:20 -0400 Subject: [PATCH 7/8] wip --- ndsampler/coco_regions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ndsampler/coco_regions.py b/ndsampler/coco_regions.py index c8f2418..d989117 100644 --- a/ndsampler/coco_regions.py +++ b/ndsampler/coco_regions.py @@ -553,6 +553,9 @@ class CocoRegions(Targets, util_misc.HashIdentifiable, ub.NiceRepr): if task == 'video_detection': sample_grid = new_video_sample_grid(dset, window_dims, window_overlap) + elif task == 'image_detection': + sample_grid = new_image_sample_grid(dset, window_dims, + window_overlap) else: raise NotImplementedError(task) -- GitLab From 68c0217b1467c8b08278a05749217f7c2388a673 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 22 Jun 2021 15:22:54 -0400 Subject: [PATCH 8/8] wip --- README.rst | 18 +++++++++--------- ndsampler/delayed.py | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 177cb50..45cdf7a 100644 --- a/README.rst +++ b/README.rst @@ -4,15 +4,15 @@ ndsampler |GitlabCIPipeline| |GitlabCICoverage| |Pypi| |Downloads| -+------------------+-----------------------------------------------+ -| Read the docs | https://ndsampler.readthedocs.io | -+------------------+-----------------------------------------------+ -| Gitlab (main) | https://gitlab.kitware.com/utils/ndsampler | -+------------------+-----------------------------------------------+ -| Github (mirror) | https://github.com/Kitware/ndsampler | -+------------------+-----------------------------------------------+ -| Pypi | https://pypi.org/project/ndsampler | -+------------------+-----------------------------------------------+ ++------------------+---------------------------------------------------------+ +| Read the docs | https://ndsampler.readthedocs.io | ++------------------+---------------------------------------------------------+ +| Gitlab (main) | https://gitlab.kitware.com/computer-vision/ndsampler | ++------------------+---------------------------------------------------------+ +| Github (mirror) | https://github.com/Kitware/ndsampler | ++------------------+---------------------------------------------------------+ +| Pypi | https://pypi.org/project/ndsampler | ++------------------+---------------------------------------------------------+ The main webpage for this project is: https://gitlab.kitware.com/computer-vision/ndsampler diff --git a/ndsampler/delayed.py b/ndsampler/delayed.py index 8220af2..f92cdf1 100644 --- a/ndsampler/delayed.py +++ b/ndsampler/delayed.py @@ -1,4 +1,7 @@ """ +DEPRECATD. THIS IS BEING MOVED TO KWCOCO FOR DEVELOPMENT AND EVENTUALLY WILL +LIVE IN KWIMAGE. + The classes in this file represent a tree of delayed operations. Proof of concept for delayed chainable transforms in Python. -- GitLab