From b513f8cb762bbc79864f00ec698050c9661fbbda Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 1 Apr 2025 19:43:54 -0400 Subject: [PATCH 01/10] Start branch for 0.8.2 --- CHANGELOG.md | 5 ++++- ndsampler/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d0fcd8..f0b9ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,10 @@ This changelog follows the specifications detailed in: [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), although we have not yet reached a `1.0.0` release. -## Version 0.8.1 - Unreleased +## Version 0.8.2 - Unreleased + + +## Version 0.8.1 - Released 2025-04-01 ### Added * Added `finalize` argument to `CocoSampler.load_sample` which allows the diff --git a/ndsampler/__init__.py b/ndsampler/__init__.py index 1ae47dd..c5fa677 100644 --- a/ndsampler/__init__.py +++ b/ndsampler/__init__.py @@ -19,7 +19,7 @@ __autogen__ = """ mkinit ~/code/ndsampler/ndsampler/__init__.py --diff mkinit ~/code/ndsampler/ndsampler/__init__.py -w """ -__version__ = '0.8.1' +__version__ = '0.8.2' from ndsampler.utils.util_misc import (HashIdentifiable,) -- GitLab From 067cba2f90c7cf9ec3b00548c2d16160dce605a1 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 1 Apr 2025 19:51:02 -0400 Subject: [PATCH 02/10] chore: remove profile decorators --- ndsampler/abstract_frames.py | 12 ------------ ndsampler/coco_frames.py | 7 ------- ndsampler/coco_regions.py | 10 ---------- ndsampler/coco_sampler.py | 12 +----------- ndsampler/frame_cache.py | 1 - ndsampler/isect_indexer.py | 7 ------- ndsampler/utils/util_gdal.py | 8 -------- 7 files changed, 1 insertion(+), 56 deletions(-) diff --git a/ndsampler/abstract_frames.py b/ndsampler/abstract_frames.py index 8f6d9cd..7fea525 100644 --- a/ndsampler/abstract_frames.py +++ b/ndsampler/abstract_frames.py @@ -21,12 +21,6 @@ from ndsampler.utils import util_lru from ndsampler.frame_cache import (_ensure_image_cog, _ensure_image_npy) -try: - from xdev import profile -except Exception: - profile = ub.identity - - class Frames(object): """ Abstract implementation of Frames. @@ -339,7 +333,6 @@ class Frames(object): image_id = self.image_ids[index] return self.load_image(image_id) - @profile def load_region(self, image_id, region=None, channels=ub.NoParam, width=None, height=None): """ @@ -390,7 +383,6 @@ class Frames(object): _lru[image_id] = imgdata return imgdata - @profile def load_image(self, image_id, channels=ub.NoParam, cache=True, noreturn=False): """ @@ -651,7 +643,6 @@ class AlignableImageData(object): self.cache_backend = cache_backend self._channel_memcache = {} - @profile def _load_native_channel(self, chan_name, cache=True): """ Load a specific auxiliary channel, optionally caching it @@ -689,7 +680,6 @@ class AlignableImageData(object): _channel_memcache[cache_key] = data return data - @profile def _load_delayed_channel(self, chan_name, cache=True): height = self.pathinfo.get('height', None) width = self.pathinfo.get('width', None) @@ -726,7 +716,6 @@ class AlignableImageData(object): channels = [default_chan] return channels - @profile def _load_prefused_region(self, img_region, channels=ub.NoParam): """ Loads crops from multiple channels in their native coordinate system @@ -773,7 +762,6 @@ class AlignableImageData(object): } return prefused - @profile def _load_fused_region(self, img_region, channels=ub.NoParam): """ Loads crops from multiple channels in aligned base coordinates. diff --git a/ndsampler/coco_frames.py b/ndsampler/coco_frames.py index a6592f1..58b9f61 100644 --- a/ndsampler/coco_frames.py +++ b/ndsampler/coco_frames.py @@ -5,12 +5,6 @@ import warnings import ubelt as ub -try: - from xdev import profile -except Exception: - profile = ub.identity - - class CocoFrames(abstract_frames.Frames, util_misc.HashIdentifiable): """ wrapper around coco-style dataset to allow for getitem syntax @@ -69,7 +63,6 @@ class CocoFrames(abstract_frames.Frames, util_misc.HashIdentifiable): return super().load_region(image_id, region, width=width, height=height, channels=channels) - @profile def _build_pathinfo(self, image_id): """ Returns: diff --git a/ndsampler/coco_regions.py b/ndsampler/coco_regions.py index 3bc657b..735d19c 100644 --- a/ndsampler/coco_regions.py +++ b/ndsampler/coco_regions.py @@ -28,12 +28,6 @@ from ndsampler.utils import util_misc from ndsampler import isect_indexer -try: - from xdev import profile -except Exception: - profile = ub.identity - - class MissingNegativePool(AssertionError): pass @@ -213,7 +207,6 @@ class CocoRegions(Targets, util_misc.HashIdentifiable, ub.NiceRepr): """ return self._lazy_isect_index() - @profile def _lazy_isect_index(self, verbose=None): if self._isect_index is None: # FIXME! Any use of cacher here should be wrapped in an @@ -295,7 +288,6 @@ class CocoRegions(Targets, util_misc.HashIdentifiable, ub.NiceRepr): self._neg_anchors = neg_anchors return self._neg_anchors - @profile def overlapping_aids(self, gid, region, visible_thresh=0.0): """ Finds the other annotations in this image that overlap a region @@ -747,7 +739,6 @@ class CocoRegions(Targets, util_misc.HashIdentifiable, ub.NiceRepr): return cacher -@profile def tabular_coco_targets(dset): """ Transforms COCO box annotations into a tabular form @@ -818,7 +809,6 @@ def tabular_coco_targets(dset): return targets -@profile def select_positive_regions(targets, window_dims=(300, 300), thresh=0.0, rng=None, verbose=0): """ diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index 5f167d0..e0b3890 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -82,11 +82,6 @@ from ndsampler.utils import util_misc from delayed_image.channel_spec import FusedChannelSpec from delayed_image.channel_spec import ChannelSpec -try: - from xdev import profile -except Exception: - profile = ub.identity - class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, ub.NiceRepr): @@ -908,7 +903,6 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, target_['vidid'] = target_['video_id'] return target_ - @profile def _infer_target_attributes(self, target, **kwargs): """ Infer unpopulated target attributes @@ -1104,13 +1098,12 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, raise NotImplementedError(ndim) return target_ - @profile def _load_slice(self, target_): """ Called by load_sample after the target dictionary has been resolved. CommandLine: - xdoctest -m ndsampler.coco_sampler CocoSampler._load_slice --profile + xdoctest -m ndsampler.coco_sampler CocoSampler._load_slice Example: >>> # sample an out of bounds target @@ -1247,7 +1240,6 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, sample['tr'] = sample['target'] return sample - @profile def _load_slice_3d(self, target_): """ Breakout the 2d vs 3d logic so they can evolve somewhat independently. @@ -1644,7 +1636,6 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, sample['params']['jagged_meta'] = jagged_meta return sample - @profile def _load_slice_2d(self, target): """ Breakout the 2d vs 3d logic so they can evolve somewhat independently. @@ -1777,7 +1768,6 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, } return sample - @profile def _populate_overlap(self, sample, visible_thresh=0.1, with_annots=True, annot_ids=None): """ diff --git a/ndsampler/frame_cache.py b/ndsampler/frame_cache.py index 0348b82..bf9df65 100644 --- a/ndsampler/frame_cache.py +++ b/ndsampler/frame_cache.py @@ -24,7 +24,6 @@ class CorruptCOG(Exception): pass -# @profile def _cog_cache_write(gpath, cache_gpath, config=None): """ CommandLine: diff --git a/ndsampler/isect_indexer.py b/ndsampler/isect_indexer.py index 4642d05..f8622ba 100644 --- a/ndsampler/isect_indexer.py +++ b/ndsampler/isect_indexer.py @@ -17,11 +17,6 @@ except ImportError: import pyqtree rtree = None -try: - from xdev import profile -except Exception: - profile = ub.identity - class FrameIntersectionIndex(ub.NiceRepr): """ @@ -89,7 +84,6 @@ class FrameIntersectionIndex(ub.NiceRepr): return self @staticmethod - @profile def _build_index(dset, verbose=0): """ """ @@ -157,7 +151,6 @@ class FrameIntersectionIndex(ub.NiceRepr): qtree.aid_to_ltrb[aid] = ltrb_box return qtrees - @profile def overlapping_aids(self, gid, box): """ Find all annotation-ids within an image that have some overlap with a diff --git a/ndsampler/utils/util_gdal.py b/ndsampler/utils/util_gdal.py index f6a2b98..f7904d5 100644 --- a/ndsampler/utils/util_gdal.py +++ b/ndsampler/utils/util_gdal.py @@ -4,13 +4,6 @@ import numpy as np import ubelt as ub -try: - import xdev - profile = xdev.profile -except ImportError: - profile = ub.identity - - def have_gdal(): try: from osgeo import gdal @@ -133,7 +126,6 @@ def _benchmark_cog_conversions(): assert not len(validate(dst_data_fpath)[1]) -@profile def _imwrite_cloud_optimized_geotiff(fpath, data, compress='auto', blocksize=256): """ -- GitLab From 7502fc502100ace7919f5cf09f7b32a38e681d05 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 1 Apr 2025 19:53:24 -0400 Subject: [PATCH 03/10] Removed unused util future code --- CHANGELOG.md | 3 + ndsampler/utils/util_futures.py | 165 +------------------------------- 2 files changed, 7 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b9ff4..f0aa04c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.8.2 - Unreleased +### Deprecated +* Unused code in `utils/util_futures.py` + ## Version 0.8.1 - Released 2025-04-01 diff --git a/ndsampler/utils/util_futures.py b/ndsampler/utils/util_futures.py index 966fabb..aaa4f1d 100644 --- a/ndsampler/utils/util_futures.py +++ b/ndsampler/utils/util_futures.py @@ -1,166 +1,9 @@ """ -TODO: Use the ubelt variants of these instead. +DEPRECATED + +Use the ubelt variants of these instead. """ -import concurrent.futures from concurrent.futures import as_completed +from ubelt import Executor, SerialExecutor __all__ = ['Executor', 'SerialExecutor', 'as_completed'] - - -# class FakeCondition(object): -# def acquire(self): -# pass - -# def release(self): -# pass - - -class SerialFuture( concurrent.futures.Future): - """ - Non-threading / multiprocessing version of future for drop in compatibility - with concurrent.futures. - """ - def __init__(self, func, *args, **kw): - super(SerialFuture, self).__init__() - self.func = func - self.args = args - self.kw = kw - # self._condition = FakeCondition() - self._run_count = 0 - # fake being finished to cause __get_result to be called - self._state = concurrent.futures._base.FINISHED - - def _run(self): - result = self.func(*self.args, **self.kw) - self.set_result(result) - self._run_count += 1 - - def set_result(self, result): - """ - Overrides the implementation to revert to pre python3.8 behavior - """ - with self._condition: - self._result = result - self._state = concurrent.futures._base.FINISHED - for waiter in self._waiters: - waiter.add_result(self) - self._condition.notify_all() - self._invoke_callbacks() - - def _Future__get_result(self): - # overrides private __getresult method - if not self._run_count: - self._run() - return self._result - - -class SerialExecutor(object): - """ - Implements the concurrent.futures API around a single-threaded backend - - Example: - >>> with SerialExecutor() as executor: - >>> futures = [] - >>> for i in range(100): - >>> f = executor.submit(lambda x: x + 1, i) - >>> futures.append(f) - >>> for f in concurrent.futures.as_completed(futures): - >>> assert f.result() > 0 - >>> for i, f in enumerate(futures): - >>> assert i + 1 == f.result() - """ - def __enter__(self): - self.max_workers = 0 - return self - - def __exit__(self, ex_type, ex_value, tb): - pass - - def submit(self, func, *args, **kw): - return SerialFuture(func, *args, **kw) - - def shutdown(self): - pass - - -class Executor(object): - """ - Wrapper around a specific executor. - - Abstracts Serial, Thread, and Process Executor via arguments. - - Args: - mode (str, default='thread'): either thread, serial, or process - max_workers (int, default=0): number of workers. If 0, serial is forced. - """ - - def __init__(self, mode='thread', max_workers=0): - from concurrent import futures - if mode == 'serial' or max_workers == 0: - backend = SerialExecutor() - elif mode == 'thread': - backend = futures.ThreadPoolExecutor(max_workers=max_workers) - elif mode == 'process': - backend = futures.ProcessPoolExecutor(max_workers=max_workers) - else: - raise KeyError(mode) - self.backend = backend - - def __enter__(self): - return self.backend.__enter__() - - def __exit__(self, ex_type, ex_value, tb): - return self.backend.__exit__(ex_type, ex_value, tb) - - def submit(self, func, *args, **kw): - return self.backend.submit(func, *args, **kw) - - def shutdown(self): - return self.backend.shutdown() - - -class JobPool(object): - """ - Abstracts away boilerplate of submitting and collecting jobs - - Example: - >>> def worker(data): - >>> return data + 1 - >>> pool = JobPool('thread', max_workers=16) - >>> import ubelt as ub - >>> with pool: - >>> for data in ub.ProgIter(range(10), desc='submit jobs'): - >>> job = pool.submit(worker, data) - >>> final = [] - >>> for job in ub.ProgIter(pool.as_completed(), total=len(pool), desc='collect jobs'): - >>> info = job.result() - >>> final.append(info) - >>> print('final = {!r}'.format(final)) - """ - def __init__(self, mode='thread', max_workers=0): - self.executor = Executor(mode=mode, max_workers=max_workers) - self.jobs = [] - - def __len__(self): - return len(self.jobs) - - def submit(self, func, *args, **kwargs): - job = self.executor.submit(func, *args, **kwargs) - self.jobs.append(job) - return job - - def __enter__(self): - self.executor.__enter__() - return self - - def __exit__(self, a, b, c): - self.executor.__exit__(a, b, c) - - def as_completed(self): - from concurrent.futures import as_completed - for job in as_completed(self.jobs): - yield job - - def __iter__(self): - for job in self.as_completed(): - yield job -- GitLab From ab66c899169b7ded01919a145be165074236daab Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 1 Apr 2025 20:17:01 -0400 Subject: [PATCH 04/10] deprecations and cleanup --- ndsampler/category_tree.py | 2 -- ndsampler/coco_dataset.py | 2 -- ndsampler/coco_sampler.py | 1 - ndsampler/coerce_data.py | 10 ++++++++-- ndsampler/toydata.py | 3 +++ ndsampler/utils/util_shape.py | 8 ++++++++ 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ndsampler/category_tree.py b/ndsampler/category_tree.py index 24e9d0c..60c4715 100644 --- a/ndsampler/category_tree.py +++ b/ndsampler/category_tree.py @@ -17,8 +17,6 @@ import kwarray import functools import networkx as nx import ubelt as ub -# import torch -# import torch.nn.functional as F import numpy as np from kwcoco import CategoryTree as KWCOCO_CategoryTree # raw category tree diff --git a/ndsampler/coco_dataset.py b/ndsampler/coco_dataset.py index e2b0196..a342c61 100644 --- a/ndsampler/coco_dataset.py +++ b/ndsampler/coco_dataset.py @@ -1,7 +1,5 @@ """ This module has moded to the kwcoco module - -mkinit kwcoco """ from kwcoco.coco_dataset import CocoDataset __all__ = ['CocoDataset'] diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index e0b3890..edb067d 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -101,7 +101,6 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, details. Defaults to None, which does not do anything fancy. Example: - #print >>> from ndsampler.coco_sampler import * >>> self = CocoSampler.demo('photos') ... diff --git a/ndsampler/coerce_data.py b/ndsampler/coerce_data.py index fb64eb7..3857943 100644 --- a/ndsampler/coerce_data.py +++ b/ndsampler/coerce_data.py @@ -1,5 +1,5 @@ """ -Moved to netharn +DEPRECATED. This module has moved to netharn. """ import ubelt as ub @@ -25,7 +25,7 @@ def coerce_datasets(config, build_hashid=False, verbose=1): >>> dsets = ndsampler.coerce_data.coerce_datasets(config) >>> print('dsets = {!r}'.format(dsets)) - >>> config = {'datasets': 'special:shapes256'} + >>> config = {'datasets': 'special:shapes8'} >>> ndsampler.coerce_data.coerce_datasets(config) >>> config = { @@ -41,6 +41,12 @@ def coerce_datasets(config, build_hashid=False, verbose=1): >>> 'test_dataset': kwcoco.CocoDataset.demo('photos'), >>> }) """ + ub.schedule_deprecation( + modname='ndsampler', name='coerce_datasets', type='function', + migration='This is unused in ndsampler and will be removed. ' + 'Vendor the function if you need it.', + deprecate='0.8.2', error='0.9.0', remove='1.0.0') + # Ideally the user specifies a standard train/vali/test split def _rectify_fpath(key): fpath = key diff --git a/ndsampler/toydata.py b/ndsampler/toydata.py index 7928c95..a1ecad8 100644 --- a/ndsampler/toydata.py +++ b/ndsampler/toydata.py @@ -1,3 +1,6 @@ +""" +DEPRECATE: this functionality has moved to kwcoco +""" import numpy as np from ndsampler import abstract_sampler from ndsampler import category_tree diff --git a/ndsampler/utils/util_shape.py b/ndsampler/utils/util_shape.py index ccada52..b621dff 100644 --- a/ndsampler/utils/util_shape.py +++ b/ndsampler/utils/util_shape.py @@ -5,12 +5,20 @@ def nestshape(data): """ Examine nested shape of the data + CommandLine: + xdoctest -m ndsampler.utils.util_shape nestshape + Example: >>> data = [np.arange(10), np.arange(13)] >>> nestshape(data) [(10,), (13,)] """ import ubelt as ub + ub.schedule_deprecation( + modname='ndsampler', name='netshape', type='function', + migration='This is unused in ndsampler and will be removed. ' + 'Vendor the function if you need it.', + deprecate='0.8.2', error='0.9.0', remove='1.0.0') def _recurse(d): try: -- GitLab From 3b50efa21ea8a542ce0a631764df4bf55462aa42 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 1 Apr 2025 20:42:28 -0400 Subject: [PATCH 05/10] Cleanup docs --- README.rst | 9 +++++++++ docs/source/conf.py | 2 ++ ndsampler/__init__.py | 6 +++++- ndsampler/abstract_frames.py | 14 ++++---------- ndsampler/coco_regions.py | 2 -- ndsampler/coco_sampler.py | 23 ++++++++++++++++++++--- ndsampler/utils/util_futures.py | 3 ++- ndsampler/utils/util_misc.py | 27 ++++++++++++++------------- 8 files changed, 56 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 6fe42f1..d6303f9 100644 --- a/README.rst +++ b/README.rst @@ -38,6 +38,15 @@ data is to access. However a faster cache can be built at the cost of disk space. Currently we have a "cog" and "npy" backend. Help is wanted to integrate backends for hdf5 and other medical / domain-specific formats. + +This module is best used with +`kwcoco `_, +`kwimage `_, and +`delayed-image `_. +It enables other modules like: +`kwcoco_dataloader `_. + + Installation ------------ diff --git a/docs/source/conf.py b/docs/source/conf.py index 1070cd4..9e92e2f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -220,6 +220,8 @@ intersphinx_mapping = { # Requires that the repo have objects.inv 'kwarray': ('https://kwarray.readthedocs.io/en/latest/', None), 'kwimage': ('https://kwimage.readthedocs.io/en/latest/', None), + 'kwcoco': ('https://kwcoco.readthedocs.io/en/latest/', None), + 'delayed_image': ('https://delayed-image.readthedocs.io/en/latest/', None), # 'kwplot': ('https://kwplot.readthedocs.io/en/latest/', None), 'ndsampler': ('https://ndsampler.readthedocs.io/en/latest/', None), 'ubelt': ('https://ubelt.readthedocs.io/en/latest/', None), diff --git a/ndsampler/__init__.py b/ndsampler/__init__.py index c5fa677..34bef68 100644 --- a/ndsampler/__init__.py +++ b/ndsampler/__init__.py @@ -11,8 +11,12 @@ The ndsampler library | Pypi | https://pypi.org/project/ndsampler | +------------------+---------------------------------------------------------+ +For a quickstart see :mod:`ndsampler.coco_sampler`. -See the gitlab README for more details. +This module is best used with :mod:`kwcoco`, :mod:`kwimage`, and +:mod:`delayed_image`. + +See the `GitLab README `_ for more details. """ __autogen__ = """ diff --git a/ndsampler/abstract_frames.py b/ndsampler/abstract_frames.py index 7fea525..00dc3ec 100644 --- a/ndsampler/abstract_frames.py +++ b/ndsampler/abstract_frames.py @@ -3,13 +3,6 @@ Fast access to subregions of images. This implements the core convert-and-cache-as-cog logic, which enables us to read from subregions of images quickly. - - -TODO: - - [X] Implement npy memmap backend - - [X] Implement gdal COG.TIFF backend - - [X] Use as COG if input file is a COG - - [X] Convert to COG if needed """ import numpy as np import ubelt as ub @@ -388,6 +381,10 @@ class Frames(object): """ Load the image data for a particular image id + Note: + CAREFUL: THIS NEEDS TO MAINTAIN A STABLE API. + OTHER PROJECTS DEPEND ON IT. + Args: image_id (int): the id of the image to load cache (bool, default=True): ensure and return the efficient backend @@ -398,9 +395,6 @@ class Frames(object): This is useful if you simply want to ensure the cached representation. - CAREFUL: THIS NEEDS TO MAINTAIN A STABLE API. - OTHER PROJECTS DEPEND ON IT. - Returns: ArrayLike: an indexable array like representation, possibly memmapped. diff --git a/ndsampler/coco_regions.py b/ndsampler/coco_regions.py index 735d19c..5de7e9f 100644 --- a/ndsampler/coco_regions.py +++ b/ndsampler/coco_regions.py @@ -742,8 +742,6 @@ class CocoRegions(Targets, util_misc.HashIdentifiable, ub.NiceRepr): def tabular_coco_targets(dset): """ Transforms COCO box annotations into a tabular form - - _ = xdev.profile_now(tabular_coco_targets)(dset) """ import warnings # TODO: better handling of non-bounding box annotations; ignore for now diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index edb067d..463f9f0 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -1,6 +1,7 @@ """ The CocoSampler is the ndsampler interface for efficiently sampling windowed -data from a :class:`kwcoco.CocoDataset`. +data from a :class:`kwcoco.CocoDataset`. The following example illustrates +basic usage: CommandLine: xdoctest -m ndsampler.coco_sampler __doc__ --show @@ -220,6 +221,15 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, """ DEPRECATED, use self.classes instead """ + ub.schedule_deprecation( + 'ndsampler', + 'catgraph', + 'property' + 'Use .classes instead', + deprecate='0.8.2', + error='0.9.0', + remove='1.0.0', + ) if self.regions is None: return None return self.regions.classes @@ -366,8 +376,9 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, keypoints, and segmentation. Defaults to True. target (Dict): Extra target arguments that update the positive target, - like window_dims, pad, etc.... See :func:`load_sample` for - details on allowed keywords. + like window_dims, pad, etc... See + :func:`CocoSampler.load_sample` for details on allowed + keywords. rng (None | int | RandomState): a seed or seeded random number generator. @@ -376,11 +387,14 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, Returns: Dict: sample: dict containing keys + im (ndarray): image data + target (dict): contains the same input items as the input target but additionally specifies inferred information like rel_cx and rel_cy, which gives the center of the target w.r.t the returned **padded** sample. + annots (dict): Dict of aids, cids, and rel/abs boxes """ if index < self.n_positives: @@ -418,10 +432,13 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, Returns: Dict: sample: dict containing keys + im (ndarray): image data + tr (dict): contains the same input items as tr but additionally specifies rel_cx and rel_cy, which gives the center of the target w.r.t the returned **padded** sample. + annots (dict): Dict of aids, cids, and rel/abs boxes Example: diff --git a/ndsampler/utils/util_futures.py b/ndsampler/utils/util_futures.py index aaa4f1d..bf417cd 100644 --- a/ndsampler/utils/util_futures.py +++ b/ndsampler/utils/util_futures.py @@ -4,6 +4,7 @@ DEPRECATED Use the ubelt variants of these instead. """ from concurrent.futures import as_completed -from ubelt import Executor, SerialExecutor +from ubelt.util_futures import SerialExecutor +from ubelt import Executor __all__ = ['Executor', 'SerialExecutor', 'as_completed'] diff --git a/ndsampler/utils/util_misc.py b/ndsampler/utils/util_misc.py index 7095dcd..110675c 100644 --- a/ndsampler/utils/util_misc.py +++ b/ndsampler/utils/util_misc.py @@ -12,19 +12,20 @@ class HashIdentifiable(object): * define `_hashid` Example: - class Base: - def __init__(self): - # commenting the next line removes cooperative inheritance - super().__init__() - self.base = 1 - - class Derived(Base, HashIdentifiable): - def __init__(self): - super().__init__() - self.defived = 1 - - self = Derived() - dir(self) + >>> from ndsampler.utils.util_misc import * # NOQA + >>> class Base: + >>> def __init__(self): + >>> # commenting the next line removes cooperative inheritance + >>> super().__init__() + >>> self.base = 1 + >>> # + >>> class Derived(Base, HashIdentifiable): + >>> def __init__(self): + >>> super().__init__() + >>> self.defived = 1 + >>> # + >>> self = Derived() + >>> assert {'_depends', 'hashid', '_make_hashid'}.issubset(set(dir(self))) """ def __init__(self, **kwargs): super(HashIdentifiable, self).__init__(**kwargs) -- GitLab From 025b28412bcd8b7e1990d04214408a4d5b0793b6 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 8 Apr 2025 11:24:43 -0400 Subject: [PATCH 06/10] doc: README RST error --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index d6303f9..eff9b60 100644 --- a/README.rst +++ b/README.rst @@ -40,11 +40,11 @@ backends for hdf5 and other medical / domain-specific formats. This module is best used with -`kwcoco `_, -`kwimage `_, and -`delayed-image `_. +`kwcoco `__, +`kwimage `__, and +`delayed-image `__. It enables other modules like: -`kwcoco_dataloader `_. +`kwcoco_dataloader `__. Installation @@ -57,9 +57,9 @@ The `ndsampler `_. package can be installe pip install ndsampler -Note that ndsampler depends on `kwimage `_, -where there is a known compatibility issue between `opencv-python `_ -and `opencv-python-headless `_. Please ensure that one +Note that ndsampler depends on `kwimage `__, +where there is a known compatibility issue between `opencv-python `__ +and `opencv-python-headless `__. Please ensure that one or the other (but not both) are installed as well: .. code-block:: bash -- GitLab From 963a3ab541b189c65851b19220b532d918d99dea Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 10 Jun 2025 18:39:54 -0400 Subject: [PATCH 07/10] Fix: issue when sampling an entire frame --- CHANGELOG.md | 3 +++ ndsampler/coco_sampler.py | 4 +++ tests/test_sampling_shapes.py | 49 +++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 tests/test_sampling_shapes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f0aa04c..142f420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Deprecated * Unused code in `utils/util_futures.py` +### Fixed +* Bug where requesting a slice the same size as an grayscale image did not return a channel dimension + ## Version 0.8.1 - Released 2025-04-01 diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index 463f9f0..907047b 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -1606,6 +1606,10 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, frame = frame.astype(dtype) else: frame = to_finalize + + # workaround bug where channel dimension is not always returned + # from delayed image if there is no crop operation. + frame = kwarray.atleast_nd(frame, 3) space_frames.append(frame) # warp_sample_from_grid_alt = delayed_crop.get_transform_from(delayed_frame) diff --git a/tests/test_sampling_shapes.py b/tests/test_sampling_shapes.py new file mode 100644 index 0000000..04d0cf1 --- /dev/null +++ b/tests/test_sampling_shapes.py @@ -0,0 +1,49 @@ +def test_whole_frame_sample_shape(): + """ + <0.8.2 had a bug where requesting a full frame did not return data with a + channel dimension. + """ + import kwcoco + import ndsampler + + dset = kwcoco.CocoDataset.demo('vidshapes1', image_size=128, num_frames=1, sensorchan='gray') + + # Hack away any transforms so we can access the data in its full dims easy + # (todo: would be nice if kwcoco had a way to not generate asset to image warps) + coco_img = dset.coco_image(1) + for asset in coco_img.assets: + asset.pop('warp_aux_to_img', None) + W = coco_img.img['width'] = asset['width'] + H = coco_img.img['height'] = asset['height'] + + sampler = ndsampler.CocoSampler(dset) + + # Sample is the full image frame, no cropping. Previously this failed. + target = { + 'space_slice': (slice(0, H), slice(0, W)), + 'gids': [1], + 'channels': 'gray', + 'verbose_ndsample': 0, + } + # image_fpath = dset.coco_image(1).primary_image_filepath() + # print(f'image_fpath={image_fpath}') + sample = sampler.load_sample(target, with_annots=False) + assert sample['im'].shape == (1, H, W, 1) + + # Also test case where the shape is less + target = { + 'space_slice': (slice(0, H - 2), slice(0, W - 2)), + 'gids': [1], + 'channels': 'gray', + } + sample = sampler.load_sample(target, with_annots=False) + assert sample['im'].shape == (1, H - 2, W - 2, 1) + + # Also test case where the shape is more + target = { + 'space_slice': (slice(0, H + 2), slice(0, W + 2)), + 'gids': [1], + 'channels': 'gray', + } + sample = sampler.load_sample(target, with_annots=False) + assert sample['im'].shape == (1, H + 2, W + 2, 1) -- GitLab From 530b2d5c0fd11978780ab84865cbc543c9c1fa88 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 11 Jun 2025 13:37:35 -0400 Subject: [PATCH 08/10] fix shape fix with xarray --- ndsampler/coco_sampler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ndsampler/coco_sampler.py b/ndsampler/coco_sampler.py index 907047b..55b0392 100644 --- a/ndsampler/coco_sampler.py +++ b/ndsampler/coco_sampler.py @@ -1609,7 +1609,8 @@ class CocoSampler(abstract_sampler.AbstractSampler, util_misc.HashIdentifiable, # workaround bug where channel dimension is not always returned # from delayed image if there is no crop operation. - frame = kwarray.atleast_nd(frame, 3) + if len(frame.shape) < 3: + frame = frame[..., None] space_frames.append(frame) # warp_sample_from_grid_alt = delayed_crop.get_transform_from(delayed_frame) -- GitLab From 35fb9afad67cf2d2acb523f4f726d950a1b61289 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 11 Jun 2025 14:17:16 -0400 Subject: [PATCH 09/10] Fix tests --- tests/test_sampling_shapes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_sampling_shapes.py b/tests/test_sampling_shapes.py index 04d0cf1..48b46ca 100644 --- a/tests/test_sampling_shapes.py +++ b/tests/test_sampling_shapes.py @@ -5,6 +5,11 @@ def test_whole_frame_sample_shape(): """ import kwcoco import ndsampler + import pytest + try: + import lark # NOQA + except ImportError: + pytest.skip('requires lark') dset = kwcoco.CocoDataset.demo('vidshapes1', image_size=128, num_frames=1, sensorchan='gray') -- GitLab From c435516a388f3b639b820b5e3e0701f08d119579 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 11 Jun 2025 14:51:58 -0400 Subject: [PATCH 10/10] require typing extensions for xarray --- requirements/runtime.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index fa478e8..03cff06 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -7,6 +7,9 @@ parse >= 1.19.0 xarray>=2023.10.0 ; python_version < '4.0' and python_version >= '3.12' # Python 3.13+ xarray>=0.17.0 ; python_version < '3.12' # Python 3.12- +# Required by xarray==2025.6.0 +typing_extensions>=4.8.0 + # tensorflow requires 1.19.3 numpy>=2.1.0 ; python_version < '4.0' and python_version >= '3.13' # Python 3.13+ numpy>=1.26.0 ; python_version < '3.13' and python_version >= '3.12' # Python 3.12 -- GitLab