diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..f916c00 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,42 @@ +name: Unit Tests + +on: + push: + branches: [master] # Only run push on default branch after merge + pull_request: # Run on all PRs + +# Cancel in-progress runs when a new commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 3.6 and 3.7 not supported on free runners Ubuntu >= 24.04 + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + tests/python-lib/requirements.txt + code-env/python/spec/requirements.txt + + - name: Install dependencies + run: | + pip install -r tests/python-lib/requirements.txt + pip install -r code-env/python/spec/requirements.txt + + - name: Run tests + env: + PYTHONPATH: python-lib + run: pytest tests/python-lib -v \ No newline at end of file diff --git a/.gitignore b/.gitignore index dff3c38..8a51b5d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ .htmlcov .DS_Store .idea +/dist +AGENTS.md \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..fe32571 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## Version 0.1.8 +- Support for python 3.9, 3.10, 3.11 and 3.12 \ No newline at end of file diff --git a/Makefile b/Makefile index bf57efb..f136236 100644 --- a/Makefile +++ b/Makefile @@ -45,3 +45,76 @@ tests: unit-tests integration-tests dist-clean: rm -rf dist% + +# Docker-based unit tests for Linux environment +# Uses --platform linux/amd64 to ensure consistent behavior on Apple Silicon +# +# Usage: +# make docker-test-py310 # Without cache (default, CI) +# make docker-test-py310 USE_CACHE=true # With pip cache (faster local iteration) + +DOCKER_IMAGE_NAME=object-detection-cpu +DOCKER_PLATFORM=linux/amd64 +USE_CACHE?=false + +define run-docker-test + @echo "[START] Running unit tests in Docker with Python $(1) (USE_CACHE=$(USE_CACHE))..." + @docker build \ + --platform $(DOCKER_PLATFORM) \ + --build-arg PYTHON_VERSION=$(1) \ + --build-arg USE_CACHE=$(USE_CACHE) \ + -t $(DOCKER_IMAGE_NAME):py$(1) \ + -f tests/docker/Dockerfile \ + . && \ + docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_IMAGE_NAME):py$(1) + @echo "[DONE] Python $(1) tests completed" +endef + + +docker-clean: + @echo "Removing Docker test images..." + @docker rmi -f $(DOCKER_IMAGE_NAME):py3.6 $(DOCKER_IMAGE_NAME):py3.7 $(DOCKER_IMAGE_NAME):py3.8 $(DOCKER_IMAGE_NAME):py3.9 $(DOCKER_IMAGE_NAME):py3.10 $(DOCKER_IMAGE_NAME):py3.11 $(DOCKER_IMAGE_NAME):py3.12 2>/dev/null || true + @echo "Docker images cleaned" + +docker-test-py36: + $(call run-docker-test,3.6) + +docker-test-py39: + $(call run-docker-test,3.9) + +docker-test-py310: + $(call run-docker-test,3.10) + +docker-test-py311: + $(call run-docker-test,3.11) + +docker-test-py312: + $(call run-docker-test,3.12) + + + +# Run all tests with summary (continues on failure, reports at end) +PYTHON_VERSIONS = 3.6 3.9 3.10 3.11 3.12 + +docker-test-all: + @failed=""; passed=""; \ + for ver in $(PYTHON_VERSIONS); do \ + echo ""; \ + echo "############################################"; \ + echo "# Testing Python $$ver"; \ + echo "############################################"; \ + target="docker-test-py$$(echo $$ver | tr -d '.')"; \ + if $(MAKE) $$target USE_CACHE=$(USE_CACHE); then \ + passed="$$passed $$ver"; \ + else \ + failed="$$failed $$ver"; \ + fi; \ + done; \ + echo ""; \ + echo "############################################"; \ + echo "# TEST SUMMARY"; \ + echo "############################################"; \ + if [ -n "$$passed" ]; then echo "PASSED:$$passed"; fi; \ + if [ -n "$$failed" ]; then echo "FAILED:$$failed"; fi; \ + echo "############################################"; \ + if [ -n "$$failed" ]; then exit 1; fi \ No newline at end of file diff --git a/code-env/python/desc.json b/code-env/python/desc.json index 6683264..a011d5e 100644 --- a/code-env/python/desc.json +++ b/code-env/python/desc.json @@ -1,6 +1,7 @@ { - "acceptedPythonInterpreters": ["PYTHON36"], + "acceptedPythonInterpreters": ["PYTHON36", "PYTHON39", "PYTHON310", "PYTHON311", "PYTHON312"], + "corePackagesSet": "AUTO", + "forceConda": false, "installCorePackages": true, - "installJupyterSupport": false, - "conda": false + "installJupyterSupport": false } \ No newline at end of file diff --git a/code-env/python/spec/requirements.txt b/code-env/python/spec/requirements.txt index 10719a9..8316c7a 100644 --- a/code-env/python/spec/requirements.txt +++ b/code-env/python/spec/requirements.txt @@ -1,9 +1,12 @@ -keras==2.2.4 -h5py==2.10.0 -opencv-python==3.4.0.12 -Pillow>=5.1 +# TensorFlow 2.x runtime for Python 3.9-3.12. +tensorflow==2.16.2 +tf-keras==2.16.0 + +h5py>=3.10,<4 +opencv-python>=4.9,<5 +Pillow>=10,<12 +scikit-learn>=1.3,<2 +boto3>=1.34,<2 + +# Legacy keras-retinanet build; runtime compatibility is handled via python-lib/keras_compat.py. git+https://github.com/fizyr/keras-retinanet.git@0d89a33bace90591cad7882cf0b1cdbf0fbced43 -pip==9.0.1 -scikit-learn>=0.19 -boto3>=1.7<1.8 -tensorflow==1.4.0 diff --git a/custom-recipes/object-detection-draw-bounding-boxes-cpu/recipe.py b/custom-recipes/object-detection-draw-bounding-boxes-cpu/recipe.py index 25a10a3..dd9e4da 100644 --- a/custom-recipes/object-detection-draw-bounding-boxes-cpu/recipe.py +++ b/custom-recipes/object-detection-draw-bounding-boxes-cpu/recipe.py @@ -1,19 +1,13 @@ # -*- coding: utf-8 -*- -import os.path as op import shutil import dataiku -from dataiku.customrecipe import * -import pandas as pd, numpy as np -from dataiku import pandasutils as pdu - +import os.path +from dataiku.customrecipe import get_input_names_for_role, get_output_names_for_role, get_recipe_config import misc_utils src_folder = dataiku.Folder(get_input_names_for_role('images')[0]) -src_folder = src_folder.get_path() - dst_folder = dataiku.Folder(get_output_names_for_role('output')[0]) -dst_folder = dst_folder.get_path() configs = get_recipe_config() @@ -29,13 +23,12 @@ for path in paths: df = bboxes[bboxes.path == path] - src_path = op.join(src_folder, path) - dst_path = op.join(dst_folder, path) + src_path = os.path.join(src_folder.get_path(), path) + dst_path = os.path.join(dst_folder.get_path(), path) if len(df) == 0: shutil.copy(src_path, dst_path) continue - - print(path) - misc_utils.draw_bboxes(src_path, dst_path, df, label_caption, confidence_caption, ids) + + misc_utils.draw_bboxes(path, src_folder, path, dst_folder, df, label_caption, confidence_caption, ids) \ No newline at end of file diff --git a/custom-recipes/object-detection-retrain-object-detection-model-cpu/recipe.py b/custom-recipes/object-detection-retrain-object-detection-model-cpu/recipe.py index e33f0f4..cee65a0 100644 --- a/custom-recipes/object-detection-retrain-object-detection-model-cpu/recipe.py +++ b/custom-recipes/object-detection-retrain-object-detection-model-cpu/recipe.py @@ -19,6 +19,20 @@ logging.basicConfig(level=logging.INFO, format='[Object Detection] %(levelname)s - %(message)s') +def _batch_iterator(generator): + """Infinite Python iterator over generator batches for tf_keras adapter.""" + while True: + for _ in range(len(generator)): + if hasattr(generator, '__getitem__'): + yield generator[_] + elif hasattr(generator, 'next'): + yield generator.next() + else: + yield next(generator) + if hasattr(generator, 'on_epoch_end'): + generator.on_epoch_end() + + images_folder = dataiku.Folder(get_input_names_for_role('images')[0]) bb_df = dataiku.Dataset(get_input_names_for_role('bounding_boxes')[0]).get_dataframe() weights_folder = dataiku.Folder(get_input_names_for_role('weights')[0]) @@ -81,6 +95,9 @@ batch_size=batch_size) if len(val_gen) == 0: val_gen = None +train_data = _batch_iterator(train_gen) +val_data = _batch_iterator(val_gen) if val_gen is not None else None + model, train_model = retinanet_model.get_model(weights, len(class_mapping), freeze=configs['freeze'], n_gpu=gpu_opts['n_gpu']) @@ -102,12 +119,12 @@ logging.info('Training model for {} epochs.'.format(configs['epochs'])) logging.info('Nb labels: {:15}.'.format(len(class_mapping))) logging.info('Nb images: {:15}.'.format(len(train_gen.image_names))) -logging.info('Nb val images: {:11}'.format(len(val_gen.image_names))) +logging.info('Nb val images: {:11}'.format(len(val_gen.image_names) if val_gen is not None else 0)) -train_model.fit_generator( - train_gen, +train_model.fit( + train_data, steps_per_epoch=len(train_gen), - validation_data=val_gen, + validation_data=val_data, validation_steps=len(val_gen) if val_gen is not None else None, callbacks=cbs, epochs=int(configs['epochs']), diff --git a/plugin.json b/plugin.json index 57cee86..16f5369 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "object-detection-cpu", - "version": "0.1.7", + "version": "0.1.8", "meta": { "label": "Object detection in images (CPU)", "description": "Detect objects in images using deep learning. \n⚠️ Starting with DSS version 10 this plugin is partially superseded by the native [object detection capabilities](https://doc.dataiku.com/dss/latest/machine-learning/computer-vision/index.html).", diff --git a/python-lib/gpu_utils.py b/python-lib/gpu_utils.py index d518912..4319977 100644 --- a/python-lib/gpu_utils.py +++ b/python-lib/gpu_utils.py @@ -1,7 +1,26 @@ import os import tensorflow as tf -from keras.backend.tensorflow_backend import set_session + +try: + # Keras 2.2.x + from keras.backend.tensorflow_backend import set_session as keras_set_session +except Exception: + try: + # TF2 compatibility path + from tensorflow.compat.v1.keras.backend import set_session as keras_set_session + except Exception: + keras_set_session = None + + +def _is_tf1(): + return tf.__version__.startswith('1.') + + +def _set_session(config): + if keras_set_session is None: + return + keras_set_session(tf.compat.v1.Session(config=config)) def load_gpu_options(should_use_gpu, list_gpu, gpu_allocation): @@ -18,12 +37,18 @@ def load_gpu_options(should_use_gpu, list_gpu, gpu_allocation): gpu_options = {} if should_use_gpu: gpu_options['n_gpu'] = len(list_gpu.split(',')) - - config = tf.ConfigProto() + os.environ["CUDA_VISIBLE_DEVICES"] = list_gpu.strip() - config.gpu_options.visible_device_list = list_gpu.strip() - config.gpu_options.per_process_gpu_memory_fraction = gpu_allocation - set_session(tf.Session(config=config)) + + if _is_tf1(): + config = tf.ConfigProto() + config.gpu_options.visible_device_list = list_gpu.strip() + config.gpu_options.per_process_gpu_memory_fraction = gpu_allocation + _set_session(config) + else: + # TF2/Keras3: keep visible devices via env var and allow dynamic growth. + for gpu in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(gpu, True) else: deactivate_gpu() gpu_options['n_gpu'] = 0 @@ -38,14 +63,17 @@ def deactivate_gpu(): def can_use_gpu(): """Check that system supports gpu.""" - # Check that 'tensorflow-gpu' is installed on the current code-env - import pip - installed_packages = pip.get_installed_distributions() - return "tensorflow-gpu" in [p.project_name for p in installed_packages] + # TF1 often used tensorflow-gpu, while TF2 usually ships GPU support in tensorflow. + try: + return len(tf.config.list_physical_devices('GPU')) > 0 + except Exception: + return False def set_gpus(gpus): """Short method to set gpu configuration.""" - config = tf.ConfigProto() - config.gpu_options.visible_device_list = gpus.strip() - set_session(tf.Session(config=config)) \ No newline at end of file + os.environ["CUDA_VISIBLE_DEVICES"] = gpus.strip() + if _is_tf1(): + config = tf.ConfigProto() + config.gpu_options.visible_device_list = gpus.strip() + _set_session(config) diff --git a/python-lib/keras_compat.py b/python-lib/keras_compat.py new file mode 100644 index 0000000..c9afeb9 --- /dev/null +++ b/python-lib/keras_compat.py @@ -0,0 +1,241 @@ +import importlib.metadata as importlib_metadata +import os +import sys + + +def _safe_version(pkg_name): + try: + return importlib_metadata.version(pkg_name) + except importlib_metadata.PackageNotFoundError: + return "not-installed" + + +def _ensure_initializer_alias(initializers, alias, target): + if not hasattr(initializers, alias): + setattr(initializers, alias, target) + + +def _enable_legacy_tf_execution_mode(): + """Force TF2 runtime into TF1-style execution for legacy keras-retinanet.""" + try: + import tensorflow as tf + except Exception as exc: + raise RuntimeError( + "Could not import tensorflow while preparing keras-retinanet compatibility." + ) from exc + + # keras-retinanet 0.5.x uses graph-oriented control-flow patterns. + if tf.executing_eagerly(): + try: + tf.compat.v1.disable_eager_execution() + except Exception: + # If eager is already initialized, continue and rely on remaining shims. + pass + + disable_cf_v2 = getattr(tf.compat.v1, "disable_control_flow_v2", None) + if callable(disable_cf_v2): + try: + disable_cf_v2() + except Exception: + pass + + +def _patch_keras_retinanet_initializers(): + try: + import keras_retinanet.initializers as kr_initializers + except Exception as exc: + raise RuntimeError( + "Could not import keras_retinanet.initializers to apply dtype compatibility patch." + ) from exc + + prior_probability = getattr(kr_initializers, "PriorProbability", None) + if prior_probability is None: + raise RuntimeError( + "keras_retinanet.initializers.PriorProbability was not found; " + "cannot apply TF dtype compatibility patch." + ) + + original_call = getattr(prior_probability, "__call__", None) + if original_call is None: + raise RuntimeError( + "PriorProbability.__call__ was not found; cannot apply dtype compatibility patch." + ) + + if getattr(original_call, "__dss_dtype_patch__", False): + return + + def _patched_call(self, shape, dtype=None): + if dtype is not None and hasattr(dtype, "as_numpy_dtype"): + dtype = dtype.as_numpy_dtype + return original_call(self, shape, dtype=dtype) + + _patched_call.__dss_dtype_patch__ = True + prior_probability.__call__ = _patched_call + + +def _patch_keras_retinanet_map_fn(): + """Patch keras-retinanet TF backend map_fn for TF2 signature compatibility.""" + try: + import tensorflow as tf + import keras_retinanet.backend.tensorflow_backend as tf_backend + except Exception as exc: + raise RuntimeError( + "Could not import keras_retinanet.backend.tensorflow_backend for map_fn patch." + ) from exc + + original_map_fn = getattr(tf_backend, "map_fn", None) + if original_map_fn is None: + raise RuntimeError("keras_retinanet.backend.tensorflow_backend.map_fn was not found.") + + if getattr(original_map_fn, "__dss_map_fn_patch__", False): + return + + def _patched_map_fn(*args, **kwargs): + if "dtype" in kwargs and "fn_output_signature" not in kwargs: + kwargs["fn_output_signature"] = kwargs.pop("dtype") + return tf.map_fn(*args, **kwargs) + + _patched_map_fn.__dss_map_fn_patch__ = True + tf_backend.map_fn = _patched_map_fn + + +def _patch_keras_retinanet_filter_detections(): + """Patch legacy filter_detections for TF2 while/map dynamic-shape compatibility.""" + try: + import tensorflow as tf + import keras + import keras_retinanet.layers.filter_detections as fd + except Exception as exc: + raise RuntimeError( + "Could not import keras_retinanet.layers.filter_detections for compatibility patch." + ) from exc + + original_fn = getattr(fd, "filter_detections", None) + if original_fn is None: + raise RuntimeError("keras_retinanet.layers.filter_detections.filter_detections was not found.") + + if getattr(original_fn, "__dss_filter_detections_patch__", False): + return + + backend_ones = keras.backend.ones + + def _patched_filter_detections(*args, **kwargs): + def _compat_ones(shape, dtype=None, name=None): + # Legacy code path uses keras.backend.ones((dynamic_tensor,), dtype=...) + # inside map_fn/while loops, which fails in modern TF graph control flow. + if isinstance(shape, (tuple, list)) and len(shape) == 1 and tf.is_tensor(shape[0]): + dim = tf.cast(shape[0], tf.int32) + out_shape = tf.reshape(dim, [1]) + target_dtype = tf.as_dtype(dtype or keras.backend.floatx()) + return tf.fill(out_shape, tf.cast(1, target_dtype), name=name) + return backend_ones(shape=shape, dtype=dtype, name=name) + + previous_ones = keras.backend.ones + keras.backend.ones = _compat_ones + try: + return original_fn(*args, **kwargs) + finally: + keras.backend.ones = previous_ones + + patched_fn = tf.autograph.experimental.do_not_convert(_patched_filter_detections) + patched_fn.__dss_filter_detections_patch__ = True + fd.filter_detections = patched_fn + + +def _patch_keras_backend_ones(): + """Allow keras.backend.ones((tensor,), ...) in TF while/map control-flow.""" + try: + import tensorflow as tf + import keras + except Exception as exc: + raise RuntimeError("Could not import tensorflow/keras for backend.ones patch.") from exc + + backend_ones = getattr(keras.backend, "ones", None) + if backend_ones is None: + raise RuntimeError("keras.backend.ones was not found.") + + if getattr(backend_ones, "__dss_ones_shape_patch__", False): + return + + def _patched_ones(shape, dtype=None, name=None): + if isinstance(shape, (tuple, list)): + dynamic_dims = [dim for dim in shape if tf.is_tensor(dim)] + if dynamic_dims: + shape = tf.stack([tf.cast(dim, tf.int32) for dim in shape], axis=0) + return backend_ones(shape=shape, dtype=dtype, name=name) + + _patched_ones.__dss_ones_shape_patch__ = True + keras.backend.ones = _patched_ones + + +def bootstrap_keras_retinanet_compat(): + """Install runtime shims required by legacy keras-retinanet builds.""" + os.environ.setdefault("TF_USE_LEGACY_KERAS", "1") + _enable_legacy_tf_execution_mode() + + try: + import tf_keras as legacy_keras + # Force all downstream "import keras" calls to use the legacy Keras 2 API. + sys.modules["keras"] = legacy_keras + except Exception: + pass + + try: + import keras + except Exception as exc: + raise RuntimeError( + "Could not import keras while preparing keras-retinanet compatibility. " + "Ensure a TensorFlow 2.x + tf-keras compatible stack is installed." + ) from exc + + try: + _ensure_initializer_alias(keras.initializers, "normal", keras.initializers.RandomNormal) + _ensure_initializer_alias(keras.initializers, "uniform", keras.initializers.RandomUniform) + except Exception as exc: + raise RuntimeError( + "Failed to install keras initializer compatibility aliases for keras-retinanet." + ) from exc + + missing_symbols = [ + symbol for symbol in ("normal", "uniform") if not hasattr(keras.initializers, symbol) + ] + if missing_symbols: + tf_version = _safe_version("tensorflow") + keras_version = _safe_version("keras") + tf_keras_version = _safe_version("tf-keras") + raise RuntimeError( + "Unsupported Keras compatibility state for keras-retinanet. " + "Missing keras.initializers symbols: {}. " + "Installed versions -> tensorflow={}, keras={}, tf-keras={}. " + "Use TensorFlow 2.x with TF_USE_LEGACY_KERAS=1 and tf-keras installed." + .format(", ".join(missing_symbols), tf_version, keras_version, tf_keras_version) + ) + + try: + _patch_keras_retinanet_initializers() + except Exception as exc: + tf_version = _safe_version("tensorflow") + keras_version = _safe_version("keras") + tf_keras_version = _safe_version("tf-keras") + numpy_version = _safe_version("numpy") + raise RuntimeError( + "Failed to patch keras_retinanet.initializers.PriorProbability for TF dtype compatibility. " + "Installed versions -> tensorflow={}, keras={}, tf-keras={}, numpy={}. " + "Ensure keras-retinanet is installed and compatible with this runtime." + .format(tf_version, keras_version, tf_keras_version, numpy_version) + ) from exc + + try: + _patch_keras_retinanet_map_fn() + _patch_keras_retinanet_filter_detections() + _patch_keras_backend_ones() + except Exception as exc: + tf_version = _safe_version("tensorflow") + keras_version = _safe_version("keras") + tf_keras_version = _safe_version("tf-keras") + raise RuntimeError( + "Failed to patch keras-retinanet filter_detections/map_fn compatibility. " + "Installed versions -> tensorflow={}, keras={}, tf-keras={}. " + "Ensure legacy keras-retinanet code paths can run on this TensorFlow runtime." + .format(tf_version, keras_version, tf_keras_version) + ) from exc diff --git a/python-lib/misc_utils.py b/python-lib/misc_utils.py index 823a5e5..0071351 100644 --- a/python-lib/misc_utils.py +++ b/python-lib/misc_utils.py @@ -1,22 +1,17 @@ import subprocess as sp import os -import random + import logging -import pandas as pd import numpy as np -from sklearn.model_selection import train_test_split -from keras import optimizers + +from keras_compat import bootstrap_keras_retinanet_compat + +# Must run before importing keras/keras-retinanet symbols. +bootstrap_keras_retinanet_compat() + from keras import callbacks -from keras.utils import multi_gpu_model -from keras.models import load_model -import tensorflow as tf import cv2 -import keras_retinanet -from keras_retinanet.models.resnet import resnet50_retinanet -from keras_retinanet.models.retinanet import retinanet_bbox -from keras_retinanet.utils.model import freeze as freeze_model -from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image from keras_retinanet.utils.visualization import draw_box from keras_retinanet.utils.colors import label_color @@ -115,16 +110,19 @@ def jaccard(a, b): def compute_metrics(true_pos, false_pos, false_neg): """Compute the precision, recall, and f1 score.""" - precision = true_pos / (true_pos + false_pos) - recall = true_pos / (true_pos + false_neg) + precision = true_pos / (true_pos + false_pos) if (true_pos + false_pos) else 0.0 + recall = true_pos / (true_pos + false_neg) if (true_pos + false_neg) else 0.0 - if precision == 0 or recall == 0: return precision, recall, f1 + if precision == 0 or recall == 0: + return precision, recall, 0.0 f1 = 2 / (1/precision + 1/recall) return precision, recall, f1 -def draw_bboxes(src_path, dst_path, df, label_cap, confidence_cap, ids): +def draw_bboxes(src_path, source_folder, dst_path, dst_folder, df, label_cap, confidence_cap, ids): + from PIL import Image + """Draw boxes on images. Args: @@ -138,13 +136,18 @@ def draw_bboxes(src_path, dst_path, df, label_cap, confidence_cap, ids): Returns: None. """ - image = read_image_bgr(src_path) + + with source_folder.get_download_stream(path=src_path) as stream: + image = np.array(Image.open(stream).convert('RGB'), copy=True) + # Convert RGB->BGR and force a writable contiguous buffer for OpenCV. + image = np.ascontiguousarray(image[:, :, ::-1].copy()) for _, row in df.iterrows(): - if isinstance(row.class_name, float): continue + if isinstance(row["class_name"], float): + continue - box = tuple(row[1:5]) - name = str(row[5]) + box = tuple(row[["x1", "y1", "x2", "y2"]]) + name = str(row["class_name"]) color = label_color(ids.index(name)) @@ -155,12 +158,18 @@ def draw_bboxes(src_path, dst_path, df, label_cap, confidence_cap, ids): if label_cap: txt = [name] if confidence_cap: - confidence = round(row[6], 2) + confidence = round(row["confidence"], 2) txt.append(str(confidence)) draw_caption(image, box, ' '.join(txt)) logging.info('Drawing {}'.format(dst_path)) - cv2.imwrite(dst_path, image) + ext = os.path.splitext(dst_path)[1].lower() or '.jpg' + ok, encoded = cv2.imencode(ext, image) + if not ok: + raise RuntimeError('Could not encode image for {}'.format(dst_path)) + + with dst_folder.get_writer(dst_path) as w: + w.write(encoded.tobytes()) def draw_caption(image, box, caption): @@ -174,4 +183,4 @@ def draw_caption(image, box, caption): """ b = np.array(box).astype(int) cv2.putText(image, caption, (b[0] + 5, b[1] + 15), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0), 2) - cv2.putText(image, caption, (b[0] + 5, b[1] + 15), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 1) \ No newline at end of file + cv2.putText(image, caption, (b[0] + 5, b[1] + 15), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 1) diff --git a/python-lib/retinanet_model.py b/python-lib/retinanet_model.py index c042d0f..6bedce4 100644 --- a/python-lib/retinanet_model.py +++ b/python-lib/retinanet_model.py @@ -1,13 +1,24 @@ import logging import os +import math import numpy as np import cv2 +from keras_compat import bootstrap_keras_retinanet_compat + +# Must run before importing tensorflow/keras/keras-retinanet symbols. +bootstrap_keras_retinanet_compat() + import tensorflow as tf +import keras from keras import optimizers from keras import callbacks -from keras.utils import multi_gpu_model -from keras.models import load_model +import inspect + +try: + from keras.utils import multi_gpu_model +except Exception: + multi_gpu_model = None import keras_retinanet from keras_retinanet.models.resnet import resnet50_retinanet from keras_retinanet.models.retinanet import retinanet_bbox @@ -21,6 +32,75 @@ logging.basicConfig(level=logging.INFO, format='[Object Detection] %(levelname)s - %(message)s') +def _ensure_legacy_keras_initializers(): + """Backwards compatibility for older keras-retinanet code paths.""" + if hasattr(keras.initializers, 'normal'): + return + + def _normal(mean=0.0, stddev=0.05, seed=None): + return keras.initializers.RandomNormal(mean=mean, stddev=stddev, seed=seed) + + keras.initializers.normal = _normal + + +def _to_numpy_dtype(dtype): + """Convert tf/keras dtypes to numpy dtypes for old keras-retinanet code.""" + if dtype is None: + return None + if isinstance(dtype, np.dtype): + return dtype + try: + return tf.as_dtype(dtype).as_numpy_dtype + except Exception: + pass + try: + return np.dtype(dtype) + except Exception: + return dtype + + +def _ensure_legacy_retinanet_prior_probability(): + """Patch keras-retinanet PriorProbability for keras/tf dtype compatibility.""" + prior_cls = getattr(keras_retinanet.initializers, 'PriorProbability', None) + if prior_cls is None: + return + + def _patched_call(self, shape, dtype=None): + np_dtype = _to_numpy_dtype(dtype) or np.float32 + return np.ones(shape, dtype=np_dtype) * -math.log((1 - self.probability) / self.probability) + + prior_cls.__call__ = _patched_call + + +def _ensure_legacy_tf_resize_images(): + """Restore removed tf.image.resize_images symbol for legacy keras-retinanet.""" + if hasattr(tf.image, 'resize_images'): + return + + legacy_resize = getattr(tf.compat.v1.image, 'resize_images', None) + if legacy_resize is not None: + tf.image.resize_images = legacy_resize + return + + def _resize_images(images, size, method=tf.image.ResizeMethod.BILINEAR, align_corners=False, + preserve_aspect_ratio=False, name=None): + del align_corners # not supported by tf.image.resize in newer TF + return tf.image.resize( + images=images, + size=size, + method=method, + preserve_aspect_ratio=preserve_aspect_ratio, + name=name + ) + + tf.image.resize_images = _resize_images + + +_ensure_legacy_keras_initializers() +_ensure_legacy_retinanet_prior_probability() +_ensure_legacy_tf_resize_images() + + def get_model(weights, num_classes, freeze=False, n_gpu=None): """Return a RetinaNet model. @@ -39,6 +119,11 @@ def get_model(weights, num_classes, freeze=False, n_gpu=None): modifier = freeze_model if freeze else None if multi_gpu: + if multi_gpu_model is None: + logging.warning('`keras.utils.multi_gpu_model` is unavailable. Falling back to single model.') + model = resnet50_retinanet(num_classes=num_classes, modifier=modifier) + model.load_weights(weights, by_name=True, skip_mismatch=True) + return model, model logging.info('Loading model in multi gpu mode.') with tf.device('/cpu:0'): model = resnet50_retinanet(num_classes=num_classes, modifier=modifier) @@ -75,10 +160,17 @@ def get_test_model(weights, num_classes): def compile_model(model, configs): """Compile retinanet.""" + lr = float(configs['lr']) if configs['optimizer'].lower() == 'adam': - opt = optimizers.adam(lr=configs['lr'], clipnorm=0.001) + try: + opt = optimizers.Adam(learning_rate=lr, clipnorm=0.001) + except Exception: + opt = optimizers.adam(lr=lr, clipnorm=0.001) else: - opt = optmizers.SGD(lr=configs['lr'], momentum=True, nesterov=True, clipnorm=0.001) + try: + opt = optimizers.SGD(learning_rate=lr, momentum=0.9, nesterov=True, clipnorm=0.001) + except Exception: + opt = optimizers.SGD(lr=lr, momentum=0.9, nesterov=True, clipnorm=0.001) model .compile( loss={ @@ -251,13 +343,17 @@ class MultiGPUModelCheckpoint(callbacks.ModelCheckpoint): def __init__(self, filepath, base_model, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1): - super(MultiGPUModelCheckpoint, self).__init__(filepath, - monitor=monitor, - verbose=verbose, - save_best_only=save_best_only, - save_weights_only=save_weights_only, - mode=mode, - period=period) + kwargs = dict( + monitor=monitor, + verbose=verbose, + save_best_only=save_best_only, + save_weights_only=save_weights_only, + mode=mode, + ) + if 'period' in inspect.signature(callbacks.ModelCheckpoint.__init__).parameters: + kwargs['period'] = period + + super(MultiGPUModelCheckpoint, self).__init__(filepath, **kwargs) self.base_model = base_model def on_epoch_end(self, epoch, logs=None): @@ -273,4 +369,4 @@ def on_epoch_end(self, epoch, logs=None): super(MultiGPUModelCheckpoint, self).on_epoch_end(epoch, logs) # Resetting model afterwards - self.model = model \ No newline at end of file + self.model = model diff --git a/resource/recipes-helper.py b/resource/recipes-helper.py index 0c4d77f..538d1dd 100644 --- a/resource/recipes-helper.py +++ b/resource/recipes-helper.py @@ -3,7 +3,6 @@ import pandas as pd import numpy as np -import tensorflow as tf from PIL import Image import dataiku @@ -11,6 +10,9 @@ import misc_utils import constants +DEFAULT_MIN_SIDE = 800 +DEFAULT_MAX_SIDE = 1333 + def get_dataset_info(inputs): label_dataset_full_name = get_input_name_from_role(inputs, 'bounding_boxes') @@ -77,13 +79,25 @@ def get_avg_side(inputs, n_first=3000): sides = [] for path in paths: path = os.path.join(folder_path, path[1:]) - with Image.open(path) as img: # PIL does not load the raster data at this point, so it's fast. - w, h = img.size - sides.append(w) - sides.append(h) + try: + # PIL does not load raster data at this point, so reading size is fast. + with Image.open(path) as img: + w, h = img.size + sides.append(w) + sides.append(h) + except Exception: + # Skip unreadable files during setup stat computation. + continue + + if len(sides) == 0: + return { + 'min_side': DEFAULT_MIN_SIDE, + 'max_side': DEFAULT_MAX_SIDE + } + sides = np.array(sides) return { 'min_side': int(np.percentile(sides, 25)), 'max_side': int(np.percentile(sides, 75)) - } \ No newline at end of file + } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 0000000..0781944 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,35 @@ +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /plugin + +# Copy requirements files first (for better layer caching) +COPY code-env/python/spec/requirements.txt /plugin/requirements-plugin.txt +COPY tests/python-lib/requirements.txt /plugin/requirements-test.txt + +# Install dependencies +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements-test.txt && \ + pip install --no-cache-dir -r requirements-plugin.txt + +# Copy source code and resources +COPY python-lib/ /plugin/python-lib/ +COPY tests/python-lib/ /plugin/tests/python/unit/ + +# Set environment variables +ENV PYTHONPATH=/plugin/python-lib + +# Run tests with system info +CMD echo "=== System Info ===" && \ + echo "OS: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'=' -f2)" && \ + echo "Architecture: $(uname -m)" && \ + echo "Python: $(python --version)" && \ + echo "Pip: $(pip --version)" && \ + echo "==================" && \ + pytest tests/python/unit -v \ No newline at end of file diff --git a/tests/python-lib/requirements.txt b/tests/python-lib/requirements.txt new file mode 100644 index 0000000..e7257e4 --- /dev/null +++ b/tests/python-lib/requirements.txt @@ -0,0 +1,2 @@ +pytest +pandas \ No newline at end of file diff --git a/tests/python-lib/test_dfgenerator.py b/tests/python-lib/test_dfgenerator.py new file mode 100644 index 0000000..10c5686 --- /dev/null +++ b/tests/python-lib/test_dfgenerator.py @@ -0,0 +1,79 @@ +import sys +import os +import unittest +from unittest.mock import MagicMock, patch +import pandas as pd +import numpy as np +import json + +# Mock dependencies +sys.modules['keras_retinanet'] = MagicMock() +sys.modules['keras_retinanet.preprocessing'] = MagicMock() +sys.modules['keras_retinanet.preprocessing.csv_generator'] = MagicMock() + +# Define a mock Generator class that DfGenerator can inherit from +class MockGenerator: + def __init__(self, **kwargs): + pass + +sys.modules['keras_retinanet.preprocessing.csv_generator'].Generator = MockGenerator +sys.modules['keras_retinanet.preprocessing.csv_generator'].CSVGenerator = MockGenerator + +# Add python-lib to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../python-lib'))) + +from dfgenerator import DfGenerator + +class TestDfGenerator(unittest.TestCase): + + def setUp(self): + self.cols = { + 'col_filename': 'path', + 'col_label': 'label', + 'col_x1': 'x1', 'col_y1': 'y1', + 'col_x2': 'x2', 'col_y2': 'y2', + 'single_column_data': False + } + self.class_mapping = {'cat': 0} + + def test_init_standard_format(self): + df = pd.DataFrame({ + 'path': ['img1.jpg'], + 'label': ['cat'], + 'x1': [10], 'y1': [10], 'x2': [100], 'y2': [100] + }) + + gen = DfGenerator(df, self.class_mapping, self.cols) + + self.assertIn('img1.jpg', gen.image_data) + self.assertEqual(len(gen.image_data['img1.jpg']), 1) + self.assertEqual(gen.image_data['img1.jpg'][0]['class'], 'cat') + + def test_init_json_format(self): + self.cols['single_column_data'] = True + label_json = json.dumps([ + {"top": 10, "left": 10, "width": 90, "height": 90, "label": "cat"} + ]) + df = pd.DataFrame({ + 'path': ['img2.jpg'], + 'label': [label_json] + }) + + gen = DfGenerator(df, self.class_mapping, self.cols) + + self.assertIn('img2.jpg', gen.image_data) + data = gen.image_data['img2.jpg'][0] + self.assertEqual(data['class'], 'cat') + self.assertEqual(data['x1'], 10) + self.assertEqual(data['y1'], 10) + self.assertEqual(data['x2'], 100) # 10+90 + self.assertEqual(data['y2'], 100) # 10+90 + + def test_len(self): + df = pd.DataFrame({ + 'path': ['img1.jpg', 'img2.jpg'], + 'label': ['cat', 'cat'], + 'x1': [0,0], 'y1': [0,0], 'x2': [1,1], 'y2': [1,1] + }) + gen = DfGenerator(df, self.class_mapping, self.cols) + self.assertEqual(len(gen), 2) diff --git a/tests/python-lib/test_download_utils.py b/tests/python-lib/test_download_utils.py new file mode 100644 index 0000000..d5e3b58 --- /dev/null +++ b/tests/python-lib/test_download_utils.py @@ -0,0 +1,68 @@ +import sys +import os +import unittest +from unittest.mock import MagicMock, patch + +# Mock dependencies +sys.modules['boto3'] = MagicMock() +sys.modules['botocore'] = MagicMock() +sys.modules['botocore.handlers'] = MagicMock() + +# Add python-lib to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../python-lib'))) + +import download_utils + +class TestDownloadUtils(unittest.TestCase): + + def test_get_s3_key(self): + key = download_utils.get_s3_key('resnet50', 'coco') + self.assertEqual(key, 'pretrained_models/image/object_detection/resnet50_coco_weights.h5') + + def test_get_s3_key_labels(self): + key = download_utils.get_s3_key_labels('coco') + self.assertEqual(key, 'pretrained_models/image/coco/labels.json') + + @patch('boto3.resource') + def test_download_labels(self, mock_resource): + mock_bucket = MagicMock() + mock_resource.return_value.Bucket.return_value = mock_bucket + + download_utils.download_labels('coco', 'labels.json') + + mock_resource.assert_called_with('s3') + mock_bucket.download_file.assert_called_with( + 'pretrained_models/image/coco/labels.json', + 'labels.json' + ) + + @patch('boto3.resource') + def test_download_model(self, mock_resource): + mock_bucket = MagicMock() + mock_resource.return_value.Bucket.return_value = mock_bucket + callback = MagicMock() + + # Mock get_obj_size to return something for ProgressTracker + mock_client = mock_resource.return_value.meta.client + mock_client.get_object.return_value = {'ContentLength': 100} + + download_utils.download_model('resnet50', 'coco', 'weights.h5', callback) + + mock_bucket.download_file.assert_called() + args, kwargs = mock_bucket.download_file.call_args + self.assertEqual(args[0], 'pretrained_models/image/object_detection/resnet50_coco_weights.h5') + self.assertEqual(args[1], 'weights.h5') + self.assertIn('Callback', kwargs) + + @patch('download_utils.get_obj_size') + def test_progress_tracker(self, mock_get_size): + mock_get_size.return_value = 100 + callback = MagicMock() + resource = MagicMock() + + tracker = download_utils.ProgressTracker(resource, 'key', callback) + tracker(10) + callback.assert_called_with(10) # 10/100 * 100 = 10% + + tracker(40) + callback.assert_called_with(50) # (10+40)/100 * 100 = 50% diff --git a/tests/python-lib/test_misc_utils.py b/tests/python-lib/test_misc_utils.py new file mode 100644 index 0000000..472460b --- /dev/null +++ b/tests/python-lib/test_misc_utils.py @@ -0,0 +1,129 @@ +import sys +import os +import unittest +from unittest.mock import MagicMock, patch, call +import pandas as pd +import numpy as np + +# Mock dependencies before importing misc_utils +sys.modules['cv2'] = MagicMock() +sys.modules['keras'] = MagicMock() +sys.modules['keras.optimizers'] = MagicMock() +sys.modules['keras.callbacks'] = MagicMock() +sys.modules['keras.utils'] = MagicMock() +sys.modules['keras.models'] = MagicMock() +sys.modules['tensorflow'] = MagicMock() +sys.modules['keras_retinanet'] = MagicMock() +sys.modules['keras_retinanet.models'] = MagicMock() +sys.modules['keras_retinanet.models.resnet'] = MagicMock() +sys.modules['keras_retinanet.models.retinanet'] = MagicMock() +sys.modules['keras_retinanet.utils'] = MagicMock() +sys.modules['keras_retinanet.utils.model'] = MagicMock() +sys.modules['keras_retinanet.utils.image'] = MagicMock() +sys.modules['keras_retinanet.utils.visualization'] = MagicMock() +sys.modules['keras_retinanet.utils.colors'] = MagicMock() + +# Add python-lib to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../python-lib'))) + +import misc_utils + +class TestMiscUtils(unittest.TestCase): + + def test_mkv_to_mp4(self): + def isfile_side_effect(path): + # test.mkv exists, test.mp4 exists only after conversion + return path in ('test.mkv', 'test.mp4') + + with patch('subprocess.call') as mock_call, \ + patch('os.path.isfile', side_effect=isfile_side_effect) as mock_isfile, \ + patch('os.remove') as mock_remove: + + # Test case 1: Basic conversion + misc_utils.mkv_to_mp4('test.mkv', remove_mkv=False, has_audio=True, quiet=True) + mock_call.assert_called() + args, _ = mock_call.call_args + # The exact string might vary depending on implementation details, check key parts + self.assertIn('ffmpeg -i test.mkv', args[0]) + self.assertIn('test.mp4', args[0]) + + # Test case 2: Remove MKV after conversion + misc_utils.mkv_to_mp4('test.mkv', remove_mkv=True, has_audio=False, quiet=False) + mock_remove.assert_called_with('test.mkv') + + def test_split_dataset(self): + df = pd.DataFrame({'path': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'], 'label': range(10)}) + train, val = misc_utils.split_dataset(df, val_split=0.8, shuffle=False) + self.assertEqual(len(train), 8) + self.assertEqual(len(val), 2) + + # Test shuffle + train_s, val_s = misc_utils.split_dataset(df, val_split=0.8, shuffle=True, seed=42) + self.assertEqual(len(train_s), 8) + self.assertEqual(len(val_s), 2) + + def test_get_cm(self): + unique_vals = ['cat', 'dog', 'bird'] + mapping = misc_utils.get_cm(unique_vals) + self.assertEqual(mapping, {'cat': 0, 'dog': 1, 'bird': 2}) + + def test_jaccard(self): + box_a = [0, 0, 10, 10] + box_b = [0, 0, 10, 10] # Identical + self.assertEqual(misc_utils.jaccard(box_a, box_b), 1.0) + + box_c = [10, 10, 20, 20] # No overlap + self.assertEqual(misc_utils.jaccard(box_a, box_c), 0.0) + + box_d = [5, 5, 15, 15] # Partial overlap + # Intersection: 5x5=25. Union: 100+100-25=175. IOU: 25/175 = 1/7 + self.assertAlmostEqual(misc_utils.jaccard(box_a, box_d), 1/7) + + def test_compute_metrics(self): + tp, fp, fn = 10, 0, 0 + p, r, f1 = misc_utils.compute_metrics(tp, fp, fn) + self.assertEqual(p, 1.0) + self.assertEqual(r, 1.0) + self.assertEqual(f1, 1.0) + + def test_compute_metrics_zero_precision_recall(self): + tp, fp, fn = 0, 10, 10 + p, r, f1 = misc_utils.compute_metrics(tp, fp, fn) + self.assertEqual(p, 0.0) + self.assertEqual(r, 0.0) + self.assertEqual(f1, 0.0) + + @patch('misc_utils.read_image_bgr') + @patch('misc_utils.draw_box') + @patch('misc_utils.draw_caption') + @patch('misc_utils.cv2.imwrite') + def test_draw_bboxes(self, mock_imwrite, mock_draw_caption, mock_draw_box, mock_read_image): + mock_read_image.return_value = np.zeros((100, 100, 3), dtype=np.uint8) + + df = pd.DataFrame({ + 'path': ['img.jpg'], + 'x1': [10], 'y1': [10], 'x2': [50], 'y2': [50], + 'class_name': ['cat'], + 'confidence': [0.9] + }) + ids = ['cat', 'dog'] + + misc_utils.draw_bboxes('src.jpg', 'dst.jpg', df, label_cap=True, confidence_cap=True, ids=ids) + + mock_read_image.assert_called_with('src.jpg') + mock_draw_box.assert_called() + mock_draw_caption.assert_called_with( + mock_read_image.return_value, + (10, 10, 50, 50), + 'cat 0.9' + ) + mock_imwrite.assert_called_with('dst.jpg', mock_read_image.return_value) + + @patch('misc_utils.cv2.putText') + def test_draw_caption(self, mock_put_text): + image = np.zeros((100, 100, 3), dtype=np.uint8) + box = [10, 10, 50, 50] + caption = "test" + + misc_utils.draw_caption(image, box, caption) + self.assertEqual(mock_put_text.call_count, 2) # Called twice for shadow effect diff --git a/tests/python-lib/test_retinanet_model.py b/tests/python-lib/test_retinanet_model.py new file mode 100644 index 0000000..4fef295 --- /dev/null +++ b/tests/python-lib/test_retinanet_model.py @@ -0,0 +1,138 @@ +import sys +import os +import unittest +from unittest.mock import MagicMock, patch +import numpy as np + +# Mock dependencies +sys.modules['cv2'] = MagicMock() +sys.modules['tensorflow'] = MagicMock() +sys.modules['keras'] = MagicMock() +sys.modules['keras.optimizers'] = MagicMock() +sys.modules['keras.callbacks'] = MagicMock() +sys.modules['keras.utils'] = MagicMock() +sys.modules['keras.models'] = MagicMock() +sys.modules['keras_retinanet'] = MagicMock() +sys.modules['keras_retinanet.models'] = MagicMock() +sys.modules['keras_retinanet.models.resnet'] = MagicMock() +sys.modules['keras_retinanet.models.retinanet'] = MagicMock() +sys.modules['keras_retinanet.utils'] = MagicMock() +sys.modules['keras_retinanet.utils.model'] = MagicMock() +sys.modules['keras_retinanet.utils.image'] = MagicMock() +sys.modules['keras_retinanet.utils.transform'] = MagicMock() + +# Add python-lib to path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../python-lib'))) + +import retinanet_model + +class TestRetinaNetModel(unittest.TestCase): + + @patch('retinanet_model.resnet50_retinanet') + @patch('retinanet_model.multi_gpu_model') + @patch('retinanet_model.tf.device') + def test_get_model(self, mock_device, mock_multi_gpu, mock_resnet): + mock_model = MagicMock() + mock_resnet.return_value = mock_model + + # Case 1: CPU (n_gpu=None) + retinanet_model.get_model('weights.h5', 10, freeze=False, n_gpu=None) + mock_resnet.assert_called() + mock_multi_gpu.assert_not_called() + mock_model.load_weights.assert_called_with('weights.h5', by_name=True, skip_mismatch=True) + + # Case 2: Multi-GPU + mock_model.load_weights.reset_mock() + retinanet_model.get_model('weights.h5', 10, freeze=False, n_gpu=2) + mock_multi_gpu.assert_called() + mock_model.load_weights.assert_called_with('weights.h5', by_name=True, skip_mismatch=True) + + @patch('retinanet_model.get_model') + @patch('retinanet_model.retinanet_bbox') + def test_get_test_model(self, mock_bbox, mock_get_model): + mock_model = MagicMock() + mock_get_model.return_value = (mock_model, mock_model) + + retinanet_model.get_test_model('weights.h5', 10) + + mock_get_model.assert_called_with('weights.h5', 10, freeze=True, n_gpu=1) + mock_bbox.assert_called_with(model=mock_model) + + def test_compile_model(self): + model = MagicMock() + configs = {'optimizer': 'adam', 'lr': 0.001} + retinanet_model.compile_model(model, configs) + model.compile.assert_called() + + @patch('retinanet_model.read_image_bgr') + @patch('retinanet_model.preprocess_image') + @patch('retinanet_model.resize_image') + def test_find_objects(self, mock_resize, mock_preprocess, mock_read): + model = MagicMock() + # Mock prediction output: boxes, scores, labels + # Output shape is usually (batch, num_boxes, coords/score/label) + model.predict_on_batch.return_value = (np.zeros((1, 300, 4)), np.zeros((1, 300)), np.zeros((1, 300))) + + mock_read.return_value = np.zeros((100, 100, 3)) + mock_preprocess.return_value = np.zeros((100, 100, 3)) + mock_resize.return_value = (np.zeros((100, 100, 3)), 1.0) + + paths = ['img1.jpg'] + boxes, scores, labels = retinanet_model.find_objects(model, paths) + + self.assertEqual(len(boxes), 1) + mock_read.assert_called_with('img1.jpg') + + @patch('cv2.VideoCapture') + @patch('cv2.VideoWriter') + @patch('retinanet_model.find_objects_single') + @patch('misc_utils.mkv_to_mp4') + @patch('misc_utils.draw_box') + def test_detect_in_video_file(self, mock_draw, mock_mkv, mock_find, mock_writer, mock_capture): + cap = mock_capture.return_value + # Loop runs while cap.isOpened(). Logic: ret, img = cap.read(). if not ret: break. + cap.isOpened.return_value = True + cap.read.side_effect = [(True, np.zeros((100,100,3))), (False, None)] + cap.get.return_value = 30 # FPS, width, height + + mock_find.return_value = ( + np.array([[[0,0,10,10]]]), + np.array([[0.9]]), + np.array([[0]]) + ) + + model = MagicMock() + retinanet_model.detect_in_video_file(model, 'in.mp4', 'out_dir', detection_rate=1) + + expected_out = os.path.join('out_dir', 'in-detected.mkv') + mock_writer.assert_called_with( + expected_out, + retinanet_model.cv2.VideoWriter_fourcc.return_value, + 30, + (30, 30) + ) + mock_find.assert_called() + mock_mkv.assert_called() + + @patch('retinanet_model.random_transform_generator') + def test_get_random_augmentator(self, mock_gen): + configs = { + 'min_rotation': 0, 'max_rotation': 1, + 'min_trans': 0, 'max_trans': 1, + 'min_shear': 0, 'max_shear': 1, + 'min_scaling': 0, 'max_scaling': 1, + 'flip_x': 0.5, 'flip_y': 0.5 + } + retinanet_model.get_random_augmentator(configs) + mock_gen.assert_called_with( + min_rotation=0.0, + max_rotation=1.0, + min_translation=(0.0, 0.0), + max_translation=(1.0, 1.0), + min_shear=0.0, + max_shear=1.0, + min_scaling=(0.0, 0.0), + max_scaling=(1.0, 1.0), + flip_x_chance=0.5, + flip_y_chance=0.5 + ) diff --git a/tests/python-runnables/create-api-service/test_runnable.py b/tests/python-runnables/create-api-service/test_runnable.py new file mode 100644 index 0000000..c1df7ad --- /dev/null +++ b/tests/python-runnables/create-api-service/test_runnable.py @@ -0,0 +1,53 @@ +import sys +import os +import unittest +from unittest.mock import MagicMock, patch + +# Mock dependencies +sys.modules['dataiku'] = MagicMock() +sys.modules['dataiku.runnables'] = MagicMock() + +# Add runnable path +runnable_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../python-runnables/create-api-service')) +sys.path.append(runnable_dir) + +import runnable + +class TestRunnable(unittest.TestCase): + + def test_get_params_create_new(self): + config = { + "model_folder_id": "folder1", + "create_new_service": True, + "service_id_new": "new_service", + "endpoint_id": "ep1", + "confidence": 0.5 + } + client = MagicMock() + project = MagicMock() + + # Mock folder list + # list_managed_folders returns list of dicts-like objects or dicts + project.list_managed_folders.return_value = [{"id": "folder1"}] + + # Mock service list (empty so new service is allowed) + project.list_api_services.return_value = [] + + params = runnable.get_params(config, client, project) + + self.assertEqual(params['service_id'], 'new_service') + self.assertEqual(params['model_folder_id'], 'folder1') + + def test_get_model_endpoint_settings(self): + params = { + "endpoint_id": "ep1", + "code_env_name": "env1", + "model_folder_id": "folder1", + "confidence": 0.5 + } + + settings = runnable.get_model_endpoint_settings(params) + + self.assertEqual(settings['id'], 'ep1') + self.assertEqual(settings['inputFolderRefs'][0]['ref'], 'folder1') + self.assertIn("score < 0.5", settings['code']) # check substitution