Python 312

.github/workflows/unit-tests.yml

@@ -0,0 +1,42 @@
+name: Unit Tests
+
+on:
+  push:
+    branches: [master]  # Only run push on default branch after merge
+  pull_request:         # Run on all PRs
+
+# Cancel in-progress runs when a new commit is pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.6 and 3.7 not supported on free runners Ubuntu >= 24.04
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          tests/python-lib/requirements.txt
+          code-env/python/spec/requirements.txt
+
+    - name: Install dependencies
+      run: |
+        pip install -r tests/python-lib/requirements.txt
+        pip install -r code-env/python/spec/requirements.txt
+
+    - name: Run tests
+      env:
+        PYTHONPATH: python-lib
+      run: pytest tests/python-lib -v

.gitignore

@@ -8,3 +8,5 @@
 .htmlcov
 .DS_Store
 .idea
+/dist
+AGENTS.md

CHANGELOG.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## Version 0.1.8
+- Support for python 3.9, 3.10, 3.11 and 3.12

Makefile

@@ -45,3 +45,76 @@ tests: unit-tests integration-tests
 
 dist-clean:
 	rm -rf dist%
+
+# Docker-based unit tests for Linux environment
+# Uses --platform linux/amd64 to ensure consistent behavior on Apple Silicon
+#
+# Usage:
+#   make docker-test-py310              # Without cache (default, CI)
+#   make docker-test-py310 USE_CACHE=true  # With pip cache (faster local iteration)
+
+DOCKER_IMAGE_NAME=object-detection-cpu
+DOCKER_PLATFORM=linux/amd64
+USE_CACHE?=false
+
+define run-docker-test
+	@echo "[START] Running unit tests in Docker with Python $(1) (USE_CACHE=$(USE_CACHE))..."
+	@docker build \
+		--platform $(DOCKER_PLATFORM) \
+		--build-arg PYTHON_VERSION=$(1) \
+		--build-arg USE_CACHE=$(USE_CACHE) \
+		-t $(DOCKER_IMAGE_NAME):py$(1) \
+		-f tests/docker/Dockerfile \
+		. && \
+	docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_IMAGE_NAME):py$(1)
+	@echo "[DONE] Python $(1) tests completed"
+endef
+
+
+docker-clean:
+	@echo "Removing Docker test images..."
+	@docker rmi -f $(DOCKER_IMAGE_NAME):py3.6 $(DOCKER_IMAGE_NAME):py3.7 $(DOCKER_IMAGE_NAME):py3.8 $(DOCKER_IMAGE_NAME):py3.9 $(DOCKER_IMAGE_NAME):py3.10 $(DOCKER_IMAGE_NAME):py3.11 $(DOCKER_IMAGE_NAME):py3.12 2>/dev/null || true
+	@echo "Docker images cleaned"
+
+docker-test-py36:
+	$(call run-docker-test,3.6)
+
+docker-test-py39:
+	$(call run-docker-test,3.9)
+
+docker-test-py310:
+	$(call run-docker-test,3.10)
+
+docker-test-py311:
+	$(call run-docker-test,3.11)
+
+docker-test-py312:
+	$(call run-docker-test,3.12)
+
+
+
+# Run all tests with summary (continues on failure, reports at end)
+PYTHON_VERSIONS = 3.6 3.9 3.10 3.11 3.12
+
+docker-test-all:
+	@failed=""; passed=""; \
+	for ver in $(PYTHON_VERSIONS); do \
+		echo ""; \
+		echo "############################################"; \
+		echo "# Testing Python $$ver"; \
+		echo "############################################"; \
+		target="docker-test-py$$(echo $$ver | tr -d '.')"; \
+		if $(MAKE) $$target USE_CACHE=$(USE_CACHE); then \
+			passed="$$passed $$ver"; \
+		else \
+			failed="$$failed $$ver"; \
+		fi; \
+	done; \
+	echo ""; \
+	echo "############################################"; \
+	echo "#              TEST SUMMARY"; \
+	echo "############################################"; \
+	if [ -n "$$passed" ]; then echo "PASSED:$$passed"; fi; \
+	if [ -n "$$failed" ]; then echo "FAILED:$$failed"; fi; \
+	echo "############################################"; \
+	if [ -n "$$failed" ]; then exit 1; fi

code-env/python/desc.json

@@ -1,6 +1,7 @@
 {
-  "acceptedPythonInterpreters": ["PYTHON36"],
+  "acceptedPythonInterpreters": ["PYTHON36", "PYTHON39", "PYTHON310", "PYTHON311", "PYTHON312"],
+  "corePackagesSet": "AUTO",
+  "forceConda": false,
   "installCorePackages": true,
-  "installJupyterSupport": false,
-  "conda": false
+  "installJupyterSupport": false
 }

code-env/python/spec/requirements.txt

@@ -1,9 +1,12 @@
-keras==2.2.4
-h5py==2.10.0
-opencv-python==3.4.0.12
-Pillow>=5.1
+# TensorFlow 2.x runtime for Python 3.9-3.12.
+tensorflow==2.16.2
+tf-keras==2.16.0
+
+h5py>=3.10,<4
+opencv-python>=4.9,<5
+Pillow>=10,<12
+scikit-learn>=1.3,<2
+boto3>=1.34,<2
+
+# Legacy keras-retinanet build; runtime compatibility is handled via python-lib/keras_compat.py.
 git+https://github.com/fizyr/keras-retinanet.git@0d89a33bace90591cad7882cf0b1cdbf0fbced43
-pip==9.0.1
-scikit-learn>=0.19
-boto3>=1.7<1.8
-tensorflow==1.4.0

custom-recipes/object-detection-draw-bounding-boxes-cpu/recipe.py

@@ -1,19 +1,13 @@
 # -*- coding: utf-8 -*-
-import os.path as op
 import shutil
 
 import dataiku
-from dataiku.customrecipe import *
-import pandas as pd, numpy as np
-from dataiku import pandasutils as pdu
-
+import os.path
+from dataiku.customrecipe import get_input_names_for_role, get_output_names_for_role, get_recipe_config
 import misc_utils
 
 src_folder = dataiku.Folder(get_input_names_for_role('images')[0])
-src_folder = src_folder.get_path()
-
 dst_folder = dataiku.Folder(get_output_names_for_role('output')[0])
-dst_folder = dst_folder.get_path()
 
 configs = get_recipe_config()
 
@@ -29,13 +23,12 @@
 for path in paths:
     df = bboxes[bboxes.path == path]
 
-    src_path = op.join(src_folder, path)
-    dst_path = op.join(dst_folder, path)
+    src_path = os.path.join(src_folder.get_path(), path)
+    dst_path = os.path.join(dst_folder.get_path(), path)
 
     if len(df) == 0:
         shutil.copy(src_path, dst_path)
         continue
-
-    print(path)
-    misc_utils.draw_bboxes(src_path, dst_path, df, label_caption, confidence_caption, ids)
+
+    misc_utils.draw_bboxes(path, src_folder, path, dst_folder, df, label_caption, confidence_caption, ids)
 

custom-recipes/object-detection-retrain-object-detection-model-cpu/recipe.py

@@ -19,6 +19,20 @@
 logging.basicConfig(level=logging.INFO, format='[Object Detection] %(levelname)s - %(message)s')
 
 
+def _batch_iterator(generator):
+    """Infinite Python iterator over generator batches for tf_keras adapter."""
+    while True:
+        for _ in range(len(generator)):
+            if hasattr(generator, '__getitem__'):
+                yield generator[_]
+            elif hasattr(generator, 'next'):
+                yield generator.next()
+            else:
+                yield next(generator)
+        if hasattr(generator, 'on_epoch_end'):
+            generator.on_epoch_end()
+
+
 images_folder = dataiku.Folder(get_input_names_for_role('images')[0])
 bb_df = dataiku.Dataset(get_input_names_for_role('bounding_boxes')[0]).get_dataframe()
 weights_folder = dataiku.Folder(get_input_names_for_role('weights')[0])
@@ -81,6 +95,9 @@
                       batch_size=batch_size)
 if len(val_gen) == 0: val_gen = None
 
+train_data = _batch_iterator(train_gen)
+val_data = _batch_iterator(val_gen) if val_gen is not None else None
+
 model, train_model = retinanet_model.get_model(weights, len(class_mapping),
                                                freeze=configs['freeze'],
                                                n_gpu=gpu_opts['n_gpu'])
@@ -102,12 +119,12 @@
 logging.info('Training model for {} epochs.'.format(configs['epochs']))
 logging.info('Nb labels: {:15}.'.format(len(class_mapping)))
 logging.info('Nb images: {:15}.'.format(len(train_gen.image_names)))
-logging.info('Nb val images: {:11}'.format(len(val_gen.image_names)))
+logging.info('Nb val images: {:11}'.format(len(val_gen.image_names) if val_gen is not None else 0))
 
-train_model.fit_generator(
-    train_gen,
+train_model.fit(
+    train_data,
     steps_per_epoch=len(train_gen),
-    validation_data=val_gen,
+    validation_data=val_data,
     validation_steps=len(val_gen) if val_gen is not None else None,
     callbacks=cbs,
     epochs=int(configs['epochs']),

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "object-detection-cpu",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "meta": {
         "label": "Object detection in images (CPU)",
         "description": "Detect objects in images using deep learning. \n⚠️ Starting with DSS version 10 this plugin is partially superseded by the native [object detection capabilities](https://doc.dataiku.com/dss/latest/machine-learning/computer-vision/index.html).",

python-lib/gpu_utils.py

@@ -1,7 +1,26 @@
 import os
 
 import tensorflow as tf
-from keras.backend.tensorflow_backend import set_session
+
+try:
+    # Keras 2.2.x
+    from keras.backend.tensorflow_backend import set_session as keras_set_session
+except Exception:
+    try:
+        # TF2 compatibility path
+        from tensorflow.compat.v1.keras.backend import set_session as keras_set_session
+    except Exception:
+        keras_set_session = None
+
+
+def _is_tf1():
+    return tf.__version__.startswith('1.')
+
+
+def _set_session(config):
+    if keras_set_session is None:
+        return
+    keras_set_session(tf.compat.v1.Session(config=config))
 
 
 def load_gpu_options(should_use_gpu, list_gpu, gpu_allocation):
@@ -18,12 +37,18 @@ def load_gpu_options(should_use_gpu, list_gpu, gpu_allocation):
     gpu_options = {}
     if should_use_gpu:
         gpu_options['n_gpu'] = len(list_gpu.split(','))
-
-        config = tf.ConfigProto()
+
         os.environ["CUDA_VISIBLE_DEVICES"] = list_gpu.strip()
-        config.gpu_options.visible_device_list = list_gpu.strip()
-        config.gpu_options.per_process_gpu_memory_fraction = gpu_allocation
-        set_session(tf.Session(config=config))
+
+        if _is_tf1():
+            config = tf.ConfigProto()
+            config.gpu_options.visible_device_list = list_gpu.strip()
+            config.gpu_options.per_process_gpu_memory_fraction = gpu_allocation
+            _set_session(config)
+        else:
+            # TF2/Keras3: keep visible devices via env var and allow dynamic growth.
+            for gpu in tf.config.list_physical_devices('GPU'):
+                tf.config.experimental.set_memory_growth(gpu, True)
     else:
         deactivate_gpu()
         gpu_options['n_gpu'] = 0
@@ -38,14 +63,17 @@ def deactivate_gpu():
 
 def can_use_gpu():
     """Check that system supports gpu."""
-    # Check that 'tensorflow-gpu' is installed on the current code-env
-    import pip
-    installed_packages = pip.get_installed_distributions()
-    return "tensorflow-gpu" in [p.project_name for p in installed_packages]
+    # TF1 often used tensorflow-gpu, while TF2 usually ships GPU support in tensorflow.
+    try:
+        return len(tf.config.list_physical_devices('GPU')) > 0
+    except Exception:
+        return False
 
 
 def set_gpus(gpus):
     """Short method to set gpu configuration."""
-    config = tf.ConfigProto()
-    config.gpu_options.visible_device_list = gpus.strip()
-    set_session(tf.Session(config=config))
+    os.environ["CUDA_VISIBLE_DEVICES"] = gpus.strip()
+    if _is_tf1():
+        config = tf.ConfigProto()
+        config.gpu_options.visible_device_list = gpus.strip()
+        _set_session(config)