Merge branch 'master' into hidden-1091-continued

Author: ethho

.devcontainer/Dockerfile

@@ -1,12 +1,13 @@
-# Note: You can use any Debian/Ubuntu based image you want. 
-FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye
-
+ARG PY_VER
+ARG DISTRO
+FROM mcr.microsoft.com/devcontainers/python:${PY_VER}-${DISTRO}
 RUN \
     apt update && \
     apt-get install bash-completion graphviz default-mysql-client -y && \
-    pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \
+    pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint jupyterlab && \
     pip uninstall datajoint -y
 
+USER root
 ENV DJ_HOST fakeservices.datajoint.io
 ENV DJ_USER root
-ENV DJ_PASS simple
+ENV DJ_PASS password

.devcontainer/devcontainer.json

@@ -1,34 +1,50 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
 {
-	"name": "Development",
-	"dockerComposeFile": "docker-compose.yaml",
+	"name": "Existing Docker Compose (Extend)",
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../LNX-docker-compose.yml",
+		"docker-compose.yml"
+	],
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
 	"service": "app",
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Use this environment variable if you need to bind mount your local source code into a new container.
-	"remoteEnv": {
-		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
-	},
-	// https://containers.dev/features
-	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:1": {},
-		"ghcr.io/guiyomh/features/vim:0": {}
-	},
-	"onCreateCommand": "pip install -e .",
-	"postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [
 		80,
 		443,
 		3306,
 		8080,
 		9000
 	],
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	"shutdownAction": "stopCompose",
+	"onCreateCommand": "python3 -m pip install -e .",
+	"features": {
+		"ghcr.io/devcontainers/features/git:1": {},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+	},
+	// Configure tool-specific properties.
 	"customizations": {
 		"vscode": {
 			"extensions": [
 				"ms-python.python"
 			]
 		}
+	},
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	}
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
 }

.devcontainer/docker-compose.yaml

@@ -0,0 +1,32 @@
+version: '2.4'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  app:
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    container_name: datajoint-python-devcontainer
+    image: datajoint/datajoint-python-devcontainer:${PY_VER:-3.11}-${DISTRO:-buster}
+    build:
+      context: .
+      dockerfile: .devcontainer/Dockerfile
+      args:
+        - PY_VER=${PY_VER:-3.11}
+        - DISTRO=${DISTRO:-buster}
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    user: root
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: /bin/sh -c "while sleep 1000; do :; done"
+

.devcontainer/docker-compose.yml

@@ -28,7 +28,7 @@ jobs:
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Validate version and release notes
         run: |
           DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
@@ -42,7 +42,7 @@ jobs:
       - name: Build pip artifacts
         run: |
           export HOST_UID=$(id -u)
-          docker-compose -f docker-compose-build.yaml up --exit-code-from app --build
+          docker compose -f docker-compose-build.yaml up --exit-code-from app --build
           echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
       - if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
         name: Add pip artifacts
@@ -58,14 +58,16 @@ jobs:
         py_ver: ["3.9"]
         mysql_ver: ["8.0", "5.7"]
         include:
+          - py_ver: "3.11"
+            mysql_ver: "8.0"
+          - py_ver: "3.10"
+            mysql_ver: "8.0"
           - py_ver: "3.8"
             mysql_ver: "5.7"
-          - py_ver: "3.7"
-            mysql_ver: "5.7"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_ver}}
       - name: Install dependencies
@@ -77,30 +79,46 @@ jobs:
       - name: Run primary tests
         env:
           PY_VER: ${{matrix.py_ver}}
-          DJ_PASS: simple
+          DJ_PASS: password
           MYSQL_VER: ${{matrix.mysql_ver}}
           DISTRO: alpine
           MINIO_VER: RELEASE.2021-09-03T03-56-13Z
           DOCKER_CLIENT_TIMEOUT: "120"
           COMPOSE_HTTP_TIMEOUT: "120"
         run: |
           export HOST_UID=$(id -u)
-          docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+          docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.11"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 black==24.2.0
+      - name: Run syntax tests
+        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run style tests
         run: |
           flake8 --ignore=E203,E722,W503 datajoint \
-                 --count --max-complexity=62 --max-line-length=127 --statistics
-          black datajoint --check -v
-          black tests --check -v
-          black tests_old --check -v
+                 --count --max-complexity=62 --max-line-length=127 --statistics \
+                 --per-file-ignores='datajoint/diagram.py:C901'
+          black --required-version '24.2.0' --check -v datajoint tests tests_old
   codespell:
     name: Check for spelling errors
     permissions:
       contents: read
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
   publish-docs:
@@ -113,7 +131,7 @@ jobs:
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Deploy docs
         run: |
           export MODE=BUILD
@@ -145,9 +163,9 @@ jobs:
     outputs:
       release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{matrix.py_ver}}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_ver}}
       - name: Determine package version
@@ -172,7 +190,7 @@ jobs:
           prerelease: false
           draft: false
       - name: Fetch pip artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4.1.7
         with:
           name: pip-datajoint-${{env.DJ_VERSION}}
           path: dist
@@ -201,7 +219,7 @@ jobs:
       - name: Publish pip release
         run: |
           export HOST_UID=$(id -u)
-          docker-compose -f docker-compose-build.yaml run \
+          docker compose -f docker-compose-build.yaml run \
             -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
             sh -c "pip install twine && python -m twine upload dist/*"
       - name: Login to DockerHub

.github/workflows/development.yaml

@@ -8,7 +8,7 @@ jobs:
       DOCKER_CLIENT_TIMEOUT: "120"
       COMPOSE_HTTP_TIMEOUT: "120"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Deploy docs
         run: |
           export MODE=BUILD

.github/workflows/docs.yaml

@@ -30,4 +30,6 @@ docs/site
 
 !.vscode/settings.json
 !.vscode/launch.json
-!.devcontainer/devcontainer.json
+!.devcontainer/devcontainer.json
+!.devcontainer/docker-compose.yml
+

.gitignore

@@ -1,13 +1,23 @@
 ## Release notes
 
-### Upcoming
+### 0.14.3 -- TBD
+- Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177)
+
+### 0.14.2 -- Aug 19, 2024
+- Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142)
 - Added - Codespell GitHub Actions workflow
 - Added - GitHub Actions workflow to manually release docs
 - Changed - Update `datajoint/nginx` to `v0.2.6`
 - Changed - Migrate docs from `https://docs.datajoint.org/python` to `https://datajoint.com/docs/core/datajoint-python`
+- Fixed - [DevContainer](https://containers.dev/) configuration - PR [#1115](https://github.com/datajoint/datajoint-python/pull/1115)
 - Fixed - Updated set_password to work on MySQL 8 - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106)
 - Added - Missing tests for set_password - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106)
 - Changed - Returning success count after the .populate() call - PR [#1050](https://github.com/datajoint/datajoint-python/pull/1050)
+- Fixed - `Autopopulate.populate` excludes `reserved` jobs in addition to `ignore` and `error` jobs
+- Fixed - Issue [#1159](https://github.com/datajoint/datajoint-python/pull/1159) (cascading delete) - PR [#1160](https://github.com/datajoint/datajoint-python/pull/1160)
+- Changed - Minimum Python version for Datajoint-Python is now 3.8 PR #1163
+- Fixed - `docker compose` commands in CI [#1164](https://github.com/datajoint/datajoint-python/pull/1164)
+- Changed - Default delete behavior now includes masters of part tables - PR [#1158](https://github.com/datajoint/datajoint-python/pull/1158)
 
 ### 0.14.1 -- Jun 02, 2023
 - Fixed - Fix altering a part table that uses the "master" keyword - PR [#991](https://github.com/datajoint/datajoint-python/pull/991)

CHANGELOG.md

@@ -7,9 +7,10 @@ x-net:
 services:
   db:
     <<: *net
-    image: datajoint/mysql:${MYSQL_VER}
+    image: datajoint/mysql:${MYSQL_VER:-8.0}
     environment:
-      - MYSQL_ROOT_PASSWORD=${DJ_PASS}
+      - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
+    command: mysqld --default-authentication-plugin=mysql_native_password
     # ports:
     #   - "3306:3306"
     # volumes:
@@ -21,7 +22,7 @@ services:
       interval: 15s
   minio:
     <<: *net
-    image: minio/minio:${MINIO_VER}
+    image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
@@ -44,7 +45,7 @@ services:
       interval: 15s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.2.6
+    image: datajoint/nginx:latest
     environment:
       - ADD_db_TYPE=DATABASE
       - ADD_db_ENDPOINT=db:3306
@@ -58,7 +59,7 @@ services:
     #   - "3306:3306"
   app:
     <<: *net
-    image: datajoint/djtest:py${PY_VER}-${DISTRO}
+    image: datajoint/djtest:py${PY_VER:-3.8}-${DISTRO:-alpine}
     depends_on:
       db:
         condition: service_healthy
@@ -69,7 +70,7 @@ services:
     environment:
       - DJ_HOST=fakeservices.datajoint.io
       - DJ_USER=root
-      - DJ_PASS
+      - DJ_PASS=password
       - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
@@ -86,14 +87,12 @@ services:
       - -c
       - |
         set -e
-        pip install --user nose nose-cov
         pip install -e .
         pip list --format=freeze | grep datajoint
         pytest -sv --cov-report term-missing --cov=datajoint tests
-        nosetests -vsw tests_old --with-coverage --cover-package=datajoint
     # ports:
     #   - "8888:8888"
-    user: ${HOST_UID}:anaconda
+    user: ${HOST_UID:-1000}:anaconda
     volumes:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw

LNX-docker-compose.yml

@@ -1,4 +1,5 @@
 """This module defines class dj.AutoPopulate"""
+
 import logging
 import datetime
 import traceback
@@ -207,12 +208,12 @@ def handler(signum, frame):
 
         keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
 
-        # exclude "error" or "ignore" jobs
+        # exclude "error", "ignore" or "reserved" jobs
         if reserve_jobs:
             exclude_key_hashes = (
                 jobs
                 & {"table_name": self.target.table_name}
-                & 'status in ("error", "ignore")'
+                & 'status in ("error", "ignore", "reserved")'
             ).fetch("key_hash")
             keys = [key for key in keys if key_hash(key) not in exclude_key_hashes]