From 1fd81ff9bc27d6f8ad56eff967c386cbd8bc787c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 5 Sep 2025 09:40:25 +0200 Subject: [PATCH 1/2] docs: Add troubleshooting guide on new S3 checksums --- .../airflow/pages/troubleshooting/index.adoc | 27 +++++++++++++++++++ docs/modules/airflow/partials/nav.adoc | 1 + 2 files changed, 28 insertions(+) create mode 100644 docs/modules/airflow/pages/troubleshooting/index.adoc diff --git a/docs/modules/airflow/pages/troubleshooting/index.adoc b/docs/modules/airflow/pages/troubleshooting/index.adoc new file mode 100644 index 00000000..9f70470c --- /dev/null +++ b/docs/modules/airflow/pages/troubleshooting/index.adoc @@ -0,0 +1,27 @@ += Troubleshooting + +== S3 Logging: An error occurred (411) when calling the PutObject operation: Length Required + +If Airflow is trying to access S3 (e.g. for remote task logging) and throws the following error + +[source,text] +---- +{s3_task_handler.py:190} ERROR - Could not write logs to s3://***/***-task-logs/dag_id=test/run_id=manual__2025-05-22T08:14:17.917519+00:00/task_id=extract/attempt=1.log +Traceback (most recent call last): +[..] +botocore.exceptions.ClientError: An error occurred (411) when calling the PutObject operation: Length Required +---- + +Your S3 store doesn't support the new AWS S3 checksums. +Ideally you upgrade your S3 store to a more recent version that support the new checksums (if there is any). +If this is not possible, you can work around this by setting the following env vars + +[source,yaml] +---- +webservers: # Also add to other roles! + envOverrides: + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required +---- + +See e.g. https://github.com/minio/minio/issues/20845[this MinIO issue] for details. diff --git a/docs/modules/airflow/partials/nav.adoc b/docs/modules/airflow/partials/nav.adoc index 864760d7..58613356 100644 --- a/docs/modules/airflow/partials/nav.adoc +++ b/docs/modules/airflow/partials/nav.adoc @@ -18,6 +18,7 @@ *** xref:airflow:usage-guide/operations/pod-placement.adoc[] *** xref:airflow:usage-guide/operations/pod-disruptions.adoc[] *** xref:airflow:usage-guide/operations/graceful-shutdown.adoc[] +* xref:nifi:troubleshooting/index.adoc[] * xref:airflow:reference/index.adoc[] ** xref:airflow:reference/crds.adoc[] *** {crd-docs}/airflow.stackable.tech/airflowcluster/v1alpha1/[AirflowCluster {external-link-icon}^] From 879856bf682e9883e91ccf4227ce977bbd505a9e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 5 Sep 2025 10:13:08 +0200 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> --- docs/modules/airflow/pages/troubleshooting/index.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/airflow/pages/troubleshooting/index.adoc b/docs/modules/airflow/pages/troubleshooting/index.adoc index 9f70470c..bb8fd12c 100644 --- a/docs/modules/airflow/pages/troubleshooting/index.adoc +++ b/docs/modules/airflow/pages/troubleshooting/index.adoc @@ -12,8 +12,8 @@ Traceback (most recent call last): botocore.exceptions.ClientError: An error occurred (411) when calling the PutObject operation: Length Required ---- -Your S3 store doesn't support the new AWS S3 checksums. -Ideally you upgrade your S3 store to a more recent version that support the new checksums (if there is any). +then this is an indication that your S3 store doesn't support the new AWS S3 checksums. +Ideally you should upgrade your S3 store to a more recent version that supports the new checksums (if there is any). If this is not possible, you can work around this by setting the following env vars [source,yaml]