diff --git a/.github/workflows/Build-Test-And-Deploy.yml b/.github/workflows/Build-Test-And-Deploy.yml index 5afc3a7c..7805eb8f 100644 --- a/.github/workflows/Build-Test-And-Deploy.yml +++ b/.github/workflows/Build-Test-And-Deploy.yml @@ -63,6 +63,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + - name: Write IndexNow key verification file + if: env.INDEXNOW_API_KEY != '' + env: + INDEXNOW_API_KEY: ${{ secrets.INDEXNOW_API_KEY }} + run: printf '%s' "$INDEXNOW_API_KEY" > "EssentialCSharp.Web/wwwroot/$INDEXNOW_API_KEY.txt" + # Only build for dev registry — prod gets the image via az acr import in deploy-production - name: Build Container Image if: github.event_name != 'pull_request_target' && github.event_name != 'pull_request' @@ -238,6 +244,49 @@ jobs: git tag -f "deployed/prod/${{ github.sha }}" git push origin "deployed/prod/${{ github.sha }}" --force + - name: Notify IndexNow of Sitemap Update + continue-on-error: true + env: + INDEXNOW_API_KEY: ${{ secrets.INDEXNOW_API_KEY }} + run: | + # IndexNow Protocol: Notifies search engines (Bing, Yandex, Naver) of content updates. + # See: https://www.indexnow.org/documentation + # + # Domain ownership is verified via a key file hosted at: + # https://essentialcsharp.com/{key}.txt (served from wwwroot/{key}.txt) + # The file must contain exactly the key value. Without it, submissions return HTTP 403. + # + # We fetch the live sitemap (app is confirmed up after smoke test) and extract all + # URLs to submit as actual content URLs per the IndexNow spec. + + # Fetch the live sitemap and extract all URLs into a newline-delimited list + SITEMAP=$(curl -sf "https://essentialcsharp.com/sitemap.xml") + if [ -z "$SITEMAP" ]; then + echo "::warning::Could not fetch sitemap; skipping IndexNow submission." + exit 0 + fi + + # Build a JSON array from ... entries + URL_ARRAY=$(echo "$SITEMAP" \ + | grep -oP '(?<=)[^<]+' \ + | jq -Rn '[inputs]') + + URL_COUNT=$(echo "$URL_ARRAY" | jq 'length') + echo "Submitting $URL_COUNT URLs to IndexNow..." + + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X POST https://api.indexnow.org/indexnow \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "{ + \"host\": \"essentialcsharp.com\", + \"key\": \"$INDEXNOW_API_KEY\", + \"urlList\": $URL_ARRAY + }") + + echo "IndexNow response: HTTP $RESPONSE" + if [ "$RESPONSE" != "200" ] && [ "$RESPONSE" != "202" ]; then + echo "::warning::IndexNow submission returned HTTP $RESPONSE (200/202 = success, 202 = key validation pending)." + fi + - name: Logout of Azure CLI if: always() uses: azure/CLI@v3 diff --git a/README.md b/README.md index 071bc6ad..f9169d63 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,37 @@ For any bugs, questions, or anything else with specifically the code found insid For information on setting up your local development environment, please see the [Getting Started Guide](docs/getting-started.md). Please use issues or discussions to report issues found. + +## IndexNow Setup (Production Deployment) + +The deployment workflow notifies search engines (Bing, Yandex, Naver) of content updates via the [IndexNow protocol](https://www.indexnow.org/). After each successful production deploy the workflow fetches the live `sitemap.xml`, extracts all content URLs, and submits them in a single batch POST to IndexNow. + +### How It Works + +1. After the smoke test confirms the app is live, the workflow fetches `https://essentialcsharp.com/sitemap.xml`. +2. All `` URLs are extracted and POSTed to `https://api.indexnow.org/indexnow`. +3. IndexNow distributes those URLs to all participating search engines (Bing, Yandex, Naver, etc.). + +### Setup Requirements + +#### 1. GitHub Secret +Generate a random hex key and store it in GitHub repository secrets as `INDEXNOW_API_KEY`. The workflow uses this value both to write the key verification file into the Docker image at build time and to authenticate IndexNow submissions. + +Go to: **GitHub repo → Settings → Secrets and variables → Actions → New repository secret** +- Name: `INDEXNOW_API_KEY` +- Value: a random hex string (e.g. generate with `openssl rand -hex 16`) + +#### 2. Key Verification File (auto-generated at build time) +IndexNow requires domain ownership proof via a publicly accessible `.txt` file: +- The workflow writes `wwwroot/{key}.txt` (containing the key value) before building the Docker image +- The file is NOT committed to git — it is generated fresh on every CI run from the GitHub Secret +- ASP.NET Core's static file middleware serves it at `https://essentialcsharp.com/{key}.txt` +- IndexNow crawls that URL to verify domain ownership before accepting submissions +- Without this file, all submissions return HTTP 403 + +The key file is intentionally public — that is by design. The "security" is that only the domain owner can host a file at that path. + +### Important Notes +- The IndexNow step uses `continue-on-error: true` — a submission failure will never block a deployment +- Submitting all sitemap URLs on every deploy is intentional; IndexNow has no hard rate limit for batch submissions and the full URL list ensures nothing is missed +- To rotate the key: generate a new hex string and update the GitHub Secret — the old key file disappears automatically on the next deploy