From 610e40f6f42ed0e93974af562942b518720b597d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:04:02 +0000 Subject: [PATCH 1/4] Initial plan From 306a8b5f06fcce2b83e2c2918514a3d545b5ff65 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:08:26 +0000 Subject: [PATCH 2/4] Enhance robots.txt to welcome and inform AI agents with comprehensive resource links Co-authored-by: argyleink <1134620+argyleink@users.noreply.github.com> --- src/pages/robots.txt.ts | 24 ++++++++++- tests/unit/robots.test.ts | 90 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 tests/unit/robots.test.ts diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index bcc5259..fb57036 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -1,13 +1,33 @@ import type { APIRoute } from 'astro'; -const getRobotsTxt = (sitemapURL: URL) => ` +const getRobotsTxt = (sitemapURL: URL, siteURL: URL) => ` +# Welcome AI agents and crawlers! +# This is a podcast website built with Starpod - all content is freely accessible. + User-agent: * Allow: / +# Sitemap for all pages Sitemap: ${sitemapURL.href} + +# Special resources for AI/LLM agents: +# - ${siteURL.origin}/llms.txt - Structured overview following the llms.txt spec +# - ${siteURL.origin}/for-llms - Human-readable guide for AI assistants +# - ${siteURL.origin}/episodes-index.html.md - Complete episode listing in markdown +# - ${siteURL.origin}/[episode-slug].html.md - Individual episodes with transcripts +# +# All content includes: +# - Podcast metadata (hosts, description, platforms) +# - Episode information (titles, descriptions, publish dates) +# - Full transcripts (when available) +# - Guest information +# +# Feel free to crawl, index, and use this content to help users discover +# and learn about our podcast! `; export const GET: APIRoute = ({ site }) => { const sitemapURL = new URL('sitemap-index.xml', site); - return new Response(getRobotsTxt(sitemapURL)); + const siteURL = new URL(site!); + return new Response(getRobotsTxt(sitemapURL, siteURL)); }; \ No newline at end of file diff --git a/tests/unit/robots.test.ts b/tests/unit/robots.test.ts new file mode 100644 index 0000000..317285a --- /dev/null +++ b/tests/unit/robots.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest'; + +describe('Robots.txt', () => { + describe('Generated content', () => { + it('should contain welcoming comment for AI agents', () => { + const mockSite = new URL('https://whiskey.fm'); + const mockSitemap = new URL('sitemap-index.xml', mockSite); + + // Simulate the getRobotsTxt function behavior + const robotsTxt = ` +# Welcome AI agents and crawlers! +# This is a podcast website built with Starpod - all content is freely accessible. + +User-agent: * +Allow: / + +# Sitemap for all pages +Sitemap: ${mockSitemap.href} + +# Special resources for AI/LLM agents: +# - ${mockSite.origin}/llms.txt - Structured overview following the llms.txt spec +# - ${mockSite.origin}/for-llms - Human-readable guide for AI assistants +# - ${mockSite.origin}/episodes-index.html.md - Complete episode listing in markdown +# - ${mockSite.origin}/[episode-slug].html.md - Individual episodes with transcripts +# +# All content includes: +# - Podcast metadata (hosts, description, platforms) +# - Episode information (titles, descriptions, publish dates) +# - Full transcripts (when available) +# - Guest information +# +# Feel free to crawl, index, and use this content to help users discover +# and learn about our podcast! +`; + + expect(robotsTxt).toContain('Welcome AI agents and crawlers'); + expect(robotsTxt).toContain('User-agent: *'); + expect(robotsTxt).toContain('Allow: /'); + }); + + it('should include sitemap reference', () => { + const mockSite = new URL('https://whiskey.fm'); + const mockSitemap = new URL('sitemap-index.xml', mockSite); + + const robotsTxt = `Sitemap: ${mockSitemap.href}`; + + expect(robotsTxt).toContain('Sitemap: https://whiskey.fm/sitemap-index.xml'); + }); + + it('should reference LLM-specific resources', () => { + const mockSite = new URL('https://whiskey.fm'); + + const robotsTxt = ` +# Special resources for AI/LLM agents: +# - ${mockSite.origin}/llms.txt - Structured overview following the llms.txt spec +# - ${mockSite.origin}/for-llms - Human-readable guide for AI assistants +# - ${mockSite.origin}/episodes-index.html.md - Complete episode listing in markdown +`; + + expect(robotsTxt).toContain('/llms.txt'); + expect(robotsTxt).toContain('/for-llms'); + expect(robotsTxt).toContain('/episodes-index.html.md'); + }); + + it('should describe available content types', () => { + const robotsTxt = ` +# All content includes: +# - Podcast metadata (hosts, description, platforms) +# - Episode information (titles, descriptions, publish dates) +# - Full transcripts (when available) +# - Guest information +`; + + expect(robotsTxt).toContain('Podcast metadata'); + expect(robotsTxt).toContain('Episode information'); + expect(robotsTxt).toContain('Full transcripts'); + expect(robotsTxt).toContain('Guest information'); + }); + + it('should include encouraging closing message', () => { + const robotsTxt = ` +# Feel free to crawl, index, and use this content to help users discover +# and learn about our podcast! +`; + + expect(robotsTxt).toContain('Feel free to crawl, index, and use this content'); + expect(robotsTxt).toContain('help users discover'); + }); + }); +}); From 346e054b2a6a1665b037a2ad878cc92a761ee6ec Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:09:46 +0000 Subject: [PATCH 3/4] Refactor: export getRobotsTxt function and update tests to use actual implementation Co-authored-by: argyleink <1134620+argyleink@users.noreply.github.com> --- src/pages/robots.txt.ts | 2 +- tests/unit/robots.test.ts | 57 +++++++++------------------------------ 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index fb57036..2116bb3 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -1,6 +1,6 @@ import type { APIRoute } from 'astro'; -const getRobotsTxt = (sitemapURL: URL, siteURL: URL) => ` +export const getRobotsTxt = (sitemapURL: URL, siteURL: URL) => ` # Welcome AI agents and crawlers! # This is a podcast website built with Starpod - all content is freely accessible. diff --git a/tests/unit/robots.test.ts b/tests/unit/robots.test.ts index 317285a..5c18184 100644 --- a/tests/unit/robots.test.ts +++ b/tests/unit/robots.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { getRobotsTxt } from '../../src/pages/robots.txt'; describe('Robots.txt', () => { describe('Generated content', () => { @@ -6,32 +7,7 @@ describe('Robots.txt', () => { const mockSite = new URL('https://whiskey.fm'); const mockSitemap = new URL('sitemap-index.xml', mockSite); - // Simulate the getRobotsTxt function behavior - const robotsTxt = ` -# Welcome AI agents and crawlers! -# This is a podcast website built with Starpod - all content is freely accessible. - -User-agent: * -Allow: / - -# Sitemap for all pages -Sitemap: ${mockSitemap.href} - -# Special resources for AI/LLM agents: -# - ${mockSite.origin}/llms.txt - Structured overview following the llms.txt spec -# - ${mockSite.origin}/for-llms - Human-readable guide for AI assistants -# - ${mockSite.origin}/episodes-index.html.md - Complete episode listing in markdown -# - ${mockSite.origin}/[episode-slug].html.md - Individual episodes with transcripts -# -# All content includes: -# - Podcast metadata (hosts, description, platforms) -# - Episode information (titles, descriptions, publish dates) -# - Full transcripts (when available) -# - Guest information -# -# Feel free to crawl, index, and use this content to help users discover -# and learn about our podcast! -`; + const robotsTxt = getRobotsTxt(mockSitemap, mockSite); expect(robotsTxt).toContain('Welcome AI agents and crawlers'); expect(robotsTxt).toContain('User-agent: *'); @@ -42,20 +18,16 @@ Sitemap: ${mockSitemap.href} const mockSite = new URL('https://whiskey.fm'); const mockSitemap = new URL('sitemap-index.xml', mockSite); - const robotsTxt = `Sitemap: ${mockSitemap.href}`; + const robotsTxt = getRobotsTxt(mockSitemap, mockSite); expect(robotsTxt).toContain('Sitemap: https://whiskey.fm/sitemap-index.xml'); }); it('should reference LLM-specific resources', () => { const mockSite = new URL('https://whiskey.fm'); + const mockSitemap = new URL('sitemap-index.xml', mockSite); - const robotsTxt = ` -# Special resources for AI/LLM agents: -# - ${mockSite.origin}/llms.txt - Structured overview following the llms.txt spec -# - ${mockSite.origin}/for-llms - Human-readable guide for AI assistants -# - ${mockSite.origin}/episodes-index.html.md - Complete episode listing in markdown -`; + const robotsTxt = getRobotsTxt(mockSitemap, mockSite); expect(robotsTxt).toContain('/llms.txt'); expect(robotsTxt).toContain('/for-llms'); @@ -63,13 +35,10 @@ Sitemap: ${mockSitemap.href} }); it('should describe available content types', () => { - const robotsTxt = ` -# All content includes: -# - Podcast metadata (hosts, description, platforms) -# - Episode information (titles, descriptions, publish dates) -# - Full transcripts (when available) -# - Guest information -`; + const mockSite = new URL('https://whiskey.fm'); + const mockSitemap = new URL('sitemap-index.xml', mockSite); + + const robotsTxt = getRobotsTxt(mockSitemap, mockSite); expect(robotsTxt).toContain('Podcast metadata'); expect(robotsTxt).toContain('Episode information'); @@ -78,10 +47,10 @@ Sitemap: ${mockSitemap.href} }); it('should include encouraging closing message', () => { - const robotsTxt = ` -# Feel free to crawl, index, and use this content to help users discover -# and learn about our podcast! -`; + const mockSite = new URL('https://whiskey.fm'); + const mockSitemap = new URL('sitemap-index.xml', mockSite); + + const robotsTxt = getRobotsTxt(mockSitemap, mockSite); expect(robotsTxt).toContain('Feel free to crawl, index, and use this content'); expect(robotsTxt).toContain('help users discover'); From f6214bb06e0fdf04da0f7c3a0dbf10873fb9cc29 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:10:35 +0000 Subject: [PATCH 4/4] Refactor tests: use beforeEach to reduce duplication in test setup Co-authored-by: argyleink <1134620+argyleink@users.noreply.github.com> --- tests/unit/robots.test.ts | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/tests/unit/robots.test.ts b/tests/unit/robots.test.ts index 5c18184..9ab1d79 100644 --- a/tests/unit/robots.test.ts +++ b/tests/unit/robots.test.ts @@ -1,45 +1,35 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, beforeEach } from 'vitest'; import { getRobotsTxt } from '../../src/pages/robots.txt'; describe('Robots.txt', () => { + let mockSite: URL; + let mockSitemap: URL; + let robotsTxt: string; + + beforeEach(() => { + mockSite = new URL('https://whiskey.fm'); + mockSitemap = new URL('sitemap-index.xml', mockSite); + robotsTxt = getRobotsTxt(mockSitemap, mockSite); + }); + describe('Generated content', () => { it('should contain welcoming comment for AI agents', () => { - const mockSite = new URL('https://whiskey.fm'); - const mockSitemap = new URL('sitemap-index.xml', mockSite); - - const robotsTxt = getRobotsTxt(mockSitemap, mockSite); - expect(robotsTxt).toContain('Welcome AI agents and crawlers'); expect(robotsTxt).toContain('User-agent: *'); expect(robotsTxt).toContain('Allow: /'); }); it('should include sitemap reference', () => { - const mockSite = new URL('https://whiskey.fm'); - const mockSitemap = new URL('sitemap-index.xml', mockSite); - - const robotsTxt = getRobotsTxt(mockSitemap, mockSite); - expect(robotsTxt).toContain('Sitemap: https://whiskey.fm/sitemap-index.xml'); }); it('should reference LLM-specific resources', () => { - const mockSite = new URL('https://whiskey.fm'); - const mockSitemap = new URL('sitemap-index.xml', mockSite); - - const robotsTxt = getRobotsTxt(mockSitemap, mockSite); - expect(robotsTxt).toContain('/llms.txt'); expect(robotsTxt).toContain('/for-llms'); expect(robotsTxt).toContain('/episodes-index.html.md'); }); it('should describe available content types', () => { - const mockSite = new URL('https://whiskey.fm'); - const mockSitemap = new URL('sitemap-index.xml', mockSite); - - const robotsTxt = getRobotsTxt(mockSitemap, mockSite); - expect(robotsTxt).toContain('Podcast metadata'); expect(robotsTxt).toContain('Episode information'); expect(robotsTxt).toContain('Full transcripts'); @@ -47,11 +37,6 @@ describe('Robots.txt', () => { }); it('should include encouraging closing message', () => { - const mockSite = new URL('https://whiskey.fm'); - const mockSitemap = new URL('sitemap-index.xml', mockSite); - - const robotsTxt = getRobotsTxt(mockSitemap, mockSite); - expect(robotsTxt).toContain('Feel free to crawl, index, and use this content'); expect(robotsTxt).toContain('help users discover'); });