feat: add GitHub commits loader and S3 media loader

README.md

@@ -10,3 +10,5 @@ Some Astro content loaders
 - [@ascorbic/mock-loader](packages/mock) - Mock data for development
 - [@ascorbic/bluesky-loader](packages/bluesky) - Load Bluesky posts
 - [@ascorbic/youtube-loader](packages/youtube) - Load YouTube videos
+- [@ascorbic/s3-media-loader](packages/s3-media) - Load media files from S3-compatible storage
+- [@ascorbic/github-commits-loader](packages/github-commits) - Load commit history from GitHub repositories

packages/airtable/package.json

@@ -45,4 +45,4 @@
 		"airtable": "^0.12.2",
 		"ts-pattern": "^5.6.2"
 	}
-}
+}

packages/bluesky/package.json

@@ -45,4 +45,4 @@
 		"@ascorbic/loader-utils": "workspace:^",
 		"@atproto/api": "^0.13.31"
 	}
-}
+}

packages/csv/package.json

@@ -46,4 +46,4 @@
 		"papaparse": "^5.5.2",
 		"pathe": "^2.0.2"
 	}
-}
+}

packages/feed/package.json

@@ -42,7 +42,7 @@
 	},
 	"homepage": "https://github.com/ascorbic/astro-loaders",
 	"dependencies": {
-		"@rowanmanning/feed-parser": "^2.0.0",
-		"@ascorbic/loader-utils": "workspace:^"
+		"@ascorbic/loader-utils": "workspace:^",
+		"@rowanmanning/feed-parser": "^2.0.0"
 	}
-}
+}

packages/github-commits/CHANGELOG.md

@@ -0,0 +1,11 @@
+# @ascorbic/github-commits
+
+## 0.0.1
+
+### Initial Release
+
+- Initial release of GitHub commits loader for Astro
+- Load commits from any GitHub repository
+- Support for fetching commit files
+- ETag caching support for efficient re-fetching
+- Configurable pagination and timeout

packages/github-commits/README.md

@@ -0,0 +1,80 @@
+# Astro GitHub Commits Loader
+
+This package provides a GitHub commits loader for Astro. It allows you to load commits from any GitHub repository and use them as content in your Astro project.
+
+## Installation
+
+```sh
+npm install @ascorbic/github-commits
+```
+
+## Usage
+
+This package requires Astro 5.0.0 or later.
+
+You can use the GitHub loader in your content collection configuration:
+
+```typescript
+// src/content/config.ts
+import { defineCollection } from "astro:content";
+import { githubLoader } from "@ascorbic/github-commits";
+
+const commits = defineCollection({
+  loader: githubLoader({
+    repo: "owner/repo",
+    token: import.meta.env.GITHUB_TOKEN,
+    perPage: 15,
+    fetchFilesFor: 5,
+  }),
+});
+
+export const collections = { commits };
+```
+
+You can then use these like any other content collection in Astro:
+
+```astro
+---
+import { getCollection } from "astro:content";
+const commits = await getCollection("commits");
+---
+
+<ul>
+  {commits.map((commit) => (
+    <li>
+      <strong>{commit.data.shortSha}</strong> - {commit.data.message}
+      <br />
+      <small>by {commit.data.author} on {commit.data.date.toDateString()}</small>
+    </li>
+  ))}
+</ul>
+```
+
+## Options
+
+The `githubLoader` function takes an object with the following options:
+
+- `repo` (required): GitHub repository in the format "owner/repo"
+- `token` (optional): GitHub personal access token for higher API rate limits
+- `perPage` (optional): Number of commits to fetch per page (default: 15)
+- `timeoutMs` (optional): Request timeout in milliseconds (default: 8000)
+- `fetchFilesFor` (optional): Number of recent commits to fetch files for (default: 0)
+
+## Data Structure
+
+Each commit item contains:
+
+- `sha`: Full commit SHA
+- `shortSha`: Shortened 7-character SHA
+- `message`: Commit message (first line)
+- `author`: Author name
+- `date`: Commit date
+- `files`: Array of changed files (if fetchFilesFor > 0)
+
+Each file contains:
+
+- `filename`: File path
+- `status`: File status (added, modified, deleted)
+- `changes`: Total changes
+- `additions`: Number of additions
+- `deletions`: Number of deletions

packages/github-commits/package.json

@@ -0,0 +1,46 @@
+{
+	"name": "@ascorbic/github-commits-loader",
+	"version": "0.0.1",
+	"description": "Load commits from a GitHub repository into Astro",
+	"type": "module",
+	"main": "dist/index.js",
+	"files": [
+		"dist"
+	],
+	"exports": {
+		".": "./dist/index.js"
+	},
+	"scripts": {
+		"build": "tsup src/index.ts --format esm --dts --clean",
+		"dev": "tsup src/index.ts --format esm --dts --watch",
+		"prepublishOnly": "node --run build",
+		"check": "publint && attw --pack --ignore-rules=cjs-resolves-to-esm",
+		"test": "vitest run",
+		"test:watch": "vitest"
+	},
+	"devDependencies": {
+		"@arethetypeswrong/cli": "^0.17.3",
+		"astro": "^5.10.1",
+		"publint": "^0.3.2",
+		"tsup": "^8.3.6",
+		"typescript": "^5.7.3"
+	},
+	"peerDependencies": {
+		"astro": "^4.14.0 || ^5.0.0"
+	},
+	"keywords": [
+		"withastro",
+		"astro-loader"
+	],
+	"author": "strangeZombies",
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com:ascorbic/astro-loaders.git",
+		"directory": "packages/github-commits"
+	},
+	"homepage": "https://github.com/ascorbic/astro-loaders",
+	"dependencies": {
+		"@ascorbic/loader-utils": "workspace:^"
+	}
+}

packages/github-commits/src/github-commits-loader.ts

@@ -0,0 +1,150 @@
+// packages/github/src/github-commits-loader.ts
+import type { Loader, LoaderContext } from "astro/loaders";
+import { getLoaderFetch } from "@ascorbic/loader-utils";
+import type { GitHubCommit, GitHubCommitFile, ProcessedCommit, GitHubLoaderOptions } from "./schema.js";
+
+const ETAG_KEY = (repo: string, perPage: number) => `gh-commits-etag:${repo}:${perPage}`;
+
+export function githubLoader(options: GitHubLoaderOptions): Loader {
+	const {
+		repo,
+		token,
+		perPage = 15,
+		timeoutMs = 8000,
+		fetchFilesFor = 0,
+	} = options;
+
+	return {
+			name: "github",
+
+			load: async ({ store, logger, parseData, meta, generateDigest }: LoaderContext) => {
+				const etagKey = ETAG_KEY(repo, perPage);
+				const fetchImpl = getLoaderFetch();
+
+				try {
+					// Test if repo is accessible
+					const testRes = await fetchImpl(`https://api.github.com/repos/${repo}`, {
+						headers: getHeaders(token),
+					});
+				if (!testRes.ok) {
+					const text = await testRes.text().catch(() => "");
+					logger.error(`Cannot access repo ${repo}: ${testRes.status} - ${text.slice(0, 200)}`);
+					return;
+				}
+
+				const url = `https://api.github.com/repos/${repo}/commits?per_page=${perPage}`;
+				const headers = getHeaders(token);
+
+				// ETag support
+				const prevEtag = meta.get(etagKey);
+				if (prevEtag) {
+					headers["If-None-Match"] = prevEtag;
+				}
+
+				const controller = new AbortController();
+				const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+					const res = await fetchImpl(url, { headers, signal: controller.signal });
+				clearTimeout(timeoutId);
+
+				if (res.status === 304) {
+					logger.info(`Commits not modified (ETag hit) → keeping existing data`);
+					return;
+				}
+
+				if (!res.ok) {
+					throw new Error(`GitHub API error: ${res.status} ${await res.text().catch(() => "")}`);
+				}
+
+				const etag = res.headers.get("ETag");
+				if (etag) meta.set(etagKey, etag);
+
+				const commits = (await res.json()) as GitHubCommit[];
+
+				logger.info(`Fetched ${commits.length} commits from ${repo}`);
+
+				const detailedCommits = await Promise.all(
+					commits.map(async (c, index) => {
+						let files: ProcessedCommit["files"] = [];
+
+						if (fetchFilesFor > 0 && index < fetchFilesFor) {
+							try {
+									const detailRes = await fetchImpl(
+										`https://api.github.com/repos/${repo}/commits/${c.sha}`,
+										{ headers }
+									);
+
+								if (detailRes.ok) {
+									const detail = (await detailRes.json()) as { files?: GitHubCommitFile[] };
+									files = (detail.files || []).map((f) => ({
+										filename: f.filename,
+										status: f.status,
+										changes: f.changes,
+										additions: f.additions,
+										deletions: f.deletions,
+									}));
+								} else if (detailRes.status === 403 || detailRes.status === 429) {
+									logger.warn(`Rate limit hit when fetching files for ${c.sha.slice(0, 7)}`);
+								}
+							} catch (e) {
+								logger.warn(`Failed to fetch files for ${c.sha.slice(0, 7)}: ${e}`);
+							}
+						}
+
+						return {
+							sha: c.sha,
+							shortSha: c.sha.slice(0, 7),
+							message: c.commit?.message?.split("\n")[0]?.trim() ?? "",
+							author: c.commit.author.name,
+							date: new Date(c.commit.author.date),
+							files,
+						};
+					})
+				);
+
+				await store.clear();
+
+				for (const commit of detailedCommits) {
+					const id = commit.shortSha;
+
+					const digest = generateDigest({
+						sha: commit.sha,
+						message: commit.message,
+						date: commit.date.toISOString(),
+						filesLength: commit.files.length,
+					});
+
+					const parsed = await parseData({
+						id,
+						data: commit,
+					});
+
+					await store.set({
+						id,
+						data: parsed,
+						digest,
+					});
+				}
+
+				logger.info(`Stored ${detailedCommits.length} commits`);
+			} catch (err: unknown) {
+				if (err instanceof Error && err.name === "AbortError") {
+					logger.error("GitHub request timeout");
+				} else {
+					logger.error(`Load failed: ${err instanceof Error ? err.message : err}`);
+				}
+			}
+		},
+	};
+}
+
+function getHeaders(token?: string): Record<string, string> {
+	const h: Record<string, string> = {
+		Accept: "application/vnd.github+json",
+		"User-Agent": "Astro-GitHub-Loader",
+	};
+	if (token) {
+		h.Authorization = `Bearer ${token}`;
+	}
+	return h;
+}

packages/github-commits/src/index.ts

@@ -0,0 +1,3 @@
+// packages/github/src/index.ts
+export { githubLoader } from "./github-commits-loader.js";
+export type { GitHubLoaderOptions, ProcessedCommit, GitHubCommit, GitHubCommitFile } from "./schema.js";

packages/github-commits/src/schema.ts

@@ -0,0 +1,43 @@
+// packages/github/src/schema.ts
+export interface GitHubCommit {
+	sha: string;
+	commit: {
+		message: string;
+		author: {
+			name: string;
+			date: string;
+		};
+	};
+	author?: { avatar_url?: string };
+}
+
+export interface GitHubCommitFile {
+	sha: string;
+	filename: string;
+	status: string;
+	additions: number;
+	deletions: number;
+	changes: number;
+}
+
+export interface ProcessedCommit {
+	sha: string;
+	shortSha: string;
+	message: string;
+	author: string;
+	date: Date;
+	files: Pick<GitHubCommitFile, "filename" | "status" | "changes" | "additions" | "deletions">[];
+}
+
+export interface GitHubLoaderOptions {
+	/** GitHub repository in the format "owner/repo" */
+	repo: string;
+	/** GitHub personal access token for higher rate limits */
+	token?: string;
+	/** Number of commits to fetch per page */
+	perPage?: number;
+	/** Request timeout in milliseconds */
+	timeoutMs?: number;
+	/** Number of recent commits to fetch files for (0 = none) */
+	fetchFilesFor?: number;
+}

packages/github-commits/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": ["src"]
+}

packages/mock/package.json

@@ -45,4 +45,4 @@
 		"@ascorbic/loader-utils": "workspace:^",
 		"@faker-js/faker": "^9.4.0"
 	}
-}
+}