perf(calendar): pre-filter ICS text before parsing

cspell.config.json

@@ -257,6 +257,7 @@
 		"pubdate",
 		"radokristof",
 		"rajniszp",
+		"RDATE",
 		"rebuilded",
 		"Reis",
 		"rejas",
@@ -364,7 +365,8 @@
 		"xxxe",
 		"Ybbet",
 		"yearmatch",
-		"yearmatchgroup"
+		"yearmatchgroup",
+		"YYYYMMDDTHHMMSSZ"
 	],
 	"ignorePaths": [
 		"css/roboto.css",

defaultmodules/calendar/calendarfetcher.js

@@ -1,6 +1,6 @@
 const ical = require("node-ical");
 const Log = require("logger");
-const CalendarFetcherUtils = require("./calendarfetcherutils");
+const { filterEvents, preFilterICSText } = require("./calendarfetcherutils");
 const HTTPFetcher = require("#http_fetcher");
 
 /**
@@ -52,11 +52,15 @@ class CalendarFetcher {
 	async #handleResponse (response) {
 		try {
 			const responseData = await response.text();
-			const parsed = await ical.async.parseICS(responseData);
+			const filteredData = preFilterICSText(responseData, {
+				includePastEvents: this.includePastEvents,
+				maximumNumberOfDays: this.maximumNumberOfDays
+			});
+			const parsed = await ical.async.parseICS(filteredData);
 
 			Log.debug(`Parsed iCal data from ${this.url} with ${Object.keys(parsed).length} entries.`);
 
-			this.events = CalendarFetcherUtils.filterEvents(parsed, {
+			this.events = filterEvents(parsed, {
 				excludedEvents: this.excludedEvents,
 				includePastEvents: this.includePastEvents,
 				maximumEntries: this.maximumEntries,

defaultmodules/calendar/calendarfetcherutils.js

@@ -6,8 +6,110 @@ const ical = require("node-ical");
 
 const Log = require("logger");
 
+/**
+ * Unfold RFC 5545 continuation lines before scanning event fields.
+ * @param {string} text Raw ICS text.
+ * @returns {string} Text with folded lines joined.
+ */
+function unfoldICS (text) {
+	return text.replace(/\r?\n[ \t]/g, "");
+}
+
+/**
+ * Get the end date implied by a DURATION value.
+ * @param {string} startYYYYMMDD Event start date.
+ * @param {string} duration ICS DURATION value.
+ * @returns {string} End date as YYYYMMDD.
+ */
+function getDurationEndDate (startYYYYMMDD, duration) {
+	const match = (/^P(?:(\d+)W)?(?:(\d+)D)?/).exec(duration);
+	if (!match) return startYYYYMMDD;
+	const days = (parseInt(match[1] || "0", 10) * 7) + parseInt(match[2] || "0", 10);
+	if (!days) return startYYYYMMDD;
+	return moment(startYYYYMMDD, "YYYYMMDD").add(days, "days").format("YYYYMMDD");
+}
+
+/**
+ * Decide whether a raw VEVENT block should be kept for parsing.
+ * @param {string} block Raw VEVENT block.
+ * @param {string} pastDateStr Inclusive lower date bound as YYYYMMDD.
+ * @param {string} futureDateStr Inclusive upper date bound as YYYYMMDD.
+ * @returns {boolean} True if the block should be parsed.
+ */
+function shouldKeepEventBlock (block, pastDateStr, futureDateStr) {
+	const unfolded = unfoldICS(block);
+	if ((/(?:^|\n)(?:RRULE|RDATE|RECURRENCE-ID)[;:]/i).test(unfolded)) return true;
+
+	// Matches DTSTART:20221215T..., DTSTART;TZID=...:20221215T..., DTSTART;VALUE=DATE:20221215
+	const startMatch = (/\nDTSTART[;:][^\r\n]*?(\d{8})/).exec(unfolded);
+	if (!startMatch) return true; // no DTSTART — let the parser deal with it
+
+	const startDateStr = startMatch[1];
+	const endMatch = (/\nDTEND[;:][^\r\n]*?(\d{8})/).exec(unfolded);
+	let endDateStr = endMatch ? endMatch[1] : startDateStr;
+	if (!endMatch) {
+		const durMatch = (/\nDURATION:([^\r\n]+)/).exec(unfolded);
+		if (durMatch) endDateStr = getDurationEndDate(startDateStr, durMatch[1].trim());
+	}
+
+	// YYYYMMDD sorts lexicographically, so plain string compare is fine.
+	return endDateStr >= pastDateStr && startDateStr <= futureDateStr;
+}
+
 const CalendarFetcherUtils = {
 
+	/**
+	 * Drops simple out-of-range VEVENT blocks before parsing the ICS text.
+	 * Recurring events and overrides are kept because their instances need expansion.
+	 * @param {string} icsText - Raw ICS text from the calendar feed.
+	 * @param {object} config - Needs includePastEvents (boolean) and maximumNumberOfDays (number).
+	 * @returns {string} Filtered ICS text, ready for ical.parseICS().
+	 */
+	preFilterICSText (icsText, config) {
+		const today = moment().startOf("day");
+		const pastDateStr = (config.includePastEvents ? today.clone().subtract(config.maximumNumberOfDays, "days") : today).format("YYYYMMDD");
+		const futureDateStr = today.clone().add(config.maximumNumberOfDays, "days").format("YYYYMMDD");
+
+		const firstEventIndex = icsText.indexOf("BEGIN:VEVENT");
+		if (firstEventIndex === -1) return icsText;
+
+		const parts = [icsText.slice(0, firstEventIndex)];
+		let pos = firstEventIndex;
+		let kept = 0;
+		let dropped = 0;
+
+		while (pos < icsText.length) {
+			const blockStart = icsText.indexOf("BEGIN:VEVENT", pos);
+			if (blockStart === -1) {
+				parts.push(icsText.slice(pos));
+				break;
+			}
+
+			const endIdx = icsText.indexOf("END:VEVENT", blockStart);
+			if (endIdx === -1) {
+				// Malformed ICS — keep the rest and let the parser surface it.
+				parts.push(icsText.slice(pos));
+				break;
+			}
+
+			const lineEnd = icsText.indexOf("\n", endIdx);
+			const blockEnd = lineEnd !== -1 ? lineEnd + 1 : icsText.length;
+			const block = icsText.slice(blockStart, blockEnd);
+
+			if (shouldKeepEventBlock(block, pastDateStr, futureDateStr)) {
+				parts.push(block);
+				kept++;
+			} else {
+				dropped++;
+			}
+
+			pos = blockEnd;
+		}
+
+		Log.debug(`ICS pre-filter: kept ${kept} events, dropped ${dropped} events.`);
+		return parts.join("");
+	},
+
 	/**
 	 * Determine based on the title of an event if it should be excluded from the list of events
 	 * @param {object} config the global config

tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js

@@ -497,4 +497,147 @@ END:VCALENDAR`);
 			expect(filteredEvents[0].location).toBe("Berlin");
 		});
 	});
+
+	describe("preFilterICSText", () => {
+		const makeICS = (events) => {
+			const header = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n";
+			const footer = "END:VCALENDAR\r\n";
+			return header + events.join("") + footer;
+		};
+
+		const makeEvent = (dtstart, dtend, summary, extra = "") => {
+			return `BEGIN:VEVENT\r\nDTSTART:${dtstart}\r\nDTEND:${dtend}\r\nSUMMARY:${summary}\r\n${extra ? `${extra}\r\n` : ""}END:VEVENT\r\n`;
+		};
+
+		// Format: YYYYMMDDTHHMMSSZ
+		const dateStr = (m) => `${m.clone().startOf("day").format("YYYYMMDD")}T120000Z`;
+
+		const now = moment();
+		const past = now.clone().subtract(400, "days");
+		const recent = now.clone().subtract(1, "days");
+		const tomorrow = now.clone().add(1, "days");
+		const future = now.clone().add(400, "days");
+
+		const preFilterConfig = { includePastEvents: false, maximumNumberOfDays: 365 };
+
+		it("should return unchanged ICS if there are no VEVENT blocks", () => {
+			const input = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n";
+			const result = CalendarFetcherUtils.preFilterICSText(input, preFilterConfig);
+			expect(result).toBe(input);
+		});
+
+		it("should keep events within the date range", () => {
+			const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Recent event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Recent event");
+		});
+
+		it("should drop non-recurring events that ended in the distant past", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old event");
+		});
+
+		it("should drop non-recurring events that start in the distant future", () => {
+			const icsText = makeICS([makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Far future event");
+		});
+
+		it("should always keep events with RRULE regardless of DTSTART", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Birthday", "RRULE:FREQ=YEARLY")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Birthday");
+		});
+
+		it("should keep recurring events even when property names are lowercase", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Lowercase recurrence", "rrule:FREQ=YEARLY")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Lowercase recurrence");
+		});
+
+		it("should always keep events with RDATE regardless of DTSTART", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "RDATE event", `RDATE:${dateStr(tomorrow)}`)]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("RDATE event");
+		});
+
+		it("should always keep events with RECURRENCE-ID", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Override", `RECURRENCE-ID:${dateStr(past)}`)]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Override");
+		});
+
+		it("should handle DTSTART with TZID parameter (out of range)", () => {
+			const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${dateOnlyStr}\r\nDTEND;TZID=America/New_York:${dateOnlyStr}\r\nSUMMARY:Old TZ event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old TZ event");
+		});
+
+		it("should handle DTSTART with TZID parameter (in range)", () => {
+			const startStr = `${tomorrow.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${startStr}\r\nDTEND;TZID=America/New_York:${startStr}\r\nSUMMARY:Upcoming TZ event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Upcoming TZ event");
+		});
+
+		it("should drop events whose DTSTART + DURATION still lies in the past", () => {
+			const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P1D\r\nSUMMARY:Short past event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).not.toContain("Short past event");
+		});
+
+		it("should keep events whose DURATION extends into the window", () => {
+			const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P800D\r\nSUMMARY:Long running event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).toContain("Long running event");
+		});
+
+		it("should handle folded DTSTART lines (RFC 5545 line folding)", () => {
+			const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			// Fold after the parameter list — a CRLF + space continues the previous line.
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=Europe/Berlin\r\n :${dateOnlyStr}\r\nSUMMARY:Folded event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).not.toContain("Folded event");
+		});
+
+		it("should handle all-day events with VALUE=DATE format", () => {
+			const recentDate = recent.clone().startOf("day").format("YYYYMMDD");
+			const event = `BEGIN:VEVENT\r\nDTSTART;VALUE=DATE:${recentDate}\r\nDTEND;VALUE=DATE:${recent.clone().add(1, "days").startOf("day").format("YYYYMMDD")}\r\nSUMMARY:All day event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("All day event");
+		});
+
+		it("should keep multiple events selectively", () => {
+			const icsText = makeICS([
+				makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event"),
+				makeEvent(dateStr(recent), dateStr(tomorrow), "Current event"),
+				makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event")
+			]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old event");
+			expect(result).toContain("Current event");
+			expect(result).not.toContain("Far future event");
+		});
+
+		it("should include past events when includePastEvents is true", () => {
+			const recentPast = now.clone().subtract(100, "days");
+			const icsText = makeICS([makeEvent(dateStr(recentPast), dateStr(recentPast.clone().add(1, "days")), "Past event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, { includePastEvents: true, maximumNumberOfDays: 365 });
+			expect(result).toContain("Past event");
+		});
+
+		it("should preserve the ICS header and footer", () => {
+			const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("BEGIN:VCALENDAR");
+			expect(result).toContain("VERSION:2.0");
+			expect(result).toContain("END:VCALENDAR");
+		});
+	});
+
 });