diff --git a/cspell.config.json b/cspell.config.json index 51af23e5ec..02f658ad90 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -257,6 +257,7 @@ "pubdate", "radokristof", "rajniszp", + "RDATE", "rebuilded", "Reis", "rejas", @@ -364,7 +365,8 @@ "xxxe", "Ybbet", "yearmatch", - "yearmatchgroup" + "yearmatchgroup", + "YYYYMMDDTHHMMSSZ" ], "ignorePaths": [ "css/roboto.css", diff --git a/defaultmodules/calendar/calendarfetcher.js b/defaultmodules/calendar/calendarfetcher.js index e746628512..b8338f3b2c 100644 --- a/defaultmodules/calendar/calendarfetcher.js +++ b/defaultmodules/calendar/calendarfetcher.js @@ -1,6 +1,6 @@ const ical = require("node-ical"); const Log = require("logger"); -const CalendarFetcherUtils = require("./calendarfetcherutils"); +const { filterEvents, preFilterICSText } = require("./calendarfetcherutils"); const HTTPFetcher = require("#http_fetcher"); /** @@ -52,11 +52,15 @@ class CalendarFetcher { async #handleResponse (response) { try { const responseData = await response.text(); - const parsed = await ical.async.parseICS(responseData); + const filteredData = preFilterICSText(responseData, { + includePastEvents: this.includePastEvents, + maximumNumberOfDays: this.maximumNumberOfDays + }); + const parsed = await ical.async.parseICS(filteredData); Log.debug(`Parsed iCal data from ${this.url} with ${Object.keys(parsed).length} entries.`); - this.events = CalendarFetcherUtils.filterEvents(parsed, { + this.events = filterEvents(parsed, { excludedEvents: this.excludedEvents, includePastEvents: this.includePastEvents, maximumEntries: this.maximumEntries, diff --git a/defaultmodules/calendar/calendarfetcherutils.js b/defaultmodules/calendar/calendarfetcherutils.js index 437c081e28..3d59a50318 100644 --- a/defaultmodules/calendar/calendarfetcherutils.js +++ b/defaultmodules/calendar/calendarfetcherutils.js @@ -6,8 +6,110 @@ const ical = require("node-ical"); const Log = require("logger"); +/** + * Unfold RFC 5545 continuation lines before scanning event fields. + * @param {string} text Raw ICS text. + * @returns {string} Text with folded lines joined. + */ +function unfoldICS (text) { + return text.replace(/\r?\n[ \t]/g, ""); +} + +/** + * Get the end date implied by a DURATION value. + * @param {string} startYYYYMMDD Event start date. + * @param {string} duration ICS DURATION value. + * @returns {string} End date as YYYYMMDD. + */ +function getDurationEndDate (startYYYYMMDD, duration) { + const match = (/^P(?:(\d+)W)?(?:(\d+)D)?/).exec(duration); + if (!match) return startYYYYMMDD; + const days = (parseInt(match[1] || "0", 10) * 7) + parseInt(match[2] || "0", 10); + if (!days) return startYYYYMMDD; + return moment(startYYYYMMDD, "YYYYMMDD").add(days, "days").format("YYYYMMDD"); +} + +/** + * Decide whether a raw VEVENT block should be kept for parsing. + * @param {string} block Raw VEVENT block. + * @param {string} pastDateStr Inclusive lower date bound as YYYYMMDD. + * @param {string} futureDateStr Inclusive upper date bound as YYYYMMDD. + * @returns {boolean} True if the block should be parsed. + */ +function shouldKeepEventBlock (block, pastDateStr, futureDateStr) { + const unfolded = unfoldICS(block); + if ((/(?:^|\n)(?:RRULE|RDATE|RECURRENCE-ID)[;:]/i).test(unfolded)) return true; + + // Matches DTSTART:20221215T..., DTSTART;TZID=...:20221215T..., DTSTART;VALUE=DATE:20221215 + const startMatch = (/\nDTSTART[;:][^\r\n]*?(\d{8})/).exec(unfolded); + if (!startMatch) return true; // no DTSTART — let the parser deal with it + + const startDateStr = startMatch[1]; + const endMatch = (/\nDTEND[;:][^\r\n]*?(\d{8})/).exec(unfolded); + let endDateStr = endMatch ? endMatch[1] : startDateStr; + if (!endMatch) { + const durMatch = (/\nDURATION:([^\r\n]+)/).exec(unfolded); + if (durMatch) endDateStr = getDurationEndDate(startDateStr, durMatch[1].trim()); + } + + // YYYYMMDD sorts lexicographically, so plain string compare is fine. + return endDateStr >= pastDateStr && startDateStr <= futureDateStr; +} + const CalendarFetcherUtils = { + /** + * Drops simple out-of-range VEVENT blocks before parsing the ICS text. + * Recurring events and overrides are kept because their instances need expansion. + * @param {string} icsText - Raw ICS text from the calendar feed. + * @param {object} config - Needs includePastEvents (boolean) and maximumNumberOfDays (number). + * @returns {string} Filtered ICS text, ready for ical.parseICS(). + */ + preFilterICSText (icsText, config) { + const today = moment().startOf("day"); + const pastDateStr = (config.includePastEvents ? today.clone().subtract(config.maximumNumberOfDays, "days") : today).format("YYYYMMDD"); + const futureDateStr = today.clone().add(config.maximumNumberOfDays, "days").format("YYYYMMDD"); + + const firstEventIndex = icsText.indexOf("BEGIN:VEVENT"); + if (firstEventIndex === -1) return icsText; + + const parts = [icsText.slice(0, firstEventIndex)]; + let pos = firstEventIndex; + let kept = 0; + let dropped = 0; + + while (pos < icsText.length) { + const blockStart = icsText.indexOf("BEGIN:VEVENT", pos); + if (blockStart === -1) { + parts.push(icsText.slice(pos)); + break; + } + + const endIdx = icsText.indexOf("END:VEVENT", blockStart); + if (endIdx === -1) { + // Malformed ICS — keep the rest and let the parser surface it. + parts.push(icsText.slice(pos)); + break; + } + + const lineEnd = icsText.indexOf("\n", endIdx); + const blockEnd = lineEnd !== -1 ? lineEnd + 1 : icsText.length; + const block = icsText.slice(blockStart, blockEnd); + + if (shouldKeepEventBlock(block, pastDateStr, futureDateStr)) { + parts.push(block); + kept++; + } else { + dropped++; + } + + pos = blockEnd; + } + + Log.debug(`ICS pre-filter: kept ${kept} events, dropped ${dropped} events.`); + return parts.join(""); + }, + /** * Determine based on the title of an event if it should be excluded from the list of events * @param {object} config the global config diff --git a/tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js b/tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js index bb7c86cac7..eb1a0e038b 100644 --- a/tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js +++ b/tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js @@ -497,4 +497,147 @@ END:VCALENDAR`); expect(filteredEvents[0].location).toBe("Berlin"); }); }); + + describe("preFilterICSText", () => { + const makeICS = (events) => { + const header = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n"; + const footer = "END:VCALENDAR\r\n"; + return header + events.join("") + footer; + }; + + const makeEvent = (dtstart, dtend, summary, extra = "") => { + return `BEGIN:VEVENT\r\nDTSTART:${dtstart}\r\nDTEND:${dtend}\r\nSUMMARY:${summary}\r\n${extra ? `${extra}\r\n` : ""}END:VEVENT\r\n`; + }; + + // Format: YYYYMMDDTHHMMSSZ + const dateStr = (m) => `${m.clone().startOf("day").format("YYYYMMDD")}T120000Z`; + + const now = moment(); + const past = now.clone().subtract(400, "days"); + const recent = now.clone().subtract(1, "days"); + const tomorrow = now.clone().add(1, "days"); + const future = now.clone().add(400, "days"); + + const preFilterConfig = { includePastEvents: false, maximumNumberOfDays: 365 }; + + it("should return unchanged ICS if there are no VEVENT blocks", () => { + const input = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n"; + const result = CalendarFetcherUtils.preFilterICSText(input, preFilterConfig); + expect(result).toBe(input); + }); + + it("should keep events within the date range", () => { + const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Recent event")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("Recent event"); + }); + + it("should drop non-recurring events that ended in the distant past", () => { + const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).not.toContain("Old event"); + }); + + it("should drop non-recurring events that start in the distant future", () => { + const icsText = makeICS([makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).not.toContain("Far future event"); + }); + + it("should always keep events with RRULE regardless of DTSTART", () => { + const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Birthday", "RRULE:FREQ=YEARLY")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("Birthday"); + }); + + it("should keep recurring events even when property names are lowercase", () => { + const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Lowercase recurrence", "rrule:FREQ=YEARLY")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("Lowercase recurrence"); + }); + + it("should always keep events with RDATE regardless of DTSTART", () => { + const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "RDATE event", `RDATE:${dateStr(tomorrow)}`)]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("RDATE event"); + }); + + it("should always keep events with RECURRENCE-ID", () => { + const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Override", `RECURRENCE-ID:${dateStr(past)}`)]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("Override"); + }); + + it("should handle DTSTART with TZID parameter (out of range)", () => { + const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`; + const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${dateOnlyStr}\r\nDTEND;TZID=America/New_York:${dateOnlyStr}\r\nSUMMARY:Old TZ event\r\nEND:VEVENT\r\n`; + const icsText = makeICS([event]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).not.toContain("Old TZ event"); + }); + + it("should handle DTSTART with TZID parameter (in range)", () => { + const startStr = `${tomorrow.clone().startOf("day").format("YYYYMMDD")}T120000`; + const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${startStr}\r\nDTEND;TZID=America/New_York:${startStr}\r\nSUMMARY:Upcoming TZ event\r\nEND:VEVENT\r\n`; + const icsText = makeICS([event]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("Upcoming TZ event"); + }); + + it("should drop events whose DTSTART + DURATION still lies in the past", () => { + const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P1D\r\nSUMMARY:Short past event\r\nEND:VEVENT\r\n`; + const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig); + expect(result).not.toContain("Short past event"); + }); + + it("should keep events whose DURATION extends into the window", () => { + const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P800D\r\nSUMMARY:Long running event\r\nEND:VEVENT\r\n`; + const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig); + expect(result).toContain("Long running event"); + }); + + it("should handle folded DTSTART lines (RFC 5545 line folding)", () => { + const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`; + // Fold after the parameter list — a CRLF + space continues the previous line. + const event = `BEGIN:VEVENT\r\nDTSTART;TZID=Europe/Berlin\r\n :${dateOnlyStr}\r\nSUMMARY:Folded event\r\nEND:VEVENT\r\n`; + const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig); + expect(result).not.toContain("Folded event"); + }); + + it("should handle all-day events with VALUE=DATE format", () => { + const recentDate = recent.clone().startOf("day").format("YYYYMMDD"); + const event = `BEGIN:VEVENT\r\nDTSTART;VALUE=DATE:${recentDate}\r\nDTEND;VALUE=DATE:${recent.clone().add(1, "days").startOf("day").format("YYYYMMDD")}\r\nSUMMARY:All day event\r\nEND:VEVENT\r\n`; + const icsText = makeICS([event]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("All day event"); + }); + + it("should keep multiple events selectively", () => { + const icsText = makeICS([ + makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event"), + makeEvent(dateStr(recent), dateStr(tomorrow), "Current event"), + makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event") + ]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).not.toContain("Old event"); + expect(result).toContain("Current event"); + expect(result).not.toContain("Far future event"); + }); + + it("should include past events when includePastEvents is true", () => { + const recentPast = now.clone().subtract(100, "days"); + const icsText = makeICS([makeEvent(dateStr(recentPast), dateStr(recentPast.clone().add(1, "days")), "Past event")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, { includePastEvents: true, maximumNumberOfDays: 365 }); + expect(result).toContain("Past event"); + }); + + it("should preserve the ICS header and footer", () => { + const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Event")]); + const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig); + expect(result).toContain("BEGIN:VCALENDAR"); + expect(result).toContain("VERSION:2.0"); + expect(result).toContain("END:VCALENDAR"); + }); + }); + });