Merge pull request #735 from sanskar-soni-9/fix/unicode-noncharacters

feat(repair): added unicode non-character repair step

src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js

@@ -0,0 +1,40 @@
+/**
+ * @copyright Copyright (c) 2024 Sanskar Soni
+ *
+ * @author Sanskar Soni <[email protected]>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+import AbstractRepairStep from '../abstractRepairStep.js'
+
+/**
+ * @class ICalendarRemoveUnicodeSpecialNoncharactersRepairStep
+ * @classdesc This repair step removes Unicode specials non-characters i.e. U+FFFE & U+FFFF
+ */
+export default class ICalendarRemoveUnicodeSpecialNoncharactersRepairStep extends AbstractRepairStep {
+
+	/**
+	 * Please see the corresponding test file for an example of broken calendar-data
+	 *
+	 * @inheritDoc
+	 */
+	repair(ics) {
+		return ics
+			.replace(/(\uFFFF|\uFFFE)/g, '')
+	}
+
+}

src/parsers/repairsteps/icalendar/index.js

@@ -26,6 +26,7 @@ import ICalendarEmptyTriggerRepairStep from './icalendarEmptyTriggerRepairStep.j
 import ICalendarIllegalCreatedRepairStep from './icalendarIllegalCreatedRepairStep.js'
 import ICalendarMultipleVCalendarBlocksRepairStep from './icalendarMultipleVCalendarBlocksRepairStep.js'
 import ICalendarRemoveXNCGroupIdRepairStep from './icalendarRemoveXNCGroupIdRepairStep.js'
+import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep from './icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js'
 
 /**
  * Get an iterator over all repair steps for iCalendar documents
@@ -38,4 +39,5 @@ export function * getRepairSteps() {
 	yield ICalendarIllegalCreatedRepairStep
 	yield ICalendarMultipleVCalendarBlocksRepairStep
 	yield ICalendarRemoveXNCGroupIdRepairStep
+	yield ICalendarRemoveUnicodeSpecialNoncharactersRepairStep
 }

tests/assets/unicode-non-character-fffe-after.ics

@@ -0,0 +1,55 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//SabreDAV//SabreDAV//EN
+X-WR-CALNAME:41424_Sigharting (Webportal Plugin)
+REFRESH-INTERVAL;VALUE=DURATION:PT4H
+X-PUBLISHED-TTL:PT4H
+BEGIN:VTIMEZONE
+TZID:Europe/Vienna
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20230405T070540Z
+DTSTAMP:20230614T183802Z
+LAST-MODIFIED:20230614T183802Z
+SEQUENCE:6
+UID:0e85e9f1-8e56-4581-b4d6-04d865152898
+DTSTART;TZID=Europe/Vienna:20230613T083000
+DTEND;TZID=Europe/Vienna:20230613T113000
+STATUS:CONFIRMED
+SUMMARY:TEST1
+LOCATION:Church
+DESCRIPTION:test spacew
+ allfahrt  desende.
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20240603T130712Z
+DTSTAMP:20240603T130749Z
+LAST-MODIFIED:20240603T130749Z
+SEQUENCE:3
+UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e
+DTSTART;VALUE=DATE:20241020
+DTEND;VALUE=DATE:20241021
+STATUS:CONFIRMED
+DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnisse. Diim text neme
+ lt liebenswerten ääääts überzeugt mit ergreidddszinie
+ ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu
+ nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie
+ lten Opern aller Zeiten avanciert ist.\n
+SUMMARY:Zauberflöte
+END:VEVENT
+END:VCALENDAR

tests/assets/unicode-non-character-fffe-before.ics

@@ -0,0 +1,55 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//SabreDAV//SabreDAV//EN
+X-WR-CALNAME:41424_Sigharting (Webportal Plugin)
+REFRESH-INTERVAL;VALUE=DURATION:PT4H
+X-PUBLISHED-TTL:PT4H
+BEGIN:VTIMEZONE
+TZID:Europe/Vienna
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20230405T070540Z
+DTSTAMP:20230614T183802Z
+LAST-MODIFIED:20230614T183802Z
+SEQUENCE:6
+UID:0e85e9f1-8e56-4581-b4d6-04d865152898
+DTSTART;TZID=Europe/Vienna:20230613T083000
+DTEND;TZID=Europe/Vienna:20230613T113000
+STATUS:CONFIRMED
+SUMMARY:TEST1
+LOCATION:Church
+DESCRIPTION:test spacew
+ allfahrt  desende.
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20240603T130712Z
+DTSTAMP:20240603T130749Z
+LAST-MODIFIED:20240603T130749Z
+SEQUENCE:3
+UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e
+DTSTART;VALUE=DATE:20241020
+DTEND;VALUE=DATE:20241021
+STATUS:CONFIRMED
+DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnis￾se. Diim text neme
+ lt lie￾benswerten ääääts über￾zeugt mit ergreidddszi￾nie
+ ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu
+ nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie
+ lten Opern aller Zeiten avanciert ist.\n
+SUMMARY:Zauberflöte
+END:VEVENT
+END:VCALENDAR

tests/assets/unicode-non-character-ffff-after.ics

@@ -0,0 +1,55 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//SabreDAV//SabreDAV//EN
+X-WR-CALNAME:41424_Sigharting (Webportal Plugin)
+REFRESH-INTERVAL;VALUE=DURATION:PT4H
+X-PUBLISHED-TTL:PT4H
+BEGIN:VTIMEZONE
+TZID:Europe/Vienna
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20230405T070540Z
+DTSTAMP:20230614T183802Z
+LAST-MODIFIED:20230614T183802Z
+SEQUENCE:6
+UID:0e85e9f1-8e56-4581-b4d6-04d865152898
+DTSTART;TZID=Europe/Vienna:20230613T083000
+DTEND;TZID=Europe/Vienna:20230613T113000
+STATUS:CONFIRMED
+SUMMARY:TEST1
+LOCATION:Church
+DESCRIPTION:test spacew
+ allfahrt  desende.
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20240603T130712Z
+DTSTAMP:20240603T130749Z
+LAST-MODIFIED:20240603T130749Z
+SEQUENCE:3
+UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e
+DTSTART;VALUE=DATE:20241020
+DTEND;VALUE=DATE:20241021
+STATUS:CONFIRMED
+DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnisse. Diim text neme
+ lt liebenswerten ääääts überzeugt mit ergreidddszinie
+ ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu
+ nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie
+ lten Opern aller Zeiten avanciert ist.\n
+SUMMARY:Zauberflöte
+END:VEVENT
+END:VCALENDAR

tests/assets/unicode-non-character-ffff-before.ics

@@ -0,0 +1,55 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+CALSCALE:GREGORIAN
+PRODID:-//SabreDAV//SabreDAV//EN
+X-WR-CALNAME:41424_Sigharting (Webportal Plugin)
+REFRESH-INTERVAL;VALUE=DURATION:PT4H
+X-PUBLISHED-TTL:PT4H
+BEGIN:VTIMEZONE
+TZID:Europe/Vienna
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20230405T070540Z
+DTSTAMP:20230614T183802Z
+LAST-MODIFIED:20230614T183802Z
+SEQUENCE:6
+UID:0e85e9f1-8e56-4581-b4d6-04d865152898
+DTSTART;TZID=Europe/Vienna:20230613T083000
+DTEND;TZID=Europe/Vienna:20230613T113000
+STATUS:CONFIRMED
+SUMMARY:TEST1
+LOCATION:Church
+DESCRIPTION:test spacew
+ allfahrt  desende.
+END:VEVENT
+BEGIN:VEVENT
+CREATED:20240603T130712Z
+DTSTAMP:20240603T130749Z
+LAST-MODIFIED:20240603T130749Z
+SEQUENCE:3
+UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e
+DTSTART;VALUE=DATE:20241020
+DTEND;VALUE=DATE:20241021
+STATUS:CONFIRMED
+DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnis￿se. Diim text neme
+ lt lie￿benswerten ääääts über￿zeugt mit ergreidddszi￿nie
+ ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu
+ nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie
+ lten Opern aller Zeiten avanciert ist.\n
+SUMMARY:Zauberflöte
+END:VEVENT
+END:VCALENDAR

tests/unit/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.test.js

@@ -0,0 +1,56 @@
+/**
+ * @copyright Copyright (c) 2019 Richard Steinmetz <[email protected]>
+ *
+ * @author 2024 Richard Steinmetz <[email protected]>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+import AbstractRepairStep from '../../../../../src/parsers/repairsteps/abstractRepairStep.js';
+import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep
+	from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js';
+
+it('The repair step should inherit from AbstractRepairStep', () => {
+	expect((new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep() instanceof AbstractRepairStep)).toEqual(true)
+})
+
+it('The repair step should have a priority', () => {
+	expect(ICalendarRemoveUnicodeSpecialNoncharactersRepairStep.priority()).toEqual(0)
+})
+
+it('The repair step should remove U+FFFE non-characters', () => {
+	const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep()
+	const brokenICS = getAsset('unicode-non-character-fffe-before')
+	const fixedICS = getAsset('unicode-non-character-fffe-after')
+
+	expect(repairStep.repair(brokenICS)).toEqual(fixedICS)
+})
+
+it('The repair step should remove U+FFFF non-characters', () => {
+	const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep()
+	const brokenICS = getAsset('unicode-non-character-ffff-before')
+	const fixedICS = getAsset('unicode-non-character-ffff-after')
+
+	expect(repairStep.repair(brokenICS)).toEqual(fixedICS)
+})
+
+it('The repair step should not change valid calendar data', () => {
+	const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep()
+	const ics = getAsset('simple-date-time-europe-berlin-dtstart-dtend')
+
+	expect(repairStep.repair(ics)).toEqual(ics)
+})

tests/unit/parsers/repairsteps/icalendar/index.test.js

@@ -34,6 +34,8 @@ import ICalendarMultipleVCalendarBlocksRepairStep
 	from '../../../../../src/parsers/repairsteps/icalendar/icalendarMultipleVCalendarBlocksRepairStep.js';
 import ICalendarRemoveXNCGroupIdRepairStep
 	from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveXNCGroupIdRepairStep.js';
+import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep
+	from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js';
 
 it('should provide an iterator over all parsers', () => {
 	const iterator = getRepairSteps()
@@ -45,5 +47,6 @@ it('should provide an iterator over all parsers', () => {
 	expect(iterator.next().value).toEqual(ICalendarIllegalCreatedRepairStep)
 	expect(iterator.next().value).toEqual(ICalendarMultipleVCalendarBlocksRepairStep)
 	expect(iterator.next().value).toEqual(ICalendarRemoveXNCGroupIdRepairStep)
+	expect(iterator.next().value).toEqual(ICalendarRemoveUnicodeSpecialNoncharactersRepairStep)
 	expect(iterator.next().value).toEqual(undefined)
 })