From dc5cc22299e7288f9cfe1a398043c515c18af14c Mon Sep 17 00:00:00 2001 From: Sanskar Soni Date: Mon, 10 Jun 2024 07:30:44 +0530 Subject: [PATCH] feat(repair): added unicode non-character repair step Signed-off-by: Sanskar Soni Signed-off-by: Richard Steinmetz --- ...veUnicodeSpecialNoncharactersRepairStep.js | 40 +++++++++++++ src/parsers/repairsteps/icalendar/index.js | 2 + .../unicode-non-character-fffe-after.ics | 55 ++++++++++++++++++ .../unicode-non-character-fffe-before.ics | 55 ++++++++++++++++++ .../unicode-non-character-ffff-after.ics | 55 ++++++++++++++++++ .../unicode-non-character-ffff-before.ics | 55 ++++++++++++++++++ ...codeSpecialNoncharactersRepairStep.test.js | 56 +++++++++++++++++++ .../repairsteps/icalendar/index.test.js | 3 + 8 files changed, 321 insertions(+) create mode 100644 src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js create mode 100644 tests/assets/unicode-non-character-fffe-after.ics create mode 100644 tests/assets/unicode-non-character-fffe-before.ics create mode 100644 tests/assets/unicode-non-character-ffff-after.ics create mode 100644 tests/assets/unicode-non-character-ffff-before.ics create mode 100644 tests/unit/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.test.js diff --git a/src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js b/src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js new file mode 100644 index 00000000..72007f84 --- /dev/null +++ b/src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js @@ -0,0 +1,40 @@ +/** + * @copyright Copyright (c) 2024 Sanskar Soni + * + * @author Sanskar Soni + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +import AbstractRepairStep from '../abstractRepairStep.js' + +/** + * @class ICalendarRemoveUnicodeSpecialNoncharactersRepairStep + * @classdesc This repair step removes Unicode specials non-characters i.e. U+FFFE & U+FFFF + */ +export default class ICalendarRemoveUnicodeSpecialNoncharactersRepairStep extends AbstractRepairStep { + + /** + * Please see the corresponding test file for an example of broken calendar-data + * + * @inheritDoc + */ + repair(ics) { + return ics + .replace(/(\uFFFF|\uFFFE)/g, '') + } + +} diff --git a/src/parsers/repairsteps/icalendar/index.js b/src/parsers/repairsteps/icalendar/index.js index 29172aaa..35f52628 100644 --- a/src/parsers/repairsteps/icalendar/index.js +++ b/src/parsers/repairsteps/icalendar/index.js @@ -26,6 +26,7 @@ import ICalendarEmptyTriggerRepairStep from './icalendarEmptyTriggerRepairStep.j import ICalendarIllegalCreatedRepairStep from './icalendarIllegalCreatedRepairStep.js' import ICalendarMultipleVCalendarBlocksRepairStep from './icalendarMultipleVCalendarBlocksRepairStep.js' import ICalendarRemoveXNCGroupIdRepairStep from './icalendarRemoveXNCGroupIdRepairStep.js' +import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep from './icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js' /** * Get an iterator over all repair steps for iCalendar documents @@ -38,4 +39,5 @@ export function * getRepairSteps() { yield ICalendarIllegalCreatedRepairStep yield ICalendarMultipleVCalendarBlocksRepairStep yield ICalendarRemoveXNCGroupIdRepairStep + yield ICalendarRemoveUnicodeSpecialNoncharactersRepairStep } diff --git a/tests/assets/unicode-non-character-fffe-after.ics b/tests/assets/unicode-non-character-fffe-after.ics new file mode 100644 index 00000000..ddc51e7f --- /dev/null +++ b/tests/assets/unicode-non-character-fffe-after.ics @@ -0,0 +1,55 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +PRODID:-//SabreDAV//SabreDAV//EN +X-WR-CALNAME:41424_Sigharting (Webportal Plugin) +REFRESH-INTERVAL;VALUE=DURATION:PT4H +X-PUBLISHED-TTL:PT4H +BEGIN:VTIMEZONE +TZID:Europe/Vienna +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20230405T070540Z +DTSTAMP:20230614T183802Z +LAST-MODIFIED:20230614T183802Z +SEQUENCE:6 +UID:0e85e9f1-8e56-4581-b4d6-04d865152898 +DTSTART;TZID=Europe/Vienna:20230613T083000 +DTEND;TZID=Europe/Vienna:20230613T113000 +STATUS:CONFIRMED +SUMMARY:TEST1 +LOCATION:Church +DESCRIPTION:test spacew + allfahrt  desende. +END:VEVENT +BEGIN:VEVENT +CREATED:20240603T130712Z +DTSTAMP:20240603T130749Z +LAST-MODIFIED:20240603T130749Z +SEQUENCE:3 +UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e +DTSTART;VALUE=DATE:20241020 +DTEND;VALUE=DATE:20241021 +STATUS:CONFIRMED +DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnisse. Diim text neme + lt liebenswerten ääääts überzeugt mit ergreidddszinie + ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu + nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie + lten Opern aller Zeiten avanciert ist.\n +SUMMARY:Zauberflöte +END:VEVENT +END:VCALENDAR diff --git a/tests/assets/unicode-non-character-fffe-before.ics b/tests/assets/unicode-non-character-fffe-before.ics new file mode 100644 index 00000000..ae62be8a --- /dev/null +++ b/tests/assets/unicode-non-character-fffe-before.ics @@ -0,0 +1,55 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +PRODID:-//SabreDAV//SabreDAV//EN +X-WR-CALNAME:41424_Sigharting (Webportal Plugin) +REFRESH-INTERVAL;VALUE=DURATION:PT4H +X-PUBLISHED-TTL:PT4H +BEGIN:VTIMEZONE +TZID:Europe/Vienna +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20230405T070540Z +DTSTAMP:20230614T183802Z +LAST-MODIFIED:20230614T183802Z +SEQUENCE:6 +UID:0e85e9f1-8e56-4581-b4d6-04d865152898 +DTSTART;TZID=Europe/Vienna:20230613T083000 +DTEND;TZID=Europe/Vienna:20230613T113000 +STATUS:CONFIRMED +SUMMARY:TEST1 +LOCATION:Church +DESCRIPTION:test spacew + allfahrt  desende. +END:VEVENT +BEGIN:VEVENT +CREATED:20240603T130712Z +DTSTAMP:20240603T130749Z +LAST-MODIFIED:20240603T130749Z +SEQUENCE:3 +UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e +DTSTART;VALUE=DATE:20241020 +DTEND;VALUE=DATE:20241021 +STATUS:CONFIRMED +DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnis￾se. Diim text neme + lt lie￾benswerten ääääts über￾zeugt mit ergreidddszi￾nie + ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu + nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie + lten Opern aller Zeiten avanciert ist.\n +SUMMARY:Zauberflöte +END:VEVENT +END:VCALENDAR diff --git a/tests/assets/unicode-non-character-ffff-after.ics b/tests/assets/unicode-non-character-ffff-after.ics new file mode 100644 index 00000000..ddc51e7f --- /dev/null +++ b/tests/assets/unicode-non-character-ffff-after.ics @@ -0,0 +1,55 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +PRODID:-//SabreDAV//SabreDAV//EN +X-WR-CALNAME:41424_Sigharting (Webportal Plugin) +REFRESH-INTERVAL;VALUE=DURATION:PT4H +X-PUBLISHED-TTL:PT4H +BEGIN:VTIMEZONE +TZID:Europe/Vienna +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20230405T070540Z +DTSTAMP:20230614T183802Z +LAST-MODIFIED:20230614T183802Z +SEQUENCE:6 +UID:0e85e9f1-8e56-4581-b4d6-04d865152898 +DTSTART;TZID=Europe/Vienna:20230613T083000 +DTEND;TZID=Europe/Vienna:20230613T113000 +STATUS:CONFIRMED +SUMMARY:TEST1 +LOCATION:Church +DESCRIPTION:test spacew + allfahrt  desende. +END:VEVENT +BEGIN:VEVENT +CREATED:20240603T130712Z +DTSTAMP:20240603T130749Z +LAST-MODIFIED:20240603T130749Z +SEQUENCE:3 +UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e +DTSTART;VALUE=DATE:20241020 +DTEND;VALUE=DATE:20241021 +STATUS:CONFIRMED +DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnisse. Diim text neme + lt liebenswerten ääääts überzeugt mit ergreidddszinie + ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu + nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie + lten Opern aller Zeiten avanciert ist.\n +SUMMARY:Zauberflöte +END:VEVENT +END:VCALENDAR diff --git a/tests/assets/unicode-non-character-ffff-before.ics b/tests/assets/unicode-non-character-ffff-before.ics new file mode 100644 index 00000000..325a7a65 --- /dev/null +++ b/tests/assets/unicode-non-character-ffff-before.ics @@ -0,0 +1,55 @@ +BEGIN:VCALENDAR +VERSION:2.0 +CALSCALE:GREGORIAN +PRODID:-//SabreDAV//SabreDAV//EN +X-WR-CALNAME:41424_Sigharting (Webportal Plugin) +REFRESH-INTERVAL;VALUE=DURATION:PT4H +X-PUBLISHED-TTL:PT4H +BEGIN:VTIMEZONE +TZID:Europe/Vienna +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20230405T070540Z +DTSTAMP:20230614T183802Z +LAST-MODIFIED:20230614T183802Z +SEQUENCE:6 +UID:0e85e9f1-8e56-4581-b4d6-04d865152898 +DTSTART;TZID=Europe/Vienna:20230613T083000 +DTEND;TZID=Europe/Vienna:20230613T113000 +STATUS:CONFIRMED +SUMMARY:TEST1 +LOCATION:Church +DESCRIPTION:test spacew + allfahrt  desende. +END:VEVENT +BEGIN:VEVENT +CREATED:20240603T130712Z +DTSTAMP:20240603T130749Z +LAST-MODIFIED:20240603T130749Z +SEQUENCE:3 +UID:1af6c9e2-76be-4e2d-80f5-90bc1b41a84e +DTSTART;VALUE=DATE:20241020 +DTEND;VALUE=DATE:20241021 +STATUS:CONFIRMED +DESCRIPTION:Mozarts "Die Zauberflöte" erlebt ... enntnis￿se. Diim text neme + lt lie￿benswerten ääääts über￿zeugt mit ergreidddszi￿nie + ltert durch ihre zeitlosen Themen von Liebe\, Freundschaft \nu + nd Selbstfindung\, wodurch sie zu einer der beliebtesten \nund meistgespie + lten Opern aller Zeiten avanciert ist.\n +SUMMARY:Zauberflöte +END:VEVENT +END:VCALENDAR diff --git a/tests/unit/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.test.js b/tests/unit/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.test.js new file mode 100644 index 00000000..2c1871d3 --- /dev/null +++ b/tests/unit/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.test.js @@ -0,0 +1,56 @@ +/** + * @copyright Copyright (c) 2019 Richard Steinmetz + * + * @author 2024 Richard Steinmetz + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +import AbstractRepairStep from '../../../../../src/parsers/repairsteps/abstractRepairStep.js'; +import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep + from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js'; + +it('The repair step should inherit from AbstractRepairStep', () => { + expect((new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep() instanceof AbstractRepairStep)).toEqual(true) +}) + +it('The repair step should have a priority', () => { + expect(ICalendarRemoveUnicodeSpecialNoncharactersRepairStep.priority()).toEqual(0) +}) + +it('The repair step should remove U+FFFE non-characters', () => { + const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep() + const brokenICS = getAsset('unicode-non-character-fffe-before') + const fixedICS = getAsset('unicode-non-character-fffe-after') + + expect(repairStep.repair(brokenICS)).toEqual(fixedICS) +}) + +it('The repair step should remove U+FFFF non-characters', () => { + const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep() + const brokenICS = getAsset('unicode-non-character-ffff-before') + const fixedICS = getAsset('unicode-non-character-ffff-after') + + expect(repairStep.repair(brokenICS)).toEqual(fixedICS) +}) + +it('The repair step should not change valid calendar data', () => { + const repairStep = new ICalendarRemoveUnicodeSpecialNoncharactersRepairStep() + const ics = getAsset('simple-date-time-europe-berlin-dtstart-dtend') + + expect(repairStep.repair(ics)).toEqual(ics) +}) diff --git a/tests/unit/parsers/repairsteps/icalendar/index.test.js b/tests/unit/parsers/repairsteps/icalendar/index.test.js index e47bd0d2..1c02e335 100644 --- a/tests/unit/parsers/repairsteps/icalendar/index.test.js +++ b/tests/unit/parsers/repairsteps/icalendar/index.test.js @@ -34,6 +34,8 @@ import ICalendarMultipleVCalendarBlocksRepairStep from '../../../../../src/parsers/repairsteps/icalendar/icalendarMultipleVCalendarBlocksRepairStep.js'; import ICalendarRemoveXNCGroupIdRepairStep from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveXNCGroupIdRepairStep.js'; +import ICalendarRemoveUnicodeSpecialNoncharactersRepairStep + from '../../../../../src/parsers/repairsteps/icalendar/icalendarRemoveUnicodeSpecialNoncharactersRepairStep.js'; it('should provide an iterator over all parsers', () => { const iterator = getRepairSteps() @@ -45,5 +47,6 @@ it('should provide an iterator over all parsers', () => { expect(iterator.next().value).toEqual(ICalendarIllegalCreatedRepairStep) expect(iterator.next().value).toEqual(ICalendarMultipleVCalendarBlocksRepairStep) expect(iterator.next().value).toEqual(ICalendarRemoveXNCGroupIdRepairStep) + expect(iterator.next().value).toEqual(ICalendarRemoveUnicodeSpecialNoncharactersRepairStep) expect(iterator.next().value).toEqual(undefined) })