Support non-ASCII tags

amake · Oct 12, 2024 · a3795a1 · a3795a1 · llcc · Oct 12, 2024
1 parent 56e1600
commit a3795a1
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 11 deletions.
diff --git a/lib/src/org/grammar.dart b/lib/src/org/grammar.dart
@@ -1,4 +1,3 @@
-import 'package:more/char_matcher.dart';
 import 'package:org_parser/src/todo/model.dart';
 import 'package:org_parser/src/util/util.dart';
 import 'package:petitparser/petitparser.dart';
@@ -73,7 +72,7 @@ class OrgGrammarDefinition extends GrammarDefinition {
       char(':') &
       lineEnd().and();
 
-  Parser tag() => pattern('a-zA-Z0-9_@#%').plusString('Tags expected');
+  Parser tag() => (alnum() | anyOf('_@#%')).plus().flatten('Tags expected');
 
   Parser content() => ref0(_content).flatten('Content expected');
 
@@ -313,14 +312,7 @@ class OrgContentGrammarDefinition extends GrammarDefinition {
       ref1(sexpList, '{}') |
       ref0(sexpList) |
       char('*') |
-      (anyOf('+-').optional() &
-          reverseId(ref0(alnum) | anyOf(r'.,\'), ref0(alnum)));
-
-  Parser alnum() => any().where((char) {
-        final c = char.codeUnitAt(0);
-        return UnicodeCharMatcher.letter().match(c) ||
-            UnicodeCharMatcher.number().match(c);
-      });
+      (anyOf('+-').optional() & reverseId(alnum() | anyOf(r'.,\'), alnum()));
 
   Parser macroReference() =>
       string('{{{') &

diff --git a/lib/src/query/grammar.dart b/lib/src/query/grammar.dart
@@ -44,7 +44,7 @@ class OrgQueryGrammarDefinition extends GrammarDefinition {
   // TODO(aaron): Support regex element: {^boss.*}
   Parser element() => ref0(propertyExpression) | ref0(tag);
 
-  Parser tag() => pattern('a-zA-Z0-9_@#%').plusString('Tag expected');
+  Parser tag() => (alnum() | anyOf('_@#%')).plus().flatten('Tag expected');
 
   Parser propertyExpression() =>
       ref0(propertyName) & ref0(op) & ref0(propertyValue);

diff --git a/lib/src/util/alnum.dart b/lib/src/util/alnum.dart
@@ -0,0 +1,8 @@
+import 'package:more/char_matcher.dart';
+import 'package:petitparser/petitparser.dart';
+
+Parser alnum() => any().where((char) {
+      final c = char.codeUnitAt(0);
+      return UnicodeCharMatcher.letter().match(c) ||
+          UnicodeCharMatcher.number().match(c);
+    });
diff --git a/lib/src/util/util.dart b/lib/src/util/util.dart
@@ -1,3 +1,4 @@
+export 'alnum.dart';
 export 'block.dart';
 export 'bof.dart';
 export 'drop.dart';

diff --git a/test/org/model/header_test.dart b/test/org/model/header_test.dart
@@ -19,5 +19,12 @@ void main() {
       final headline = result.value as OrgHeadline;
       expect(headline.toMarkup(), markup);
     });
+    test('non-ASCII tags', () {
+      final markup = '* TODO foo :あ:';
+      final result = parser.parse(markup);
+      final headline = result.value as OrgHeadline;
+      expect(headline.tags!.values, ['あ']);
+      expect(headline.toMarkup(), markup);
+    });
   });
 }
diff --git a/test/query/parser_test.dart b/test/query/parser_test.dart
@@ -13,6 +13,10 @@ void main() {
         OrgQueryNotMatcher(OrgQueryTagMatcher('foo')),
       );
     });
+    test('non-ASCII tag', () {
+      expect(orgQuery.parse('あ').value, OrgQueryTagMatcher('あ'));
+      expect(orgQuery.parse('+あ').value, OrgQueryTagMatcher('あ'));
+    });
     test('implicit and', () {
       expect(
         orgQuery.parse('foo+bar').value,