From 517a0e71164cc765037e48d47865866763e335d0 Mon Sep 17 00:00:00 2001
From: Cameron Bytheway <bytheway.cameron@gmail.com>
Date: Tue, 5 Nov 2024 12:25:29 -0700
Subject: [PATCH] fix(duvet): remove BOM markers (#132)

---
 duvet/src/specification/ietf/snapshots.tar.gz | 4 ++--
 duvet/src/specification/ietf/tokenizer.rs     | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/duvet/src/specification/ietf/snapshots.tar.gz b/duvet/src/specification/ietf/snapshots.tar.gz
index 142dc3f..3be58b3 100644
--- a/duvet/src/specification/ietf/snapshots.tar.gz
+++ b/duvet/src/specification/ietf/snapshots.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fec55dd607d2bc099af6940745ba04fc5c31a917340c301084ad1b85fafa7844
-size 573480960
+oid sha256:139f84b4377e51be6089a905ae7f23a837fd56fac22c1759c2230d0e53aa54cd
+size 574074880
diff --git a/duvet/src/specification/ietf/tokenizer.rs b/duvet/src/specification/ietf/tokenizer.rs
index d3e308c..39c7c5e 100644
--- a/duvet/src/specification/ietf/tokenizer.rs
+++ b/duvet/src/specification/ietf/tokenizer.rs
@@ -128,7 +128,12 @@ macro_rules! expect_contents {
 
 /// Transforms file contents into lines
 fn lines(contents: &SourceFile) -> impl Iterator<Item = Token> + '_ {
-    contents.lines().enumerate().map(move |(line, value)| {
+    contents.lines().enumerate().map(move |(line, mut value)| {
+        // look for Byte Order Mark and filter it out
+        if line == 0 {
+            value = value.trim_start_matches("\u{feff}");
+        }
+
         // line numbers start at 1
         let line = line + 1;
         let value = contents.substr(value).unwrap();