feat: update core & tests

And fix remaining issues

.github/workflows/ci.yml

@@ -34,16 +34,11 @@ jobs:
         with:
           python-version: ${{matrix.python}}
       - name: Lint
+        continue-on-error: true
         run: pipx run ruff check .
       - name: Build
-        run: pip install -e .
+        run: pip install -v -e .[tests]
         env:
-          CFLAGS: "-O0 -g"
+          CFLAGS: -Wextra -O0 -g
       - name: Test
-        shell: python
-        # run: python -Wignore:::tree_sitter -munittest
-        run: |-
-          try: __import__('tree_sitter').Language(1048576)
-          except RuntimeError as err: print(err)
-        env:
-          PYTHONFAULTHANDLER: 1
+        run: python -munittest -v

.gitmodules

@@ -1,21 +1,3 @@
 [submodule "tree-sitter"]
 	path = tree_sitter/core
 	url = https://github.com/tree-sitter/tree-sitter
-[submodule "tree-sitter-embedded-template"]
-	path = tests/fixtures/tree-sitter-embedded-template
-	url = https://github.com/tree-sitter/tree-sitter-embedded-template
-[submodule "tree-sitter-html"]
-	path = tests/fixtures/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-[submodule "tree-sitter-javascript"]
-	path = tests/fixtures/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-[submodule "tree-sitter-json"]
-	path = tests/fixtures/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-[submodule "tree-sitter-python"]
-	path = tests/fixtures/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-[submodule "tree-sitter-rust"]
-	path = tests/fixtures/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust

pyproject.toml

@@ -31,6 +31,13 @@ email = "[email protected]"
 
 [project.optional-dependencies]
 docs = ["sphinx~=7.3", "sphinx-book-theme"]
+tests = [
+    "tree-sitter-html @ git+https://github.com/tree-sitter/tree-sitter-html@refs/pull/93/head",
+    "tree-sitter-javascript",
+    "tree-sitter-json",
+    "tree-sitter-python",
+    "tree-sitter-rust",
+]
 
 [tool.ruff]
 target-version = "py39"
@@ -39,7 +46,6 @@ indent-width = 4
 extend-exclude = [
     ".github",
     "__pycache__",
-    "tests/fixtures",
     "tree_sitter/core",
 ]
 
@@ -49,6 +55,7 @@ indent-style = "space"
 
 [tool.cibuildwheel]
 build-frontend = "build"
+test-extras = ["tests"]
 test-command = "python -munittest discover -s {project}/tests"
 
 [tool.cibuildwheel.environment]

setup.py

@@ -1,8 +1,6 @@
-"""Py-Tree-sitter"""
-
 from platform import system
 
-from setuptools import Extension, setup
+from setuptools import Extension, setup  # type: ignore
 
 setup(
     packages=["tree_sitter"],
@@ -41,8 +39,12 @@
             extra_compile_args=[
                 "-std=c11",
                 "-fvisibility=hidden",
+                "-Wno-cast-function-type",
                 "-Werror=implicit-function-declaration",
-            ] if system() != "Windows" else None
+            ] if system() != "Windows" else [
+                "/std:c11",
+                "/wd4244",
+            ]
         )
     ]
 )

tests/test_language.py

@@ -0,0 +1,86 @@
+from unittest import TestCase
+
+from tree_sitter import Language, Query
+
+import tree_sitter_html
+import tree_sitter_javascript
+import tree_sitter_json
+import tree_sitter_python
+import tree_sitter_rust
+
+
+class TestLanguage(TestCase):
+    def setUp(self):
+        self.html = tree_sitter_html.language()
+        self.javascript = tree_sitter_javascript.language()
+        self.json = tree_sitter_json.language()
+        self.python = tree_sitter_python.language()
+        self.rust = tree_sitter_rust.language()
+
+    def test_init_not_positive(self):
+        self.assertRaises(ValueError, Language, -1)
+
+    def test_init_segv(self):
+        self.assertRaises(RuntimeError, Language, 1024)
+
+    def test_properties(self):
+        lang = Language(self.python)
+        self.assertEqual(lang.version, 14)
+        self.assertEqual(lang.node_kind_count, 274)
+        self.assertEqual(lang.parse_state_count, 2831)
+        self.assertEqual(lang.field_count, 32)
+
+    def test_node_kind_for_id(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.node_kind_for_id(1), "{")
+        self.assertEqual(lang.node_kind_for_id(3), "}")
+
+    def test_id_for_node_kind(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.id_for_node_kind(":", False), 4)
+        self.assertEqual(lang.id_for_node_kind("string", True), 20)
+
+    def test_node_kind_is_named(self):
+        lang = Language(self.json)
+        self.assertFalse(lang.node_kind_is_named(4))
+        self.assertTrue(lang.node_kind_is_named(20))
+
+    def test_node_kind_is_visible(self):
+        lang = Language(self.json)
+        self.assertTrue(lang.node_kind_is_visible(2))
+
+    def test_field_name_for_id(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.field_name_for_id(1), "key")
+        self.assertEqual(lang.field_name_for_id(2), "value")
+
+    def test_field_id_for_name(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.field_id_for_name("key"), 1)
+        self.assertEqual(lang.field_id_for_name("value"), 2)
+
+    def test_next_state(self):
+        lang = Language(self.javascript)
+        self.assertNotEqual(lang.next_state(1, 1), 0)
+
+    def test_lookahead_iterator(self):
+        lang = Language(self.javascript)
+        self.assertIsNotNone(lang.lookahead_iterator(0))
+        self.assertIsNone(lang.lookahead_iterator(9999))
+
+    def test_query(self):
+        lang = Language(self.json)
+        query = lang.query("(string) @string")
+        self.assertIsInstance(query, Query)
+
+    def test_eq(self):
+        self.assertEqual(Language(self.json), Language(self.json))
+        self.assertNotEqual(Language(self.rust), Language(self.html))
+
+    def test_int(self):
+        for name in ["html", "javascript", "json", "python", "rust"]:
+            with self.subTest(language=name):
+                ptr = getattr(self, name)
+                lang = Language(ptr)
+                self.assertEqual(int(lang), ptr)
+                self.assertEqual(hash(lang), ptr)

tests/test_lookahead_iterator.py

@@ -0,0 +1,43 @@
+from unittest import TestCase
+
+from tree_sitter import Language, Parser
+
+import tree_sitter_rust
+
+class TestLookaheadIterator(TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.rust = Language(tree_sitter_rust.language())
+
+    def test_lookahead_iterator(self):
+        parser = Parser(self.rust)
+        cursor = parser.parse(b"struct Stuff{}").walk()
+
+        self.assertEqual(cursor.goto_first_child(), True)  # struct
+        self.assertEqual(cursor.goto_first_child(), True)  # struct keyword
+
+        next_state = cursor.node.next_parse_state
+
+        self.assertNotEqual(next_state, 0)
+        self.assertEqual(
+            next_state, self.rust.next_state(cursor.node.parse_state, cursor.node.grammar_id)
+        )
+        self.assertLess(next_state, self.rust.parse_state_count)
+        self.assertEqual(cursor.goto_next_sibling(), True)  # type_identifier
+        self.assertEqual(next_state, cursor.node.parse_state)
+        self.assertEqual(cursor.node.grammar_name, "identifier")
+        self.assertNotEqual(cursor.node.grammar_id, cursor.node.kind_id)
+
+        expected_symbols = ["//", "/*", "identifier", "line_comment", "block_comment"]
+        lookahead = self.rust.lookahead_iterator(next_state)
+        self.assertEqual(lookahead.language, self.rust)
+        self.assertListEqual(list(lookahead.iter_names()), expected_symbols)
+
+        lookahead.reset_state(next_state)
+        self.assertListEqual(list(lookahead.iter_names()), expected_symbols)
+
+        lookahead.reset_state(next_state, self.rust)
+        self.assertListEqual(
+            list(map(self.rust.node_kind_for_id, list(iter(lookahead)))),
+            expected_symbols
+        )