[USING, DON'T MERGE THO] Enable Lexical-style parsing

.github/workflows/wheels.yml

@@ -15,82 +15,43 @@ env:
 jobs:
   macos:
     runs-on: macos-latest
+    strategy:
+      matrix:
+        target: [x86_64]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
           architecture: x64
       - uses: dtolnay/rust-toolchain@stable
-      - name: Build wheels - x86_64
+      - name: Build wheels - ${{ matrix.target }}
         uses: PyO3/maturin-action@v1
         with:
-          target: x86_64
+          target: ${{ matrix.target }}
           args: --release --out dist --sdist -i 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9
-      - name: Test built wheel - x86_64
-        run: |
-          pip install y-py --no-index --find-links dist --force-reinstall
-          pip install pytest
-          pytest
       - name: Build wheels - universal2
         uses: PyO3/maturin-action@v1
         with:
           args: --release --target universal2-apple-darwin --out dist -i 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9
-      - name: Test built wheel - universal2
-        run: |
-          pip install y-py --no-index --find-links dist --force-reinstall
-          pytest
-      - name: Upload wheels
-        uses: actions/upload-artifact@v2
-        with:
-          name: wheels
-          path: dist
-
-  windows:
-    runs-on: windows-latest
-    name: windows (${{ matrix.platform.target }})
-    strategy:
-      matrix:
-        platform:
-          - target: x64
-            interpreter: 3.7 3.8 3.9 3.10 3.11
-          - target: x86
-            interpreter: 3.7 3.8 3.9 3.10 3.11
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          architecture: ${{ matrix.platform.target }}
-      - uses: dtolnay/rust-toolchain@stable
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i ${{ matrix.platform.interpreter }}
-      - name: Test built wheel
-        run: |
-          pip install y-py --no-index --find-links dist --force-reinstall
-          pip install pytest
-          pytest
       - name: Upload wheels
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4.6.0
         with:
-          name: wheels
+          name: wheels-macos-${{ matrix.target }}
           path: dist
 
   linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: [x86_64, i686]
+        target: [x86_64]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
           architecture: x64
-      - name: Build wheels
+      - name: Build wheels - ${{ matrix.target }}
         uses: PyO3/maturin-action@v1
         with:
           rust-toolchain: stable
@@ -100,13 +61,13 @@ jobs:
       - name: Test built wheel
         if: matrix.target == 'x86_64'
         run: |
-          pip install y-py --no-index --find-links dist --force-reinstall
+          pip install y-py-dart --no-index --find-links dist --force-reinstall
           pip install pytest
           pytest
       - name: Upload wheels
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4.6.0
         with:
-          name: wheels
+          name: wheels-linux-${{ matrix.target }}
           path: dist
 
   linux-cross:
@@ -118,121 +79,39 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
-      - name: Build wheels
+          python-version: "3.11"
+      - name: Build wheels - ${{ matrix.target }}
         uses: PyO3/maturin-action@v1
         with:
           rust-toolchain: stable
           target: ${{ matrix.target }}
           manylinux: auto
           args: --release --out dist -i 3.7 3.8 3.9 3.10 3.11 3.12 pypy3.8 pypy3.9
-
-      - uses: uraimo/[email protected]
-        if: matrix.target != 'ppc64'
-        name: Test built wheel
-        with:
-          arch: ${{ matrix.target }}
-          distro: ubuntu20.04
-          githubToken: ${{ github.token }}
-          install: |
-            apt-get update
-            apt-get install -y --no-install-recommends python3 python3-pip
-            pip3 install -U pip pytest
-          run: |
-            pip3 install y-py --no-index --find-links dist/ --force-reinstall
-            pytest
-
       - name: Upload wheels
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.6.0
         with:
-          name: wheels
+          name: wheels-linux-cross-${{ matrix.target }}
           path: dist
 
-  wasm:
-    runs-on: ubuntu-20.04
-    # Pyodide began supporting `micropip.install` from emscripten-compiled binary wheels
-    # in Pyodide 0.21.0 (Aug 2022), so no need to build wheels for versions before then.
-    # As of Nov 2022, the matrix for emscripten/python versions since then is simple. 
-    # Update this matrix when new Pyodide versions come out that bump the Python interpreter
-    # or emscripten version. Ref: https://pyodide.org/en/stable/project/changelog.html
-    strategy:
-      matrix:
-        python-version: ["3.10.2"]
-        emscripten-version: ["3.1.14"]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - name: Install Rust
-        # if this isn't here, wasm wheel build fails with error: failed to run `rustc` to learn about target-specific information
-        uses: actions-rs/toolchain@v1
-        with:
-            # maturin build will generate args -Z link-native-libraries=no which is only accepted on nightly compiler
-            toolchain: nightly
-            profile: minimal
-            override: true
-      - name: Rustup add wasm32 target
-        # maturin build --target wasm32-unknown-emscripten requires rust to add that target first
-        run: rustup target add wasm32-unknown-emscripten
-      - name: Setup emsdk
-        uses: mymindstorm/setup-emsdk@v11
-        with: 
-          version: ${{ matrix.emscripten-version }}
-      - name: Build wheels
-        uses: messense/maturin-action@v1
-        with:
-          target: wasm32-unknown-emscripten
-          args: --release --out wasm_wheel --find-interpreter
-      - name: Upload wheels
-        uses: actions/upload-artifact@v2
-        with:
-          name: wasm_wheel
-          path: wasm_wheel
-
   pypi-release:
-    name: Publish to Pypi on Release
+    name: Publish to PyPI on Release
     runs-on: ubuntu-latest
     needs:
       - macos
-      - windows
       - linux
       - linux-cross
     if: startsWith(github.ref, 'refs/tags/')
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4.1.8
         with:
-          name: wheels
+          merge-multiple: true
+          path: dist
+          pattern: wheels-*
       - uses: actions/setup-python@v2
-      - name: Publish to PyPi
+      - name: Publish to PyPI
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           pip install --upgrade twine
-          twine upload --skip-existing *
-
-  # Can't upload emscripten wheels to Pypi, see https://github.com/pypi/warehouse/issues/10416.
-  # For now, this will attach the binary wheels to the Release page in Github. Users can 
-  # download those into a pyodide environment and micropip.install from there.
-  wasm-release:
-    name: Attach wasm wheel to Release
-    runs-on: ubuntu-latest
-    needs:
-      - wasm
-    if: startsWith(github.ref, 'refs/tags/')
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: wasm_wheel
-
-      - name: Attach assets to Release
-        uses: softprops/action-gh-release@v1
-        with:
-          files: '*.whl'
-          # 'name' (release name) defaults to the tag ref in this action.
-          # If release naming pattern changes, will need to configure 'name' here.
-
-
-
+          twine upload --skip-existing dist/*

.gitignore

@@ -9,4 +9,6 @@ Cargo.lock
 docs/_*
 .DS_Store
 *.pyc
-dist
+dist
+node_modules
+.venv

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "y-py"
-version = "0.7.0-alpha.1"
+version = "0.6.3-alpha.19"
 rust-version = "1.72"
 edition = "2021"
 
@@ -12,8 +12,17 @@ crate-type = ["cdylib"]
 
 [dependencies]
 lib0 = "0.16.10"
-yrs = "0.16.10"
+yrs = { git = "https://github.com/its-dart/y-crdt.git", branch = "v0.16.10" }
+
+[dev-dependencies]
+assert-json-diff = "2.0.2"
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.79", features = ["unbounded_depth"] }
 
 [dependencies.pyo3]
-version = "0.19.2"
+version = "0.21.0"
+# Make the extension-module feature optional and default.
+# See https://pyo3.rs/v0.21.2/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror for more details
+# extension-module = ["pyo3/extension-module"]
+# default = ["extension-module"]
 features = ["extension-module"]

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["maturin>=1.2.3,<2"]
 build-backend = "maturin"
 
 [project]
-name = "y-py"
+name = "y-py-dart"
 description = "Python bindings for the Y-CRDT built from yrs (Rust)"
 license = { file = "LICENSE" }
 authors = [
@@ -30,10 +30,10 @@ classifiers = [
 ]
 
 [project.urls]
-Homepage = "https://github.com/y-crdt/ypy"
-Source = "https://github.com/y-crdt/ypy"
-Issues = "https://github.com/y-crdt/ypy/issues"
-Pypi = "https://pypi.org/project/y-py"
+Homepage = "https://github.com/its-dart/ypy"
+Source = "https://github.com/its-dart/ypy"
+Issues = "https://github.com/its-dart/ypy/issues"
+Pypi = "https://pypi.org/project/y-py-dart"
 
 [tool.hatch.envs.test]
 dependencies = ["pytest", "maturin"]

src/json_builder.rs

@@ -60,6 +60,11 @@ impl<'a> JsonBuildable for CompatiblePyType<'a> {
 
                 buffer.push_str("]");
             }
+            CompatiblePyType::ByteArray(bytes) => {
+                return Err(PyTypeError::new_err(
+                    "Binary objects cannot be converted to JSON format"
+                ));
+            }
             CompatiblePyType::Dict(dict) => {
                 buffer.push_str("{");
                 let length = dict.len();

src/lib.rs

@@ -1,3 +1,5 @@
+#![recursion_limit = "512"]
+
 use pyo3::prelude::*;
 use pyo3::wrap_pyfunction;
 mod json_builder;

src/shared_types.rs

@@ -65,6 +65,7 @@ pub enum CompatiblePyType<'a> {
     String(&'a pytypes::PyString),
     List(&'a pytypes::PyList),
     Dict(&'a pytypes::PyDict),
+    ByteArray(&'a pytypes::PyByteArray),
     YType(YPyType<'a>),
     None,
 }

src/type_conversions.rs

@@ -77,6 +77,8 @@ impl<'a> TryFrom<&'a PyAny> for CompatiblePyType<'a> {
             Ok(Self::List(list))
         } else if let Ok(dict) = py_any.downcast::<pytypes::PyDict>() {
             Ok(Self::Dict(dict))
+        } else if let Ok(bytes) = py_any.downcast::<pytypes::PyByteArray>() {
+            Ok(Self::ByteArray(bytes))
         } else if let Ok(v) = YPyType::try_from(py_any) {
             Ok(Self::YType(v))
         } else {
@@ -259,6 +261,7 @@ impl<'a> From<CompatiblePyType<'a>> for PyObject {
             CompatiblePyType::String(s) => s.into(),
             CompatiblePyType::List(list) => list.into(),
             CompatiblePyType::Dict(dict) => dict.into(),
+            CompatiblePyType::ByteArray(bytes) => bytes.into(),
             CompatiblePyType::YType(y_type) => y_type.into(),
             CompatiblePyType::None => Python::with_gil(|py| py.None()),
         }
@@ -339,6 +342,10 @@ impl<'a> TryFrom<CompatiblePyType<'a>> for Any {
                     .collect();
                 result.map(|res| Any::Map(Box::new(res)))
             },
+            CompatiblePyType::ByteArray(b) => {
+                let bytes: Vec<u8> = b.extract()?;
+                Ok(Any::Buffer(bytes.into_boxed_slice()))
+            }
             CompatiblePyType::None => Ok(Any::Null),
             CompatiblePyType::YType(v) => Err(MultipleIntegrationError::new_err(format!(
                     "Cannot integrate a nested Ypy object because is already integrated into a YDoc: {v}"