feat:(ast) add fallback api on not-amd64 env (#341)

* feat:(ast) add  fallback api on `not-amd64` env

* test: add native `linux-arm64` CI

* opt: just skip number chars whne `decodeFloat64`

* fmt

* fix: check EOF

.github/workflows/benchmark-linux-arm64.yml

@@ -0,0 +1,27 @@
+name: Benchmark Linux-ARM
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: [arm]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Check Branch
+        run: ./check_branch_name.sh ${{ github.head_ref }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.1
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Benchmark sonic
+        run: sh bench-arm.sh

.github/workflows/benchmark-linux-amd64.yml → .github/workflows/benchmark-linux-x64.yml

@@ -1,4 +1,4 @@
-name: Pull Request Benchmark
+name: Benchmark Linux-X64
 
 on: pull_request
 
@@ -24,10 +24,4 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Benchmark sonic
-        run: sh bench.sh
-
-      # - name: Benchmark third-party
-      #   run: go test -benchmem -run=^$ -bench . -v ./generic_test
-
-      # - name: Diff
-      #   run: ./bench.py -b '"^Benchmark.*Sonic"' -c
+        run: sh bench.sh

.github/workflows/push-check-go118.yml

@@ -1,4 +1,4 @@
-name: Push Check Go1.18
+name: Push Check Go1.18-Linux-X64
 
 on: push
 

.github/workflows/push-check-linux-arm64.yml

@@ -0,0 +1,31 @@
+name: Push Check Linux-ARM
+
+on: push
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.19.x]
+        os: [arm]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Compatibility Test - main
+        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race github.com/bytedance/sonic
+
+      - name: Compatibility Test - ast
+        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race github.com/bytedance/sonic/ast

.github/workflows/push-check-linux-amd64.yml → .github/workflows/push-check-linux-x64.yml

@@ -1,4 +1,4 @@
-name: Push Check All
+name: Push Check Linux-X64
 
 on: push
 

.github/workflows/push-check-arm.yml → .github/workflows/push-check-qemu.yml

@@ -1,4 +1,4 @@
-name: Push Check ARM
+name: Push Check Linux-Qemu
 
 on: push
 
@@ -24,10 +24,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - name: Compatibility Test
+      - name: Compatibility Test - qemu
         run: |
           printf ' #!/bin/bash\n if [ ! -x "/usr/bin/qemu-x86_64" ];then\n sudo apt-get update\n sudo apt-get -y install make gcc g++ libglib2.0-dev libpixman-1-dev libfdt-dev python3-pip ninja-build\n sudo pip3 install meson\n wget https://download.qemu.org/qemu-6.2.0.tar.xz\n tar -xvf qemu-6.2.0.tar.xz\n cd qemu-6.2.0\n sudo ./configure\n sudo make -j 4\n sudo make install\n cd ..\n cp /usr/local/bin/qemu-x86_64  /usr/bin/qemu-x86_64\n fi\n' > qemu_install.sh
           chmod +x qemu_install.sh
           ./qemu_install.sh
           GOARCH=amd64 go test -gcflags=-d=checkptr=0 -c .
-          qemu-x86_64 -cpu max ./sonic.test -test.v
+          qemu-x86_64 -cpu max ./sonic.test -test.v

.github/workflows/push-check-windows.yml

@@ -1,4 +1,4 @@
-name: Push Check Windows
+name: Push Check Windows-X64
 
 on: push
 

ast/api_amd64.go

@@ -0,0 +1,91 @@
+//go:build amd64
+// +build amd64
+
+package ast
+
+import (
+    `runtime`
+    `unsafe`
+
+    `github.com/bytedance/sonic/encoder`
+    `github.com/bytedance/sonic/internal/native`
+    `github.com/bytedance/sonic/internal/native/types`
+    `github.com/bytedance/sonic/internal/rt`
+    uq `github.com/bytedance/sonic/unquote`
+    `github.com/chenzhuoyu/base64x`
+)
+
+var typeByte = rt.UnpackEface(byte(0)).Type
+
+func quote(buf *[]byte, val string) {
+    *buf = append(*buf, '"')
+    if len(val) == 0 {
+        *buf = append(*buf, '"')
+    }
+
+    sp := rt.IndexChar(val, 0)
+    nb := len(val)
+    b := (*rt.GoSlice)(unsafe.Pointer(buf))
+
+    // input buffer
+    for nb > 0 {
+        // output buffer
+        dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len))
+        dn := b.Cap - b.Len
+        // call native.Quote, dn is byte count it outputs
+        ret := native.Quote(sp, nb, dp, &dn, 0)
+        // update *buf length
+        b.Len += dn
+
+        // no need more output
+        if ret >= 0 {
+            break
+        }
+
+        // double buf size
+        *b = growslice(typeByte, *b, b.Cap*2)
+        // ret is the complement of consumed input
+        ret = ^ret
+        // update input buffer
+        nb -= ret
+        sp = unsafe.Pointer(uintptr(sp) + uintptr(ret))
+    }
+
+    runtime.KeepAlive(buf)
+    runtime.KeepAlive(sp)
+    *buf = append(*buf, '"')
+}
+
+func unquote(src string) (string, types.ParsingError) {
+    return uq.String(src)
+}
+
+func decodeBase64(src string) ([]byte, error) {
+    return base64x.StdEncoding.DecodeString(src)
+}
+
+func encodeBase64(src []byte) string {
+    return base64x.StdEncoding.EncodeToString(src)
+}
+
+func (self *Parser) decodeValue() (val types.JsonState) {
+    sv := (*rt.GoString)(unsafe.Pointer(&self.s))
+    self.p = native.Value(sv.Ptr, sv.Len, self.p, &val, 0)
+    return
+}
+
+func (self *Parser) skip() (int, types.ParsingError) {
+    fsm := types.NewStateMachine()
+    start := native.SkipOne(&self.s, &self.p, fsm, 0)
+    types.FreeStateMachine(fsm)
+
+    if start < 0 {
+        return self.p, types.ParsingError(-start)
+    }
+    return start, 0
+}
+
+func (self *Node) encodeInterface(buf *[]byte) error {
+    //WARN: NOT compatible with json.Encoder
+    return encoder.EncodeInto(buf, self.packAny(), 0)
+}

ast/api_amd64_test.go

@@ -0,0 +1,34 @@
+//go:build amd64
+// +build amd64
+
+package ast
+
+import (
+    `testing`
+
+    `github.com/bytedance/sonic/encoder`
+    `github.com/stretchr/testify/assert`
+)
+
+func TestSortNodeTwitter(t *testing.T) {root, err := NewSearcher(_TwitterJson).GetByPath()
+    if err != nil {
+        t.Fatal(err)
+    }
+    obj, err := root.MapUseNumber()
+    if err != nil {
+        t.Fatal(err)
+    }
+    exp, err := encoder.Encode(obj, encoder.SortMapKeys)
+    if err != nil {
+        t.Fatal(err)
+    }
+    if err := root.SortKeys(true); err != nil {
+        t.Fatal(err)
+    }
+    act, err := root.MarshalJSON()
+    if err != nil {
+        t.Fatal(err)
+    }
+    assert.Equal(t, len(exp), len(act))
+    assert.Equal(t, string(exp), string(act))
+}

ast/api_compat.go

@@ -0,0 +1,62 @@
+//go:build !amd64
+// +build !amd64
+
+package ast
+
+import (
+    `encoding/base64`
+    `encoding/json`
+
+    `github.com/bytedance/sonic/internal/native/types`
+    `github.com/bytedance/sonic/internal/rt`
+)
+
+func quote(buf *[]byte, val string) {
+    quoteString(buf, val)
+}
+
+func unquote(src string) (string, types.ParsingError) {
+    sp := rt.IndexChar(src, -1)
+    out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2))
+    if !ok {
+        return "", types.ERR_INVALID_ESCAPE
+    }
+    return rt.Mem2Str(out), 0
+}
+
+
+
+func decodeBase64(src string) ([]byte, error) {
+    return base64.StdEncoding.DecodeString(src)
+}
+
+func encodeBase64(src []byte) string {
+    return base64.StdEncoding.EncodeToString(src)
+}
+
+func (self *Parser) decodeValue() (val types.JsonState) {
+    e, v := decodeValue(self.s, self.p)
+    if e < 0 {
+        return v
+    }
+    self.p = e
+    return v
+}
+
+func (self *Parser) skip() (int, types.ParsingError) {
+    e, s := skipValue(self.s, self.p)
+    if e < 0 {
+        return self.p, types.ParsingError(-e)
+    }
+    self.p = e
+    return s, 0
+}
+
+func (self *Node) encodeInterface(buf *[]byte) error {
+    out, err := json.Marshal(self.packAny())
+    if err != nil {
+        return err
+    }
+    *buf = append(*buf, out...)
+    return nil
+}