Make remote rule sets support multiple URLs

.github/workflows/debug.yml

@@ -33,26 +33,6 @@ jobs:
       - name: Run Test
         run: |
           go test -v ./...
-  build_go118:
-    name: Debug build (Go 1.18)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
-        with:
-          fetch-depth: 0
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ~1.18
-      - name: Cache go module
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/go/pkg/mod
-          key: go118-${{ hashFiles('**/go.sum') }}
-      - name: Run Test
-        run: make ci_build_go118
   build_go120:
     name: Debug build (Go 1.20)
     runs-on: ubuntu-latest

Makefile

@@ -1,7 +1,6 @@
 NAME = sing-box
 COMMIT = $(shell git rev-parse --short HEAD)
-TAGS_GO118 = with_gvisor,with_dhcp,with_wireguard,with_reality_server,with_clash_api
-TAGS_GO120 = with_quic,with_utls
+TAGS_GO120 = with_gvisor,with_dhcp,with_wireguard,with_reality_server,with_clash_api,with_quic,with_utls
 TAGS_GO121 = with_ech
 TAGS ?= $(TAGS_GO118),$(TAGS_GO120),$(TAGS_GO121)
 TAGS_TEST ?= with_gvisor,with_quic,with_wireguard,with_grpc,with_ech,with_utls,with_reality_server
@@ -20,13 +19,9 @@ PREFIX ?= $(shell go env GOPATH)
 build:
 	go build $(MAIN_PARAMS) $(MAIN)
 
-ci_build_go118:
-	go build $(PARAMS) $(MAIN)
-	go build $(PARAMS) -tags "$(TAGS_GO118)" $(MAIN)
-
 ci_build_go120:
 	go build $(PARAMS) $(MAIN)
-	go build $(PARAMS) -tags "$(TAGS_GO118),$(TAGS_GO120)" $(MAIN)
+	go build $(PARAMS) -tags "$(TAGS_GO120)" $(MAIN)
 
 ci_build:
 	go build $(PARAMS) $(MAIN)

adapter/experimental.go

@@ -4,14 +4,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
-	"io"
 	"net"
 	"time"
 
 	"github.com/sagernet/sing-box/common/urltest"
 	"github.com/sagernet/sing-dns"
 	N "github.com/sagernet/sing/common/network"
-	"github.com/sagernet/sing/common/rw"
+	"github.com/sagernet/sing/common/varbin"
 )
 
 type ClashServer interface {
@@ -56,16 +55,15 @@ func (s *SavedRuleSet) MarshalBinary() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = rw.WriteUVariant(&buffer, uint64(len(s.Content)))
+	err = varbin.Write(&buffer, binary.BigEndian, s.Content)
 	if err != nil {
 		return nil, err
 	}
-	buffer.Write(s.Content)
 	err = binary.Write(&buffer, binary.BigEndian, s.LastUpdated.Unix())
 	if err != nil {
 		return nil, err
 	}
-	err = rw.WriteVString(&buffer, s.LastEtag)
+	err = varbin.Write(&buffer, binary.BigEndian, s.LastEtag)
 	if err != nil {
 		return nil, err
 	}
@@ -79,12 +77,7 @@ func (s *SavedRuleSet) UnmarshalBinary(data []byte) error {
 	if err != nil {
 		return err
 	}
-	contentLen, err := rw.ReadUVariant(reader)
-	if err != nil {
-		return err
-	}
-	s.Content = make([]byte, contentLen)
-	_, err = io.ReadFull(reader, s.Content)
+	err = varbin.Read(reader, binary.BigEndian, &s.Content)
 	if err != nil {
 		return err
 	}
@@ -94,7 +87,7 @@ func (s *SavedRuleSet) UnmarshalBinary(data []byte) error {
 		return err
 	}
 	s.LastUpdated = time.Unix(lastUpdated, 0)
-	s.LastEtag, err = rw.ReadVString(reader)
+	err = varbin.Read(reader, binary.BigEndian, &s.LastEtag)
 	if err != nil {
 		return err
 	}

adapter/inbound.go

@@ -31,11 +31,16 @@ type InboundContext struct {
 	Network     string
 	Source      M.Socksaddr
 	Destination M.Socksaddr
-	Domain      string
-	Protocol    string
 	User        string
 	Outbound    string
 
+	// sniffer
+
+	Protocol     string
+	Domain       string
+	Client       string
+	SniffContext any
+
 	// cache
 
 	InboundDetour        string
@@ -51,7 +56,9 @@ type InboundContext struct {
 
 	// rule cache
 
-	IPCIDRMatchSource            bool
+	IPCIDRMatchSource bool
+	IPCIDRAcceptEmpty bool
+
 	SourceAddressMatch           bool
 	SourcePortMatch              bool
 	DestinationAddressMatch      bool
@@ -62,6 +69,7 @@ type InboundContext struct {
 
 func (c *InboundContext) ResetRuleCache() {
 	c.IPCIDRMatchSource = false
+	c.IPCIDRAcceptEmpty = false
 	c.SourceAddressMatch = false
 	c.SourcePortMatch = false
 	c.DestinationAddressMatch = false

adapter/router.go

@@ -10,15 +10,18 @@ import (
 	"github.com/sagernet/sing-tun"
 	"github.com/sagernet/sing/common/control"
 	N "github.com/sagernet/sing/common/network"
+	"github.com/sagernet/sing/common/x/list"
 	"github.com/sagernet/sing/service"
 
 	mdns "github.com/miekg/dns"
+	"go4.org/netipx"
 )
 
 type Router interface {
 	Service
 	PreStarter
 	PostStarter
+	Cleanup() error
 
 	Outbounds() []Outbound
 	Outbound(tag string) (Outbound, bool)
@@ -45,7 +48,9 @@ type Router interface {
 	DefaultInterface() string
 	AutoDetectInterface() bool
 	AutoDetectInterfaceFunc() control.Func
-	DefaultMark() int
+	DefaultMark() uint32
+	RegisterAutoRedirectOutputMark(mark uint32) error
+	AutoRedirectOutputMark() uint32
 	NetworkMonitor() tun.NetworkUpdateMonitor
 	InterfaceMonitor() tun.DefaultInterfaceMonitor
 	PackageManager() tun.PackageManager
@@ -92,12 +97,22 @@ type DNSRule interface {
 }
 
 type RuleSet interface {
+	Name() string
 	StartContext(ctx context.Context, startContext RuleSetStartContext) error
+	PostStart() error
 	Metadata() RuleSetMetadata
+	ExtractIPSet() []*netipx.IPSet
+	IncRef()
+	DecRef()
+	Cleanup()
+	RegisterCallback(callback RuleSetUpdateCallback) *list.Element[RuleSetUpdateCallback]
+	UnregisterCallback(element *list.Element[RuleSetUpdateCallback])
 	Close() error
 	HeadlessRule
 }
 
+type RuleSetUpdateCallback func(it RuleSet)
+
 type RuleSetMetadata struct {
 	ContainsProcessRule bool
 	ContainsWIFIRule    bool

box.go

@@ -111,6 +111,7 @@ func New(options Options) (*Box, error) {
 			ctx,
 			router,
 			logFactory.NewLogger(F.ToString("inbound/", inboundOptions.Type, "[", tag, "]")),
+			tag,
 			inboundOptions,
 			options.PlatformInterface,
 		)
@@ -302,7 +303,11 @@ func (s *Box) start() error {
 			return E.Cause(err, "initialize inbound/", in.Type(), "[", tag, "]")
 		}
 	}
-	return s.postStart()
+	err = s.postStart()
+	if err != nil {
+		return err
+	}
+	return s.router.Cleanup()
 }
 
 func (s *Box) postStart() error {
@@ -312,16 +317,28 @@ func (s *Box) postStart() error {
 			return E.Cause(err, "start ", serviceName)
 		}
 	}
-	for _, outbound := range s.outbounds {
-		if lateOutbound, isLateOutbound := outbound.(adapter.PostStarter); isLateOutbound {
+	// TODO: reorganize ALL start order
+	for _, out := range s.outbounds {
+		if lateOutbound, isLateOutbound := out.(adapter.PostStarter); isLateOutbound {
 			err := lateOutbound.PostStart()
 			if err != nil {
-				return E.Cause(err, "post-start outbound/", outbound.Tag())
+				return E.Cause(err, "post-start outbound/", out.Tag())
 			}
 		}
 	}
-
-	return s.router.PostStart()
+	err := s.router.PostStart()
+	if err != nil {
+		return err
+	}
+	for _, in := range s.inbounds {
+		if lateInbound, isLateInbound := in.(adapter.PostStarter); isLateInbound {
+			err = lateInbound.PostStart()
+			if err != nil {
+				return E.Cause(err, "post-start inbound/", in.Tag())
+			}
+		}
+	}
+	return nil
 }
 
 func (s *Box) Close() error {

box_outbound.go

@@ -45,7 +45,9 @@ func (s *Box) startOutbounds() error {
 			}
 			started[outboundTag] = true
 			canContinue = true
-			if starter, isStarter := outboundToStart.(common.Starter); isStarter {
+			if starter, isStarter := outboundToStart.(interface {
+				Start() error
+			}); isStarter {
 				monitor.Start("initialize outbound/", outboundToStart.Type(), "[", outboundTag, "]")
 				err := starter.Start()
 				monitor.Finish()

cmd/internal/build_libbox/main.go

@@ -93,7 +93,7 @@ func buildAndroid() {
 
 	const name = "libbox.aar"
 	copyPath := filepath.Join("..", "sing-box-for-android", "app", "libs")
-	if rw.FileExists(copyPath) {
+	if rw.IsDir(copyPath) {
 		copyPath, _ = filepath.Abs(copyPath)
 		err = rw.CopyFile(name, filepath.Join(copyPath, name))
 		if err != nil {
@@ -134,7 +134,7 @@ func buildiOS() {
 	}
 
 	copyPath := filepath.Join("..", "sing-box-for-apple")
-	if rw.FileExists(copyPath) {
+	if rw.IsDir(copyPath) {
 		targetDir := filepath.Join(copyPath, "Libbox.xcframework")
 		targetDir, _ = filepath.Abs(targetDir)
 		os.RemoveAll(targetDir)

cmd/internal/build_shared/sdk.go

@@ -30,7 +30,7 @@ func FindSDK() {
 	}
 	for _, path := range searchPath {
 		path = os.ExpandEnv(path)
-		if rw.FileExists(filepath.Join(path, "licenses", "android-sdk-license")) {
+		if rw.IsFile(filepath.Join(path, "licenses", "android-sdk-license")) {
 			androidSDKPath = path
 			break
 		}
@@ -60,7 +60,7 @@ func FindSDK() {
 func findNDK() bool {
 	const fixedVersion = "26.2.11394342"
 	const versionFile = "source.properties"
-	if fixedPath := filepath.Join(androidSDKPath, "ndk", fixedVersion); rw.FileExists(filepath.Join(fixedPath, versionFile)) {
+	if fixedPath := filepath.Join(androidSDKPath, "ndk", fixedVersion); rw.IsFile(filepath.Join(fixedPath, versionFile)) {
 		androidNDKPath = fixedPath
 		return true
 	}
@@ -86,7 +86,7 @@ func findNDK() bool {
 	})
 	for _, versionName := range versionNames {
 		currentNDKPath := filepath.Join(androidSDKPath, "ndk", versionName)
-		if rw.FileExists(filepath.Join(androidSDKPath, versionFile)) {
+		if rw.IsFile(filepath.Join(androidSDKPath, versionFile)) {
 			androidNDKPath = currentNDKPath
 			log.Warn("reproducibility warning: using NDK version " + versionName + " instead of " + fixedVersion)
 			return true
@@ -100,11 +100,11 @@ var GoBinPath string
 func FindMobile() {
 	goBin := filepath.Join(build.Default.GOPATH, "bin")
 	if runtime.GOOS == "windows" {
-		if !rw.FileExists(filepath.Join(goBin, "gobind.exe")) {
+		if !rw.IsFile(filepath.Join(goBin, "gobind.exe")) {
 			log.Fatal("missing gomobile installation")
 		}
 	} else {
-		if !rw.FileExists(filepath.Join(goBin, "gobind")) {
+		if !rw.IsFile(filepath.Join(goBin, "gobind")) {
 			log.Fatal("missing gomobile installation")
 		}
 	}

cmd/sing-box/cmd_geoip_export.go

@@ -87,7 +87,7 @@ func geoipExport(countryCode string) error {
 		headlessRule.IPCIDR = append(headlessRule.IPCIDR, cidr.String())
 	}
 	var plainRuleSet option.PlainRuleSetCompat
-	plainRuleSet.Version = C.RuleSetVersion1
+	plainRuleSet.Version = C.RuleSetVersion2
 	plainRuleSet.Options.Rules = []option.HeadlessRule{
 		{
 			Type:           C.RuleTypeDefault,

cmd/sing-box/cmd_geosite_export.go

@@ -70,7 +70,7 @@ func geositeExport(category string) error {
 	headlessRule.DomainKeyword = defaultRule.DomainKeyword
 	headlessRule.DomainRegex = defaultRule.DomainRegex
 	var plainRuleSet option.PlainRuleSetCompat
-	plainRuleSet.Version = C.RuleSetVersion1
+	plainRuleSet.Version = C.RuleSetVersion2
 	plainRuleSet.Options.Rules = []option.HeadlessRule{
 		{
 			Type:           C.RuleTypeDefault,

cmd/sing-box/cmd_merge.go

@@ -54,7 +54,11 @@ func merge(outputPath string) error {
 			return nil
 		}
 	}
-	err = rw.WriteFile(outputPath, buffer.Bytes())
+	err = rw.MkdirParent(outputPath)
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(outputPath, buffer.Bytes(), 0o644)
 	if err != nil {
 		return err
 	}

cmd/sing-box/cmd_rule_set.go

@@ -6,7 +6,7 @@ import (
 
 var commandRuleSet = &cobra.Command{
 	Use:   "rule-set",
-	Short: "Manage rule sets",
+	Short: "Manage rule-sets",
 }
 
 func init() {

cmd/sing-box/cmd_rule_set_compile.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/sagernet/sing-box/common/srs"
+	C "github.com/sagernet/sing-box/constant"
 	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/option"
 	"github.com/sagernet/sing/common/json"
@@ -55,10 +56,10 @@ func compileRuleSet(sourcePath string) error {
 	if err != nil {
 		return err
 	}
+	ruleSet, err := plainRuleSet.Upgrade()
 	if err != nil {
 		return err
 	}
-	ruleSet := plainRuleSet.Upgrade()
 	var outputPath string
 	if flagRuleSetCompileOutput == flagRuleSetCompileDefaultOutput {
 		if strings.HasSuffix(sourcePath, ".json") {
@@ -73,7 +74,7 @@ func compileRuleSet(sourcePath string) error {
 	if err != nil {
 		return err
 	}
-	err = srs.Write(outputFile, ruleSet)
+	err = srs.Write(outputFile, ruleSet, plainRuleSet.Version == C.RuleSetVersion2)
 	if err != nil {
 		outputFile.Close()
 		os.Remove(outputPath)