diff --git a/.gitignore b/.gitignore index 6630f428ce..55bdab3a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.idea/ /vendor/ /*.json +/*.srs /*.db /site/ /bin/ diff --git a/adapter/experimental.go b/adapter/experimental.go index 87eb936c4f..2a6776cd0e 100644 --- a/adapter/experimental.go +++ b/adapter/experimental.go @@ -1,11 +1,16 @@ package adapter import ( + "bytes" "context" + "encoding/binary" + "io" "net" + "time" "github.com/sagernet/sing-box/common/urltest" N "github.com/sagernet/sing/common/network" + "github.com/sagernet/sing/common/rw" ) type ClashServer interface { @@ -13,22 +18,83 @@ type ClashServer interface { PreStarter Mode() string ModeList() []string - StoreSelected() bool - StoreFakeIP() bool - CacheFile() ClashCacheFile HistoryStorage() *urltest.HistoryStorage RoutedConnection(ctx context.Context, conn net.Conn, metadata InboundContext, matchedRule Rule) (net.Conn, Tracker) RoutedPacketConnection(ctx context.Context, conn N.PacketConn, metadata InboundContext, matchedRule Rule) (N.PacketConn, Tracker) } -type ClashCacheFile interface { +type CacheFile interface { + Service + PreStarter + + StoreFakeIP() bool + FakeIPStorage + LoadMode() string StoreMode(mode string) error LoadSelected(group string) string StoreSelected(group string, selected string) error LoadGroupExpand(group string) (isExpand bool, loaded bool) StoreGroupExpand(group string, expand bool) error - FakeIPStorage + LoadRuleSet(tag string) *SavedRuleSet + SaveRuleSet(tag string, set *SavedRuleSet) error +} + +type SavedRuleSet struct { + Content []byte + LastUpdated time.Time + LastEtag string +} + +func (s *SavedRuleSet) MarshalBinary() ([]byte, error) { + var buffer bytes.Buffer + err := binary.Write(&buffer, binary.BigEndian, uint8(1)) + if err != nil { + return nil, err + } + err = rw.WriteUVariant(&buffer, uint64(len(s.Content))) + if err != nil { + return nil, err + } + buffer.Write(s.Content) + err = binary.Write(&buffer, binary.BigEndian, s.LastUpdated.Unix()) + if err != nil { + return nil, err + } + err = rw.WriteVString(&buffer, s.LastEtag) + if err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + +func (s *SavedRuleSet) UnmarshalBinary(data []byte) error { + reader := bytes.NewReader(data) + var version uint8 + err := binary.Read(reader, binary.BigEndian, &version) + if err != nil { + return err + } + contentLen, err := rw.ReadUVariant(reader) + if err != nil { + return err + } + s.Content = make([]byte, contentLen) + _, err = io.ReadFull(reader, s.Content) + if err != nil { + return err + } + var lastUpdated int64 + err = binary.Read(reader, binary.BigEndian, &lastUpdated) + if err != nil { + return err + } + s.LastUpdated = time.Unix(lastUpdated, 0) + s.LastEtag, err = rw.ReadVString(reader) + if err != nil { + return err + } + return nil } type Tracker interface { diff --git a/adapter/inbound.go b/adapter/inbound.go index 2d24083c4a..0e5a1c4a27 100644 --- a/adapter/inbound.go +++ b/adapter/inbound.go @@ -46,11 +46,25 @@ type InboundContext struct { SourceGeoIPCode string GeoIPCode string ProcessInfo *process.Info + QueryType uint16 FakeIP bool + IPCIDRMatchSource bool - // dns cache + // rule cache - QueryType uint16 + IPCIDRMatchSource bool + SourceAddressMatch bool + SourcePortMatch bool + DestinationAddressMatch bool + DestinationPortMatch bool +} + +func (c *InboundContext) ResetRuleCache() { + c.IPCIDRMatchSource = false + c.SourceAddressMatch = false + c.SourcePortMatch = false + c.DestinationAddressMatch = false + c.DestinationPortMatch = false } type inboundContextKey struct{} diff --git a/adapter/router.go b/adapter/router.go index 3d18eb387f..b4bdd143a5 100644 --- a/adapter/router.go +++ b/adapter/router.go @@ -2,12 +2,14 @@ package adapter import ( "context" + "net/http" "net/netip" "github.com/sagernet/sing-box/common/geoip" "github.com/sagernet/sing-dns" "github.com/sagernet/sing-tun" "github.com/sagernet/sing/common/control" + N "github.com/sagernet/sing/common/network" "github.com/sagernet/sing/service" mdns "github.com/miekg/dns" @@ -19,7 +21,7 @@ type Router interface { Outbounds() []Outbound Outbound(tag string) (Outbound, bool) - DefaultOutbound(network string) Outbound + DefaultOutbound(network string) (Outbound, error) FakeIPStore() FakeIPStore @@ -28,6 +30,8 @@ type Router interface { GeoIPReader() *geoip.Reader LoadGeosite(code string) (Rule, error) + RuleSet(tag string) (RuleSet, bool) + Exchange(ctx context.Context, message *mdns.Msg) (*mdns.Msg, error) Lookup(ctx context.Context, domain string, strategy dns.DomainStrategy) ([]netip.Addr, error) LookupDefault(ctx context.Context, domain string) ([]netip.Addr, error) @@ -62,11 +66,15 @@ func RouterFromContext(ctx context.Context) Router { return service.FromContext[Router](ctx) } +type HeadlessRule interface { + Match(metadata *InboundContext) bool +} + type Rule interface { + HeadlessRule Service Type() string UpdateGeosite() error - Match(metadata *InboundContext) bool Outbound() string String() string } @@ -77,6 +85,24 @@ type DNSRule interface { RewriteTTL() *uint32 } +type RuleSet interface { + StartContext(ctx context.Context, startContext RuleSetStartContext) error + PostStart() error + Metadata() RuleSetMetadata + Close() error + HeadlessRule +} + +type RuleSetMetadata struct { + ContainsProcessRule bool + ContainsWIFIRule bool +} + +type RuleSetStartContext interface { + HTTPClient(detour string, dialer N.Dialer) *http.Client + Close() +} + type InterfaceUpdateListener interface { InterfaceUpdated() } diff --git a/box.go b/box.go index 3c0479c770..72189d9443 100644 --- a/box.go +++ b/box.go @@ -9,7 +9,10 @@ import ( "time" "github.com/sagernet/sing-box/adapter" + "github.com/sagernet/sing-box/common/taskmonitor" + C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/experimental" + "github.com/sagernet/sing-box/experimental/cachefile" "github.com/sagernet/sing-box/experimental/libbox/platform" "github.com/sagernet/sing-box/inbound" "github.com/sagernet/sing-box/log" @@ -32,7 +35,8 @@ type Box struct { outbounds []adapter.Outbound logFactory log.Factory logger log.ContextLogger - preServices map[string]adapter.Service + preServices1 map[string]adapter.Service + preServices2 map[string]adapter.Service postServices map[string]adapter.Service done chan struct{} } @@ -45,17 +49,21 @@ type Options struct { } func New(options Options) (*Box, error) { + createdAt := time.Now() ctx := options.Context if ctx == nil { ctx = context.Background() } ctx = service.ContextWithDefaultRegistry(ctx) ctx = pause.ContextWithDefaultManager(ctx) - createdAt := time.Now() experimentalOptions := common.PtrValueOrDefault(options.Experimental) applyDebugOptions(common.PtrValueOrDefault(experimentalOptions.Debug)) + var needCacheFile bool var needClashAPI bool var needV2RayAPI bool + if experimentalOptions.CacheFile != nil && experimentalOptions.CacheFile.Enabled || options.PlatformLogWriter != nil { + needCacheFile = true + } if experimentalOptions.ClashAPI != nil || options.PlatformLogWriter != nil { needClashAPI = true } @@ -145,8 +153,14 @@ func New(options Options) (*Box, error) { return nil, E.Cause(err, "initialize platform interface") } } - preServices := make(map[string]adapter.Service) + preServices1 := make(map[string]adapter.Service) + preServices2 := make(map[string]adapter.Service) postServices := make(map[string]adapter.Service) + if needCacheFile { + cacheFile := cachefile.New(ctx, common.PtrValueOrDefault(experimentalOptions.CacheFile)) + preServices1["cache file"] = cacheFile + service.MustRegister[adapter.CacheFile](ctx, cacheFile) + } if needClashAPI { clashAPIOptions := common.PtrValueOrDefault(experimentalOptions.ClashAPI) clashAPIOptions.ModeList = experimental.CalculateClashModeList(options.Options) @@ -155,7 +169,7 @@ func New(options Options) (*Box, error) { return nil, E.Cause(err, "create clash api server") } router.SetClashServer(clashServer) - preServices["clash api"] = clashServer + preServices2["clash api"] = clashServer } if needV2RayAPI { v2rayServer, err := experimental.NewV2RayServer(logFactory.NewLogger("v2ray-api"), common.PtrValueOrDefault(experimentalOptions.V2RayAPI)) @@ -163,7 +177,7 @@ func New(options Options) (*Box, error) { return nil, E.Cause(err, "create v2ray api server") } router.SetV2RayServer(v2rayServer) - preServices["v2ray api"] = v2rayServer + preServices2["v2ray api"] = v2rayServer } return &Box{ router: router, @@ -172,7 +186,8 @@ func New(options Options) (*Box, error) { createdAt: createdAt, logFactory: logFactory, logger: logFactory.Logger(), - preServices: preServices, + preServices1: preServices1, + preServices2: preServices2, postServices: postServices, done: make(chan struct{}), }, nil @@ -217,16 +232,34 @@ func (s *Box) Start() error { } func (s *Box) preStart() error { - for serviceName, service := range s.preServices { + monitor := taskmonitor.New(s.logger, C.DefaultStartTimeout) + monitor.Start("start logger") + err := s.logFactory.Start() + monitor.Finish() + if err != nil { + return E.Cause(err, "start logger") + } + for serviceName, service := range s.preServices1 { + if preService, isPreService := service.(adapter.PreStarter); isPreService { + monitor.Start("pre-start ", serviceName) + err := preService.PreStart() + monitor.Finish() + if err != nil { + return E.Cause(err, "pre-start ", serviceName) + } + } + } + for serviceName, service := range s.preServices2 { if preService, isPreService := service.(adapter.PreStarter); isPreService { - s.logger.Trace("pre-start ", serviceName) + monitor.Start("pre-start ", serviceName) err := preService.PreStart() + monitor.Finish() if err != nil { - return E.Cause(err, "pre-starting ", serviceName) + return E.Cause(err, "pre-start ", serviceName) } } } - err := s.startOutbounds() + err = s.startOutbounds() if err != nil { return err } @@ -238,8 +271,13 @@ func (s *Box) start() error { if err != nil { return err } - for serviceName, service := range s.preServices { - s.logger.Trace("starting ", serviceName) + for serviceName, service := range s.preServices1 { + err = service.Start() + if err != nil { + return E.Cause(err, "start ", serviceName) + } + } + for serviceName, service := range s.preServices2 { err = service.Start() if err != nil { return E.Cause(err, "start ", serviceName) @@ -252,7 +290,6 @@ func (s *Box) start() error { } else { tag = in.Tag() } - s.logger.Trace("initializing inbound/", in.Type(), "[", tag, "]") err = in.Start() if err != nil { return E.Cause(err, "initialize inbound/", in.Type(), "[", tag, "]") @@ -263,7 +300,6 @@ func (s *Box) start() error { func (s *Box) postStart() error { for serviceName, service := range s.postServices { - s.logger.Trace("starting ", service) err := service.Start() if err != nil { return E.Cause(err, "start ", serviceName) @@ -271,14 +307,16 @@ func (s *Box) postStart() error { } for _, outbound := range s.outbounds { if lateOutbound, isLateOutbound := outbound.(adapter.PostStarter); isLateOutbound { - s.logger.Trace("post-starting outbound/", outbound.Tag()) err := lateOutbound.PostStart() if err != nil { return E.Cause(err, "post-start outbound/", outbound.Tag()) } } } - s.logger.Trace("post-starting router") + err := s.router.PostStart() + if err != nil { + return E.Cause(err, "post-start router") + } return s.router.PostStart() } @@ -289,41 +327,53 @@ func (s *Box) Close() error { default: close(s.done) } + monitor := taskmonitor.New(s.logger, C.DefaultStopTimeout) var errors error for serviceName, service := range s.postServices { - s.logger.Trace("closing ", serviceName) + monitor.Start("close ", serviceName) errors = E.Append(errors, service.Close(), func(err error) error { return E.Cause(err, "close ", serviceName) }) + monitor.Finish() } for i, in := range s.inbounds { - s.logger.Trace("closing inbound/", in.Type(), "[", i, "]") + monitor.Start("close inbound/", in.Type(), "[", i, "]") errors = E.Append(errors, in.Close(), func(err error) error { return E.Cause(err, "close inbound/", in.Type(), "[", i, "]") }) + monitor.Finish() } for i, out := range s.outbounds { - s.logger.Trace("closing outbound/", out.Type(), "[", i, "]") + monitor.Start("close outbound/", out.Type(), "[", i, "]") errors = E.Append(errors, common.Close(out), func(err error) error { return E.Cause(err, "close outbound/", out.Type(), "[", i, "]") }) + monitor.Finish() } - s.logger.Trace("closing router") + monitor.Start("close router") if err := common.Close(s.router); err != nil { errors = E.Append(errors, err, func(err error) error { return E.Cause(err, "close router") }) } - for serviceName, service := range s.preServices { - s.logger.Trace("closing ", serviceName) + monitor.Finish() + for serviceName, service := range s.preServices1 { + monitor.Start("close ", serviceName) + errors = E.Append(errors, service.Close(), func(err error) error { + return E.Cause(err, "close ", serviceName) + }) + monitor.Finish() + } + for serviceName, service := range s.preServices2 { + monitor.Start("close ", serviceName) errors = E.Append(errors, service.Close(), func(err error) error { return E.Cause(err, "close ", serviceName) }) + monitor.Finish() } - s.logger.Trace("closing log factory") if err := common.Close(s.logFactory); err != nil { errors = E.Append(errors, err, func(err error) error { - return E.Cause(err, "close log factory") + return E.Cause(err, "close logger") }) } return errors diff --git a/box_outbound.go b/box_outbound.go index 676ae7af72..95ba950ad4 100644 --- a/box_outbound.go +++ b/box_outbound.go @@ -4,12 +4,15 @@ import ( "strings" "github.com/sagernet/sing-box/adapter" + "github.com/sagernet/sing-box/common/taskmonitor" + C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing/common" E "github.com/sagernet/sing/common/exceptions" F "github.com/sagernet/sing/common/format" ) func (s *Box) startOutbounds() error { + monitor := taskmonitor.New(s.logger, C.DefaultStartTimeout) outboundTags := make(map[adapter.Outbound]string) outbounds := make(map[string]adapter.Outbound) for i, outboundToStart := range s.outbounds { @@ -43,8 +46,9 @@ func (s *Box) startOutbounds() error { started[outboundTag] = true canContinue = true if starter, isStarter := outboundToStart.(common.Starter); isStarter { - s.logger.Trace("initializing outbound/", outboundToStart.Type(), "[", outboundTag, "]") + monitor.Start("initialize outbound/", outboundToStart.Type(), "[", outboundTag, "]") err := starter.Start() + monitor.Finish() if err != nil { return E.Cause(err, "initialize outbound/", outboundToStart.Type(), "[", outboundTag, "]") } diff --git a/cmd/sing-box/cmd_format.go b/cmd/sing-box/cmd_format.go index 10a5497cd4..c5e939e4a9 100644 --- a/cmd/sing-box/cmd_format.go +++ b/cmd/sing-box/cmd_format.go @@ -7,7 +7,6 @@ import ( "github.com/sagernet/sing-box/common/json" "github.com/sagernet/sing-box/log" - "github.com/sagernet/sing-box/option" E "github.com/sagernet/sing/common/exceptions" "github.com/spf13/cobra" @@ -69,41 +68,3 @@ func format() error { } return nil } - -func formatOne(configPath string) error { - configContent, err := os.ReadFile(configPath) - if err != nil { - return E.Cause(err, "read config") - } - var options option.Options - err = options.UnmarshalJSON(configContent) - if err != nil { - return E.Cause(err, "decode config") - } - buffer := new(bytes.Buffer) - encoder := json.NewEncoder(buffer) - encoder.SetIndent("", " ") - err = encoder.Encode(options) - if err != nil { - return E.Cause(err, "encode config") - } - if !commandFormatFlagWrite { - os.Stdout.WriteString(buffer.String() + "\n") - return nil - } - if bytes.Equal(configContent, buffer.Bytes()) { - return nil - } - output, err := os.Create(configPath) - if err != nil { - return E.Cause(err, "open output") - } - _, err = output.Write(buffer.Bytes()) - output.Close() - if err != nil { - return E.Cause(err, "write output") - } - outputPath, _ := filepath.Abs(configPath) - os.Stderr.WriteString(outputPath + "\n") - return nil -} diff --git a/cmd/sing-box/cmd_geoip.go b/cmd/sing-box/cmd_geoip.go new file mode 100644 index 0000000000..dbbbff135e --- /dev/null +++ b/cmd/sing-box/cmd_geoip.go @@ -0,0 +1,43 @@ +package main + +import ( + "github.com/sagernet/sing-box/log" + E "github.com/sagernet/sing/common/exceptions" + + "github.com/oschwald/maxminddb-golang" + "github.com/spf13/cobra" +) + +var ( + geoipReader *maxminddb.Reader + commandGeoIPFlagFile string +) + +var commandGeoip = &cobra.Command{ + Use: "geoip", + Short: "GeoIP tools", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + err := geoipPreRun() + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoip.PersistentFlags().StringVarP(&commandGeoIPFlagFile, "file", "f", "geoip.db", "geoip file") + mainCommand.AddCommand(commandGeoip) +} + +func geoipPreRun() error { + reader, err := maxminddb.Open(commandGeoIPFlagFile) + if err != nil { + return err + } + if reader.Metadata.DatabaseType != "sing-geoip" { + reader.Close() + return E.New("incorrect database type, expected sing-geoip, got ", reader.Metadata.DatabaseType) + } + geoipReader = reader + return nil +} diff --git a/cmd/sing-box/cmd_geoip_export.go b/cmd/sing-box/cmd_geoip_export.go new file mode 100644 index 0000000000..d170d10b72 --- /dev/null +++ b/cmd/sing-box/cmd_geoip_export.go @@ -0,0 +1,98 @@ +package main + +import ( + "io" + "net" + "os" + "strings" + + "github.com/sagernet/sing-box/common/json" + C "github.com/sagernet/sing-box/constant" + "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing-box/option" + E "github.com/sagernet/sing/common/exceptions" + + "github.com/oschwald/maxminddb-golang" + "github.com/spf13/cobra" +) + +var flagGeoipExportOutput string + +const flagGeoipExportDefaultOutput = "geoip-.srs" + +var commandGeoipExport = &cobra.Command{ + Use: "export ", + Short: "Export geoip country as rule-set", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := geoipExport(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoipExport.Flags().StringVarP(&flagGeoipExportOutput, "output", "o", flagGeoipExportDefaultOutput, "Output path") + commandGeoip.AddCommand(commandGeoipExport) +} + +func geoipExport(countryCode string) error { + networks := geoipReader.Networks(maxminddb.SkipAliasedNetworks) + countryMap := make(map[string][]*net.IPNet) + var ( + ipNet *net.IPNet + nextCountryCode string + err error + ) + for networks.Next() { + ipNet, err = networks.Network(&nextCountryCode) + if err != nil { + return err + } + countryMap[nextCountryCode] = append(countryMap[nextCountryCode], ipNet) + } + ipNets := countryMap[strings.ToLower(countryCode)] + if len(ipNets) == 0 { + return E.New("country code not found: ", countryCode) + } + + var ( + outputFile *os.File + outputWriter io.Writer + ) + if flagGeoipExportOutput == "stdout" { + outputWriter = os.Stdout + } else if flagGeoipExportOutput == flagGeoipExportDefaultOutput { + outputFile, err = os.Create("geoip-" + countryCode + ".json") + if err != nil { + return err + } + defer outputFile.Close() + outputWriter = outputFile + } else { + outputFile, err = os.Create(flagGeoipExportOutput) + if err != nil { + return err + } + defer outputFile.Close() + outputWriter = outputFile + } + + encoder := json.NewEncoder(outputWriter) + encoder.SetIndent("", " ") + var headlessRule option.DefaultHeadlessRule + headlessRule.IPCIDR = make([]string, 0, len(ipNets)) + for _, cidr := range ipNets { + headlessRule.IPCIDR = append(headlessRule.IPCIDR, cidr.String()) + } + var plainRuleSet option.PlainRuleSetCompat + plainRuleSet.Version = C.RuleSetVersion1 + plainRuleSet.Options.Rules = []option.HeadlessRule{ + { + Type: C.RuleTypeDefault, + DefaultOptions: headlessRule, + }, + } + return encoder.Encode(plainRuleSet) +} diff --git a/cmd/sing-box/cmd_geoip_list.go b/cmd/sing-box/cmd_geoip_list.go new file mode 100644 index 0000000000..54dd426ea9 --- /dev/null +++ b/cmd/sing-box/cmd_geoip_list.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + + "github.com/sagernet/sing-box/log" + + "github.com/spf13/cobra" +) + +var commandGeoipList = &cobra.Command{ + Use: "list", + Short: "List geoip country codes", + Run: func(cmd *cobra.Command, args []string) { + err := listGeoip() + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoip.AddCommand(commandGeoipList) +} + +func listGeoip() error { + for _, code := range geoipReader.Metadata.Languages { + os.Stdout.WriteString(code + "\n") + } + return nil +} diff --git a/cmd/sing-box/cmd_geoip_lookup.go b/cmd/sing-box/cmd_geoip_lookup.go new file mode 100644 index 0000000000..d5157bb404 --- /dev/null +++ b/cmd/sing-box/cmd_geoip_lookup.go @@ -0,0 +1,47 @@ +package main + +import ( + "net/netip" + "os" + + "github.com/sagernet/sing-box/log" + E "github.com/sagernet/sing/common/exceptions" + N "github.com/sagernet/sing/common/network" + + "github.com/spf13/cobra" +) + +var commandGeoipLookup = &cobra.Command{ + Use: "lookup
", + Short: "Lookup if an IP address is contained in the GeoIP database", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := geoipLookup(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoip.AddCommand(commandGeoipLookup) +} + +func geoipLookup(address string) error { + addr, err := netip.ParseAddr(address) + if err != nil { + return E.Cause(err, "parse address") + } + if !N.IsPublicAddr(addr) { + os.Stdout.WriteString("private\n") + return nil + } + var code string + _ = geoipReader.Lookup(addr.AsSlice(), &code) + if code != "" { + os.Stdout.WriteString(code + "\n") + return nil + } + os.Stdout.WriteString("unknown\n") + return nil +} diff --git a/cmd/sing-box/cmd_geosite.go b/cmd/sing-box/cmd_geosite.go new file mode 100644 index 0000000000..95db935797 --- /dev/null +++ b/cmd/sing-box/cmd_geosite.go @@ -0,0 +1,41 @@ +package main + +import ( + "github.com/sagernet/sing-box/common/geosite" + "github.com/sagernet/sing-box/log" + E "github.com/sagernet/sing/common/exceptions" + + "github.com/spf13/cobra" +) + +var ( + commandGeoSiteFlagFile string + geositeReader *geosite.Reader + geositeCodeList []string +) + +var commandGeoSite = &cobra.Command{ + Use: "geosite", + Short: "Geosite tools", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + err := geositePreRun() + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoSite.PersistentFlags().StringVarP(&commandGeoSiteFlagFile, "file", "f", "geosite.db", "geosite file") + mainCommand.AddCommand(commandGeoSite) +} + +func geositePreRun() error { + reader, codeList, err := geosite.Open(commandGeoSiteFlagFile) + if err != nil { + return E.Cause(err, "open geosite file") + } + geositeReader = reader + geositeCodeList = codeList + return nil +} diff --git a/cmd/sing-box/cmd_geosite_export.go b/cmd/sing-box/cmd_geosite_export.go new file mode 100644 index 0000000000..18163daeaf --- /dev/null +++ b/cmd/sing-box/cmd_geosite_export.go @@ -0,0 +1,81 @@ +package main + +import ( + "io" + "os" + + "github.com/sagernet/sing-box/common/geosite" + "github.com/sagernet/sing-box/common/json" + C "github.com/sagernet/sing-box/constant" + "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing-box/option" + + "github.com/spf13/cobra" +) + +var commandGeositeExportOutput string + +const commandGeositeExportDefaultOutput = "geosite-.json" + +var commandGeositeExport = &cobra.Command{ + Use: "export ", + Short: "Export geosite category as rule-set", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := geositeExport(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeositeExport.Flags().StringVarP(&commandGeositeExportOutput, "output", "o", commandGeositeExportDefaultOutput, "Output path") + commandGeoSite.AddCommand(commandGeositeExport) +} + +func geositeExport(category string) error { + sourceSet, err := geositeReader.Read(category) + if err != nil { + return err + } + var ( + outputFile *os.File + outputWriter io.Writer + ) + if commandGeositeExportOutput == "stdout" { + outputWriter = os.Stdout + } else if commandGeositeExportOutput == commandGeositeExportDefaultOutput { + outputFile, err = os.Create("geosite-" + category + ".json") + if err != nil { + return err + } + defer outputFile.Close() + outputWriter = outputFile + } else { + outputFile, err = os.Create(commandGeositeExportOutput) + if err != nil { + return err + } + defer outputFile.Close() + outputWriter = outputFile + } + + encoder := json.NewEncoder(outputWriter) + encoder.SetIndent("", " ") + var headlessRule option.DefaultHeadlessRule + defaultRule := geosite.Compile(sourceSet) + headlessRule.Domain = defaultRule.Domain + headlessRule.DomainSuffix = defaultRule.DomainSuffix + headlessRule.DomainKeyword = defaultRule.DomainKeyword + headlessRule.DomainRegex = defaultRule.DomainRegex + var plainRuleSet option.PlainRuleSetCompat + plainRuleSet.Version = C.RuleSetVersion1 + plainRuleSet.Options.Rules = []option.HeadlessRule{ + { + Type: C.RuleTypeDefault, + DefaultOptions: headlessRule, + }, + } + return encoder.Encode(plainRuleSet) +} diff --git a/cmd/sing-box/cmd_geosite_list.go b/cmd/sing-box/cmd_geosite_list.go new file mode 100644 index 0000000000..cedb7adfd2 --- /dev/null +++ b/cmd/sing-box/cmd_geosite_list.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + "sort" + + "github.com/sagernet/sing-box/log" + F "github.com/sagernet/sing/common/format" + + "github.com/spf13/cobra" +) + +var commandGeositeList = &cobra.Command{ + Use: "list ", + Short: "List geosite categories", + Run: func(cmd *cobra.Command, args []string) { + err := geositeList() + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoSite.AddCommand(commandGeositeList) +} + +func geositeList() error { + var geositeEntry []struct { + category string + items int + } + for _, category := range geositeCodeList { + sourceSet, err := geositeReader.Read(category) + if err != nil { + return err + } + geositeEntry = append(geositeEntry, struct { + category string + items int + }{category, len(sourceSet)}) + } + sort.SliceStable(geositeEntry, func(i, j int) bool { + return geositeEntry[i].items < geositeEntry[j].items + }) + for _, entry := range geositeEntry { + os.Stdout.WriteString(F.ToString(entry.category, " (", entry.items, ")\n")) + } + return nil +} diff --git a/cmd/sing-box/cmd_geosite_lookup.go b/cmd/sing-box/cmd_geosite_lookup.go new file mode 100644 index 0000000000..f648ce62a4 --- /dev/null +++ b/cmd/sing-box/cmd_geosite_lookup.go @@ -0,0 +1,97 @@ +package main + +import ( + "os" + "sort" + + "github.com/sagernet/sing-box/log" + E "github.com/sagernet/sing/common/exceptions" + + "github.com/spf13/cobra" +) + +var commandGeositeLookup = &cobra.Command{ + Use: "lookup [category] ", + Short: "Check if a domain is in the geosite", + Args: cobra.RangeArgs(1, 2), + Run: func(cmd *cobra.Command, args []string) { + var ( + source string + target string + ) + switch len(args) { + case 1: + target = args[0] + case 2: + source = args[0] + target = args[1] + } + err := geositeLookup(source, target) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandGeoSite.AddCommand(commandGeositeLookup) +} + +func geositeLookup(source string, target string) error { + var sourceMatcherList []struct { + code string + matcher *searchGeositeMatcher + } + if source != "" { + sourceSet, err := geositeReader.Read(source) + if err != nil { + return err + } + sourceMatcher, err := newSearchGeositeMatcher(sourceSet) + if err != nil { + return E.Cause(err, "compile code: "+source) + } + sourceMatcherList = []struct { + code string + matcher *searchGeositeMatcher + }{ + { + code: source, + matcher: sourceMatcher, + }, + } + + } else { + for _, code := range geositeCodeList { + sourceSet, err := geositeReader.Read(code) + if err != nil { + return err + } + sourceMatcher, err := newSearchGeositeMatcher(sourceSet) + if err != nil { + return E.Cause(err, "compile code: "+code) + } + sourceMatcherList = append(sourceMatcherList, struct { + code string + matcher *searchGeositeMatcher + }{ + code: code, + matcher: sourceMatcher, + }) + } + } + sort.SliceStable(sourceMatcherList, func(i, j int) bool { + return sourceMatcherList[i].code < sourceMatcherList[j].code + }) + + for _, matcherItem := range sourceMatcherList { + if matchRule := matcherItem.matcher.Match(target); matchRule != "" { + os.Stdout.WriteString("Match code (") + os.Stdout.WriteString(matcherItem.code) + os.Stdout.WriteString(") ") + os.Stdout.WriteString(matchRule) + os.Stdout.WriteString("\n") + } + } + return nil +} diff --git a/cmd/sing-box/cmd_geosite_matcher.go b/cmd/sing-box/cmd_geosite_matcher.go new file mode 100644 index 0000000000..791dba2499 --- /dev/null +++ b/cmd/sing-box/cmd_geosite_matcher.go @@ -0,0 +1,56 @@ +package main + +import ( + "regexp" + "strings" + + "github.com/sagernet/sing-box/common/geosite" +) + +type searchGeositeMatcher struct { + domainMap map[string]bool + suffixList []string + keywordList []string + regexList []string +} + +func newSearchGeositeMatcher(items []geosite.Item) (*searchGeositeMatcher, error) { + options := geosite.Compile(items) + domainMap := make(map[string]bool) + for _, domain := range options.Domain { + domainMap[domain] = true + } + rule := &searchGeositeMatcher{ + domainMap: domainMap, + suffixList: options.DomainSuffix, + keywordList: options.DomainKeyword, + regexList: options.DomainRegex, + } + return rule, nil +} + +func (r *searchGeositeMatcher) Match(domain string) string { + if r.domainMap[domain] { + return "domain=" + domain + } + for _, suffix := range r.suffixList { + if strings.HasSuffix(domain, suffix) { + return "domain_suffix=" + suffix + } + } + for _, keyword := range r.keywordList { + if strings.Contains(domain, keyword) { + return "domain_keyword=" + keyword + } + } + for _, regexStr := range r.regexList { + regex, err := regexp.Compile(regexStr) + if err != nil { + continue + } + if regex.MatchString(domain) { + return "domain_regex=" + regexStr + } + } + return "" +} diff --git a/cmd/sing-box/cmd_merge.go b/cmd/sing-box/cmd_merge.go index 0aff750182..4fb07b8688 100644 --- a/cmd/sing-box/cmd_merge.go +++ b/cmd/sing-box/cmd_merge.go @@ -18,7 +18,7 @@ import ( ) var commandMerge = &cobra.Command{ - Use: "merge [output]", + Use: "merge ", Short: "Merge configurations", Run: func(cmd *cobra.Command, args []string) { err := merge(args[0]) diff --git a/cmd/sing-box/cmd_rule_set.go b/cmd/sing-box/cmd_rule_set.go new file mode 100644 index 0000000000..f4112a087b --- /dev/null +++ b/cmd/sing-box/cmd_rule_set.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var commandRuleSet = &cobra.Command{ + Use: "rule-set", + Short: "Manage rule sets", +} + +func init() { + mainCommand.AddCommand(commandRuleSet) +} diff --git a/cmd/sing-box/cmd_rule_set_compile.go b/cmd/sing-box/cmd_rule_set_compile.go new file mode 100644 index 0000000000..de318095ac --- /dev/null +++ b/cmd/sing-box/cmd_rule_set_compile.go @@ -0,0 +1,80 @@ +package main + +import ( + "io" + "os" + "strings" + + "github.com/sagernet/sing-box/common/json" + "github.com/sagernet/sing-box/common/srs" + "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing-box/option" + + "github.com/spf13/cobra" +) + +var flagRuleSetCompileOutput string + +const flagRuleSetCompileDefaultOutput = ".srs" + +var commandRuleSetCompile = &cobra.Command{ + Use: "compile [source-path]", + Short: "Compile rule-set json to binary", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := compileRuleSet(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandRuleSet.AddCommand(commandRuleSetCompile) + commandRuleSetCompile.Flags().StringVarP(&flagRuleSetCompileOutput, "output", "o", flagRuleSetCompileDefaultOutput, "Output file") +} + +func compileRuleSet(sourcePath string) error { + var ( + reader io.Reader + err error + ) + if sourcePath == "stdin" { + reader = os.Stdin + } else { + reader, err = os.Open(sourcePath) + if err != nil { + return err + } + } + decoder := json.NewDecoder(json.NewCommentFilter(reader)) + decoder.DisallowUnknownFields() + var plainRuleSet option.PlainRuleSetCompat + err = decoder.Decode(&plainRuleSet) + if err != nil { + return err + } + ruleSet := plainRuleSet.Upgrade() + var outputPath string + if flagRuleSetCompileOutput == flagRuleSetCompileDefaultOutput { + if strings.HasSuffix(sourcePath, ".json") { + outputPath = sourcePath[:len(sourcePath)-5] + ".srs" + } else { + outputPath = sourcePath + ".srs" + } + } else { + outputPath = flagRuleSetCompileOutput + } + outputFile, err := os.Create(outputPath) + if err != nil { + return err + } + err = srs.Write(outputFile, ruleSet) + if err != nil { + outputFile.Close() + os.Remove(outputPath) + return err + } + outputFile.Close() + return nil +} diff --git a/cmd/sing-box/cmd_rule_set_format.go b/cmd/sing-box/cmd_rule_set_format.go new file mode 100644 index 0000000000..dc3ee6aabd --- /dev/null +++ b/cmd/sing-box/cmd_rule_set_format.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "io" + "os" + "path/filepath" + + "github.com/sagernet/sing-box/common/json" + "github.com/sagernet/sing-box/log" + "github.com/sagernet/sing-box/option" + E "github.com/sagernet/sing/common/exceptions" + + "github.com/spf13/cobra" +) + +var commandRuleSetFormatFlagWrite bool + +var commandRuleSetFormat = &cobra.Command{ + Use: "format ", + Short: "Format rule-set json", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + err := formatRuleSet(args[0]) + if err != nil { + log.Fatal(err) + } + }, +} + +func init() { + commandRuleSetFormat.Flags().BoolVarP(&commandRuleSetFormatFlagWrite, "write", "w", false, "write result to (source) file instead of stdout") + commandRuleSet.AddCommand(commandRuleSetFormat) +} + +func formatRuleSet(sourcePath string) error { + var ( + reader io.Reader + err error + ) + if sourcePath == "stdin" { + reader = os.Stdin + } else { + reader, err = os.Open(sourcePath) + if err != nil { + return err + } + } + content, err := io.ReadAll(reader) + if err != nil { + return err + } + decoder := json.NewDecoder(json.NewCommentFilter(bytes.NewReader(content))) + decoder.DisallowUnknownFields() + var plainRuleSet option.PlainRuleSetCompat + err = decoder.Decode(&plainRuleSet) + if err != nil { + return err + } + ruleSet := plainRuleSet.Upgrade() + buffer := new(bytes.Buffer) + encoder := json.NewEncoder(buffer) + encoder.SetIndent("", " ") + err = encoder.Encode(ruleSet) + if err != nil { + return E.Cause(err, "encode config") + } + outputPath, _ := filepath.Abs(sourcePath) + if !commandRuleSetFormatFlagWrite || sourcePath == "stdin" { + os.Stdout.WriteString(buffer.String() + "\n") + return nil + } + if bytes.Equal(content, buffer.Bytes()) { + return nil + } + output, err := os.Create(sourcePath) + if err != nil { + return E.Cause(err, "open output") + } + _, err = output.Write(buffer.Bytes()) + output.Close() + if err != nil { + return E.Cause(err, "write output") + } + os.Stderr.WriteString(outputPath + "\n") + return nil +} diff --git a/cmd/sing-box/cmd_run.go b/cmd/sing-box/cmd_run.go index 46f3495fc9..f28b39ae5f 100644 --- a/cmd/sing-box/cmd_run.go +++ b/cmd/sing-box/cmd_run.go @@ -14,6 +14,7 @@ import ( "github.com/sagernet/sing-box" "github.com/sagernet/sing-box/common/badjsonmerge" + C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/option" E "github.com/sagernet/sing/common/exceptions" @@ -193,7 +194,7 @@ func run() error { } func closeMonitor(ctx context.Context) { - time.Sleep(3 * time.Second) + time.Sleep(C.DefaultStopFatalTimeout) select { case <-ctx.Done(): return diff --git a/cmd/sing-box/cmd_tools.go b/cmd/sing-box/cmd_tools.go index 460a50cd1c..c45f585576 100644 --- a/cmd/sing-box/cmd_tools.go +++ b/cmd/sing-box/cmd_tools.go @@ -38,11 +38,7 @@ func createPreStartedClient() (*box.Box, error) { func createDialer(instance *box.Box, network string, outboundTag string) (N.Dialer, error) { if outboundTag == "" { - outbound := instance.Router().DefaultOutbound(N.NetworkName(network)) - if outbound == nil { - return nil, E.New("missing default outbound") - } - return outbound, nil + return instance.Router().DefaultOutbound(N.NetworkName(network)) } else { outbound, loaded := instance.Router().Outbound(outboundTag) if !loaded { diff --git a/cmd/sing-box/cmd_tools_connect.go b/cmd/sing-box/cmd_tools_connect.go index b904ebc9f5..3ea04bcd40 100644 --- a/cmd/sing-box/cmd_tools_connect.go +++ b/cmd/sing-box/cmd_tools_connect.go @@ -18,7 +18,7 @@ import ( var commandConnectFlagNetwork string var commandConnect = &cobra.Command{ - Use: "connect [address]", + Use: "connect
", Short: "Connect to an address", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/sing-box/main.go b/cmd/sing-box/main.go index f7974d2cc6..1880d1cbff 100644 --- a/cmd/sing-box/main.go +++ b/cmd/sing-box/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "time" @@ -37,7 +38,7 @@ func main() { func preRun(cmd *cobra.Command, args []string) { if disableColor { - log.SetStdLogger(log.NewFactory(log.Formatter{BaseTime: time.Now(), DisableColors: true}, os.Stderr, nil).Logger()) + log.SetStdLogger(log.NewDefaultFactory(context.Background(), log.Formatter{BaseTime: time.Now(), DisableColors: true}, os.Stderr, "", nil, false).Logger()) } if workingDir != "" { _, err := os.Stat(workingDir) diff --git a/common/badjsonmerge/merge.go b/common/badjsonmerge/merge.go index 39635e6632..9c928698a8 100644 --- a/common/badjsonmerge/merge.go +++ b/common/badjsonmerge/merge.go @@ -1,10 +1,10 @@ package badjsonmerge import ( - "encoding/json" "reflect" "github.com/sagernet/sing-box/common/badjson" + "github.com/sagernet/sing-box/common/json" "github.com/sagernet/sing-box/option" E "github.com/sagernet/sing/common/exceptions" ) diff --git a/common/contextjson/README.md b/common/contextjson/README.md new file mode 100644 index 0000000000..da656b79fc --- /dev/null +++ b/common/contextjson/README.md @@ -0,0 +1,3 @@ +# contextjson + +mod from go1.21.4 diff --git a/common/contextjson/decode.go b/common/contextjson/decode.go new file mode 100644 index 0000000000..8457171e09 --- /dev/null +++ b/common/contextjson/decode.go @@ -0,0 +1,1325 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "encoding" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a value implementing the Unmarshaler interface, +// Unmarshal calls that value's UnmarshalJSON method, including +// when the input is a JSON null. +// Otherwise, if the value implements encoding.TextUnmarshaler +// and the input is a JSON quoted string, Unmarshal calls that value's +// UnmarshalText method with the unquoted form of the string. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. By +// default, object keys which don't have a corresponding struct field are +// ignored (see Decoder.DisallowUnknownFields for an alternative). +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. +// +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be any string type, an integer, implement json.Unmarshaler, or +// implement encoding.TextUnmarshaler. +// +// If the JSON-encoded data contain a syntax error, Unmarshal returns a SyntaxError. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshaling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// “not present,” unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +func Unmarshal(data []byte, v any) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by types +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of Unmarshal itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // the full path from root node to the field +} + +func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// +// Deprecated: No longer used; kept for compatibility. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Pointer { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v any) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + d.scanWhile(scanSkipSpace) + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + err := d.value(rv) + if err != nil { + return d.addErrorContext(err) + } + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// An errorContext provides context for type errors during decoding. +type errorContext struct { + Struct reflect.Type + FieldStack []string +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // next read offset in data + opcode int // last read result + scan scanner + errorContext *errorContext + savedError error + useNumber bool + disallowUnknownFields bool + context *decodeContext +} + +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + +// phasePanicMsg is used as a panic message when we end up with something that +// shouldn't happen. It can indicate a bug in the JSON decoder, or that +// something is editing the data slice while the decoder executes. +const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?" + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + if d.errorContext != nil { + d.errorContext.Struct = nil + // Reuse the allocated space for the FieldStack slice. + d.errorContext.FieldStack = d.errorContext.FieldStack[:0] + } + return d +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + if d.context != nil { + d.savedError = d.addErrorContext(&contextError{err, d.formatContext(), d.context.key == ""}) + } else { + d.savedError = d.addErrorContext(err) + } + } +} + +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext != nil && (d.errorContext.Struct != nil || len(d.errorContext.FieldStack) > 0) { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct.Name() + err.Field = strings.Join(d.errorContext.FieldStack, ".") + } + } + return err +} + +// skip scans to the end of what was started. +func (d *decodeState) skip() { + s, data, i := &d.scan, d.data, d.off + depth := len(s.parseState) + for { + op := s.step(s, data[i]) + i++ + if len(s.parseState) < depth { + d.off = i + d.opcode = op + return + } + } +} + +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + if d.off < len(d.data) { + d.opcode = d.scan.step(&d.scan, d.data[d.off]) + d.off++ + } else { + d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(data) { + newOp := s.step(s, data[i]) + i++ + if newOp != op { + d.opcode = newOp + d.off = i + return + } + } + + d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} + +// rescanLiteral is similar to scanWhile(scanContinue), but it specialises the +// common case where we're decoding a literal. The decoder scans the input +// twice, once for syntax errors and to check the length of the value, and the +// second to perform the decoding. +// +// Only in the second step do we use decodeState to tokenize literals, so we +// know there aren't any syntax errors. We can take advantage of that knowledge, +// and scan a literal's bytes much more quickly. +func (d *decodeState) rescanLiteral() { + data, i := d.data, d.off +Switch: + switch data[i-1] { + case '"': // string + for ; i < len(data); i++ { + switch data[i] { + case '\\': + i++ // escaped char + case '"': + i++ // tokenize the closing quote too + break Switch + } + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number + for ; i < len(data); i++ { + switch data[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '.', 'e', 'E', '+', '-': + default: + break Switch + } + } + case 't': // true + i += len("rue") + case 'f': // false + i += len("alse") + case 'n': // null + i += len("ull") + } + if i < len(data) { + d.opcode = stateEndValue(&d.scan, data[i]) + } else { + d.opcode = scanEnd + } + d.off = i + 1 +} + +// value consumes a JSON value from d.data[d.off-1:], decoding into v, and +// reads the following byte ahead. If v is invalid, the value is discarded. +// The first byte of the value has been read already. +func (d *decodeState) value(v reflect.Value) error { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray: + if v.IsValid() { + if err := d.array(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginObject: + if v.IsValid() { + if err := d.object(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginLiteral: + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + if v.IsValid() { + if err := d.literalStore(d.data[start:d.readIndex()], v, false); err != nil { + return err + } + } + } + return nil +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() any { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray, scanBeginObject: + d.skip() + d.scanNext() + + case scanBeginLiteral: + v := d.literalInterface() + switch v.(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// If it encounters an Unmarshaler, indirect stops and returns that. +// If decodingNull is true, indirect stops at the first settable pointer so it +// can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // Issue #24153 indicates that it is generally not a guaranteed property + // that you may round-trip a reflect.Value by calling Value.Addr().Elem() + // and expect the value to still be settable for values derived from + // unexported embedded struct fields. + // + // The logic below effectively does this when it first addresses the value + // (to satisfy possible pointer methods) and continues to dereference + // subsequent pointers as necessary. + // + // After the first round-trip, we set v back to the original value to + // preserve the original RW flags contained in reflect.Value. + v0 := v + haveAddr := false + + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Pointer && v.Type().Name() != "" && v.CanAddr() { + haveAddr = true + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Pointer && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Pointer) { + haveAddr = false + v = e + continue + } + } + + if v.Kind() != reflect.Pointer { + break + } + + if decodingNull && v.CanSet() { + break + } + + // Prevent infinite loop if v is an interface pointing to its own address: + // var v interface{} + // v = &v + if v.Elem().Kind() == reflect.Interface && v.Elem().Elem() == v { + v = v.Elem() + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 && v.CanInterface() { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + } + + if haveAddr { + v = v0 // restore original value after round-trip Value.Addr().Elem() + haveAddr = false + } else { + v = v.Elem() + } + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into v. +// The first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + err := u.UnmarshalJSON(d.data[start:d.off]) + if err != nil { + d.saveError(err) + } + return nil + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + ai := d.arrayInterface() + v.Set(reflect.ValueOf(ai)) + return nil + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + case reflect.Array, reflect.Slice: + break + } + + i := 0 + d.context = &decodeContext{parent: d.context} + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + // Expand slice length, growing the slice if necessary. + if v.Kind() == reflect.Slice { + if i >= v.Cap() { + v.Grow(1) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + if err := d.value(v.Index(i)); err != nil { + return err + } + } else { + // Ran out of fixed array: skip. + if err := d.value(reflect.Value{}); err != nil { + return err + } + } + i++ + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + d.context.index++ + } + + d.context = d.context.parent + + if i < v.Len() { + if v.Kind() == reflect.Array { + for ; i < v.Len(); i++ { + v.Index(i).SetZero() // zero remainder of array + } + } else { + v.SetLen(i) // truncate the slice + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + return nil +} + +var ( + nullLiteral = []byte("null") + textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() +) + +// object consumes an object from d.data[d.off-1:], decoding into v. +// The first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + err := u.UnmarshalJSON(d.data[start:d.off]) + if err != nil { + d.saveError(err) + } + return nil + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + t := v.Type() + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + oi := d.objectInterface() + v.Set(reflect.ValueOf(oi)) + return nil + } + + var fields structFields + + // Check type of target: + // struct or + // map[T1]T2 where T1 is string, an integer type, + // or an encoding.TextUnmarshaler + switch v.Kind() { + case reflect.Map: + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !reflect.PointerTo(t.Key()).Implements(textUnmarshalerType) { + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + fields = cachedTypeFields(t) + // ok + default: + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + + var mapElem reflect.Value + var origErrorContext errorContext + if d.errorContext != nil { + origErrorContext = *d.errorContext + } + + d.context = &decodeContext{parent: d.context} + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquoteBytes(item) + if !ok { + panic(phasePanicMsg) + } + d.context.key = string(key) + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := t.Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.SetZero() + } + subv = mapElem + } else { + f := fields.byExactName[string(key)] + if f == nil { + f = fields.byFoldedName[string(foldName(key))] + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Pointer { + if subv.IsNil() { + // If a struct embeds a pointer to an unexported type, + // it is not possible to set a newly allocated value + // since the field is unexported. + // + // See https://golang.org/issue/21357 + if !subv.CanSet() { + d.saveError(fmt.Errorf("json: cannot set embedded pointer to unexported struct: %v", subv.Type().Elem())) + // Invalidate subv to ensure d.value(subv) skips over + // the JSON value without assigning it to subv. + subv = reflect.Value{} + destring = false + break + } + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + if d.errorContext == nil { + d.errorContext = new(errorContext) + } + d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) + d.errorContext.Struct = t + } else if d.disallowUnknownFields { + d.saveError(fmt.Errorf("json: unknown field %q", key)) + } + d.context.index++ + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + if err := d.literalStore(nullLiteral, subv, false); err != nil { + return err + } + case string: + if err := d.literalStore([]byte(qv), subv, true); err != nil { + return err + } + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) + } + } else { + if err := d.value(subv); err != nil { + return err + } + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kt := t.Key() + var kv reflect.Value + switch { + case reflect.PointerTo(kt).Implements(textUnmarshalerType): + kv = reflect.New(kt) + if err := d.literalStore(item, kv, true); err != nil { + return err + } + kv = kv.Elem() + case kt.Kind() == reflect.String: + kv = reflect.ValueOf(key).Convert(kt) + default: + switch kt.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := string(key) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + s := string(key) + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + default: + panic("json: Unexpected key type") // should never occur + } + } + if kv.IsValid() { + v.SetMapIndex(kv, subv) + } + } + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.errorContext != nil { + // Reset errorContext to its original state. + // Keep the same underlying array for FieldStack, to reuse the + // space and avoid unnecessary allocs. + d.errorContext.FieldStack = d.errorContext.FieldStack[:len(origErrorContext.FieldStack)] + d.errorContext.Struct = origErrorContext.Struct + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + d.context = d.context.parent + return nil +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (any, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeOf(0.0), Offset: int64(d.off)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) error { + // Check for unmarshaler. + if len(item) == 0 { + // Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + isNull := item[0] == 'n' // null + u, ut, pv := indirect(v, isNull) + if u != nil { + err := u.UnmarshalJSON(item) + if err != nil { + d.saveError(err) + } + return nil + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + val := "number" + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) + return nil + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + return ut.UnmarshalText(s) + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + case reflect.Interface, reflect.Pointer, reflect.Map, reflect.Slice: + v.SetZero() + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.SetBytes(b[:n]) + case reflect.String: + if v.Type() == numberType && !isValidNumber(string(s)) { + return fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item) + } + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + s := string(item) + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + // s must be a valid number, because it's + // already been tokenized. + v.SetString(s) + break + } + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetFloat(n) + } + } + return nil +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() (val any) { + switch d.opcode { + default: + panic(phasePanicMsg) + case scanBeginArray: + val = d.arrayInterface() + d.scanNext() + case scanBeginObject: + val = d.objectInterface() + d.scanNext() + case scanBeginLiteral: + val = d.literalInterface() + } + return +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []any { + v := make([]any, 0) + d.context = &decodeContext{parent: d.context} + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + d.context.index++ + } + d.context = d.context.parent + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]any { + m := make(map[string]any) + d.context = &decodeContext{parent: d.context} + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read string key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + d.context.key = key + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + d.context.index++ + } + d.context = d.context.parent + return m +} + +// literalInterface consumes and returns a literal from d.data[d.off-1:] and +// it reads the following byte ahead. The first byte of the literal has been +// read already (that's how the caller knows it's a literal). +func (d *decodeState) literalInterface() any { + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + item := d.data[start:d.readIndex()] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + panic(phasePanicMsg) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var r rune + for _, c := range s[2:6] { + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = c - 'a' + 10 + case 'A' <= c && c <= 'F': + c = c - 'A' + 10 + default: + return -1 + } + r = r*16 + rune(c) + } + return r +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/common/contextjson/decode_context.go b/common/contextjson/decode_context.go new file mode 100644 index 0000000000..3000926aa0 --- /dev/null +++ b/common/contextjson/decode_context.go @@ -0,0 +1,49 @@ +package json + +import "strconv" + +type decodeContext struct { + parent *decodeContext + index int + key string +} + +func (d *decodeState) formatContext() string { + var description string + context := d.context + var appendDot bool + for context != nil { + if appendDot { + description = "." + description + } + if context.key != "" { + description = context.key + description + appendDot = true + } else { + description = "[" + strconv.Itoa(context.index) + "]" + description + appendDot = false + } + context = context.parent + } + return description +} + +type contextError struct { + parent error + context string + index bool +} + +func (c *contextError) Unwrap() error { + return c.parent +} + +func (c *contextError) Error() string { + //goland:noinspection GoTypeAssertionOnErrors + switch c.parent.(type) { + case *contextError: + return c.context + "." + c.parent.Error() + default: + return c.context + ": " + c.parent.Error() + } +} diff --git a/common/contextjson/encode.go b/common/contextjson/encode.go new file mode 100644 index 0000000000..6da0bd9c4a --- /dev/null +++ b/common/contextjson/encode.go @@ -0,0 +1,1283 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON as defined in +// RFC 7159. The mapping between JSON and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. If no MarshalJSON method is present but the +// value implements encoding.TextMarshaler instead, Marshal calls +// its MarshalText method and encodes the result as a JSON string. +// The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// NaN and +/-Inf values will return an [UnsupportedValueError]. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// So that the JSON will be safe to embed inside HTML