From d1fffcfc102ea9fc011f304b84f6044e2efbb5b9 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Fri, 4 Aug 2023 14:52:40 -0400 Subject: [PATCH] Use flowmatic --- go.mod | 2 +- go.sum | 4 ++-- linkcheck/linkcheck.go | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index d28bb1b..12413c6 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( github.com/carlmjohnson/be v0.22.5 github.com/carlmjohnson/exitcode v0.20.2 github.com/carlmjohnson/flagx v0.22.2 + github.com/carlmjohnson/flowmatic v0.23.3 github.com/carlmjohnson/requests v0.23.2 github.com/carlmjohnson/versioninfo v0.22.4 - github.com/carlmjohnson/workgroup v0.23.1 github.com/getsentry/sentry-go v0.18.0 golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/net v0.5.0 diff --git a/go.sum b/go.sum index 6edb146..aab6e41 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,12 @@ github.com/carlmjohnson/exitcode v0.20.2 h1:vE6rmkCGNA4kO4m1qwWIa77PKlUBVg46cNjs github.com/carlmjohnson/exitcode v0.20.2/go.mod h1:MZ6ThCDx517DQcrpYnnns1pLh8onjFl+B/AsrOrdmpc= github.com/carlmjohnson/flagx v0.22.2 h1:UXf7gL4Ffv5RIH/HKp8CGNzDyopgezFLrDO1m4F8jWc= github.com/carlmjohnson/flagx v0.22.2/go.mod h1:obobISvBnxgEXPLBITVXhRUOlSlzza1SGt34M64CPJc= +github.com/carlmjohnson/flowmatic v0.23.3 h1:yBwzGm7dYQqjRmJiOXBcGvwLI1DGAsL03bnYHOI0Umk= +github.com/carlmjohnson/flowmatic v0.23.3/go.mod h1:A4bYrUo5oG59wPEk/ewwdyV+UkQRFR1KiaFMBKuCVWg= github.com/carlmjohnson/requests v0.23.2 h1:SzaY+/5v8QOvt++7HTXe1xgmIb3wc/bYf2QJmrO73sM= github.com/carlmjohnson/requests v0.23.2/go.mod h1:09VwhOaRQYCraJcByjEuvuOGO1jxUjIx6vnAEkt2ges= github.com/carlmjohnson/versioninfo v0.22.4 h1:AucUHDSKmk6j7Yx3dECGUxaowGHOAN0Zx5/EBtsXn4Y= github.com/carlmjohnson/versioninfo v0.22.4/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= -github.com/carlmjohnson/workgroup v0.23.1 h1:mlWwtGDeaqPEXZhivbpQHtGnUqQKTA68CiJmxkPVG40= -github.com/carlmjohnson/workgroup v0.23.1/go.mod h1:mAOZgF/WPlX7lbWl38MRns+Y0W6flCghmNV/CznLb9Y= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= diff --git a/linkcheck/linkcheck.go b/linkcheck/linkcheck.go index 10270f8..0a4b5de 100644 --- a/linkcheck/linkcheck.go +++ b/linkcheck/linkcheck.go @@ -26,9 +26,9 @@ import ( "github.com/carlmjohnson/exitcode" "github.com/carlmjohnson/flagx" + "github.com/carlmjohnson/flowmatic" "github.com/carlmjohnson/requests" "github.com/carlmjohnson/versioninfo" - "github.com/carlmjohnson/workgroup" sentry "github.com/getsentry/sentry-go" "github.com/spotlightpa/linkrot/xhtml" "golang.org/x/exp/slog" @@ -65,7 +65,7 @@ Options: fmt.Fprintf(os.Stderr, usage, versioninfo.Version, versioninfo.Revision) fl.PrintDefaults() } - + versioninfo.AddFlag(fl) verbose := fl.Bool("verbose", false, "verbose") crawlers := fl.Int("crawlers", runtime.NumCPU(), "number of concurrent crawlers") timeout := fl.Duration("timeout", 10*time.Second, "timeout for requesting a URL") @@ -196,10 +196,10 @@ func (c *crawler) crawl() (crawled crawledPages, cancelled bool) { queued := make(map[string]bool) crawled = make(crawledPages) - manager := func(url string, info *pageInfo, err error) ([]string, error) { + manager := func(url string, info *pageInfo, err error) ([]string, bool) { crawled[url] = info if err != nil { - return nil, nil + return nil, true } var addToQueue []string for link := range info.links { @@ -209,10 +209,10 @@ func (c *crawler) crawl() (crawled crawledPages, cancelled bool) { queued[link] = true } } - return addToQueue, nil + return addToQueue, true } - _ = workgroup.Do(c.workers, task, manager, c.base) + flowmatic.ManageTasks(c.workers, task, manager, c.base) cancelled = ctx.Err() != nil