From ccd7bafab349c3fa633c934bee4de8fd9a0a4ccf Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Fri, 7 Apr 2023 10:21:05 +0800 Subject: [PATCH 01/38] Bump lia version to v0.1.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3b7f2acff..971aa3d61 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/cheggaaa/pb/v3 v3.0.8 github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c github.com/fatih/color v1.13.0 - github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b + github.com/iawia002/lia v0.1.0 github.com/itchyny/gojq v0.12.7 github.com/json-iterator/go v1.1.12 github.com/kkdai/youtube/v2 v2.7.18 diff --git a/go.sum b/go.sum index 3568a6f85..ebc3ab0a0 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b h1:rCLzCJKes3J0QCtEyPy6iefW28PCPSMwkPWhslaJLBQ= -github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b/go.mod h1:aL35R6u/LH1owTr+DdEOFtUMGsED9DhVddvN6KIyPbU= +github.com/iawia002/lia v0.1.0 h1:IzgR5pnOEt3bABB3TtcK5UCXkzcuc7+3GYxT0cVpYfU= +github.com/iawia002/lia v0.1.0/go.mod h1:Jxu7iNh5z17HWuLedH3jwh09aa5SO3g2BI2Ct87aXpY= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= From 901950b047fb689a8dbedf5d3ff53016f44cafaf Mon Sep 17 00:00:00 2001 From: baiyufei Date: Wed, 19 Apr 2023 22:35:58 +0800 Subject: [PATCH 02/38] fix: bilibili extractBangumi(#1213) --- extractors/bilibili/bilibili.go | 50 +++++++++++++++++++++++++++++---- extractors/bilibili/types.go | 12 ++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index cf28d8f21..607e71993 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -3,6 +3,7 @@ package bilibili import ( "encoding/json" "fmt" + "sort" "strconv" "strings" "time" @@ -92,20 +93,51 @@ type bilibiliOptions struct { } func extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { - dataString := utils.MatchOneOf(html, `window.__INITIAL_STATE__=(.+?);\(function`)[1] - var data bangumiData - err := json.Unmarshal([]byte(dataString), &data) + dataString := utils.MatchOneOf(html, `(.*?)`)[1] + epMapString := utils.MatchOneOf(dataString, `"epMap"\s*:\s*(.+?)\s*,\s*"initEpList"`)[1] + fullVideoIdString := utils.MatchOneOf(dataString, `"videoId"\s*:\s*"(ep|ss)(\d+)"`) + epSsString := fullVideoIdString[1] // "ep" or "ss" + videoIdString := fullVideoIdString[2] + + var epMap map[string]json.RawMessage + err := json.Unmarshal([]byte(epMapString), &epMap) if err != nil { return nil, errors.WithStack(err) } + var data bangumiData + for idString, jsonByte := range epMap { + var epInfo bangumiEpData + err := json.Unmarshal(jsonByte, &epInfo) + if err != nil { + return nil, errors.WithStack(err) + } + epID, err := strconv.ParseInt(idString, 10, 0) + if err != nil { + return nil, errors.WithStack(err) + } + epInfo.EpID = int(epID) + if idString == videoIdString || (epSsString == "ss" && epInfo.TitleFormat == "第1话") { + data.EpInfo = epInfo + } + data.EpList = append(data.EpList, epInfo) + } + + sort.Slice(data.EpList, func(i, j int) bool { + return data.EpList[i].EpID < data.EpList[j].EpID + }) + if !extractOption.Playlist { aid := data.EpInfo.Aid cid := data.EpInfo.Cid bvid := data.EpInfo.BVid + titleFormat := data.EpInfo.TitleFormat + longTitle := data.EpInfo.LongTitle if aid <= 0 || cid <= 0 || bvid == "" { aid = data.EpList[0].Aid cid = data.EpList[0].Cid bvid = data.EpList[0].BVid + titleFormat = data.EpList[0].TitleFormat + longTitle = data.EpList[0].LongTitle } options := bilibiliOptions{ url: url, @@ -114,6 +146,8 @@ func extractBangumi(url, html string, extractOption extractors.Options) ([]*extr aid: aid, cid: cid, bvid: bvid, + + subtitle: fmt.Sprintf("%s %s", titleFormat, longTitle), } return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil } @@ -139,6 +173,8 @@ func extractBangumi(url, html string, extractOption extractors.Options) ([]*extr aid: u.Aid, cid: u.Cid, bvid: u.BVid, + + subtitle: fmt.Sprintf("%s %s", u.TitleFormat, u.LongTitle), } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() @@ -425,10 +461,14 @@ func bilibiliDownload(options bilibiliOptions, extractOption extractors.Options) } title := parser.Title(doc) if options.subtitle != "" { + pageString := "" + if options.page > 0 { + pageString = fmt.Sprintf("P%d ", options.page) + } if extractOption.EpisodeTitleOnly { - title = fmt.Sprintf("P%d %s", options.page, options.subtitle) + title = fmt.Sprintf("%s%s", pageString, options.subtitle) } else { - title = fmt.Sprintf("%s P%d %s", title, options.page, options.subtitle) + title = fmt.Sprintf("%s %s%s", title, pageString, options.subtitle) } } diff --git a/extractors/bilibili/types.go b/extractors/bilibili/types.go index ba8533269..815cf9932 100644 --- a/extractors/bilibili/types.go +++ b/extractors/bilibili/types.go @@ -13,11 +13,13 @@ type token struct { } type bangumiEpData struct { - Aid int `json:"aid"` - Cid int `json:"cid"` - BVid string `json:"bvid"` - ID int `json:"id"` - EpID int `json:"ep_id"` + Aid int `json:"aid"` + Cid int `json:"cid"` + BVid string `json:"bvid"` + ID int `json:"id"` + EpID int `json:"ep_id"` + TitleFormat string `json:"titleFormat"` + LongTitle string `json:"long_title"` } type bangumiData struct { From 976f05c19d7e0cb6d5b74773030385ec606e64ef Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Thu, 20 Apr 2023 10:39:03 +0800 Subject: [PATCH 03/38] extractors/bilibili: update test --- extractors/bilibili/bilibili_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractors/bilibili/bilibili_test.go b/extractors/bilibili/bilibili_test.go index 7505cdf72..8688f3b44 100644 --- a/extractors/bilibili/bilibili_test.go +++ b/extractors/bilibili/bilibili_test.go @@ -26,7 +26,6 @@ func TestBilibili(t *testing.T) { args: test.Args{ URL: "https://www.bilibili.com/video/av41301960", Title: "【英雄联盟】2019赛季CG 《觉醒》", - Size: 70696896, }, playlist: false, }, @@ -34,7 +33,7 @@ func TestBilibili(t *testing.T) { name: "bangumi test", args: test.Args{ URL: "https://www.bilibili.com/bangumi/play/ep167000", - Title: "狐妖小红娘：第70话 苏苏智商上线", + Title: "狐妖小红娘 第70话 苏苏智商上线", }, }, { From e0729b067aff8b1115a9a78d76216e0f86a22ce3 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Thu, 20 Apr 2023 10:46:52 +0800 Subject: [PATCH 04/38] v0.18.0 [ci skip] --- app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app.go b/app/app.go index e2e4bfbe1..f6fc323a5 100644 --- a/app/app.go +++ b/app/app.go @@ -20,7 +20,7 @@ import ( const ( // Name is the name of this app. Name = "lux" - version = "v0.17.2" + version = "v0.18.0" ) func init() { From 9142f2aee8373f6aace3604a5cc055ca7ea56c04 Mon Sep 17 00:00:00 2001 From: Shavit Date: Mon, 8 May 2023 22:42:14 -0400 Subject: [PATCH 05/38] Add Rumble extractor (#1232) * Add Rumble extractor * Remove unused cookies * Update Supported Sites * Fetch live playlist --- .github/workflows/stream_rumble.yml | 31 +++ README.md | 1 + app/register.go | 1 + extractors/rumble/rumble.go | 320 ++++++++++++++++++++++++++++ extractors/rumble/rumble_test.go | 28 +++ 5 files changed, 381 insertions(+) create mode 100644 .github/workflows/stream_rumble.yml create mode 100644 extractors/rumble/rumble.go create mode 100644 extractors/rumble/rumble_test.go diff --git a/.github/workflows/stream_rumble.yml b/.github/workflows/stream_rumble.yml new file mode 100644 index 000000000..425aff8e7 --- /dev/null +++ b/.github/workflows/stream_rumble.yml @@ -0,0 +1,31 @@ +name: rumble + +on: + push: + paths: + - "extractors/rumble/*.go" + - ".github/workflows/stream_rumble.yml" + pull_request: + paths: + - "extractors/rumble/*.go" + - ".github/workflows/stream_rumble.yml" + schedule: + # run ci weekly + - cron: "0 0 * * 0" + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + go: ["1.20"] + os: [ubuntu-latest] + name: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Test + run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/rumble diff --git a/README.md b/README.md index 439184282..509296a4f 100644 --- a/README.md +++ b/README.md @@ -641,6 +641,7 @@ $ lux -j "https://www.bilibili.com/video/av20203945" | Reddit | | ✓ | ✓ | | | | [![reddit](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml) | | VKontakte | | ✓ | | | | | [![vk](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/) | | 知乎 | | ✓ | | | | | [![zhihu](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/) | +| Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) | ## Known issues diff --git a/app/register.go b/app/register.go index b0392472d..2f63c9c8e 100644 --- a/app/register.go +++ b/app/register.go @@ -23,6 +23,7 @@ import ( _ "github.com/iawia002/lux/extractors/pornhub" _ "github.com/iawia002/lux/extractors/qq" _ "github.com/iawia002/lux/extractors/reddit" + _ "github.com/iawia002/lux/extractors/rumble" _ "github.com/iawia002/lux/extractors/streamtape" _ "github.com/iawia002/lux/extractors/tangdou" _ "github.com/iawia002/lux/extractors/tiktok" diff --git a/extractors/rumble/rumble.go b/extractors/rumble/rumble.go new file mode 100644 index 000000000..d078fc2fc --- /dev/null +++ b/extractors/rumble/rumble.go @@ -0,0 +1,320 @@ +package rumble + +import ( + "compress/flate" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + + "github.com/pkg/errors" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/request" + "github.com/iawia002/lux/utils" +) + +func init() { + extractors.Register("rumble", New()) +} + +type extractor struct{} + +// New returns a rumble extractor. +func New() extractors.Extractor { + return &extractor{} +} + +type rumbleData struct { + Format string `json:"format"` + Name string `json:"name"` + EmbedURL string `json:"embedUrl"` + ThumbnailURL string `json:"thumbnailUrl"` + Type string `json:"@type"` + VideoURL string `json:"videoUrl"` + Quality string `json:"quality"` +} + +// Extract is the main function to extract the data. +func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { + res, err := request.Request(http.MethodGet, url, nil, nil) + if err != nil { + return nil, errors.WithStack(err) + } + + defer res.Body.Close() // nolint + + var reader io.ReadCloser + switch res.Header.Get("Content-Encoding") { + case "gzip": + reader, _ = gzip.NewReader(res.Body) + case "deflate": + reader = flate.NewReader(res.Body) + default: + reader = res.Body + } + defer reader.Close() // nolint + + b, err := io.ReadAll(reader) + if err != nil { + return nil, errors.WithStack(err) + } + + html := string(b) + var title string + matchTitle := utils.MatchOneOf(html, `(.+?)`) + if len(matchTitle) > 1 { + title = matchTitle[1] + } else { + title = "rumble video" + } + + payload, err := readPayload(html) + if err != nil { + return nil, errors.WithStack(err) + } + + videoID, err := getVideoID(payload.EmbedURL) + if err != nil { + return nil, errors.WithStack(err) + } + + streams, err := fetchVideoQuality(videoID) + if err != nil { + return nil, errors.WithStack(err) + } + + return []*extractors.Data{ + { + Site: "Rumble rumble.com", + Title: title, + Type: extractors.DataTypeVideo, + Streams: streams, + URL: url, + }, + }, nil +} + +// Read JSON object from the video webpage +func readPayload(html string) (*rumbleData, error) { + matchPayload := utils.MatchOneOf(html, `\(.+?)\<\/script>`) + if len(matchPayload) < 1 { + return nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed) + } + + rumbles := make([]rumbleData, 0) + if err := json.Unmarshal([]byte(matchPayload[1]), &rumbles); err != nil { + return nil, errors.WithStack(err) + } + + for _, it := range rumbles { + if it.Type == "VideoObject" { + return &it, nil + } + } + + return nil, errors.WithStack(extractors.ErrURLParseFailed) +} + +func getVideoID(embedURL string) (string, error) { + u, err := url.Parse(embedURL) + if err != nil { + return "", errors.WithStack(extractors.ErrURLParseFailed) + } + + return path.Base(u.Path), nil +} + +// Rumble response contains the streams in `rumbleStreams` +type rumbleResponse struct { + Streams *json.RawMessage `json:"ua"` +} + +// Common video meta data +type streamInfo struct { + URL string `json:"url"` + Meta struct { + Bitrate uint16 `json:"bitrate"` + Size int64 `json:"size"` + Width uint16 `json:"w"` + Height uint16 `json:"h"` + } `json:"meta"` +} + +// common video qualities for `mp4`, `webm` +type videoQualities struct { + Q240 struct{ streamInfo } `json:"240"` + Q360 struct{ streamInfo } `json:"360"` + Q480 struct{ streamInfo } `json:"480"` + Q720 struct{ streamInfo } `json:"720"` + Q1080 struct{ streamInfo } `json:"1080"` + Q1440 struct{ streamInfo } `json:"1440"` + Q2160 struct{ streamInfo } `json:"2160"` + Q2161 struct{ streamInfo } `json:"2161"` +} + +// Video payload for adaptive stream and different qualities +type rumbleStreams struct { + FMp4 struct { + videoQualities + } `json:"mp4"` + FWebm struct { + videoQualities + } `json:"webm"` + FHLS struct { + QAuto struct{ streamInfo } `json:"auto"` + } `json:"hls"` +} + +// Unmarshall the video response +// Some properties like `mp4`, `webm` are either array or an object +func (r *rumbleStreams) UnmarshalJSON(b []byte) error { + var resp *rumbleResponse + if err := json.Unmarshal(b, &resp); err != nil { + return errors.WithStack(extractors.ErrURLParseFailed) + } + + // Get individual stream from the response + var obj map[string]*json.RawMessage + if err := json.Unmarshal(*resp.Streams, &obj); err != nil { + return errors.WithStack(extractors.ErrURLParseFailed) + } + + if v, ok := obj["mp4"]; ok { + _ = json.Unmarshal(*v, &r.FMp4) + } + if v, ok := obj["webm"]; ok { + _ = json.Unmarshal(*v, &r.FWebm) + } + if v, ok := obj["hls"]; ok { + _ = json.Unmarshal(*v, &r.FHLS) + } + + return nil +} + +// Use this to create all the streams for `mp4`, `webm` +func (rs *rumbleStreams) makeAllVODStreams(m map[string]*extractors.Stream) { + m["webm"] = makeStreamMeta("480", "webm", &rs.FWebm.Q480.streamInfo) + m["240"] = makeStreamMeta("240", "mp4", &rs.FMp4.Q240.streamInfo) + m["360"] = makeStreamMeta("360", "mp4", &rs.FMp4.Q360.streamInfo) + m["480"] = makeStreamMeta("480", "mp4", &rs.FMp4.Q480.streamInfo) + m["720"] = makeStreamMeta("720", "mp4", &rs.FMp4.Q720.streamInfo) + m["1080"] = makeStreamMeta("1080", "mp4", &rs.FMp4.Q1080.streamInfo) + m["1440"] = makeStreamMeta("1440", "mp4", &rs.FMp4.Q1440.streamInfo) + m["2160"] = makeStreamMeta("2160", "mp4", &rs.FMp4.Q2160.streamInfo) + m["2161"] = makeStreamMeta("2161", "mp4", &rs.FMp4.Q2161.streamInfo) +} + +var reResolution = regexp.MustCompile(`_(\d{3,4})p\/`) // ex. _720p/ + +// Use this to create all the streams for live videos +func (rs *rumbleStreams) makeAllLiveStreams(m map[string]*extractors.Stream) error { + playlists, err := utils.M3u8URLs(rs.FHLS.QAuto.URL) + if err != nil { + return errors.WithStack(err) + } + + if len(playlists) == 0 { + return errors.WithStack(extractors.ErrURLParseFailed) + } + + // Find the highest resolution + playlistURL := playlists[0] + maxRes := 0 + for _, x := range playlists { + matched := reResolution.FindStringSubmatch(x) + if len(matched) == 0 { + continue + } + res, err := strconv.Atoi(matched[1]) + if err != nil { + continue + } + + if maxRes < res { + maxRes = res + playlistURL = x + } + } + + tsURLs, err := utils.M3u8URLs(playlistURL) + if err != nil { + return errors.WithStack(err) + } + + var parts []*extractors.Part + for _, x := range tsURLs { + part := &extractors.Part{ + URL: x, + Size: rs.FHLS.QAuto.streamInfo.Meta.Size, + Ext: "ts", + } + parts = append(parts, part) + } + + m["hls"] = &extractors.Stream{ + Parts: parts, + Size: rs.FHLS.QAuto.streamInfo.Meta.Size, + Quality: strconv.Itoa(maxRes), + } + + return nil +} + +// Request video formats and qualities +func fetchVideoQuality(videoID string) (map[string]*extractors.Stream, error) { + reqURL := fmt.Sprintf(`https://rumble.com/embedJS/u3/?request=video&ver=2&v=%s&ext={"ad_count":null}&ad_wt=0`, videoID) + + res, err := request.Request(http.MethodGet, reqURL, nil, nil) + if err != nil { + return nil, errors.WithStack(err) + } + defer res.Body.Close() // nolint + + var reader io.ReadCloser + switch res.Header.Get("Content-Encoding") { + case "gzip": + reader, _ = gzip.NewReader(res.Body) + case "deflate": + reader = flate.NewReader(res.Body) + default: + reader = res.Body + } + defer reader.Close() // nolint + + b, err := io.ReadAll(reader) + if err != nil { + return nil, errors.WithStack(err) + } + + var rs rumbleStreams + if err := json.Unmarshal(b, &rs); err != nil { + return nil, errors.WithStack(err) + } + + streams := make(map[string]*extractors.Stream, 9) + rs.makeAllVODStreams(streams) + _ = rs.makeAllLiveStreams(streams) + + return streams, nil +} + +func makeStreamMeta(q, ext string, info *streamInfo) *extractors.Stream { + urlMeta := &extractors.Part{ + URL: info.URL, + Size: info.Meta.Size, + Ext: ext, + } + + return &extractors.Stream{ + Parts: []*extractors.Part{urlMeta}, + Size: info.Meta.Size, + Quality: q, + } +} diff --git a/extractors/rumble/rumble_test.go b/extractors/rumble/rumble_test.go new file mode 100644 index 000000000..622d8e617 --- /dev/null +++ b/extractors/rumble/rumble_test.go @@ -0,0 +1,28 @@ +package rumble + +import ( + "testing" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/test" +) + +func TestRumble(t *testing.T) { + tests := []struct { + name string + args test.Args + }{ + { + name: "normal test", + args: test.Args{ + URL: "https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html", + Title: "Just Say YES to Climate Lockdowns!", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + New().Extract(tt.args.URL, extractors.Options{}) + }) + } +} From 37a325a3e30548ecf84957e6a3196a4546233b86 Mon Sep 17 00:00:00 2001 From: sunyoupan Date: Fri, 2 Jun 2023 22:29:58 +0800 Subject: [PATCH 06/38] 1. fix panic if xsrf not exists 2. get more available quality videos --- extractors/weibo/weibo.go | 59 +++++++++++++++++++++++----------- extractors/weibo/weibo_test.go | 2 +- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/extractors/weibo/weibo.go b/extractors/weibo/weibo.go index ddfaf714b..c852de924 100644 --- a/extractors/weibo/weibo.go +++ b/extractors/weibo/weibo.go @@ -55,8 +55,15 @@ func getXSRFToken() (string, error) { } defer res.Body.Close() // nolint - token := utils.MatchOneOf(res.Header.Get("Set-Cookie"), `XSRF-TOKEN=(.+?);`)[1] - return token, nil + cookie := res.Header.Get("Set-Cookie") + if cookie == "" { + return "", nil + } + xsrfTokens := utils.MatchOneOf(cookie, `XSRF-TOKEN=(.+?);`) + if xsrfTokens == nil || len(xsrfTokens) != 2 { + return "", nil + } + return xsrfTokens[1], nil } func downloadWeiboVideo(url string) ([]*extractors.Data, error) { @@ -144,10 +151,13 @@ func downloadWeiboTV(url string) ([]*extractors.Data, error) { return nil, errors.WithStack(err) } headers := map[string]string{ - "Cookie": "SUB=_2AkMpogLYf8NxqwJRmP0XxG7kbo10ww_EieKf_vMDJRMxHRl-yj_nqm4NtRB6AiIsKFFGRY4-UuGD5B1-Kf9glz3sp7Ii; XSRF-TOKEN=" + token, + "Cookie": "SUB=_2AkMpogLYf8NxqwJRmP0XxG7kbo10ww_EieKf_vMDJRMxHRl-yj_nqm4NtRB6AiIsKFFGRY4-UuGD5B1-Kf9glz3sp7Ii", "Referer": utils.MatchOneOf(url, `^([^?]+)`)[1], "content-type": `application/x-www-form-urlencoded`, - "x-xsrf-token": token, + } + if token != "" { + headers["Cookie"] += "; XSRF-TOKEN=" + token + headers["x-xsrf-token"] = token } oid := utils.MatchOneOf(url, `tv/show/([^?]+)`)[1] postData := "data=" + netURL.QueryEscape("{\"Component_Play_Playinfo\":{\"oid\":\""+oid+"\"}}") @@ -241,28 +251,39 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor } title := titles[1] - realURLs := utils.MatchOneOf( - html, `"stream_url_hd": "(.+?)"`, `"stream_url": "(.+?)"`, + urlsJsonStrs := utils.MatchOneOf( + html, `"urls": (\{[^\}]+\})`, ) - if realURLs == nil || len(realURLs) < 2 { + if urlsJsonStrs == nil || len(urlsJsonStrs) < 2 { return nil, errors.WithStack(extractors.ErrURLParseFailed) } - realURL := realURLs[1] - - size, err := request.Size(realURL, url) + urlsJson := urlsJsonStrs[1] + var qualityUrls map[string]string + err = json.Unmarshal([]byte(urlsJson), &qualityUrls) if err != nil { - return nil, errors.WithStack(err) - } - urlData := &extractors.Part{ - URL: realURL, - Size: size, - Ext: "mp4", + return nil, errors.WithStack(extractors.ErrURLParseFailed) } - streams := map[string]*extractors.Stream{ - "default": { + + streams := make(map[string]*extractors.Stream) + var size int64 + for quality, realURL := range qualityUrls { + streamId := quality + size, err = request.Size(realURL, url) + if err != nil { + continue + } + urlData := &extractors.Part{ + URL: realURL, + Size: size, + Ext: "mp4", + } + streams[streamId] = &extractors.Stream{ Parts: []*extractors.Part{urlData}, Size: size, - }, + } + } + if err != nil || len(streams) <= 0 { + return nil, errors.WithStack(err) } return []*extractors.Data{ diff --git a/extractors/weibo/weibo_test.go b/extractors/weibo/weibo_test.go index 9f6e7016d..2d59a71af 100644 --- a/extractors/weibo/weibo_test.go +++ b/extractors/weibo/weibo_test.go @@ -23,7 +23,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://m.weibo.cn/status/4237529215145705", Title: `近日，日本视错觉大师、明治大学特任教授\"杉原厚吉的“错觉箭头“作品又引起世界人民的关注。反射，透视和视角的巧妙结合产生了这种惊人的幻觉：箭头向右？转过来还是向右？\n\n引用杉原教授的经典描述：“我们看外面的世界的方式——也就是我们的知觉——都是由大脑机制间接产生的，所以所有知觉在某`, - Size: 1125984, + Size: 2005728, }, }, { From 3da4af3722f9964602cce738e130fa6c84ce7260 Mon Sep 17 00:00:00 2001 From: Bilal Hameed <68427058+LinuxSploit@users.noreply.github.com> Date: Wed, 5 Jul 2023 02:04:27 +0000 Subject: [PATCH 07/38] Added Pinterest video downloading feature (#1253) * Added Pinterest video downloading feature * Update pinterest.go Fixed title extractor --- .github/workflows/stream_pinterest.yml | 31 ++++++++++ README.md | 1 + app/register.go | 1 + extractors/pinterest/pinterest.go | 85 ++++++++++++++++++++++++++ extractors/pinterest/pinterest_test.go | 39 ++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 .github/workflows/stream_pinterest.yml create mode 100644 extractors/pinterest/pinterest.go create mode 100644 extractors/pinterest/pinterest_test.go diff --git a/.github/workflows/stream_pinterest.yml b/.github/workflows/stream_pinterest.yml new file mode 100644 index 000000000..330cddfe1 --- /dev/null +++ b/.github/workflows/stream_pinterest.yml @@ -0,0 +1,31 @@ +name: pinterest + +on: + push: + paths: + - "extractors/pinterest/*.go" + - ".github/workflows/stream_pinterest.yml" + pull_request: + paths: + - "extractors/tiktok/*.go" + - ".github/workflows/stream_pinterest.yml" + schedule: + # run ci weekly + - cron: "0 0 * * 0" + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + go: ["1.20"] + os: [ubuntu-latest] + name: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Test + run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/pinterest diff --git a/README.md b/README.md index 509296a4f..7d58a39b9 100644 --- a/README.md +++ b/README.md @@ -630,6 +630,7 @@ $ lux -j "https://www.bilibili.com/video/av20203945" | XVIDEOS | | ✓ | | | | | [![xvideos](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml) | | 聯合新聞網 | | ✓ | | | | | [![udn](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml) | | TikTok | | ✓ | | | | | [![tiktok](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml) | +| Pinterest | | ✓ | | | | | [![pinterest](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml) | | 好看视频 | | ✓ | | | | | [![haokan](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml) | | AcFun | | ✓ | | | ✓ | | [![acfun](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml) | | Eporner | | ✓ | | | | | [![eporner](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml) | diff --git a/app/register.go b/app/register.go index 2f63c9c8e..5b1391d10 100644 --- a/app/register.go +++ b/app/register.go @@ -19,6 +19,7 @@ import ( _ "github.com/iawia002/lux/extractors/mgtv" _ "github.com/iawia002/lux/extractors/miaopai" _ "github.com/iawia002/lux/extractors/netease" + _ "github.com/iawia002/lux/extractors/pinterest" _ "github.com/iawia002/lux/extractors/pixivision" _ "github.com/iawia002/lux/extractors/pornhub" _ "github.com/iawia002/lux/extractors/qq" diff --git a/extractors/pinterest/pinterest.go b/extractors/pinterest/pinterest.go new file mode 100644 index 000000000..b91c6fee9 --- /dev/null +++ b/extractors/pinterest/pinterest.go @@ -0,0 +1,85 @@ +package pinterest + +import ( + "regexp" + "strings" + + "github.com/pkg/errors" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/request" +) + +func init() { + extractors.Register("pinterest", New()) +} + +type extractor struct{} + +// New returns a pinterest extractor. +func New() extractors.Extractor { + return &extractor{} +} + +// Extract is the main function to extract the data. +func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { + html, err := request.Get(url, url, map[string]string{ + // pinterest require a user agent + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0", + }) + if err != nil { + return nil, errors.WithStack(err) + } + + urlMatcherRegExp := regexp.MustCompile(`"contentUrl":"https:\/\/v1\.pinimg\.com\/videos\/mc\/720p\/[a-zA-Z0-9\/]+\.mp4`) + + downloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html) + + if len(downloadURLMatcher) == 0 { + return nil, errors.WithStack(extractors.ErrURLParseFailed) + } + + videoURL := strings.ReplaceAll(downloadURLMatcher[0], `"contentUrl":"`, "") + + titleMatcherRegExp := regexp.MustCompile(`]*>([^<]+)`) + + titleMatcher := titleMatcherRegExp.FindStringSubmatch(html) + + if len(titleMatcher) == 0 { + return nil, errors.WithStack(extractors.ErrURLParseFailed) + } + + title := strings.ReplaceAll(strings.ReplaceAll(titleMatcher[0], "", ""), "", "") + + titleArr := strings.Split(title, "|") + + if len(titleArr) > 0 { + title = titleArr[0] + } + + streams := make(map[string]*extractors.Stream) + + size, err := request.Size(videoURL, url) + if err != nil { + return nil, errors.WithStack(err) + } + urlData := &extractors.Part{ + URL: videoURL, + Size: size, + Ext: "mp4", + } + streams["default"] = &extractors.Stream{ + Parts: []*extractors.Part{urlData}, + Size: size, + } + + return []*extractors.Data{ + { + Site: "Pinterest pinterest.com", + Title: title, + Type: extractors.DataTypeVideo, + Streams: streams, + URL: url, + }, + }, nil +} diff --git a/extractors/pinterest/pinterest_test.go b/extractors/pinterest/pinterest_test.go new file mode 100644 index 000000000..7e249b1bd --- /dev/null +++ b/extractors/pinterest/pinterest_test.go @@ -0,0 +1,39 @@ +package pinterest + +import ( + "testing" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/test" +) + +func TestDownload(t *testing.T) { + tests := []struct { + name string + args test.Args + }{ + { + name: "normal test 1", + args: test.Args{ + URL: "https://www.pinterest.com/pin/creamy-cheesy-pretzel-bites-video--368450813272292084/", + Title: "Creamy Cheesy Pretzel Bites [Video] ", + Size: 30247497, + }, + }, + { + name: "normal test 2", + args: test.Args{ + URL: "https://www.pinterest.com/pin/532198880988430823/", + Title: "Pin on TikTok ~ The world of food", + Size: 4676927, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := New().Extract(tt.args.URL, extractors.Options{}) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + }) + } +} From 99f80937cd2fc7144dbd8a1306bb119bdbaf810a Mon Sep 17 00:00:00 2001 From: Shavit Date: Wed, 5 Jul 2023 22:21:15 -0400 Subject: [PATCH 08/38] 1228 fix instagram download (#1252) * Extract Instagram payload Read the payload of stories and images * Extract images and videos * Update the file sizes in Instagram tests * go mod tidy --------- Co-authored-by: Xinzhao Xu --- extractors/errors.go | 1 + extractors/instagram/instagram.go | 214 ++++++++++++++----------- extractors/instagram/instagram_test.go | 6 +- go.mod | 2 +- 4 files changed, 129 insertions(+), 94 deletions(-) diff --git a/extractors/errors.go b/extractors/errors.go index 9b57c107c..8759033bb 100644 --- a/extractors/errors.go +++ b/extractors/errors.go @@ -9,4 +9,5 @@ var ( ErrURLParseFailed = errors.New("url parse failed") ErrInvalidRegularExpression = errors.New("invalid regular expression") ErrURLQueryParamsParseFailed = errors.New("url query params parse failed") + ErrBodyParseFailed = errors.New("body parse failed") ) diff --git a/extractors/instagram/instagram.go b/extractors/instagram/instagram.go index 7c7f2902a..3477726de 100644 --- a/extractors/instagram/instagram.go +++ b/extractors/instagram/instagram.go @@ -3,13 +3,12 @@ package instagram import ( "encoding/json" netURL "net/url" - "path" "strings" "github.com/pkg/errors" + "golang.org/x/net/html" "github.com/iawia002/lux/extractors" - "github.com/iawia002/lux/parser" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) @@ -18,18 +17,30 @@ func init() { extractors.Register("instagram", New()) } -type instagram struct { - ShortcodeMedia struct { - EdgeSidecar struct { - Edges []struct { - Node struct { - DisplayURL string `json:"display_url"` - IsVideo bool `json:"is_video"` - VideoURL string `json:"video_url"` - } `json:"node"` - } `json:"edges"` - } `json:"edge_sidecar_to_children"` - } `json:"shortcode_media"` +type instagramPayload struct { + ArticleBody string `json:"articleBody"` + Author struct { + Image string `json:"image"` + Name string `json:"name"` + AlternativeName string `json:"alternativeName"` + Url string `json:"url"` + } `json:"author"` + Videos []struct { + UploadData string `json:"string"` + Description string `json:"description"` + Name string `json:"name"` + Caption string `json:"caption"` + Height string `json:"height"` + Width string `json:"width"` + ContentURL string `json:"contentUrl"` + ThumbnailURL string `json:"thumbnailUrl"` + } `json:"video"` + Images []struct { + Caption string `json:"caption"` + Height string `json:"height"` + Width string `json:"width"` + URL string `json:"url"` + } `json:"image"` } type extractor struct{} @@ -39,104 +50,65 @@ func New() extractors.Extractor { return &extractor{} } -func extractImageFromPage(html, url string) (map[string]*extractors.Stream, error) { - _, realURLs, err := parser.GetImages(html, "EmbeddedMediaImage", nil) +// Extract is the main function to extract the data. +func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { + u, err := netURL.Parse(url) if err != nil { return nil, errors.WithStack(err) } - urls := make([]*extractors.Part, 0, len(realURLs)) - var totalSize int64 - for _, realURL := range realURLs { - size, err := request.Size(realURL, url) - if err != nil { - return nil, errors.WithStack(err) - } - urlData := &extractors.Part{ - URL: realURL, - Size: size, - Ext: "jpg", - } - urls = append(urls, urlData) - totalSize += size + htmlResp, err := request.Get(u.String(), url, nil) + if err != nil { + return nil, errors.WithStack(err) } - return map[string]*extractors.Stream{ - "default": { - Parts: urls, - Size: totalSize, - }, - }, nil -} + reader := strings.NewReader(htmlResp) + htmlRoot, err := html.Parse(reader) + if err != nil { + return nil, errors.WithStack(err) + } -func extractFromData(dataString, url string) (map[string]*extractors.Stream, error) { - var data instagram - if err := json.Unmarshal([]byte(dataString), &data); err != nil { + sNode, err := dfsFindScript(htmlRoot) + if err != nil { + return nil, errors.WithStack(err) + } + + var payload instagramPayload + if err = json.Unmarshal([]byte(sNode.Data), &payload); err != nil { return nil, errors.WithStack(err) } - urls := make([]*extractors.Part, 0, len(data.ShortcodeMedia.EdgeSidecar.Edges)) var totalSize int64 - for _, u := range data.ShortcodeMedia.EdgeSidecar.Edges { - // Image - realURL := u.Node.DisplayURL - ext := "jpg" - if u.Node.IsVideo { - // Video - realURL = u.Node.VideoURL - ext = "mp4" + var parts []*extractors.Part + if len(payload.Videos) > 0 { + videoParts, err := createPartVideos(&payload, url) + if err != nil { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) } - size, err := request.Size(realURL, url) + parts = append(parts, videoParts...) + } + if len(payload.Images) > 0 { + imageParts, err := createPartImages(&payload, url) if err != nil { - return nil, errors.WithStack(err) - } - urlData := &extractors.Part{ - URL: realURL, - Size: size, - Ext: ext, + return nil, errors.WithStack(extractors.ErrBodyParseFailed) } - urls = append(urls, urlData) - totalSize += size + + parts = append(parts, imageParts...) + } + + for _, part := range parts { + totalSize += part.Size } - return map[string]*extractors.Stream{ + streams := map[string]*extractors.Stream{ "default": { - Parts: urls, + Parts: parts, Size: totalSize, }, - }, nil -} - -// Extract is the main function to extract the data. -func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { - // Instagram is forcing a login to access the page, so we use the embed page to bypass that. - u, err := netURL.Parse(url) - if err != nil { - return nil, errors.WithStack(err) } - id := u.Path[strings.LastIndex(u.Path, "/")+1:] - u.Path = path.Join(u.Path, "embed") - html, err := request.Get(u.String(), url, nil) - if err != nil { - return nil, errors.WithStack(err) - } - dataStrings := utils.MatchOneOf(html, `window\.__additionalDataLoaded\('graphql',(.*)\);`) - if dataStrings == nil || len(dataStrings) < 2 { - return nil, errors.WithStack(extractors.ErrURLParseFailed) - } - dataString := dataStrings[1] - - var streams map[string]*extractors.Stream - if dataString == "" || dataString == "null" { - streams, err = extractImageFromPage(html, url) - } else { - streams, err = extractFromData(dataString, url) - } - if err != nil { - return nil, errors.WithStack(err) - } + id := u.Path[strings.LastIndex(u.Path, "/")+1:] return []*extractors.Data{ { @@ -148,3 +120,65 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor }, }, nil } + +func dfsFindScript(n *html.Node) (*html.Node, error) { + if n.Type == html.ElementNode && n.Data == "script" { + for _, attr := range n.Attr { + if attr.Key == "type" && attr.Val == "application/ld+json" { + return n.FirstChild, nil + } + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if ret, err := dfsFindScript(c); err == nil { + return ret, nil + } + } + + return nil, errors.WithStack(extractors.ErrBodyParseFailed) +} + +func createPartVideos(payload *instagramPayload, ref string) (parts []*extractors.Part, err error) { + for _, it := range payload.Videos { + _, ext, err := utils.GetNameAndExt(it.ContentURL) + if err != nil { + return parts, errors.WithStack(err) + } + filesize, err := request.Size(it.ContentURL, ref) + if err != nil { + return parts, errors.WithStack(err) + } + + part := &extractors.Part{ + URL: it.ContentURL, + Size: filesize, + Ext: ext, + } + parts = append(parts, part) + } + + return parts, err +} + +func createPartImages(payload *instagramPayload, ref string) (parts []*extractors.Part, err error) { + for _, it := range payload.Images { + _, ext, err := utils.GetNameAndExt(it.URL) + if err != nil { + return parts, errors.WithStack(err) + } + filesize, err := request.Size(it.URL, ref) + if err != nil { + return parts, errors.WithStack(err) + } + + part := &extractors.Part{ + URL: it.URL, + Size: filesize, + Ext: ext, + } + parts = append(parts, part) + } + + return parts, err +} diff --git a/extractors/instagram/instagram_test.go b/extractors/instagram/instagram_test.go index 0bd4b9bde..7105ed068 100644 --- a/extractors/instagram/instagram_test.go +++ b/extractors/instagram/instagram_test.go @@ -17,7 +17,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/BlIka1ZFCNr", Title: "Instagram BlIka1ZFCNr", - Size: 3003662, + Size: 577298, }, }, { @@ -25,7 +25,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/Bl5oVUyl9Yx", Title: "Instagram Bl5oVUyl9Yx", - Size: 250596, + Size: 101611, }, }, { @@ -33,7 +33,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/Bjyr-gxF4Rb", Title: "Instagram Bjyr-gxF4Rb", - Size: 4599909, + Size: 241466, }, }, } diff --git a/go.mod b/go.mod index 971aa3d61..f28d0b6c6 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f github.com/urfave/cli/v2 v2.6.0 + golang.org/x/net v0.7.0 ) require ( @@ -37,7 +38,6 @@ require ( github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf // indirect - golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect From e8e6bd4ae41f8fc011d7c84ebeb1eb24c060e293 Mon Sep 17 00:00:00 2001 From: Shavit Date: Wed, 5 Jul 2023 22:32:27 -0400 Subject: [PATCH 09/38] Update Cask URL (#1251) * Update Cask URL * Update Cask URL --------- Co-authored-by: Xinzhao Xu --- Cask.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cask.toml b/Cask.toml index e32fb3c28..ce5e4112a 100644 --- a/Cask.toml +++ b/Cask.toml @@ -9,12 +9,12 @@ description = """ """ [darwin] -x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_macOS_64-bit.tar.gz" } -aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_macOS_ARM64.tar.gz" } +x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_x86_64.tar.gz" } +aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Darwin_arm64.tar.gz" } [windows] -x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Windows_64-bit.zip" } +x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Windows_x86_64.zip" } [linux] -x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_64-bit.tar.gz" } -aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_ARM64.tar.gz" } +x86_64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_x86_64.tar.gz" } +aarch64 = { url = "https://github.com/iawia002/lux/releases/download/v{version}/lux_{version}_Linux_arm64.tar.gz" } From 092bc0f6c18f81e1441ac17b2d053e74d7e6ddfe Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Thu, 6 Jul 2023 10:33:24 +0800 Subject: [PATCH 10/38] v0.19.0 [ci skip] --- app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app.go b/app/app.go index f6fc323a5..e689e7c6f 100644 --- a/app/app.go +++ b/app/app.go @@ -20,7 +20,7 @@ import ( const ( // Name is the name of this app. Name = "lux" - version = "v0.18.0" + version = "v0.19.0" ) func init() { From b9cf2efc9ce0d90ff88a21572105bde4f8956096 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Mon, 14 Aug 2023 10:34:27 +0800 Subject: [PATCH 11/38] Bump go version to 1.21 --- .github/workflows/builder.yml | 14 +++++++------- .github/workflows/ci.yml | 8 ++++---- .github/workflows/stream_acfun.yml | 2 +- .github/workflows/stream_bcy.yml | 2 +- .github/workflows/stream_bilibili.yml | 2 +- .github/workflows/stream_douyin.yml | 2 +- .github/workflows/stream_douyu.yml | 2 +- .github/workflows/stream_eporner.yml | 2 +- .github/workflows/stream_facebook.yml | 2 +- .github/workflows/stream_geekbang.yml | 2 +- .github/workflows/stream_haokan.yml | 2 +- .github/workflows/stream_hupu.yml | 2 +- .github/workflows/stream_huya.yml | 2 +- .github/workflows/stream_instagram.yml | 2 +- .github/workflows/stream_iqiyi.yml | 2 +- .github/workflows/stream_ixigua.yml | 2 +- .github/workflows/stream_kuaishou.yml | 2 +- .github/workflows/stream_mgtv.yml | 2 +- .github/workflows/stream_miaopai.yml | 2 +- .github/workflows/stream_netease.yml | 2 +- .github/workflows/stream_pinterest.yml | 4 ++-- .github/workflows/stream_pixivision.yml | 2 +- .github/workflows/stream_pornhub.yml | 2 +- .github/workflows/stream_qq.yml | 2 +- .github/workflows/stream_reddit.yml | 2 +- .github/workflows/stream_rumble.yml | 2 +- .github/workflows/stream_streamtape.yml | 2 +- .github/workflows/stream_tangdou.yml | 2 +- .github/workflows/stream_tiktok.yml | 2 +- .github/workflows/stream_tumblr.yml | 2 +- .github/workflows/stream_twitter.yml | 2 +- .github/workflows/stream_udn.yml | 2 +- .github/workflows/stream_vimeo.yml | 2 +- .github/workflows/stream_vk.yml | 2 +- .github/workflows/stream_weibo.yml | 2 +- .github/workflows/stream_ximalaya.yml | 2 +- .github/workflows/stream_xinpianchang.yml | 2 +- .github/workflows/stream_xvideos.yml | 2 +- .github/workflows/stream_yinyuetai.yml | 2 +- .github/workflows/stream_youku.yml | 2 +- .github/workflows/stream_youtube.yml | 2 +- .github/workflows/stream_zhihu.yml | 2 +- .golangci.yml | 2 +- extractors/bilibili/bilibili.go | 8 ++++---- extractors/miaopai/miaopai.go | 3 --- extractors/qq/qq.go | 4 ++-- extractors/youtube/youtube.go | 4 ++-- go.mod | 4 +--- go.sum | 7 +++---- script/github_action_template.yml | 2 +- utils/utils.go | 4 ++-- 51 files changed, 68 insertions(+), 74 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 1746503f6..24ec79736 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -87,26 +87,26 @@ jobs: GOMIPS64: ${{ matrix.mips64 }} GOMIPSLE: ${{ matrix.mipsle }} steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v4 with: - go-version: ^1.20 + go-version: 1.21 - name: Check out code base if: github.event_name == 'push' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Check out code base if: github.event_name == 'pull_request' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Cache go module - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -133,7 +133,7 @@ jobs: echo "::set-output name=filename::${{ env.PRODUCT }}-${ARGS}" - name: Upload binary artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ steps.builder.outputs.filename }} path: ./bin/${{ env.PRODUCT }}* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a7bb8f66..a0c8a2f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,12 +11,12 @@ jobs: timeout-minutes: 30 strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest, macOS-latest] name: Go ${{ matrix.go }} in ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} - name: Environment @@ -26,7 +26,7 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v3 with: - version: v1.51.2 + version: v1.54.1 only-new-issues: true - name: Test env: diff --git a/.github/workflows/stream_acfun.yml b/.github/workflows/stream_acfun.yml index a8b15041c..44888182b 100644 --- a/.github/workflows/stream_acfun.yml +++ b/.github/workflows/stream_acfun.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_bcy.yml b/.github/workflows/stream_bcy.yml index a3c0d191e..8b34d8984 100644 --- a/.github/workflows/stream_bcy.yml +++ b/.github/workflows/stream_bcy.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_bilibili.yml b/.github/workflows/stream_bilibili.yml index 58cc145db..1c7431451 100644 --- a/.github/workflows/stream_bilibili.yml +++ b/.github/workflows/stream_bilibili.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_douyin.yml b/.github/workflows/stream_douyin.yml index 079d5845b..70673f77e 100644 --- a/.github/workflows/stream_douyin.yml +++ b/.github/workflows/stream_douyin.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_douyu.yml b/.github/workflows/stream_douyu.yml index fd5442c5c..064fea5e2 100644 --- a/.github/workflows/stream_douyu.yml +++ b/.github/workflows/stream_douyu.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_eporner.yml b/.github/workflows/stream_eporner.yml index bc11c940c..8072e659f 100644 --- a/.github/workflows/stream_eporner.yml +++ b/.github/workflows/stream_eporner.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_facebook.yml b/.github/workflows/stream_facebook.yml index 439c2529c..64ca8f0ab 100644 --- a/.github/workflows/stream_facebook.yml +++ b/.github/workflows/stream_facebook.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_geekbang.yml b/.github/workflows/stream_geekbang.yml index c941912b2..bfa44e471 100644 --- a/.github/workflows/stream_geekbang.yml +++ b/.github/workflows/stream_geekbang.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_haokan.yml b/.github/workflows/stream_haokan.yml index 240220a5e..02b35848f 100644 --- a/.github/workflows/stream_haokan.yml +++ b/.github/workflows/stream_haokan.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_hupu.yml b/.github/workflows/stream_hupu.yml index f0f34d6f8..5aa857b74 100644 --- a/.github/workflows/stream_hupu.yml +++ b/.github/workflows/stream_hupu.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_huya.yml b/.github/workflows/stream_huya.yml index c29a3a8af..61fedecdd 100644 --- a/.github/workflows/stream_huya.yml +++ b/.github/workflows/stream_huya.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_instagram.yml b/.github/workflows/stream_instagram.yml index dceef149b..cdec32bc9 100644 --- a/.github/workflows/stream_instagram.yml +++ b/.github/workflows/stream_instagram.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_iqiyi.yml b/.github/workflows/stream_iqiyi.yml index f28d791df..5fb1f3e5d 100644 --- a/.github/workflows/stream_iqiyi.yml +++ b/.github/workflows/stream_iqiyi.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_ixigua.yml b/.github/workflows/stream_ixigua.yml index 5e4d5e109..66eb4f56e 100644 --- a/.github/workflows/stream_ixigua.yml +++ b/.github/workflows/stream_ixigua.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_kuaishou.yml b/.github/workflows/stream_kuaishou.yml index 7fd48399f..43c16a451 100644 --- a/.github/workflows/stream_kuaishou.yml +++ b/.github/workflows/stream_kuaishou.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_mgtv.yml b/.github/workflows/stream_mgtv.yml index 713119f2a..c3c790136 100644 --- a/.github/workflows/stream_mgtv.yml +++ b/.github/workflows/stream_mgtv.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_miaopai.yml b/.github/workflows/stream_miaopai.yml index 75f5505bb..d65244bce 100644 --- a/.github/workflows/stream_miaopai.yml +++ b/.github/workflows/stream_miaopai.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_netease.yml b/.github/workflows/stream_netease.yml index 8f297aa6b..450de6668 100644 --- a/.github/workflows/stream_netease.yml +++ b/.github/workflows/stream_netease.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_pinterest.yml b/.github/workflows/stream_pinterest.yml index 330cddfe1..d4bdfe656 100644 --- a/.github/workflows/stream_pinterest.yml +++ b/.github/workflows/stream_pinterest.yml @@ -7,7 +7,7 @@ on: - ".github/workflows/stream_pinterest.yml" pull_request: paths: - - "extractors/tiktok/*.go" + - "extractors/pinterest/*.go" - ".github/workflows/stream_pinterest.yml" schedule: # run ci weekly @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_pixivision.yml b/.github/workflows/stream_pixivision.yml index 212340d5b..0a75be799 100644 --- a/.github/workflows/stream_pixivision.yml +++ b/.github/workflows/stream_pixivision.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_pornhub.yml b/.github/workflows/stream_pornhub.yml index ae24c42da..302e87a5b 100644 --- a/.github/workflows/stream_pornhub.yml +++ b/.github/workflows/stream_pornhub.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_qq.yml b/.github/workflows/stream_qq.yml index 7801d52b6..43c97676f 100644 --- a/.github/workflows/stream_qq.yml +++ b/.github/workflows/stream_qq.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_reddit.yml b/.github/workflows/stream_reddit.yml index 408e0d8c6..c365da690 100644 --- a/.github/workflows/stream_reddit.yml +++ b/.github/workflows/stream_reddit.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_rumble.yml b/.github/workflows/stream_rumble.yml index 425aff8e7..4aff7a32f 100644 --- a/.github/workflows/stream_rumble.yml +++ b/.github/workflows/stream_rumble.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_streamtape.yml b/.github/workflows/stream_streamtape.yml index 4b8fd9827..545040441 100644 --- a/.github/workflows/stream_streamtape.yml +++ b/.github/workflows/stream_streamtape.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_tangdou.yml b/.github/workflows/stream_tangdou.yml index a43ff2b94..d0a5dae59 100644 --- a/.github/workflows/stream_tangdou.yml +++ b/.github/workflows/stream_tangdou.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_tiktok.yml b/.github/workflows/stream_tiktok.yml index 7f1797a29..ed72302b8 100644 --- a/.github/workflows/stream_tiktok.yml +++ b/.github/workflows/stream_tiktok.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_tumblr.yml b/.github/workflows/stream_tumblr.yml index d621b8b9a..568969d2e 100644 --- a/.github/workflows/stream_tumblr.yml +++ b/.github/workflows/stream_tumblr.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_twitter.yml b/.github/workflows/stream_twitter.yml index 714211975..8c3aa70eb 100644 --- a/.github/workflows/stream_twitter.yml +++ b/.github/workflows/stream_twitter.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_udn.yml b/.github/workflows/stream_udn.yml index 75a2e99c7..8a8f481ad 100644 --- a/.github/workflows/stream_udn.yml +++ b/.github/workflows/stream_udn.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_vimeo.yml b/.github/workflows/stream_vimeo.yml index b8b5d19de..933511330 100644 --- a/.github/workflows/stream_vimeo.yml +++ b/.github/workflows/stream_vimeo.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_vk.yml b/.github/workflows/stream_vk.yml index d243ae821..a8b7c7d31 100644 --- a/.github/workflows/stream_vk.yml +++ b/.github/workflows/stream_vk.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_weibo.yml b/.github/workflows/stream_weibo.yml index 97f40d5bf..9d60def9d 100644 --- a/.github/workflows/stream_weibo.yml +++ b/.github/workflows/stream_weibo.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_ximalaya.yml b/.github/workflows/stream_ximalaya.yml index ad741ea4e..801e29e69 100644 --- a/.github/workflows/stream_ximalaya.yml +++ b/.github/workflows/stream_ximalaya.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_xinpianchang.yml b/.github/workflows/stream_xinpianchang.yml index 34a319f68..0666b6431 100644 --- a/.github/workflows/stream_xinpianchang.yml +++ b/.github/workflows/stream_xinpianchang.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_xvideos.yml b/.github/workflows/stream_xvideos.yml index ffcacb4f8..7ea96fab8 100644 --- a/.github/workflows/stream_xvideos.yml +++ b/.github/workflows/stream_xvideos.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_yinyuetai.yml b/.github/workflows/stream_yinyuetai.yml index 72e498ad7..25bd9fbe1 100644 --- a/.github/workflows/stream_yinyuetai.yml +++ b/.github/workflows/stream_yinyuetai.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_youku.yml b/.github/workflows/stream_youku.yml index 8780b0368..7f736fe12 100644 --- a/.github/workflows/stream_youku.yml +++ b/.github/workflows/stream_youku.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_youtube.yml b/.github/workflows/stream_youtube.yml index ec65160ed..ace30b984 100644 --- a/.github/workflows/stream_youtube.yml +++ b/.github/workflows/stream_youtube.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.github/workflows/stream_zhihu.yml b/.github/workflows/stream_zhihu.yml index 5f1daec2b..09aca92f1 100644 --- a/.github/workflows/stream_zhihu.yml +++ b/.github/workflows/stream_zhihu.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/.golangci.yml b/.golangci.yml index 44db4be46..8d4b882ae 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ run: concurrency: 2 timeout: 5m - go: 1.20 + go: 1.21 linter-settings: goconst: diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index 607e71993..b07a87c6d 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -3,12 +3,12 @@ package bilibili import ( "encoding/json" "fmt" + "slices" "sort" "strconv" "strings" "time" - "github.com/iawia002/lia/array" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" @@ -158,7 +158,7 @@ func extractBangumi(url, html string, extractOption extractors.Options) ([]*extr wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range data.EpList { - if !array.ItemInArray(index+1, needDownloadItems) { + if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() @@ -259,7 +259,7 @@ func multiEpisodeDownload(url, html string, extractOption extractors.Options, pa wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range pageData.Sections[0].Episodes { - if !array.ItemInArray(index+1, needDownloadItems) { + if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() @@ -288,7 +288,7 @@ func multiPageDownload(url, html string, extractOption extractors.Options, pageD wgp := utils.NewWaitGroupPool(extractOption.ThreadNumber) dataIndex := 0 for index, u := range pageData.VideoData.Pages { - if !array.ItemInArray(index+1, needDownloadItems) { + if !slices.Contains(needDownloadItems, index+1) { continue } wgp.Add() diff --git a/extractors/miaopai/miaopai.go b/extractors/miaopai/miaopai.go index 26ed4a244..e8ebbf602 100644 --- a/extractors/miaopai/miaopai.go +++ b/extractors/miaopai/miaopai.go @@ -5,7 +5,6 @@ import ( "fmt" "math/rand" "strings" - "time" "github.com/pkg/errors" @@ -30,8 +29,6 @@ type miaopaiData struct { } func getRandomString(l int) string { - rand.Seed(time.Now().UnixNano()) - s := make([]string, 0) chars := []string{ "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "n", "m", "o", "p", "q", "r", "s", "t", "u", "v", diff --git a/extractors/qq/qq.go b/extractors/qq/qq.go index b7929a349..e21530fba 100644 --- a/extractors/qq/qq.go +++ b/extractors/qq/qq.go @@ -3,10 +3,10 @@ package qq import ( "encoding/json" "fmt" + "slices" "strconv" "strings" - "github.com/iawia002/lia/array" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" @@ -85,7 +85,7 @@ func genStreams(vid, cdn string, data qqVideoInfo) (map[string]*extractors.Strea for _, fi := range data.Fl.Fi { var fmtIDPrefix string var fns []string - if array.ItemInArray(fi.Name, []string{"shd", "fhd"}) { + if slices.Contains([]string{"shd", "fhd"}, fi.Name) { fmtIDPrefix = "p" fmtIDName := fmt.Sprintf("%s%d", fmtIDPrefix, fi.ID%10000) fns = []string{strings.Split(data.Vl.Vi[0].Fn, ".")[0], fmtIDName, "mp4"} diff --git a/extractors/youtube/youtube.go b/extractors/youtube/youtube.go index d9ccde421..c0ba43a2b 100644 --- a/extractors/youtube/youtube.go +++ b/extractors/youtube/youtube.go @@ -3,9 +3,9 @@ package youtube import ( "fmt" "net/http" + "slices" "strconv" - "github.com/iawia002/lia/array" "github.com/kkdai/youtube/v2" "github.com/pkg/errors" @@ -59,7 +59,7 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor wgp := utils.NewWaitGroupPool(option.ThreadNumber) dataIndex := 0 for index, videoEntry := range playlist.Videos { - if !array.ItemInArray(index+1, needDownloadItems) { + if !slices.Contains(needDownloadItems, index+1) { continue } diff --git a/go.mod b/go.mod index f28d0b6c6..1787dbb5c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/iawia002/lux -go 1.20 +go 1.21 require ( github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 @@ -8,7 +8,6 @@ require ( github.com/cheggaaa/pb/v3 v3.0.8 github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c github.com/fatih/color v1.13.0 - github.com/iawia002/lia v0.1.0 github.com/itchyny/gojq v0.12.7 github.com/json-iterator/go v1.1.12 github.com/kkdai/youtube/v2 v2.7.18 @@ -37,7 +36,6 @@ require ( github.com/rivo/uniseg v0.4.3 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index ebc3ab0a0..39042ee97 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -41,8 +42,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/iawia002/lia v0.1.0 h1:IzgR5pnOEt3bABB3TtcK5UCXkzcuc7+3GYxT0cVpYfU= -github.com/iawia002/lia v0.1.0/go.mod h1:Jxu7iNh5z17HWuLedH3jwh09aa5SO3g2BI2Ct87aXpY= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -94,13 +93,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/urfave/cli/v2 v2.6.0 h1:yj2Drkflh8X/zUrkWlWlUjZYHyWN7WMmpVxyxXIUyv8= github.com/urfave/cli/v2 v2.6.0/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf h1:oXVg4h2qJDd9htKxb5SCpFBHLipW6hXmL3qpUixS2jw= -golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -151,3 +149,4 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/script/github_action_template.yml b/script/github_action_template.yml index bb1fe22d0..5cea30750 100644 --- a/script/github_action_template.yml +++ b/script/github_action_template.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.20"] + go: ["1.21"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: diff --git a/utils/utils.go b/utils/utils.go index f997fd83d..4c6091ba5 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -11,9 +11,9 @@ import ( "path/filepath" "regexp" "runtime" + "slices" "strings" - "github.com/iawia002/lia/array" "github.com/pkg/errors" "github.com/iawia002/lux/request" @@ -152,7 +152,7 @@ func ParseInputFile(r io.Reader, items string, itemStart, itemEnd int) []string itemList := make([]string, 0, len(wantedItems)) for i, item := range temp { - if array.ItemInArray(i+1, wantedItems) { + if slices.Contains(wantedItems, i+1) { itemList = append(itemList, item) } } From 08e790db8427c0357d7d09831298016a1a295974 Mon Sep 17 00:00:00 2001 From: gmfan <3530553117@qq.com> Date: Wed, 13 Sep 2023 14:58:17 +0800 Subject: [PATCH 12/38] feat: support video download for xiaohognshu.com --- app/register.go | 1 + extractors/xiaohongshu/xiaohongshu.go | 92 ++++++++++++++++++++++ extractors/xiaohongshu/xiaohongshu_test.go | 32 ++++++++ 3 files changed, 125 insertions(+) create mode 100644 extractors/xiaohongshu/xiaohongshu.go create mode 100644 extractors/xiaohongshu/xiaohongshu_test.go diff --git a/app/register.go b/app/register.go index 5b1391d10..07c72897c 100644 --- a/app/register.go +++ b/app/register.go @@ -35,6 +35,7 @@ import ( _ "github.com/iawia002/lux/extractors/vimeo" _ "github.com/iawia002/lux/extractors/vk" _ "github.com/iawia002/lux/extractors/weibo" + _ "github.com/iawia002/lux/extractors/xiaohongshu" _ "github.com/iawia002/lux/extractors/ximalaya" _ "github.com/iawia002/lux/extractors/xinpianchang" _ "github.com/iawia002/lux/extractors/xvideos" diff --git a/extractors/xiaohongshu/xiaohongshu.go b/extractors/xiaohongshu/xiaohongshu.go new file mode 100644 index 000000000..0a255c909 --- /dev/null +++ b/extractors/xiaohongshu/xiaohongshu.go @@ -0,0 +1,92 @@ +package xiaohongshu + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/iawia002/lux/config" + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/request" + "github.com/iawia002/lux/utils" +) + +func init() { + extractors.Register("xiaohongshu", New()) +} + +type extractor struct{} + +// New returns a xiaohognshu extractor. +func New() extractors.Extractor { + return &extractor{} +} + +const mp4VideoType = "mp4" + +// Extract is the main function to extract the data. +func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { + html, err := request.Get(url, url, config.FakeHeaders) + if err != nil { + return nil, errors.WithStack(err) + } + + // title + titles := utils.MatchOneOf(html, `,"title":"(.+?)",`) + if titles == nil || len(titles) != 2 { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) + } + title := titles[1] + + // video url + urlsJSON := utils.MatchOneOf(html, `"backupUrls":(\[.+?\])`) + if urlsJSON == nil || len(urlsJSON) != 2 { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) + } + var urls []string + err = json.Unmarshal([]byte(urlsJSON[1]), &urls) + if err != nil { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) + } + + // streams + streams := make(map[string]*extractors.Stream) + var size int64 + for i, u := range urls { + if !strings.Contains(u, mp4VideoType) { + continue + } + size, err = request.Size(u, u) + if err != nil { + continue + } + streams[strconv.Itoa(i)] = &extractors.Stream{ + Parts: []*extractors.Part{ + { + URL: u, + Size: size, + Ext: mp4VideoType, + }, + }, + Size: size, + } + } + if err != nil { + return nil, errors.WithStack(err) + } + if len(streams) == 0 { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) + } + + return []*extractors.Data{ + { + Site: "小红书 xiaohongshu.com", + Title: title, + Type: extractors.DataTypeVideo, + Streams: streams, + URL: url, + }, + }, nil +} diff --git a/extractors/xiaohongshu/xiaohongshu_test.go b/extractors/xiaohongshu/xiaohongshu_test.go new file mode 100644 index 000000000..ea709673d --- /dev/null +++ b/extractors/xiaohongshu/xiaohongshu_test.go @@ -0,0 +1,32 @@ +package xiaohongshu + +import ( + "testing" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/test" +) + +func TestDownload(t *testing.T) { + tests := []struct { + name string + args test.Args + }{ + { + name: "normal test", + args: test.Args{ + URL: "https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f?m_source=pinpai", + Title: "七星级大厨都不会告诉你的，五花肉的8种做法", + Size: 59410194, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := New().Extract(tt.args.URL, extractors.Options{}) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + }) + } +} From 14bbe6b2f1345974da244444c69e8de9a8b9e2eb Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Wed, 13 Sep 2023 15:53:09 +0800 Subject: [PATCH 13/38] Update README.md and workflows --- .github/workflows/stream_xiaohongshu.yml | 31 ++++++++++++++++++++++++ README.md | 5 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/stream_xiaohongshu.yml diff --git a/.github/workflows/stream_xiaohongshu.yml b/.github/workflows/stream_xiaohongshu.yml new file mode 100644 index 000000000..43b52ee37 --- /dev/null +++ b/.github/workflows/stream_xiaohongshu.yml @@ -0,0 +1,31 @@ +name: xiaohongshu + +on: + push: + paths: + - "extractors/xiaohongshu/*.go" + - ".github/workflows/stream_xiaohongshu.yml" + pull_request: + paths: + - "extractors/xiaohongshu/*.go" + - ".github/workflows/stream_xiaohongshu.yml" + schedule: + # run ci weekly + - cron: "0 0 * * 0" + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + go: ["1.21"] + os: [ubuntu-latest] + name: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Test + run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/xiaohongshu diff --git a/README.md b/README.md index 7d58a39b9..0203ef154 100644 --- a/README.md +++ b/README.md @@ -630,7 +630,7 @@ $ lux -j "https://www.bilibili.com/video/av20203945" | XVIDEOS | | ✓ | | | | | [![xvideos](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xvideos.yml) | | 聯合新聞網 | | ✓ | | | | | [![udn](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_udn.yml) | | TikTok | | ✓ | | | | | [![tiktok](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_tiktok.yml) | -| Pinterest | | ✓ | | | | | [![pinterest](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml) | +| Pinterest | | ✓ | | | | | [![pinterest](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_pinterest.yml) | | 好看视频 | | ✓ | | | | | [![haokan](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_haokan.yml) | | AcFun | | ✓ | | | ✓ | | [![acfun](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_acfun.yml) | | Eporner | | ✓ | | | | | [![eporner](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_eporner.yml) | @@ -642,7 +642,8 @@ $ lux -j "https://www.bilibili.com/video/av20203945" | Reddit | | ✓ | ✓ | | | | [![reddit](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_reddit.yml) | | VKontakte | | ✓ | | | | | [![vk](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_vk.yml/) | | 知乎 | | ✓ | | | | | [![zhihu](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/) | -| Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) | +| Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) | +| 小红书 | | ✓ | | | | | [![xiaohongshu](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/) | ## Known issues From 6ae8fc8371a4d9f3d79757408bbb38d8476e857f Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Wed, 13 Sep 2023 16:55:37 +0800 Subject: [PATCH 14/38] v0.20.0 [ci skip] --- app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/app.go b/app/app.go index e689e7c6f..c01dfb10f 100644 --- a/app/app.go +++ b/app/app.go @@ -20,7 +20,7 @@ import ( const ( // Name is the name of this app. Name = "lux" - version = "v0.19.0" + version = "v0.20.0" ) func init() { From 2456ad9ba7031867b6eecacb13a12ac02cf4915f Mon Sep 17 00:00:00 2001 From: Ha Tien Loi Date: Mon, 9 Oct 2023 13:16:04 +0700 Subject: [PATCH 15/38] feat: add zingmp3 extractor (#1280) * feat: add zingmp3 * adhoc: temp commit * register the extractor --------- Co-authored-by: Xinzhao Xu --- .github/workflows/stream_zingmp3.yml | 31 ++++ README.md | 1 + app/register.go | 1 + extractors/zingmp3/zingmp3.go | 220 +++++++++++++++++++++++++++ extractors/zingmp3/zingmp3_test.go | 44 ++++++ go.mod | 1 + go.sum | 2 + utils/utils.go | 2 +- 8 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/stream_zingmp3.yml create mode 100644 extractors/zingmp3/zingmp3.go create mode 100644 extractors/zingmp3/zingmp3_test.go diff --git a/.github/workflows/stream_zingmp3.yml b/.github/workflows/stream_zingmp3.yml new file mode 100644 index 000000000..e912e4cd2 --- /dev/null +++ b/.github/workflows/stream_zingmp3.yml @@ -0,0 +1,31 @@ +name: zingmp3 + +on: + push: + paths: + - "extractors/zingmp3/*.go" + - ".github/workflows/stream_zingmp3.yml" + pull_request: + paths: + - "extractors/zingmp3/*.go" + - ".github/workflows/stream_zingmp3.yml" + schedule: + # run ci weekly + - cron: "0 0 * * 0" + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + go: ["1.21"] + os: [ubuntu-latest] + name: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Test + run: go test -timeout 5m -race -coverpkg=./... -coverprofile=coverage.txt github.com/iawia002/lux/extractors/zingmp3 diff --git a/README.md b/README.md index 0203ef154..7e820f713 100644 --- a/README.md +++ b/README.md @@ -644,6 +644,7 @@ $ lux -j "https://www.bilibili.com/video/av20203945" | 知乎 | | ✓ | | | | | [![zhihu](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zhihu.yml/) | | Rumble | | ✓ | | | | | [![rumble](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_rumble.yml/) | | 小红书 | | ✓ | | | | | [![xiaohongshu](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_xiaohongshu.yml/) | +| Zing MP3 | | ✓ | | ✓ | | | [![zingmp3](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/badge.svg)](https://github.com/iawia002/lux/actions/workflows/stream_zingmp3.yml/) | ## Known issues diff --git a/app/register.go b/app/register.go index 07c72897c..e78df4805 100644 --- a/app/register.go +++ b/app/register.go @@ -43,4 +43,5 @@ import ( _ "github.com/iawia002/lux/extractors/youku" _ "github.com/iawia002/lux/extractors/youtube" _ "github.com/iawia002/lux/extractors/zhihu" + _ "github.com/iawia002/lux/extractors/zingmp3" ) diff --git a/extractors/zingmp3/zingmp3.go b/extractors/zingmp3/zingmp3.go new file mode 100644 index 000000000..0ca86365b --- /dev/null +++ b/extractors/zingmp3/zingmp3.go @@ -0,0 +1,220 @@ +package zingmp3 + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "fmt" + "maps" + "net/http" + neturl "net/url" + "regexp" + "sort" + + "github.com/buger/jsonparser" + "github.com/pkg/errors" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/request" + "github.com/iawia002/lux/utils" +) + +func init() { + zingmp3Extractor := New() + extractors.Register("zingmp3", zingmp3Extractor) + extractors.Register("zing", zingmp3Extractor) +} + +type extractor struct{} + +// New returns a zingmp3 extractor. +func New() extractors.Extractor { + return &extractor{} +} + +type params map[string]string + +var ApiSlugs = map[string]string{ + "bai-hat": "/api/v2/page/get/song", + "embed": "/api/v2/page/get/song", + "video-clip": "/api/v2/page/get/video", + "lyric": "/api/v2/lyric/get/lyric", + "song-streaming": "/api/v2/song/get/streaming", +} + +const Domain = "https://zingmp3.vn" + +// Extract is the main function to extract the data. +func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { + urlRegExp := regexp.MustCompile(`https?://(?:mp3\.zing|zingmp3)\.vn/(?P(?:bai-hat|video-clip|embed))/[^/?#]+/(?P\w+)(?:\.html|\?)`) + urlMatcher := urlRegExp.FindStringSubmatch(url) + if len(urlMatcher) == 0 { + return nil, errors.WithStack(extractors.ErrURLParseFailed) + } + urlType := urlMatcher[1] + id := urlMatcher[2] + if err := updatingCookies(); err != nil { + return nil, errors.WithStack(err) + } + data := callApi(urlType, params{"id": id}) + title, _ := jsonparser.GetString(data, "title") + var contentType extractors.DataType + var source []byte + if urlType == "video-clip" { + source, _, _, _ = jsonparser.Get(data, "streaming") + api := fmt.Sprintf(`http://api.mp3.zing.vn/api/mobile/video/getvideoinfo?requestdata={"id":"%s"}`, id) + res, _ := request.Get(api, api, nil) + newSource, _, _, _ := jsonparser.Get([]byte(res), "source") + source, _ = jsonparser.Set(source, newSource, "mp4") + contentType = extractors.DataTypeVideo + } else { + contentType = extractors.DataTypeAudio + source = callApi("song-streaming", params{"id": id}) + } + streams := make(map[string]*extractors.Stream) + if err := jsonparser.ObjectEach(source, func(k []byte, v []byte, dataType jsonparser.ValueType, offset int) error { + key := string(k) + value := string(v) + if value == "" || value == "VIP" { + return nil + } + + // Handle for audio + if key != "mp4" && key != "hls" { + size, _ := request.Size(value, url) + urlData := &extractors.Part{ + URL: value, + Ext: "mp3", + Size: size, + } + streams["default"] = &extractors.Stream{ + Parts: []*extractors.Part{urlData}, + } + return nil + } + + // Handle for video + return jsonparser.ObjectEach(v, func(kSource []byte, vSource []byte, _ jsonparser.ValueType, _ int) error { + resolution := string(kSource) + videoUrl := string(vSource) + if resolution == "" { + return nil + } + if resolution == "hls" { + urls, _ := utils.M3u8URLs(videoUrl) + parts := make([]*extractors.Part, 0) + for _, u := range urls { + parts = append(parts, &extractors.Part{ + URL: u, + Ext: "ts", + }) + } + streams[resolution] = &extractors.Stream{ + ID: resolution, + Parts: parts, + NeedMux: false, + } + return nil + } + size, _ := request.Size(videoUrl, url) + streams[fmt.Sprintf("mp4-%s", resolution)] = &extractors.Stream{ + Parts: []*extractors.Part{{ + URL: videoUrl, + Ext: "mp4", + Size: size, + }}, + } + return nil + }) + }); err != nil { + return nil, errors.WithStack(err) + } + + return []*extractors.Data{ + { + Site: "Zing MP3 zingmp3.vn", + Title: title, + Type: contentType, + Streams: streams, + URL: url, + }, + }, nil +} + +func callApi(urlType string, p params) []byte { + api := generateApi(urlType, p) + res, _ := request.GetByte(api, api, nil) + data, _, _, _ := jsonparser.Get(res, "data") + return data +} + +func updatingCookies() error { + // For the first time. We need to call the temp API to get cookies and set cookies to for next request + // But sometime zingmp3 doesn't return cookies. We need to retry get and set cookies again (only allow 5 time) + for i := 0; i < 5; i++ { + api := generateApi("bai-hat", params{"id": ""}) + res, err := request.Request(http.MethodGet, api, nil, nil) + if err != nil { + return err + } + cookies := "" + for _, value := range res.Cookies() { + cookies += value.String() + } + res.Body.Close() // nolint + if cookies != "" { + request.SetOptions(request.Options{ + Cookie: cookies, + }) + return nil + } + } + return nil +} + +func generateApi(urlType string, p params) string { + slugApi := ApiSlugs[urlType] + maps.Copy(p, params{"ctime": "1"}) + + sortedParams := sortedParams(p) + sig := generateSig(slugApi, sortedParams) + maps.Copy(sortedParams, params{ + "apiKey": "X5BM3w8N7MKozC0B85o4KMlzLZKhV00y", + "sig": sig, + }) + + urlParams := neturl.Values{} + for key, value := range sortedParams { + urlParams.Add(key, value) + } + return fmt.Sprintf("%s%s?%s", Domain, slugApi, urlParams.Encode()) +} + +func generateSig(slugApi string, p params) string { + str := "" + for key, value := range p { + str += fmt.Sprintf("%s=%s", key, value) + } + h := sha256.New() + h.Write([]byte(str)) + sha256Value := hex.EncodeToString(h.Sum(nil)) + var passwordBytes = []byte(fmt.Sprintf("%s%s", slugApi, sha256Value)) + salt := []byte("acOrvUS15XRW2o9JksiK1KgQ6Vbds8ZW") + hmacHashed := hmac.New(sha512.New, salt) + hmacHashed.Write(passwordBytes) + return hex.EncodeToString(hmacHashed.Sum(nil)) +} + +func sortedParams(p params) params { + keys := make([]string, 0, len(p)) + for k := range p { + keys = append(keys, k) + } + sort.Strings(keys) + sortedParams := params{} + for _, k := range keys { + sortedParams[k] = p[k] + } + return sortedParams +} diff --git a/extractors/zingmp3/zingmp3_test.go b/extractors/zingmp3/zingmp3_test.go new file mode 100644 index 000000000..39ca97c4d --- /dev/null +++ b/extractors/zingmp3/zingmp3_test.go @@ -0,0 +1,44 @@ +package zingmp3 + +import ( + "testing" + + "github.com/iawia002/lux/extractors" + "github.com/iawia002/lux/test" +) + +func TestDownload(t *testing.T) { + tests := []struct { + name string + args test.Args + }{ + { + name: "Host is mp3.zing.vn", + args: test.Args{ + URL: "https://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html", + Title: "Xa Mãi Xa", + }, + }, + { + name: "Host is zingmp3.vn", + args: test.Args{ + URL: "https://zingmp3.vn/bai-hat/SOLO-JENNIE/ZW9FID6Z.html", + Title: "SOLO", + }, + }, + { + name: "Video clip", + args: test.Args{ + URL: "https://zingmp3.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html", + Title: "Sương Hoa Đưa Lối", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := New().Extract(tt.args.URL, extractors.Options{}) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + }) + } +} diff --git a/go.mod b/go.mod index 1787dbb5c..61ca062dc 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 github.com/PuerkitoBio/goquery v1.8.0 + github.com/buger/jsonparser v1.1.1 github.com/cheggaaa/pb/v3 v3.0.8 github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c github.com/fatih/color v1.13.0 diff --git a/go.sum b/go.sum index 39042ee97..2f55bf070 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkN github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= diff --git a/utils/utils.go b/utils/utils.go index 4c6091ba5..623ac7a33 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -62,7 +62,7 @@ func Domain(url string) string { domainPattern := `([a-z0-9][-a-z0-9]{0,62})\.` + `(com\.cn|com\.hk|` + `cn|com|net|edu|gov|biz|org|info|pro|name|xxx|xyz|be|` + - `me|top|cc|tv|tt)` + `me|top|cc|tv|tt|vn)` domain := MatchOneOf(url, domainPattern) if domain != nil { return domain[1] From e71df48e81d3c979b6f000f6e85dd789ac5b8274 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Mon, 9 Oct 2023 15:15:46 +0800 Subject: [PATCH 16/38] Add goreleaser action --- .github/workflows/goreleaser.yml | 27 +++++++++++++++++++++++++++ .goreleaser.yml | 21 +++------------------ app/app.go | 10 +++++----- 3 files changed, 35 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/goreleaser.yml diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml new file mode 100644 index 000000000..4165aa6b9 --- /dev/null +++ b/.github/workflows/goreleaser.yml @@ -0,0 +1,27 @@ +name: goreleaser +on: + push: + tags: + - '*' +permissions: + contents: write +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.21 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: ${{ env.GITHUB_REF_NAME }} + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} diff --git a/.goreleaser.yml b/.goreleaser.yml index 2ca048451..4ad2fd231 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,13 +1,13 @@ project_name: lux env: - GO111MODULE=on + - CGO_ENABLED=0 before: hooks: - go mod download builds: -- env: - - CGO_ENABLED=0 - binary: lux +- binary: lux + ldflags: -s -w -X github.com/iawia002/lux/app.version={{ .RawVersion }} goos: - windows - darwin @@ -29,17 +29,6 @@ builds: - goos: openbsd goarch: arm goarm: 6 -env_files: - github_token: ./github_token -changelog: - sort: asc - use: github - filters: - exclude: - - '^docs' - - '^tests' - - Merge pull request - - Merge branch archives: - name_template: >- {{ .ProjectName }}_ @@ -56,7 +45,3 @@ archives: files: - none* wrap_in_directory: false -release: - draft: true - footer: | - **Full Changelog**: https://github.com/iawia002/lux/compare/{{ .PreviousTag }}...{{ .Tag }} diff --git a/app/app.go b/app/app.go index c01dfb10f..05167995f 100644 --- a/app/app.go +++ b/app/app.go @@ -17,11 +17,11 @@ import ( "github.com/iawia002/lux/utils" ) -const ( - // Name is the name of this app. - Name = "lux" - version = "v0.20.0" -) +// Name is the name of this app. +const Name = "lux" + +// This value will be injected into the corresponding git tag value at build time using `-ldflags`. +var version = "v0.0.0" func init() { cli.VersionPrinter = func(c *cli.Context) { From 01f062d26a8a6644b7842d63299c0ec9f58c76ab Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Mon, 9 Oct 2023 15:58:18 +0800 Subject: [PATCH 17/38] Update goreleaser action --- .github/workflows/goreleaser.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index 4165aa6b9..55d449896 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Go @@ -18,10 +18,9 @@ jobs: with: go-version: 1.21 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v5 with: - distribution: goreleaser - version: ${{ env.GITHUB_REF_NAME }} + version: latest args: release --clean env: GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} From a5a67fb065ec04070443d04a63936c93607dcee4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 23:23:15 +0000 Subject: [PATCH 18/38] build(deps): bump golang.org/x/net from 0.7.0 to 0.17.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.7.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.7.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 61ca062dc..c0cf5c6d2 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f github.com/urfave/cli/v2 v2.6.0 - golang.org/x/net v0.7.0 + golang.org/x/net v0.17.0 ) require ( @@ -37,7 +37,7 @@ require ( github.com/rivo/uniseg v0.4.3 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index 2f55bf070..3d2b3b69f 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -123,8 +123,8 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -132,8 +132,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From bff4e21754272bfc984f4a05396dd7fe7016fb71 Mon Sep 17 00:00:00 2001 From: Tianyu Wu Date: Sat, 14 Oct 2023 22:45:14 -0400 Subject: [PATCH 19/38] Updated API for requesting subtitles --- extractors/bilibili/bilibili.go | 4 ++-- extractors/bilibili/types.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index b07a87c6d..b65a1ba45 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -500,7 +500,7 @@ func getExtFromMimeType(mimeType string) string { func getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart { jsonString, err := request.Get( - fmt.Sprintf("http://api.bilibili.com/x/web-interface/view?aid=%d&cid=%d", aid, cid), referer, nil, + fmt.Sprintf("http://api.bilibili.com/x/player/wbi/v2?aid=%d&cid=%d", aid, cid), referer, nil, ) if err != nil { return nil @@ -512,7 +512,7 @@ func getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart { } return &extractors.CaptionPart{ Part: extractors.Part{ - URL: stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl, + URL: fmt.Sprintf("https:%s", stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl), Ext: "srt", }, Transform: subtitleTransform, diff --git a/extractors/bilibili/types.go b/extractors/bilibili/types.go index 815cf9932..5f90b5eb9 100644 --- a/extractors/bilibili/types.go +++ b/extractors/bilibili/types.go @@ -131,7 +131,7 @@ type subtitleProperty struct { type subtitleInfo struct { AllowSubmit bool `json:"allow_submit"` - SubtitleList []subtitleProperty `json:"list"` + SubtitleList []subtitleProperty `json:"subtitles"` } type bilibiliWebInterfaceData struct { From d529cbb949d78bbaf283c7d035c8a7f475477ef8 Mon Sep 17 00:00:00 2001 From: long2ice Date: Mon, 23 Oct 2023 09:30:41 +0800 Subject: [PATCH 20/38] feat: support bilibili b23.tv domain (#1287) * feat: support bilibili b23.tv domain * Use the same instance --------- Co-authored-by: Xinzhao Xu --- extractors/bilibili/bilibili.go | 4 +++- extractors/bilibili/bilibili_test.go | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index b65a1ba45..f1e2f8346 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -18,7 +18,9 @@ import ( ) func init() { - extractors.Register("bilibili", New()) + bilibiliExtractor := New() + extractors.Register("bilibili", bilibiliExtractor) + extractors.Register("b23", bilibiliExtractor) } const ( diff --git a/extractors/bilibili/bilibili_test.go b/extractors/bilibili/bilibili_test.go index 8688f3b44..471861f74 100644 --- a/extractors/bilibili/bilibili_test.go +++ b/extractors/bilibili/bilibili_test.go @@ -59,6 +59,13 @@ func TestBilibili(t *testing.T) { Title: "【8K演示片】B站首发！你的设备还顶得住吗？", }, }, + { + name: "b23 test", + args: test.Args{ + URL: "https://b23.tv/Fc9i7QF", + Title: "【十年榜】2000-2009年最强华语金曲TOP100 P1 100爱转角-罗志祥", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From ce85102347ce4dcf9ebf39ade43fd0a5aa24ffa4 Mon Sep 17 00:00:00 2001 From: Tianyu Wu Date: Sun, 5 Nov 2023 23:15:59 -0500 Subject: [PATCH 21/38] Updated Bilibili bangumi extractor due to an API change --- extractors/bilibili/bilibili.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index f1e2f8346..ca8cf5edb 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -96,29 +96,28 @@ type bilibiliOptions struct { func extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { dataString := utils.MatchOneOf(html, `(.*?)`)[1] - epMapString := utils.MatchOneOf(dataString, `"epMap"\s*:\s*(.+?)\s*,\s*"initEpList"`)[1] + epArrayString := utils.MatchOneOf(dataString, `"episodes"\s*:\s*(.+?)\s*,\s*"user_status"`)[1] fullVideoIdString := utils.MatchOneOf(dataString, `"videoId"\s*:\s*"(ep|ss)(\d+)"`) epSsString := fullVideoIdString[1] // "ep" or "ss" videoIdString := fullVideoIdString[2] - var epMap map[string]json.RawMessage - err := json.Unmarshal([]byte(epMapString), &epMap) + var epArray []json.RawMessage + err := json.Unmarshal([]byte(epArrayString), &epArray) if err != nil { return nil, errors.WithStack(err) } var data bangumiData - for idString, jsonByte := range epMap { + for _, jsonByte := range epArray { var epInfo bangumiEpData err := json.Unmarshal(jsonByte, &epInfo) if err != nil { return nil, errors.WithStack(err) } - epID, err := strconv.ParseInt(idString, 10, 0) + videoId, err := strconv.ParseInt(videoIdString, 10, 0) if err != nil { return nil, errors.WithStack(err) } - epInfo.EpID = int(epID) - if idString == videoIdString || (epSsString == "ss" && epInfo.TitleFormat == "第1话") { + if epInfo.ID == int(videoId) || (epSsString == "ss" && epInfo.TitleFormat == "第1话") { data.EpInfo = epInfo } data.EpList = append(data.EpList, epInfo) From 3d3f2a0fe8e4753adb347084f8ffb23495910dab Mon Sep 17 00:00:00 2001 From: maijver Date: Wed, 27 Dec 2023 10:05:20 +0800 Subject: [PATCH 22/38] Update douyinData types.go (#1299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update types.go "https://v.douyin.com/i8rV35QW/" Interval 不一定都是int * fix code format --------- Co-authored-by: Xinzhao Xu --- extractors/douyin/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractors/douyin/types.go b/extractors/douyin/types.go index b3433bac3..a77763e1f 100644 --- a/extractors/douyin/types.go +++ b/extractors/douyin/types.go @@ -395,7 +395,7 @@ type douyinData struct { ImgXSize int `json:"img_x_size"` ImgYLen int `json:"img_y_len"` ImgYSize int `json:"img_y_size"` - Interval int `json:"interval"` + Interval float64 `json:"interval"` URI string `json:"uri"` } `json:"big_thumbs"` BitRate []struct { From ed7b658f19c032be3f9cfe91426a4d4c42d8b997 Mon Sep 17 00:00:00 2001 From: Sanjeev Kumar Date: Fri, 29 Dec 2023 23:03:36 +0000 Subject: [PATCH 23/38] Update README.md Added a link to a similar tool. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7e820f713..06f11651a 100644 --- a/README.md +++ b/README.md @@ -669,6 +669,7 @@ Code with ❤️ by [iawia002](https://github.com/iawia002) and lovely [contribu ## Similar projects +- [youtube](https://github.com/kkdai/youtube) - [youtube-dl](https://github.com/rg3/youtube-dl) - [you-get](https://github.com/soimort/you-get) - [ytdl](https://github.com/rylio/ytdl) From bd142b2c33806ee70c6f472f5fd9930c050ec807 Mon Sep 17 00:00:00 2001 From: Shaun_Sheep <54682807+GNUSheep@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:46:08 +0100 Subject: [PATCH 24/38] Add --audio-only flag (#1304) * Add --audio-only flag * Resolve all conflicts * Fix go lint errors --- app/app.go | 6 ++++++ downloader/downloader.go | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/app.go b/app/app.go index 05167995f..13a3da127 100644 --- a/app/app.go +++ b/app/app.go @@ -89,6 +89,11 @@ func New() *cli.App { Aliases: []string{"f"}, Usage: "Select specific stream to download", }, + &cli.BoolFlag{ + Name: "audio-only", + Aliases: []string{"ao"}, + Usage: "Download audio only at best quality", + }, &cli.StringFlag{ Name: "file", Aliases: []string{"F"}, @@ -300,6 +305,7 @@ func download(c *cli.Context, videoURL string) error { Silent: c.Bool("silent"), InfoOnly: c.Bool("info"), Stream: c.String("stream-format"), + AudioOnly: c.Bool("audio-only"), Refer: c.String("refer"), OutputPath: c.String("output-path"), OutputName: c.String("output-name"), diff --git a/downloader/downloader.go b/downloader/downloader.go index 8537ab487..c70e71787 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -28,6 +28,7 @@ type Options struct { InfoOnly bool Silent bool Stream string + AudioOnly bool Refer string OutputPath string OutputName string @@ -568,6 +569,26 @@ func (downloader *Downloader) Download(data *extractors.Data) error { return errors.Errorf("no stream named %s", streamName) } + if downloader.option.AudioOnly { + var isFound bool + reg, err := regexp.Compile("audio+") + if err != nil { + return err + } + + for _, s := range sortedStreams { + // Looking for the best quality + if reg.MatchString(s.Quality) { + isFound = true + stream = data.Streams[s.ID] + break + } + } + if !isFound { + return errors.Errorf("No audio stream found") + } + } + if !downloader.option.Silent { printStreamInfo(data, stream) } From 1178e5f278350312a6501c50787c4300107112be Mon Sep 17 00:00:00 2001 From: Shavit Date: Sun, 18 Feb 2024 21:11:39 -0500 Subject: [PATCH 25/38] Bilibili Festival (#1315) * Bilibili festival This change will work with festival pages. * Linter recommendations --- downloader/downloader.go | 10 +++++--- extractors/bilibili/bilibili.go | 34 ++++++++++++++++++++++++++-- extractors/bilibili/bilibili_test.go | 7 ++++++ extractors/bilibili/types.go | 27 ++++++++++++++++++++++ 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index c70e71787..924785b08 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -52,6 +52,10 @@ type Downloader struct { option Options } +const ( + DOWNLOAD_FILE_EXT = ".download" +) + func progressBar(size int64) *pb.ProgressBar { tmpl := `{{counters .}} {{bar . "[" "=" ">" "-" "]"}} {{speed .}} {{percent . | green}} {{rtime .}}` return pb.New64(size). @@ -135,7 +139,7 @@ func (downloader *Downloader) save(part *extractors.Part, refer, fileName string return nil } - tempFilePath := filePath + ".download" + tempFilePath := filePath + DOWNLOAD_FILE_EXT tempFileSize, _, err := utils.FileSize(tempFilePath) if err != nil { return err @@ -233,7 +237,7 @@ func (downloader *Downloader) multiThreadSave(dataPart *extractors.Part, refer, downloader.bar.Add64(fileSize) return nil } - tmpFilePath := filePath + ".download" + tmpFilePath := filePath + DOWNLOAD_FILE_EXT tmpFileSize, tmpExists, err := utils.FileSize(tmpFilePath) if err != nil { return err @@ -469,7 +473,7 @@ func writeFilePartMeta(file *os.File, meta *FilePartMeta) error { } func mergeMultiPart(filepath string, parts []*FilePartMeta) error { - tempFilePath := filepath + ".download" + tempFilePath := filepath + DOWNLOAD_FILE_EXT tempFile, err := os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index ca8cf5edb..f0da13ef1 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -202,6 +202,33 @@ func getMultiPageData(html string) (*multiPage, error) { return &data, nil } +func extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { + matches := utils.MatchAll(html, "<\\s*script[^>]*>\\s*window\\.__INITIAL_STATE__=([\\s\\S]*?);\\s?\\(function[\\s\\S]*?<\\/\\s*script\\s*>") + if len(matches) < 1 { + return nil, errors.WithStack(extractors.ErrURLParseFailed) + } + if len(matches[0]) < 2 { + return nil, errors.New("could not find video in page") + } + + var festivalData festival + err := json.Unmarshal([]byte(matches[0][1]), &festivalData) + if err != nil { + return nil, errors.WithStack(err) + } + + options := bilibiliOptions{ + url: url, + html: html, + aid: festivalData.VideoInfo.Aid, + bvid: festivalData.VideoInfo.BVid, + cid: festivalData.VideoInfo.Cid, + page: 0, + } + + return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil +} + func extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { pageData, err := getMultiPageData(html) if err != nil { @@ -333,9 +360,12 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor if strings.Contains(url, "bangumi") { // handle bangumi return extractBangumi(url, html, option) + } else if strings.Contains(url, "festival") { + return extractFestival(url, html, option) + } else { + // handle normal video + return extractNormalVideo(url, html, option) } - // handle normal video - return extractNormalVideo(url, html, option) } // bilibiliDownload is the download function for a single URL diff --git a/extractors/bilibili/bilibili_test.go b/extractors/bilibili/bilibili_test.go index 471861f74..f1ad0ecbc 100644 --- a/extractors/bilibili/bilibili_test.go +++ b/extractors/bilibili/bilibili_test.go @@ -66,6 +66,13 @@ func TestBilibili(t *testing.T) { Title: "【十年榜】2000-2009年最强华语金曲TOP100 P1 100爱转角-罗志祥", }, }, + { + name: "festival test", + args: test.Args{ + URL: "https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt", + Title: "洛天依十周年官方演唱会", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/extractors/bilibili/types.go b/extractors/bilibili/types.go index 5f90b5eb9..844b1a05e 100644 --- a/extractors/bilibili/types.go +++ b/extractors/bilibili/types.go @@ -143,3 +143,30 @@ type bilibiliWebInterface struct { Code int `json:"code"` Data bilibiliWebInterfaceData `json:"data"` } + +type festival struct { + VideoSections []struct { + Id int64 `json:"id"` + Title string `json:"title"` + Type int `json:"type"` + } `json:"videoSections"` + Episodes []episode `json:"episodes"` + VideoInfo struct { + Aid int `json:"aid"` + BVid string `json:"bvid"` + Cid int `json:"cid"` + Title string `json:"title"` + Desc string `json:"desc"` + Pages []struct { + Cid int `json:"cid"` + Duration int `json:"duration"` + Page int `json:"page"` + Part string `json:"part"` + Dimension struct { + Width int `json:"width"` + Height int `json:"height"` + Rotate int `json:"rotate"` + } `json:"dimension"` + } `json:"pages"` + } `json:"videoInfo"` +} From b808f36fc0b466020357a742310bcc70207d9482 Mon Sep 17 00:00:00 2001 From: leehow1988 Date: Tue, 20 Feb 2024 17:24:36 +0800 Subject: [PATCH 26/38] =?UTF-8?q?=E8=A7=A3=E5=86=B3=20bilibili=20=E4=B8=8B?= =?UTF-8?q?=E8=BD=BDplaylist=E6=97=B6=20LUX=20will=20skip=20files=20with?= =?UTF-8?q?=20same=20names=20(#1316)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bilibili playlist下载时 LUX will skip files with same names * 下载playlist列表的时候添加 P1 P2 ... 后缀 --- extractors/bilibili/bilibili.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index f0da13ef1..f391cb64f 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -297,7 +297,7 @@ func multiEpisodeDownload(url, html string, extractOption extractors.Options, pa aid: u.Aid, bvid: u.BVid, cid: u.Cid, - subtitle: u.Title, + subtitle: fmt.Sprintf("%s P%d", u.Title, dataIndex+1), } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() From 37870921d38f3218e79df184ca134d0c6afb21d1 Mon Sep 17 00:00:00 2001 From: leehow1988 Date: Wed, 21 Feb 2024 09:47:42 +0800 Subject: [PATCH 27/38] =?UTF-8?q?=E9=92=88=E5=AF=B9=E6=9C=89-items?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E6=97=B6=EF=BC=8C=E8=A7=86=E9=A2=91=E7=9A=84?= =?UTF-8?q?=E5=BA=8F=E5=8F=B7=E5=BA=94=E8=AF=A5=E5=8F=96=E6=95=B4=E4=B8=AA?= =?UTF-8?q?=E5=90=88=E9=9B=86=E4=B8=AD=E7=9A=84=E5=BA=8F=E5=8F=B7=EF=BC=8C?= =?UTF-8?q?=E8=80=8C=E4=B8=8D=E6=98=AF=E5=8F=96=E8=A2=AB=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E7=9A=84=E7=AC=AC=E5=87=A0=E4=B8=AA=20(#1317)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bilibili playlist下载时 LUX will skip files with same names * 下载playlist列表的时候添加 P1 P2 ... 后缀 * Update bilibili.go 针对有-items参数时，视频的序号应该取整个合集中的序号，而不是取被下载的第几个 --- extractors/bilibili/bilibili.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index f391cb64f..26c790ea8 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -297,7 +297,7 @@ func multiEpisodeDownload(url, html string, extractOption extractors.Options, pa aid: u.Aid, bvid: u.BVid, cid: u.Cid, - subtitle: fmt.Sprintf("%s P%d", u.Title, dataIndex+1), + subtitle: fmt.Sprintf("%s P%d", u.Title, index+1), } go func(index int, options bilibiliOptions, extractedData []*extractors.Data) { defer wgp.Done() From 6719d8ba70eada30405c38b36e68532b3d136be7 Mon Sep 17 00:00:00 2001 From: Dean Roker Date: Fri, 1 Mar 2024 23:57:09 +0000 Subject: [PATCH 28/38] fixes rumble loop --- request/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/request/request.go b/request/request.go index a887b73b4..89d3bf27d 100644 --- a/request/request.go +++ b/request/request.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/http/cookiejar" "strconv" "strings" "time" @@ -54,9 +55,14 @@ func Request(method, url string, body io.Reader, headers map[string]string) (*ht TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } + jar, err := cookiejar.New(nil) + if err != nil { + return nil, errors.WithStack(err) + } client := &http.Client{ Transport: transport, Timeout: 15 * time.Minute, + Jar: jar, } req, err := http.NewRequest(method, url, body) From 155ff4cc050e3c1b9ae00cf8fb62ba474d0f6312 Mon Sep 17 00:00:00 2001 From: "ken.hou" Date: Wed, 13 Mar 2024 18:32:24 +0800 Subject: [PATCH 29/38] =?UTF-8?q?=E8=A7=A3=E6=B1=BA:=20windows=20os=20?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E5=98=97=E8=A9=A6=E5=9C=A8=E7=95=B6=E5=89=8D?= =?UTF-8?q?=E7=9B=AE=E9=8C=84=E4=B8=AD=E6=9F=A5=E6=89=BEffmpeg=E5=A4=B1?= =?UTF-8?q?=E6=95=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/ffmpeg.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/utils/ffmpeg.go b/utils/ffmpeg.go index 0bca2940b..66f731c52 100644 --- a/utils/ffmpeg.go +++ b/utils/ffmpeg.go @@ -5,10 +5,28 @@ import ( "fmt" "os" "os/exec" + "path/filepath" + "runtime" "github.com/pkg/errors" ) +func findFFmpegExecutable() string { + ffmpegFileName := "ffmpeg" + if runtime.GOOS == "windows" { + ffmpegFileName = "ffmpeg.exe" // 在Windows上添加.exe擴展名 + } + // 嘗試在當前目錄中查找ffmpeg + matches, err := filepath.Glob("./" + ffmpegFileName) + if err == nil && len(matches) > 0 { + // 如果在當前目錄找到了ffmpeg，直接返回這個路徑 + return "./" + ffmpegFileName + } + + // 返回從PATH中找到的ffmpeg路徑 + return ffmpegFileName +} + func runMergeCmd(cmd *exec.Cmd, paths []string, mergeFilePath string) error { var stderr bytes.Buffer cmd.Stderr = &stderr @@ -37,7 +55,8 @@ func MergeFilesWithSameExtension(paths []string, mergedFilePath string) error { cmds = append(cmds, "-i", path) } cmds = append(cmds, "-c:v", "copy", "-c:a", "copy", mergedFilePath) - return runMergeCmd(exec.Command("ffmpeg", cmds...), paths, "") + + return runMergeCmd(exec.Command(findFFmpegExecutable(), cmds...), paths, "") } // MergeToMP4 merges video parts to an MP4 file. @@ -52,7 +71,7 @@ func MergeToMP4(paths []string, mergedFilePath string, filename string) error { mergeFile.Close() // nolint cmd := exec.Command( - "ffmpeg", "-y", "-f", "concat", "-safe", "0", + findFFmpegExecutable(), "-y", "-f", "concat", "-safe", "0", "-i", mergeFilePath, "-c", "copy", "-bsf:a", "aac_adtstoasc", mergedFilePath, ) return runMergeCmd(cmd, paths, mergeFilePath) From aaf3892b10019c33ed0746fe6ea67a9e63f09053 Mon Sep 17 00:00:00 2001 From: omegaatt36 Date: Sun, 24 Mar 2024 17:31:10 +0800 Subject: [PATCH 30/38] fix: refactor instagram extractor by using embedding. --- extractors/instagram/instagram.go | 243 +++++++++++++------------ extractors/instagram/instagram_test.go | 6 +- go.mod | 15 +- go.sum | 102 +++++++++++ 4 files changed, 250 insertions(+), 116 deletions(-) diff --git a/extractors/instagram/instagram.go b/extractors/instagram/instagram.go index 3477726de..f02679a85 100644 --- a/extractors/instagram/instagram.go +++ b/extractors/instagram/instagram.go @@ -2,45 +2,137 @@ package instagram import ( "encoding/json" + "fmt" + "net" + "net/http" netURL "net/url" + "regexp" "strings" + "time" + browser "github.com/EDDYCJY/fake-useragent" + "github.com/gocolly/colly/v2" "github.com/pkg/errors" - "golang.org/x/net/html" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" "github.com/iawia002/lux/utils" ) +var client *http.Client + func init() { extractors.Register("instagram", New()) + client = &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + TLSHandshakeTimeout: 5 * time.Second, + }, + } +} + +// sliderItemNode contains information about the Instagram post +type sliderItemNode struct { + DisplayURL string `json:"display_url"` // URL of the Media (resolution is dynamic) + + IsVideo bool `json:"is_video"` // Is type of the Media equals to video + VideoURL string `json:"video_url"` // Direct URL to the Video +} + +func (s sliderItemNode) extractMediaURL() string { + if s.IsVideo { + return s.VideoURL + } + + return s.DisplayURL } type instagramPayload struct { - ArticleBody string `json:"articleBody"` - Author struct { - Image string `json:"image"` - Name string `json:"name"` - AlternativeName string `json:"alternativeName"` - Url string `json:"url"` - } `json:"author"` - Videos []struct { - UploadData string `json:"string"` - Description string `json:"description"` - Name string `json:"name"` - Caption string `json:"caption"` - Height string `json:"height"` - Width string `json:"width"` - ContentURL string `json:"contentUrl"` - ThumbnailURL string `json:"thumbnailUrl"` - } `json:"video"` - Images []struct { - Caption string `json:"caption"` - Height string `json:"height"` - Width string `json:"width"` - URL string `json:"url"` - } `json:"image"` + Media struct { + ID string `json:"id"` // Unique ID of the Media + SliderItems struct { + Edges []struct { + Node sliderItemNode `json:"node"` + } `json:"edges"` + } `json:"edge_sidecar_to_children"` // Children of the Media + } `json:"shortcode_media"` // Media +} + +func (s instagramPayload) isEmpty() bool { + return s.Media.ID == "" +} + +func getPostWithCode(code string) ([]string, error) { + URL := fmt.Sprintf("https://www.instagram.com/p/%v/embed/captioned/", code) + + var embeddedMediaImage string + var embedResponse = instagramPayload{} + collector := colly.NewCollector() + collector.SetClient(client) + var collectorErr error + + collector.OnHTML("img.EmbeddedMediaImage", func(e *colly.HTMLElement) { + embeddedMediaImage = e.Attr("src") + }) + + collector.OnHTML("script", func(e *colly.HTMLElement) { + r := regexp.MustCompile(`\\\"gql_data\\\":([\s\S]*)\}\"\}\]\]\,\[\"NavigationMetrics`) + match := r.FindStringSubmatch(e.Text) + + if len(match) < 2 { + return + } + + s := strings.ReplaceAll(match[1], `\"`, `"`) + s = strings.ReplaceAll(s, `\\/`, `/`) + s = strings.ReplaceAll(s, `\\`, `\`) + + err := json.Unmarshal([]byte(s), &embedResponse) + if err != nil { + collectorErr = err + } + }) + + collector.OnRequest(func(r *colly.Request) { + r.Headers.Set("User-Agent", browser.Chrome()) + }) + + if err := collector.Visit(URL); err != nil { + return nil, fmt.Errorf("failed to send HTTP request to the Instagram: %v", err) + } + + if collectorErr != nil { + return nil, fmt.Errorf("failed to parse the Instagram response: %v", collectorErr) + } + + // If the method one which is JSON parsing didn't fail + if !embedResponse.isEmpty() { + result := make([]string, 0, len(embedResponse.Media.SliderItems.Edges)) + for _, item := range embedResponse.Media.SliderItems.Edges { + result = append(result, item.Node.extractMediaURL()) + } + + return result, nil + } + + if embeddedMediaImage != "" { + return []string{embeddedMediaImage}, nil + } + + // If every two methods have failed, then return an error + return nil, errors.New("failed to fetch the post, the page might be \"private\", or the link is completely wrong") +} + +func extractShortCodeFromLink(link string) (string, error) { + values := regexp.MustCompile(`(p|tv|reel|reels\/videos)\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link) + if len(values) != 3 { + return "", errors.New("couldn't extract the media short code from the link") + } + + return values[2], nil } type extractor struct{} @@ -57,44 +149,35 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor return nil, errors.WithStack(err) } - htmlResp, err := request.Get(u.String(), url, nil) + shortCode, err := extractShortCodeFromLink(u.String()) if err != nil { return nil, errors.WithStack(err) } - reader := strings.NewReader(htmlResp) - htmlRoot, err := html.Parse(reader) + urls, err := getPostWithCode(shortCode) if err != nil { return nil, errors.WithStack(err) } - sNode, err := dfsFindScript(htmlRoot) - if err != nil { - return nil, errors.WithStack(err) - } - - var payload instagramPayload - if err = json.Unmarshal([]byte(sNode.Data), &payload); err != nil { - return nil, errors.WithStack(err) - } - var totalSize int64 var parts []*extractors.Part - if len(payload.Videos) > 0 { - videoParts, err := createPartVideos(&payload, url) + + for _, u := range urls { + _, ext, err := utils.GetNameAndExt(u) if err != nil { - return nil, errors.WithStack(extractors.ErrBodyParseFailed) + return nil, errors.WithStack(err) } - - parts = append(parts, videoParts...) - } - if len(payload.Images) > 0 { - imageParts, err := createPartImages(&payload, url) + fileSize, err := request.Size(u, url) if err != nil { - return nil, errors.WithStack(extractors.ErrBodyParseFailed) + return nil, errors.WithStack(err) } - parts = append(parts, imageParts...) + part := &extractors.Part{ + URL: u, + Size: fileSize, + Ext: ext, + } + parts = append(parts, part) } for _, part := range parts { @@ -108,77 +191,13 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor }, } - id := u.Path[strings.LastIndex(u.Path, "/")+1:] - return []*extractors.Data{ { Site: "Instagram instagram.com", - Title: "Instagram " + id, + Title: "Instagram " + shortCode, Type: extractors.DataTypeImage, Streams: streams, URL: url, }, }, nil } - -func dfsFindScript(n *html.Node) (*html.Node, error) { - if n.Type == html.ElementNode && n.Data == "script" { - for _, attr := range n.Attr { - if attr.Key == "type" && attr.Val == "application/ld+json" { - return n.FirstChild, nil - } - } - } - - for c := n.FirstChild; c != nil; c = c.NextSibling { - if ret, err := dfsFindScript(c); err == nil { - return ret, nil - } - } - - return nil, errors.WithStack(extractors.ErrBodyParseFailed) -} - -func createPartVideos(payload *instagramPayload, ref string) (parts []*extractors.Part, err error) { - for _, it := range payload.Videos { - _, ext, err := utils.GetNameAndExt(it.ContentURL) - if err != nil { - return parts, errors.WithStack(err) - } - filesize, err := request.Size(it.ContentURL, ref) - if err != nil { - return parts, errors.WithStack(err) - } - - part := &extractors.Part{ - URL: it.ContentURL, - Size: filesize, - Ext: ext, - } - parts = append(parts, part) - } - - return parts, err -} - -func createPartImages(payload *instagramPayload, ref string) (parts []*extractors.Part, err error) { - for _, it := range payload.Images { - _, ext, err := utils.GetNameAndExt(it.URL) - if err != nil { - return parts, errors.WithStack(err) - } - filesize, err := request.Size(it.URL, ref) - if err != nil { - return parts, errors.WithStack(err) - } - - part := &extractors.Part{ - URL: it.URL, - Size: filesize, - Ext: ext, - } - parts = append(parts, part) - } - - return parts, err -} diff --git a/extractors/instagram/instagram_test.go b/extractors/instagram/instagram_test.go index 7105ed068..f9ed2bcb5 100644 --- a/extractors/instagram/instagram_test.go +++ b/extractors/instagram/instagram_test.go @@ -17,7 +17,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/BlIka1ZFCNr", Title: "Instagram BlIka1ZFCNr", - Size: 577298, + Size: 992330, }, }, { @@ -25,7 +25,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/Bl5oVUyl9Yx", Title: "Instagram Bl5oVUyl9Yx", - Size: 101611, + Size: 250596, }, }, { @@ -33,7 +33,7 @@ func TestDownload(t *testing.T) { args: test.Args{ URL: "https://www.instagram.com/p/Bjyr-gxF4Rb", Title: "Instagram Bjyr-gxF4Rb", - Size: 241466, + Size: 656476, }, }, } diff --git a/go.mod b/go.mod index c0cf5c6d2..89dc89d3c 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/iawia002/lux go 1.21 require ( + github.com/EDDYCJY/fake-useragent v0.2.0 github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 github.com/PuerkitoBio/goquery v1.8.0 github.com/buger/jsonparser v1.1.1 github.com/cheggaaa/pb/v3 v3.0.8 github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c github.com/fatih/color v1.13.0 + github.com/gocolly/colly/v2 v2.1.0 github.com/itchyny/gojq v0.12.7 github.com/json-iterator/go v1.1.12 github.com/kkdai/youtube/v2 v2.7.18 @@ -16,18 +18,24 @@ require ( github.com/pkg/errors v0.9.1 github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f github.com/urfave/cli/v2 v2.6.0 - golang.org/x/net v0.17.0 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.2.3 // indirect + github.com/antchfx/xmlquery v1.2.4 // indirect + github.com/antchfx/xpath v1.1.8 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.2 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -37,7 +45,12 @@ require ( github.com/rivo/uniseg v0.4.3 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/temoto/robotstxt v1.1.1 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect + google.golang.org/appengine v1.6.6 // indirect + google.golang.org/protobuf v1.24.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index 3d2b3b69f..619f13959 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,33 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2YdcpvJBs= +github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc= github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 h1:EtZwYyLbkEcIt+B//6sujwRCnHuTEK3qiSypAX5aJeM= github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403/go.mod h1:mM6WvakkX2m+NgMiPCfFFjwfH4KzENC07zeGEqq9U7s= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= +github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= +github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= +github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= +github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= +github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -20,6 +35,7 @@ github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4M github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -34,11 +50,38 @@ github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c h1:/utv6nmTctV6OVgfk5+ github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= +github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= @@ -48,8 +91,11 @@ github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kkdai/youtube/v2 v2.7.18 h1:bVP60bULmCZg5H9GsLLeBx0ONHEsZwdSrzPrVCVWJ1k= github.com/kkdai/youtube/v2 v2.7.18/go.mod h1:CeUYFc227iiNNpEaipwmJIYPNsuH6rb/8H3HpWEG63U= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -80,6 +126,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= @@ -91,28 +138,54 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= +github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/urfave/cli/v2 v2.6.0 h1:yj2Drkflh8X/zUrkWlWlUjZYHyWN7WMmpVxyxXIUyv8= github.com/urfave/cli/v2 v2.6.0/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -128,6 +201,7 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -135,10 +209,36 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -152,3 +252,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From fc5e4e4851bde2aa2550d162f2e15e076c2249b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 01:29:50 +0000 Subject: [PATCH 31/38] build(deps): bump google.golang.org/protobuf from 1.24.0 to 1.33.0 Bumps google.golang.org/protobuf from 1.24.0 to 1.33.0. --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 89dc89d3c..f81f71785 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.4.2 // indirect + github.com/golang/protobuf v1.5.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect @@ -51,6 +51,6 @@ require ( golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.6 // indirect - google.golang.org/protobuf v1.24.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index 619f13959..2f99d9c54 100644 --- a/go.sum +++ b/go.sum @@ -75,14 +75,16 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= @@ -237,8 +239,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 2656e8a157ca2947bb5fdfbac8e98ccf9ba8f107 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 01:30:27 +0000 Subject: [PATCH 32/38] build(deps): bump github.com/antchfx/xmlquery from 1.2.4 to 1.3.1 Bumps [github.com/antchfx/xmlquery](https://github.com/antchfx/xmlquery) from 1.2.4 to 1.3.1. - [Release notes](https://github.com/antchfx/xmlquery/releases) - [Commits](https://github.com/antchfx/xmlquery/compare/v1.2.4...v1.3.1) --- updated-dependencies: - dependency-name: github.com/antchfx/xmlquery dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 89dc89d3c..000a78bae 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,8 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/antchfx/htmlquery v1.2.3 // indirect - github.com/antchfx/xmlquery v1.2.4 // indirect - github.com/antchfx/xpath v1.1.8 // indirect + github.com/antchfx/xmlquery v1.3.1 // indirect + github.com/antchfx/xpath v1.1.10 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect diff --git a/go.sum b/go.sum index 619f13959..636616300 100644 --- a/go.sum +++ b/go.sum @@ -16,11 +16,13 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= +github.com/antchfx/xmlquery v1.3.1 h1:nIKWdtnhrXtj0/IRUAAw2I7TfpHUa3zMnHvNmPXFg+w= +github.com/antchfx/xmlquery v1.3.1/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.10 h1:cJ0pOvEdN/WvYXxvRrzQH9x5QWKpzHacYO8qzCcDYAg= +github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= @@ -153,6 +155,7 @@ github.com/urfave/cli/v2 v2.6.0/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDq github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -170,6 +173,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= From 0840d9f6857c5b5d87d55f219d4c23200d240a43 Mon Sep 17 00:00:00 2001 From: cuibuwei Date: Sat, 13 Apr 2024 16:50:00 +0800 Subject: [PATCH 33/38] chore: fix some typos in comments Signed-off-by: cuibuwei --- downloader/downloader.go | 2 +- downloader/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index 924785b08..b374dc30f 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -457,7 +457,7 @@ func parseFilePartMeta(filepath string, fileSize int64) (*FilePartMeta, error) { return nil, errors.WithStack(err) } if readSize < size { - return nil, errors.Errorf("the file has been broked, please delete all part files and re-download") + return nil, errors.Errorf("the file has been broken, please delete all part files and re-download") } err = binary.Read(bytes.NewBuffer(buf[:size]), binary.LittleEndian, meta) if err != nil { diff --git a/downloader/types.go b/downloader/types.go index b0f0c9f18..030d28dc4 100644 --- a/downloader/types.go +++ b/downloader/types.go @@ -6,7 +6,7 @@ type Aria2RPCData struct { // https://aria2.github.io/manual/en/html/aria2c.html#rpc-interface JSONRPC string `json:"jsonrpc"` ID string `json:"id"` - // For a simple download, only inplemented `addUri` + // For a simple download, only implemented `addUri` Method string `json:"method"` // secret, uris, options Params [3]interface{} `json:"params"` From 85bcc2f344ac922f76af5915b1cb5e012e2d543d Mon Sep 17 00:00:00 2001 From: asuma <770878450@qq.com> Date: Wed, 17 Apr 2024 14:17:27 +0800 Subject: [PATCH 34/38] =?UTF-8?q?=E4=BC=98=E5=8C=96xiaohongshu=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/extractors.go | 2 ++ extractors/xiaohongshu/xiaohongshu.go | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extractors/extractors.go b/extractors/extractors.go index 2954d32ed..76937d83a 100644 --- a/extractors/extractors.go +++ b/extractors/extractors.go @@ -41,6 +41,8 @@ func Extract(u string, option Options) ([]*Data, error) { } if u.Host == "haokan.baidu.com" { domain = "haokan" + } else if u.Host == "xhslink.com" { + domain = "xiaohongshu" } else { domain = utils.Domain(u.Host) } diff --git a/extractors/xiaohongshu/xiaohongshu.go b/extractors/xiaohongshu/xiaohongshu.go index 0a255c909..103b79be0 100644 --- a/extractors/xiaohongshu/xiaohongshu.go +++ b/extractors/xiaohongshu/xiaohongshu.go @@ -2,6 +2,7 @@ package xiaohongshu import ( "encoding/json" + neturl "net/url" "strconv" "strings" @@ -34,7 +35,7 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor } // title - titles := utils.MatchOneOf(html, `,"title":"(.+?)",`) + titles := utils.MatchOneOf(html, `(.*?)`) if titles == nil || len(titles) != 2 { return nil, errors.WithStack(extractors.ErrBodyParseFailed) } @@ -51,6 +52,11 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor return nil, errors.WithStack(extractors.ErrBodyParseFailed) } + pUrl, err := neturl.ParseRequestURI(url) + if err != nil { + return nil, errors.WithStack(err) + } + // streams streams := make(map[string]*extractors.Stream) var size int64 @@ -62,6 +68,10 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor if err != nil { continue } + + if pUrl.Host == "xhslink.com" && strings.Contains(u, "sns-video-qc") { + size += 1 // Make sure the link is downloadable and sort the link first with the same size + } streams[strconv.Itoa(i)] = &extractors.Stream{ Parts: []*extractors.Part{ { From 3f3e647c5303e216bb980f14352f48e0750b7026 Mon Sep 17 00:00:00 2001 From: asuma <770878450@qq.com> Date: Fri, 19 Apr 2024 15:50:23 +0800 Subject: [PATCH 35/38] =?UTF-8?q?=E9=87=8D=E5=86=99iXigua(Toutiao)?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E8=8E=B7=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++ extractors/ixigua/ixigua.go | 64 +++++++++------------- extractors/ixigua/types.go | 36 ++++++++++++ extractors/xiaohongshu/xiaohongshu_test.go | 2 +- 4 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 extractors/ixigua/types.go diff --git a/README.md b/README.md index 06f11651a..b8f520f0d 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,11 @@ As a text file: $ lux -c cookies.txt "https://www.bilibili.com/video/av20203945" ``` +Another example +``` +$ lux -c "msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C..." "https://m.toutiao.com/is/iYbTfJ79/" +``` + ### Proxy You can set the HTTP/SOCKS5 proxy using environment variables: @@ -655,6 +660,9 @@ $ lux -j "https://www.bilibili.com/video/av20203945" 最好是每次下载都附带登录过的 Cookie 以避免部分 `ccode` 的问题 +### 西瓜&头条 +西瓜&头条视频必须带Cookie才能下载成功，西瓜和头条可共用西瓜视频的Cookie，Cookie的有效期可能较短，下载失败就更新Cookie尝试（2024-04-19） + ## Contributing Lux is an open source project and built on the top of open-source projects. Check out the [Contributing Guide](./CONTRIBUTING.md) to get started. diff --git a/extractors/ixigua/ixigua.go b/extractors/ixigua/ixigua.go index f3b472a3e..793d9c628 100644 --- a/extractors/ixigua/ixigua.go +++ b/extractors/ixigua/ixigua.go @@ -4,11 +4,12 @@ import ( "encoding/base64" "encoding/json" "fmt" + browser "github.com/EDDYCJY/fake-useragent" + "github.com/iawia002/lux/utils" "net/http" "regexp" "strings" - "github.com/itchyny/gojq" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" @@ -40,8 +41,8 @@ func New() extractors.Extractor { // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) { headers := map[string]string{ - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0", - "Content-Type": "application/json", + "User-Agent": browser.Chrome(), + "Cookie": option.Cookie, } // ixigua 有三种格式的 URL @@ -66,60 +67,45 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor finalURL = resp.Request.URL.String() } - finalURL = strings.ReplaceAll(finalURL, "https://www.toutiao.com/a", "https://www.ixigua.com/") + finalURL = strings.ReplaceAll(finalURL, "https://www.toutiao.com/video/", "https://www.ixigua.com/") r := regexp.MustCompile(`(ixigua.com/)(\w+)?`) id := r.FindSubmatch([]byte(finalURL))[2] - url2 := fmt.Sprintf("https://www.ixigua.com/api/public/videov2/brief/details?group_id=%s", string(id)) + //url2 := fmt.Sprintf("https://www.ixigua.com/api/public/videov2/brief/details?group_id=%s", string(id)) + url2 := fmt.Sprintf("https://www.ixigua.com/%s", string(id)) body, err := request.Get(url2, url, headers) if err != nil { return nil, errors.WithStack(err) } - var m interface{} - err = json.Unmarshal([]byte(body), &m) - if err != nil { - return nil, errors.WithStack(err) + videoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\{.*?\})\<\/script\>`) + if videoListJson == nil || len(videoListJson) != 2 { + return nil, errors.WithStack(extractors.ErrBodyParseFailed) } - query, err := gojq.Parse("{title: .data.title} + {qualities: [.data.videoResource.normal.video_list | .[] | {url: .main_url, size: .size, ext: .vtype, quality: .definition}]}") + videoUrl := videoListJson[1] + videoUrl = strings.Replace(videoUrl, ":undefined", ":\"undefined\"", -1) + + // 解析JSON字符串 + var xiguanData xiguanData + err = json.Unmarshal([]byte(videoUrl), &xiguanData) if err != nil { return nil, errors.WithStack(err) } - video := Video{} - - iter := query.Run(m) - for { - v, ok := iter.Next() - if !ok { - break - } - if err, ok := v.(error); ok { - return nil, errors.WithStack(err) - } - - jsonbody, err := json.Marshal(v) - if err != nil { - return nil, errors.WithStack(err) - } - - if err := json.Unmarshal(jsonbody, &video); err != nil { - return nil, errors.WithStack(err) - } - } + title := xiguanData.AnyVideo.GidInformation.PackerData.Video.Title + videoList := xiguanData.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList streams := make(map[string]*extractors.Stream) - for _, quality := range video.Qualities { - streams[quality.Quality] = &extractors.Stream{ - Size: quality.Size, - Quality: quality.Quality, + for _, v := range videoList { + streams[v.Definition] = &extractors.Stream{ + Quality: v.Definition, Parts: []*extractors.Part{ { - URL: base64Decode(quality.URL), - Size: quality.Size, - Ext: quality.Ext, + URL: base64Decode(v.MainUrl), + Size: v.Size, + Ext: v.Vtype, }, }, } @@ -128,7 +114,7 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor return []*extractors.Data{ { Site: "西瓜视频 ixigua.com", - Title: video.Title, + Title: title, Type: extractors.DataTypeVideo, Streams: streams, URL: url, diff --git a/extractors/ixigua/types.go b/extractors/ixigua/types.go new file mode 100644 index 000000000..bdc718342 --- /dev/null +++ b/extractors/ixigua/types.go @@ -0,0 +1,36 @@ +package ixigua + +type xiguanData struct { + AnyVideo struct { + GidInformation struct { + Gid string `json:"gid"` + PackerData struct { + Video struct { + Title string `json:"title"` + PosterUrl string `json:"poster_url"` + VideoResource struct { + Vid string `json:"vid"` + Normal struct { + VideoId string `json:"video_id"` + VideoList map[string]struct { + Definition string `json:"definition"` + Quality string `json:"quality"` + Vtype string `json:"vtype"` + Vwidth int `json:"vwidth"` + Vheight int `json:"vheight"` + Bitrate int64 `json:"bitrate"` + RealBitrate int64 `json:"real_bitrate"` + Fps int `json:"fps"` + CodecType string `json:"codec_type"` + Size int64 `json:"size"` + MainUrl string `json:"main_url"` + BackupUrl1 string `json:"backup_url_1"` + } `json:"video_list"` + } `json:"normal"` + } `json:"videoResource"` + } `json:"video"` + Key string `json:"key"` + } `json:"packerData"` + } `json:"gidInformation"` + } `json:"anyVideo"` +} diff --git a/extractors/xiaohongshu/xiaohongshu_test.go b/extractors/xiaohongshu/xiaohongshu_test.go index ea709673d..d7f8f3895 100644 --- a/extractors/xiaohongshu/xiaohongshu_test.go +++ b/extractors/xiaohongshu/xiaohongshu_test.go @@ -15,7 +15,7 @@ func TestDownload(t *testing.T) { { name: "normal test", args: test.Args{ - URL: "https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f?m_source=pinpai", + URL: "https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f", Title: "七星级大厨都不会告诉你的，五花肉的8种做法", Size: 59410194, }, From 4c772e5aa5adcf98bcadd175d78d80b94844bcfc Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Fri, 19 Apr 2024 16:13:17 +0800 Subject: [PATCH 36/38] code format --- README.md | 13 ++++++------- extractors/ixigua/ixigua.go | 15 ++++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b8f520f0d..8507d8b40 100644 --- a/README.md +++ b/README.md @@ -340,11 +340,6 @@ As a text file: $ lux -c cookies.txt "https://www.bilibili.com/video/av20203945" ``` -Another example -``` -$ lux -c "msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C..." "https://m.toutiao.com/is/iYbTfJ79/" -``` - ### Proxy You can set the HTTP/SOCKS5 proxy using environment variables: @@ -660,8 +655,12 @@ $ lux -j "https://www.bilibili.com/video/av20203945" 最好是每次下载都附带登录过的 Cookie 以避免部分 `ccode` 的问题 -### 西瓜&头条 -西瓜&头条视频必须带Cookie才能下载成功，西瓜和头条可共用西瓜视频的Cookie，Cookie的有效期可能较短，下载失败就更新Cookie尝试（2024-04-19） +### 西瓜/头条视频 +西瓜/头条视频必须带 Cookie 才能下载成功，西瓜和头条可共用西瓜视频的 Cookie，Cookie 的有效期可能较短，下载失败就更新 Cookie 尝试： + +``` +$ lux -c "msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C..." "https://m.toutiao.com/is/iYbTfJ79/" +``` ## Contributing diff --git a/extractors/ixigua/ixigua.go b/extractors/ixigua/ixigua.go index 793d9c628..616255fde 100644 --- a/extractors/ixigua/ixigua.go +++ b/extractors/ixigua/ixigua.go @@ -4,16 +4,16 @@ import ( "encoding/base64" "encoding/json" "fmt" - browser "github.com/EDDYCJY/fake-useragent" - "github.com/iawia002/lux/utils" "net/http" "regexp" "strings" + browser "github.com/EDDYCJY/fake-useragent" "github.com/pkg/errors" "github.com/iawia002/lux/extractors" "github.com/iawia002/lux/request" + "github.com/iawia002/lux/utils" ) func init() { @@ -71,7 +71,6 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor r := regexp.MustCompile(`(ixigua.com/)(\w+)?`) id := r.FindSubmatch([]byte(finalURL))[2] - //url2 := fmt.Sprintf("https://www.ixigua.com/api/public/videov2/brief/details?group_id=%s", string(id)) url2 := fmt.Sprintf("https://www.ixigua.com/%s", string(id)) body, err := request.Get(url2, url, headers) @@ -87,15 +86,13 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor videoUrl := videoListJson[1] videoUrl = strings.Replace(videoUrl, ":undefined", ":\"undefined\"", -1) - // 解析JSON字符串 - var xiguanData xiguanData - err = json.Unmarshal([]byte(videoUrl), &xiguanData) - if err != nil { + var data xiguanData + if err = json.Unmarshal([]byte(videoUrl), &data); err != nil { return nil, errors.WithStack(err) } - title := xiguanData.AnyVideo.GidInformation.PackerData.Video.Title - videoList := xiguanData.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList + title := data.AnyVideo.GidInformation.PackerData.Video.Title + videoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList streams := make(map[string]*extractors.Stream) for _, v := range videoList { From 2e5da10bc7293614a826f002f126693b449972f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:13:46 +0000 Subject: [PATCH 37/38] build(deps): bump golang.org/x/net from 0.17.0 to 0.23.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.23.0. - [Commits](https://github.com/golang/net/compare/v0.17.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index fe1a1f528..02a00acc4 100644 --- a/go.mod +++ b/go.mod @@ -47,9 +47,9 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/temoto/robotstxt v1.1.1 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index 5f8b1d90b..a09f14e68 100644 --- a/go.sum +++ b/go.sum @@ -179,8 +179,8 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -202,8 +202,8 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -212,8 +212,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 831a256858403c1fbb7ad45db4b9f7e0762c3ed5 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Mon, 6 May 2024 09:19:45 +0800 Subject: [PATCH 38/38] Bump go version to 1.22 --- .github/workflows/builder.yml | 10 +++++----- .github/workflows/ci.yml | 8 ++++---- .github/workflows/goreleaser.yml | 4 ++-- .github/workflows/stream_acfun.yml | 6 +++--- .github/workflows/stream_bcy.yml | 6 +++--- .github/workflows/stream_bilibili.yml | 6 +++--- .github/workflows/stream_douyin.yml | 6 +++--- .github/workflows/stream_douyu.yml | 6 +++--- .github/workflows/stream_eporner.yml | 6 +++--- .github/workflows/stream_facebook.yml | 6 +++--- .github/workflows/stream_geekbang.yml | 6 +++--- .github/workflows/stream_haokan.yml | 6 +++--- .github/workflows/stream_hupu.yml | 6 +++--- .github/workflows/stream_huya.yml | 6 +++--- .github/workflows/stream_instagram.yml | 6 +++--- .github/workflows/stream_iqiyi.yml | 6 +++--- .github/workflows/stream_ixigua.yml | 6 +++--- .github/workflows/stream_kuaishou.yml | 6 +++--- .github/workflows/stream_mgtv.yml | 6 +++--- .github/workflows/stream_miaopai.yml | 6 +++--- .github/workflows/stream_netease.yml | 6 +++--- .github/workflows/stream_pinterest.yml | 6 +++--- .github/workflows/stream_pixivision.yml | 6 +++--- .github/workflows/stream_pornhub.yml | 6 +++--- .github/workflows/stream_qq.yml | 6 +++--- .github/workflows/stream_reddit.yml | 6 +++--- .github/workflows/stream_rumble.yml | 6 +++--- .github/workflows/stream_streamtape.yml | 6 +++--- .github/workflows/stream_tangdou.yml | 6 +++--- .github/workflows/stream_tiktok.yml | 6 +++--- .github/workflows/stream_tumblr.yml | 6 +++--- .github/workflows/stream_twitter.yml | 6 +++--- .github/workflows/stream_udn.yml | 6 +++--- .github/workflows/stream_vimeo.yml | 6 +++--- .github/workflows/stream_vk.yml | 6 +++--- .github/workflows/stream_weibo.yml | 6 +++--- .github/workflows/stream_xiaohongshu.yml | 6 +++--- .github/workflows/stream_ximalaya.yml | 6 +++--- .github/workflows/stream_xinpianchang.yml | 6 +++--- .github/workflows/stream_xvideos.yml | 6 +++--- .github/workflows/stream_yinyuetai.yml | 6 +++--- .github/workflows/stream_youku.yml | 6 +++--- .github/workflows/stream_youtube.yml | 6 +++--- .github/workflows/stream_zhihu.yml | 6 +++--- .github/workflows/stream_zingmp3.yml | 6 +++--- .golangci.yml | 2 +- go.mod | 2 +- script/github_action_template.yml | 6 +++--- 48 files changed, 142 insertions(+), 142 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 24ec79736..ea56f044b 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -88,25 +88,25 @@ jobs: GOMIPSLE: ${{ matrix.mipsle }} steps: - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: 1.22 - name: Check out code base if: github.event_name == 'push' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check out code base if: github.event_name == 'pull_request' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Cache go module - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c8a2f43..9d6bbbf8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,12 +11,12 @@ jobs: timeout-minutes: 30 strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest, macOS-latest] name: Go ${{ matrix.go }} in ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Environment @@ -26,7 +26,7 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v3 with: - version: v1.54.1 + version: v1.58.0 only-new-issues: true - name: Test env: diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index 55d449896..321b4e0af 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -14,9 +14,9 @@ jobs: with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: 1.22 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v5 with: diff --git a/.github/workflows/stream_acfun.yml b/.github/workflows/stream_acfun.yml index 44888182b..a29ab19b1 100644 --- a/.github/workflows/stream_acfun.yml +++ b/.github/workflows/stream_acfun.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_bcy.yml b/.github/workflows/stream_bcy.yml index 8b34d8984..cb05d5e8a 100644 --- a/.github/workflows/stream_bcy.yml +++ b/.github/workflows/stream_bcy.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_bilibili.yml b/.github/workflows/stream_bilibili.yml index 1c7431451..82f638765 100644 --- a/.github/workflows/stream_bilibili.yml +++ b/.github/workflows/stream_bilibili.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_douyin.yml b/.github/workflows/stream_douyin.yml index 70673f77e..073d55a31 100644 --- a/.github/workflows/stream_douyin.yml +++ b/.github/workflows/stream_douyin.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_douyu.yml b/.github/workflows/stream_douyu.yml index 064fea5e2..036a82e7f 100644 --- a/.github/workflows/stream_douyu.yml +++ b/.github/workflows/stream_douyu.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_eporner.yml b/.github/workflows/stream_eporner.yml index 8072e659f..c106ca3b5 100644 --- a/.github/workflows/stream_eporner.yml +++ b/.github/workflows/stream_eporner.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_facebook.yml b/.github/workflows/stream_facebook.yml index 64ca8f0ab..895cfe11a 100644 --- a/.github/workflows/stream_facebook.yml +++ b/.github/workflows/stream_facebook.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_geekbang.yml b/.github/workflows/stream_geekbang.yml index bfa44e471..cf7da8558 100644 --- a/.github/workflows/stream_geekbang.yml +++ b/.github/workflows/stream_geekbang.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_haokan.yml b/.github/workflows/stream_haokan.yml index 02b35848f..58ac92687 100644 --- a/.github/workflows/stream_haokan.yml +++ b/.github/workflows/stream_haokan.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_hupu.yml b/.github/workflows/stream_hupu.yml index 5aa857b74..b30e8676d 100644 --- a/.github/workflows/stream_hupu.yml +++ b/.github/workflows/stream_hupu.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_huya.yml b/.github/workflows/stream_huya.yml index 61fedecdd..b9fa949c3 100644 --- a/.github/workflows/stream_huya.yml +++ b/.github/workflows/stream_huya.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_instagram.yml b/.github/workflows/stream_instagram.yml index cdec32bc9..7f91d06e8 100644 --- a/.github/workflows/stream_instagram.yml +++ b/.github/workflows/stream_instagram.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_iqiyi.yml b/.github/workflows/stream_iqiyi.yml index 5fb1f3e5d..0629fc78d 100644 --- a/.github/workflows/stream_iqiyi.yml +++ b/.github/workflows/stream_iqiyi.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_ixigua.yml b/.github/workflows/stream_ixigua.yml index 66eb4f56e..c1b84af79 100644 --- a/.github/workflows/stream_ixigua.yml +++ b/.github/workflows/stream_ixigua.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_kuaishou.yml b/.github/workflows/stream_kuaishou.yml index 43c16a451..448e64559 100644 --- a/.github/workflows/stream_kuaishou.yml +++ b/.github/workflows/stream_kuaishou.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_mgtv.yml b/.github/workflows/stream_mgtv.yml index c3c790136..ad1f5d9af 100644 --- a/.github/workflows/stream_mgtv.yml +++ b/.github/workflows/stream_mgtv.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_miaopai.yml b/.github/workflows/stream_miaopai.yml index d65244bce..4f5c82b48 100644 --- a/.github/workflows/stream_miaopai.yml +++ b/.github/workflows/stream_miaopai.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_netease.yml b/.github/workflows/stream_netease.yml index 450de6668..8d2ffe909 100644 --- a/.github/workflows/stream_netease.yml +++ b/.github/workflows/stream_netease.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_pinterest.yml b/.github/workflows/stream_pinterest.yml index d4bdfe656..b220bbb9a 100644 --- a/.github/workflows/stream_pinterest.yml +++ b/.github/workflows/stream_pinterest.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_pixivision.yml b/.github/workflows/stream_pixivision.yml index 0a75be799..d1f415926 100644 --- a/.github/workflows/stream_pixivision.yml +++ b/.github/workflows/stream_pixivision.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_pornhub.yml b/.github/workflows/stream_pornhub.yml index 302e87a5b..fce5557ef 100644 --- a/.github/workflows/stream_pornhub.yml +++ b/.github/workflows/stream_pornhub.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_qq.yml b/.github/workflows/stream_qq.yml index 43c97676f..31acfc610 100644 --- a/.github/workflows/stream_qq.yml +++ b/.github/workflows/stream_qq.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_reddit.yml b/.github/workflows/stream_reddit.yml index c365da690..1e59b7331 100644 --- a/.github/workflows/stream_reddit.yml +++ b/.github/workflows/stream_reddit.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_rumble.yml b/.github/workflows/stream_rumble.yml index 4aff7a32f..5842e15c8 100644 --- a/.github/workflows/stream_rumble.yml +++ b/.github/workflows/stream_rumble.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_streamtape.yml b/.github/workflows/stream_streamtape.yml index 545040441..1cb7638c1 100644 --- a/.github/workflows/stream_streamtape.yml +++ b/.github/workflows/stream_streamtape.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_tangdou.yml b/.github/workflows/stream_tangdou.yml index d0a5dae59..e11903c2b 100644 --- a/.github/workflows/stream_tangdou.yml +++ b/.github/workflows/stream_tangdou.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_tiktok.yml b/.github/workflows/stream_tiktok.yml index ed72302b8..4f511b15a 100644 --- a/.github/workflows/stream_tiktok.yml +++ b/.github/workflows/stream_tiktok.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_tumblr.yml b/.github/workflows/stream_tumblr.yml index 568969d2e..87ea659aa 100644 --- a/.github/workflows/stream_tumblr.yml +++ b/.github/workflows/stream_tumblr.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_twitter.yml b/.github/workflows/stream_twitter.yml index 8c3aa70eb..7342bbfc2 100644 --- a/.github/workflows/stream_twitter.yml +++ b/.github/workflows/stream_twitter.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_udn.yml b/.github/workflows/stream_udn.yml index 8a8f481ad..86c7a361c 100644 --- a/.github/workflows/stream_udn.yml +++ b/.github/workflows/stream_udn.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_vimeo.yml b/.github/workflows/stream_vimeo.yml index 933511330..0d1121144 100644 --- a/.github/workflows/stream_vimeo.yml +++ b/.github/workflows/stream_vimeo.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_vk.yml b/.github/workflows/stream_vk.yml index a8b7c7d31..674b3c0ec 100644 --- a/.github/workflows/stream_vk.yml +++ b/.github/workflows/stream_vk.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_weibo.yml b/.github/workflows/stream_weibo.yml index 9d60def9d..151c408ed 100644 --- a/.github/workflows/stream_weibo.yml +++ b/.github/workflows/stream_weibo.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_xiaohongshu.yml b/.github/workflows/stream_xiaohongshu.yml index 43b52ee37..77462f53c 100644 --- a/.github/workflows/stream_xiaohongshu.yml +++ b/.github/workflows/stream_xiaohongshu.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_ximalaya.yml b/.github/workflows/stream_ximalaya.yml index 801e29e69..bd958d39c 100644 --- a/.github/workflows/stream_ximalaya.yml +++ b/.github/workflows/stream_ximalaya.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_xinpianchang.yml b/.github/workflows/stream_xinpianchang.yml index 0666b6431..cc3139257 100644 --- a/.github/workflows/stream_xinpianchang.yml +++ b/.github/workflows/stream_xinpianchang.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_xvideos.yml b/.github/workflows/stream_xvideos.yml index 7ea96fab8..1b5ce958c 100644 --- a/.github/workflows/stream_xvideos.yml +++ b/.github/workflows/stream_xvideos.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_yinyuetai.yml b/.github/workflows/stream_yinyuetai.yml index 25bd9fbe1..13dfba943 100644 --- a/.github/workflows/stream_yinyuetai.yml +++ b/.github/workflows/stream_yinyuetai.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_youku.yml b/.github/workflows/stream_youku.yml index 7f736fe12..f20e68de0 100644 --- a/.github/workflows/stream_youku.yml +++ b/.github/workflows/stream_youku.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_youtube.yml b/.github/workflows/stream_youtube.yml index ace30b984..bcd3f8e18 100644 --- a/.github/workflows/stream_youtube.yml +++ b/.github/workflows/stream_youtube.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_zhihu.yml b/.github/workflows/stream_zhihu.yml index 09aca92f1..b63aea4f1 100644 --- a/.github/workflows/stream_zhihu.yml +++ b/.github/workflows/stream_zhihu.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/stream_zingmp3.yml b/.github/workflows/stream_zingmp3.yml index e912e4cd2..6ec2cd37f 100644 --- a/.github/workflows/stream_zingmp3.yml +++ b/.github/workflows/stream_zingmp3.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} diff --git a/.golangci.yml b/.golangci.yml index 8d4b882ae..f071db719 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ run: concurrency: 2 timeout: 5m - go: 1.21 + go: 1.22 linter-settings: goconst: diff --git a/go.mod b/go.mod index 02a00acc4..92ce339bb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/iawia002/lux -go 1.21 +go 1.22 require ( github.com/EDDYCJY/fake-useragent v0.2.0 diff --git a/script/github_action_template.yml b/script/github_action_template.yml index 5cea30750..b3ab8c5b5 100644 --- a/script/github_action_template.yml +++ b/script/github_action_template.yml @@ -18,12 +18,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go: ["1.21"] + go: ["1.22"] os: [ubuntu-latest] name: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }}