Skip to content

Commit

Permalink
fix using new aws news api. no more html parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@circa10a
circa10a committed Mar 19, 2022
1 parent 9c136af commit 7ff4ce9
Show file tree
Hide file tree
Showing 9 changed files with 195 additions and 7,733 deletions.
10 changes: 5 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@ module github.com/circa10a/go-aws-news
go 1.13

require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/aws/aws-lambda-go v1.27.0
github.com/aws/aws-sdk-go v1.42.11
github.com/aws/aws-lambda-go v1.28.0
github.com/aws/aws-sdk-go v1.43.21
github.com/go-resty/resty/v2 v2.7.0
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/olekukonko/tablewriter v0.0.5
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.6.1
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 // indirect
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.4.0
)
32 changes: 17 additions & 15 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/aws/aws-lambda-go v1.27.0 h1:aLzrJwdyHoF1A18YeVdJjX8Ixkd+bpogdxVInvHcWjM=
github.com/aws/aws-lambda-go v1.27.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go v1.42.11 h1:5wfKuNcbch3IFZth5+j2Ud/+UOxCR0zfgLGPoiK1p4s=
github.com/aws/aws-sdk-go v1.42.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-lambda-go v1.28.0 h1:fZiik1PZqW2IyAN4rj+Y0UBaO1IDFlsNo9Zz/XnArK4=
github.com/aws/aws-lambda-go v1.28.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go v1.43.21 h1:E4S2eX3d2gKJyI/ISrcIrSwXwqjIvCK85gtBMt4sAPE=
github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
Expand Down Expand Up @@ -42,18 +40,22 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9 h1:0qxwC5n+ttVOINCBeRHO0nq9X7uy8SDsPoi5OaCdIEI=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab h1:rfJ1bsoJQQIAoAxTxB7bme+vHrNkRw8CqfsYh9w54cw=
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
Expand Down
1 change: 1 addition & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func mainExec() error {
if err != nil {
return err
}

if len(news) == 0 {
log.Info("No news fetched. Skipping notifications.")
return nil
Expand Down
201 changes: 147 additions & 54 deletions news/announcements.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,99 @@ package news
import (
"encoding/json"
"fmt"
"net/url"
"os"
"regexp"
"strings"
"time"

"github.com/PuerkitoBio/goquery"
"github.com/go-resty/resty/v2"
"github.com/olekukonko/tablewriter"
)

const (
awsWhatsNewBaseURL = "https://aws.amazon.com/api/dirs/items/search"
awsWhatsNewPostBaseURL = "https://aws.amazon.com"
)

type AWSNewsItemsResponse struct {
FieldTypes struct {
RelatedBlog string `json:"relatedBlog"`
PostBody string `json:"postBody"`
ModifiedDate string `json:"modifiedDate"`
HeadlineURL string `json:"headlineUrl"`
PostDateTime string `json:"postDateTime"`
PostSummary string `json:"postSummary"`
Headline string `json:"headline"`
ContentType string `json:"contentType"`
} `json:"fieldTypes"`
Items []struct {
Item struct {
AdditionalFields struct {
PostBody string `json:"postBody"`
ModifiedDate time.Time `json:"modifiedDate"`
HeadlineURL string `json:"headlineUrl"`
PostDateTime time.Time `json:"postDateTime"`
PostSummary string `json:"postSummary"`
ContentType string `json:"contentType"`
Headline string `json:"headline"`
} `json:"additionalFields"`
ID string `json:"id"`
Locale string `json:"locale"`
DirectoryID string `json:"directoryId"`
Name string `json:"name"`
CreatedBy string `json:"createdBy"`
LastUpdatedBy string `json:"lastUpdatedBy"`
DateCreated string `json:"dateCreated"`
DateUpdated string `json:"dateUpdated"`
Author string `json:"author"`
NumImpressions int `json:"numImpressions"`
} `json:"item"`
Tags []struct {
ID string `json:"id"`
Locale string `json:"locale"`
TagNamespaceID string `json:"tagNamespaceId"`
Name string `json:"name"`
Description string `json:"description"`
CreatedBy string `json:"createdBy"`
LastUpdatedBy string `json:"lastUpdatedBy"`
DateCreated string `json:"dateCreated"`
DateUpdated string `json:"dateUpdated"`
} `json:"tags"`
} `json:"items"`
Metadata struct {
Count int `json:"count"`
TotalHits int `json:"totalHits"`
} `json:"metadata"`
}

func getItemsYear(year int) (*AWSNewsItemsResponse, error) {
results := &AWSNewsItemsResponse{}
client := resty.New()

resp, err := client.SetBaseURL(awsWhatsNewBaseURL).R().
SetResult(results).
SetQueryParams(map[string]string{
"size": "2000", // 2000 seems to be the max or no results return
"item.directoryId": "whats-new",
"sort_by": "item.additionalFields.postDateTime",
"sort_order": "desc",
"item.locale": "en_US",
"tags.id": fmt.Sprintf("whats-new#year#%d", year),
}).
SetHeader("Accept", "application/json").
Get("/")

if err != nil {
return results, err
}

if resp.StatusCode() > 399 {
return results, fmt.Errorf("Received response code: %d", resp.StatusCode())
}

return results, nil
}

// Announcements Represents a slice containing all of the AWS announcements for a given time period.
type Announcements []Announcement

Expand All @@ -22,95 +106,104 @@ type Announcement struct {
PostDate string
}

func (d newsDoc) GetAnnouncements() (Announcements, error) {
// Create []Announcement
var announcements Announcements
d.getSelectionItems().Each(func(i int, s *goquery.Selection) {
title := d.getSelectionTitle(s)
link := fmt.Sprintf("https:%v", d.getSelectionItemLink(s))
date := parseDate(d.getSelectionItemDate(s))
announcements = append(announcements, Announcement{Title: title, Link: link, PostDate: date})
})
return announcements, nil
}

// Fetch gets all of the announcements for the specified year/month that was input.
func Fetch(year int, month int) (Announcements, error) {
doc, err := getNewsDocMonth(year, month)
announcements := Announcements{}
items, err := getItemsYear(year)
if err != nil {
return Announcements{}, err
}
return newsDoc{doc}.GetAnnouncements()

for _, item := range items.Items {
announcement := Announcement{}
_, postDateMonth, _ := item.Item.AdditionalFields.PostDateTime.Date()
if postDateMonth == time.Now().Month() {
announcement.Link = awsWhatsNewPostBaseURL + item.Item.Name
announcement.PostDate = item.Item.AdditionalFields.PostDateTime.Format(time.RFC3339)
announcement.Title = item.Item.AdditionalFields.Headline
announcements = append(announcements, announcement)
}
}

return announcements, nil
}

// FetchYear gets all of the announcements for the specified year that was input.
func FetchYear(year int) (Announcements, error) {
doc, err := getNewsDocYear(year)
announcements := Announcements{}
items, err := getItemsYear(year)
if err != nil {
return Announcements{}, err
return announcements, err
}

for _, item := range items.Items {
announcement := Announcement{}
announcement.Link = fmt.Sprintf("%s/%s", awsWhatsNewPostBaseURL, item.Item.Name)
announcement.PostDate = item.Item.DateCreated
announcement.Title = item.Item.AdditionalFields.Headline
announcements = append(announcements, announcement)
}
return newsDoc{doc}.GetAnnouncements()

return announcements, nil
}

// ThisMonth gets the current month's AWS announcements.
func ThisMonth() (Announcements, error) {
var thisMonthsAnnouncements Announcements
currentTime := time.Now()
news, err := FetchYear(currentTime.Year())
for _, announcement := range news {
if announcement.PostDate[:3] == currentTime.Month().String()[:3] {
thisMonthsAnnouncements = append(thisMonthsAnnouncements, announcement)
}
}
items, err := Fetch(currentTime.Year(), int(currentTime.Month()))
if err != nil {
return news, err
return items, err
}
return thisMonthsAnnouncements, nil
return items, nil
}

// Today gets today's AWS announcements.
func Today() (Announcements, error) {
var todaysAnnouncements Announcements
news, err := FetchYear(time.Now().Year())
todaysAnnouncements := Announcements{}
currentTime := time.Now()

items, err := Fetch(currentTime.Year(), int(currentTime.Month()))
if err != nil {
return news, err
return todaysAnnouncements, err
}
for _, announcement := range news {
postDate, _ := time.Parse("Jan 2, 2006", announcement.PostDate)
if dateEqual(postDate, time.Now()) {

for _, announcement := range items {
announcementPostDate, err := time.Parse(time.RFC3339, announcement.PostDate)
if err != nil {
return todaysAnnouncements, err
}

if dateEqual(announcementPostDate, currentTime) {
todaysAnnouncements = append(todaysAnnouncements, announcement)
}
}

return todaysAnnouncements, nil
}

// Yesterday gets yesterday's AWS announcments.
func Yesterday() (Announcements, error) {
var yesterdaysAnnouncements Announcements
news, err := FetchYear(time.Now().Year())
yesterdaysAnnouncements := Announcements{}
currentTime := time.Now()
yesterday := currentTime.AddDate(0, 0, -1)

items, err := Fetch(currentTime.Year(), int(currentTime.Month()))
if err != nil {
return news, err
return yesterdaysAnnouncements, err
}
for _, announcement := range news {
postDate, _ := time.Parse("Jan 2, 2006", announcement.PostDate)
if dateEqual(postDate, time.Now().AddDate(0, 0, -1)) {

for _, announcement := range items {
announcementPostDate, err := time.Parse(time.RFC3339, announcement.PostDate)
if err != nil {
return yesterdaysAnnouncements, err
}

if dateEqual(announcementPostDate, yesterday) {
yesterdaysAnnouncements = append(yesterdaysAnnouncements, announcement)
}
}
return yesterdaysAnnouncements, nil
}

// Extract date from amazon's format
// Input: Posted on: Jan 7, 2020
// Output: Jan 7, 2020
// parseDate Extracts a standarized date format from the AWS html document.
func parseDate(postDate string) string {
r := regexp.MustCompile(`[A-Z][a-z]{2}\s[0-9]{1,2},\s[0-9]{4}`)
// AWS sometimes doesn't have a post date
if len(r.FindStringSubmatch(postDate)) > 0 {
return r.FindStringSubmatch(postDate)[0]
}
return "No posted date"
return yesterdaysAnnouncements, nil
}

// Print Prints out an ASCII table of your selection of AWS announcements.
Expand Down Expand Up @@ -167,7 +260,7 @@ func (a Announcements) HTML() string {
var html strings.Builder
html.WriteString("<ul>")
for _, v := range a {
html.WriteString(fmt.Sprintf("<li><a href='%v'>%v</a></li>", v.Link, v.Title))
html.WriteString(fmt.Sprintf("<li><a href='%v'>%v</a></li>", url.QueryEscape(v.Link), url.QueryEscape(v.Title)))
}
html.WriteString("</ul>")
return html.String()
Expand Down
Loading

0 comments on commit 7ff4ce9

Please sign in to comment.