Skip to content

Commit e36dedf

Browse files
motatoesmotatoes
authored
Fix plan artefact storage with gcp buckets (#1251)
* Fixing of bug in GCP plan artefact storage
1 parent 137b173 commit e36dedf

File tree

11 files changed

+160
-85
lines changed

11 files changed

+160
-85
lines changed

‎.github/workflows/cli_test.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,11 @@ jobs:
2727

2828
- name: Deps
2929
run: |
30-
pwd
3130
go get -v ./...
3231
working-directory: cli
3332

3433
- name: Build
3534
run: |
36-
pwd
3735
go build -v ./cmd/digger
3836
working-directory: cli
3937

‎.github/workflows/cli_test_e2e.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Cli e2e tests
2+
on:
3+
push:
4+
pull_request:
5+
types: [opened, reopened]
6+
7+
jobs:
8+
9+
build:
10+
permissions:
11+
contents: 'read'
12+
id-token: 'write'
13+
14+
name: Build
15+
runs-on: ubuntu-latest
16+
steps:
17+
18+
- name: Check out code into the Go module directory
19+
uses: actions/checkout@v4
20+
21+
- name: Download Go
22+
uses: actions/setup-go@v5
23+
with:
24+
go-version: 1.22.0
25+
id: go
26+
27+
- name: Check out code into the Go module directory
28+
uses: actions/checkout@v4
29+
30+
- name: Deps cli
31+
run: |
32+
go get -v ./...
33+
working-directory: cli
34+
35+
- name: Deps e2e
36+
run: |
37+
go get -v ./...
38+
working-directory: cli_e2e
39+
40+
- name: Test
41+
run: |
42+
echo '${{ secrets.GCP_CREDENTIALS }}' > /tmp/gcp.json
43+
go test -v ./...
44+
working-directory: cli_e2e
45+
env:
46+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp.json
47+
GOOGLE_STORAGE_BUCKET: gcp-plan-artefacts
48+
49+
50+

‎cli/pkg/core/execution/execution.go

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"os"
77
"path"
88
"regexp"
9+
"strconv"
910
"strings"
1011

1112
"github.com/diggerhq/digger/cli/pkg/core/locking"
@@ -118,10 +119,10 @@ type PlanPathProvider interface {
118119
LocalPlanFilePath() string
119120
StoredPlanFilePath() string
120121
ArtifactName() string
121-
PlanFileName() string
122122
}
123123

124124
type ProjectPathProvider struct {
125+
PRNumber *int
125126
ProjectPath string
126127
ProjectNamespace string
127128
ProjectName string
@@ -131,29 +132,31 @@ func (d ProjectPathProvider) ArtifactName() string {
131132
return d.ProjectName
132133
}
133134

134-
func (d ProjectPathProvider) PlanFileName() string {
135-
return strings.ReplaceAll(d.ProjectNamespace, "/", "-") + "-" + d.ProjectName + ".tfplan"
136-
}
135+
func (d ProjectPathProvider) StoredPlanFilePath() string {
136+
if d.PRNumber != nil {
137+
prNumber := strconv.Itoa(*d.PRNumber)
138+
return strings.ReplaceAll(d.ProjectNamespace, "/", "-") + "-" + prNumber + "-" + d.ProjectName + ".tfplan"
139+
} else {
140+
return strings.ReplaceAll(d.ProjectNamespace, "/", "-") + "-" + d.ProjectName + ".tfplan"
141+
}
137142

138-
func (d ProjectPathProvider) LocalPlanFilePath() string {
139-
return path.Join(d.ProjectPath, d.PlanFileName())
140143
}
141144

142-
func (d ProjectPathProvider) StoredPlanFilePath() string {
143-
return path.Join(d.ProjectNamespace, d.PlanFileName())
145+
func (d ProjectPathProvider) LocalPlanFilePath() string {
146+
return path.Join(d.ProjectPath, d.StoredPlanFilePath())
144147
}
145148

146149
func (d DiggerExecutor) RetrievePlanJson() (string, error) {
147150
executor := d
148151
planStorage := executor.PlanStorage
149152
planPathProvider := executor.PlanPathProvider
150-
storedPlanExists, err := planStorage.PlanExists(planPathProvider.ArtifactName())
153+
storedPlanExists, err := planStorage.PlanExists(planPathProvider.ArtifactName(), planPathProvider.StoredPlanFilePath())
151154
if err != nil {
152155
return "", fmt.Errorf("failed to check if stored plan exists. %v", err)
153156
}
154157
if storedPlanExists {
155158
log.Printf("Pre-apply plan retrieval: stored plan exists in artefact, retrieving")
156-
storedPlanPath, err := planStorage.RetrievePlan(planPathProvider.LocalPlanFilePath(), planPathProvider.ArtifactName())
159+
storedPlanPath, err := planStorage.RetrievePlan(planPathProvider.LocalPlanFilePath(), planPathProvider.ArtifactName(), planPathProvider.StoredPlanFilePath())
157160
if err != nil {
158161
return "", fmt.Errorf("failed to retrieve stored plan path. %v", err)
159162
}
@@ -218,7 +221,7 @@ func (d DiggerExecutor) Plan() (*terraform.PlanSummary, bool, bool, string, stri
218221
}
219222

220223
if !isEmptyPlan {
221-
nonEmptyPlanFilepath := strings.Replace(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.PlanFileName(), "isNonEmptyPlan.txt", 1)
224+
nonEmptyPlanFilepath := strings.Replace(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.StoredPlanFilePath(), "isNonEmptyPlan.txt", 1)
222225
file, err := os.Create(nonEmptyPlanFilepath)
223226
if err != nil {
224227
return nil, false, false, "", "", fmt.Errorf("unable to create file: %v", err)
@@ -237,7 +240,7 @@ func (d DiggerExecutor) Plan() (*terraform.PlanSummary, bool, bool, string, stri
237240
return nil, false, false, "", "", fmt.Errorf("error reading file bytes: %v", err)
238241
}
239242

240-
err = d.PlanStorage.StorePlanFile(fileBytes, d.PlanPathProvider.ArtifactName(), d.PlanPathProvider.PlanFileName())
243+
err = d.PlanStorage.StorePlanFile(fileBytes, d.PlanPathProvider.ArtifactName(), d.PlanPathProvider.StoredPlanFilePath())
241244
if err != nil {
242245
fmt.Println("Error storing artifact file:", err)
243246
return nil, false, false, "", "", fmt.Errorf("error storing artifact file: %v", err)
@@ -284,7 +287,7 @@ func (d DiggerExecutor) Apply() (bool, string, error) {
284287
var plansFilename *string
285288
if d.PlanStorage != nil {
286289
var err error
287-
plansFilename, err = d.PlanStorage.RetrievePlan(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.ArtifactName())
290+
plansFilename, err = d.PlanStorage.RetrievePlan(d.PlanPathProvider.LocalPlanFilePath(), d.PlanPathProvider.ArtifactName(), d.PlanPathProvider.StoredPlanFilePath())
288291
if err != nil {
289292
return false, "", fmt.Errorf("error retrieving plan: %v", err)
290293
}

‎cli/pkg/core/storage/plan_storage.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
package storage
22

33
type PlanStorage interface {
4-
StorePlan(localPlanFilePath string, storedPlanFilePath string) error
5-
StorePlanFile(fileContents []byte, artifactName string, fileName string) error
6-
RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error)
7-
DeleteStoredPlan(storedPlanFilePath string) error
8-
PlanExists(storedPlanFilePath string) (bool, error)
4+
StorePlanFile(fileContents []byte, artifactName string, storedPlanFilePath string) error
5+
RetrievePlan(localPlanFilePath string, artifactName string, storedPlanFilePath string) (*string, error)
6+
DeleteStoredPlan(artifactName string, storedPlanFilePath string) error
7+
PlanExists(artifactName string, storedPlanFilePath string) (bool, error)
98
}

‎cli/pkg/digger/digger.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func RunJobs(
106106
continue
107107
}
108108

109-
executorResult, output, err := run(command, job, policyChecker, orgService, SCMOrganisation, SCMrepository, job.RequestedBy, reporter, lock, prService, job.Namespace, workingDir, planStorage, appliesPerProject)
109+
executorResult, output, err := run(command, job, policyChecker, orgService, SCMOrganisation, SCMrepository, job.PullRequestNumber, job.RequestedBy, reporter, lock, prService, job.Namespace, workingDir, planStorage, appliesPerProject)
110110
if err != nil {
111111
reportErr := backendApi.ReportProjectRun(SCMOrganisation+"-"+SCMrepository, job.ProjectName, runStartedAt, time.Now(), "FAILED", command, output)
112112
if reportErr != nil {
@@ -181,7 +181,7 @@ func reportPolicyError(projectName string, command string, requestedBy string, r
181181
return msg
182182
}
183183

184-
func run(command string, job orchestrator.Job, policyChecker policy.Checker, orgService orchestrator.OrgService, SCMOrganisation string, SCMrepository string, requestedBy string, reporter core_reporting.Reporter, lock core_locking.Lock, prService orchestrator.PullRequestService, projectNamespace string, workingDir string, planStorage storage.PlanStorage, appliesPerProject map[string]bool) (*execution.DiggerExecutorResult, string, error) {
184+
func run(command string, job orchestrator.Job, policyChecker policy.Checker, orgService orchestrator.OrgService, SCMOrganisation string, SCMrepository string, PRNumber *int, requestedBy string, reporter core_reporting.Reporter, lock core_locking.Lock, prService orchestrator.PullRequestService, projectNamespace string, workingDir string, planStorage storage.PlanStorage, appliesPerProject map[string]bool) (*execution.DiggerExecutorResult, string, error) {
185185
log.Printf("Running '%s' for project '%s' (workflow: %s)\n", command, job.ProjectName, job.ProjectWorkflow)
186186

187187
allowedToPerformCommand, err := policyChecker.CheckAccessPolicy(orgService, &prService, SCMOrganisation, SCMrepository, job.ProjectName, command, job.PullRequestNumber, requestedBy, []string{})
@@ -225,6 +225,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
225225
ProjectPath: projectPath,
226226
ProjectNamespace: projectNamespace,
227227
ProjectName: job.ProjectName,
228+
PRNumber: PRNumber,
228229
}
229230

230231
diggerExecutor := execution.LockingExecutorWrapper{
@@ -442,7 +443,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
442443
}
443444

444445
if planStorage != nil {
445-
err = planStorage.DeleteStoredPlan(planPathProvider.StoredPlanFilePath())
446+
err = planStorage.DeleteStoredPlan(planPathProvider.ArtifactName(), planPathProvider.StoredPlanFilePath())
446447
if err != nil {
447448
log.Printf("failed to delete stored plan file '%v': %v", planPathProvider.StoredPlanFilePath(), err)
448449
}
@@ -560,6 +561,7 @@ func RunJob(
560561
ProjectPath: projectPath,
561562
ProjectNamespace: repo,
562563
ProjectName: job.ProjectName,
564+
PRNumber: job.PullRequestNumber,
563565
}
564566

565567
diggerExecutor := execution.DiggerExecutor{

‎cli/pkg/digger/digger_test.go

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -191,27 +191,22 @@ type MockPlanStorage struct {
191191
Commands []RunInfo
192192
}
193193

194-
func (m *MockPlanStorage) StorePlan(localPlanFilePath string, storedPlanFilePath string) error {
195-
m.Commands = append(m.Commands, RunInfo{"StorePlan", localPlanFilePath, time.Now()})
196-
return nil
197-
}
198-
199194
func (m *MockPlanStorage) StorePlanFile(fileContents []byte, artifactName string, fileName string) error {
200195
m.Commands = append(m.Commands, RunInfo{"StorePlanFile", artifactName, time.Now()})
201196
return nil
202197
}
203198

204-
func (m *MockPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) {
199+
func (m *MockPlanStorage) RetrievePlan(localPlanFilePath string, artifactName string, storedPlanFilePath string) (*string, error) {
205200
m.Commands = append(m.Commands, RunInfo{"RetrievePlan", localPlanFilePath, time.Now()})
206201
return nil, nil
207202
}
208203

209-
func (m *MockPlanStorage) DeleteStoredPlan(storedPlanFilePath string) error {
204+
func (m *MockPlanStorage) DeleteStoredPlan(artifactName string, storedPlanFilePath string) error {
210205
m.Commands = append(m.Commands, RunInfo{"DeleteStoredPlan", storedPlanFilePath, time.Now()})
211206
return nil
212207
}
213208

214-
func (m *MockPlanStorage) PlanExists(storedPlanFilePath string) (bool, error) {
209+
func (m *MockPlanStorage) PlanExists(artifactName string, storedPlanFilePath string) (bool, error) {
215210
m.Commands = append(m.Commands, RunInfo{"PlanExists", storedPlanFilePath, time.Now()})
216211
return false, nil
217212
}
@@ -225,8 +220,8 @@ func (m MockPlanPathProvider) ArtifactName() string {
225220
return "plan"
226221
}
227222

228-
func (m MockPlanPathProvider) PlanFileName() string {
229-
m.Commands = append(m.Commands, RunInfo{"PlanFileName", "", time.Now()})
223+
func (m MockPlanPathProvider) StoredPlanFilePath() string {
224+
m.Commands = append(m.Commands, RunInfo{"StoredPlanFilePath", "", time.Now()})
230225
return "plan"
231226
}
232227

@@ -235,11 +230,6 @@ func (m MockPlanPathProvider) LocalPlanFilePath() string {
235230
return "plan"
236231
}
237232

238-
func (m MockPlanPathProvider) StoredPlanFilePath() string {
239-
m.Commands = append(m.Commands, RunInfo{"StoredPlanFilePath", "", time.Now()})
240-
return "plan"
241-
}
242-
243233
func TestCorrectCommandExecutionWhenApplying(t *testing.T) {
244234

245235
commandRunner := &MockCommandRunner{}

‎cli/pkg/storage/plan_storage.go

Lines changed: 18 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ type GithubPlanStorage struct {
3232
ZipManager utils.Zipper
3333
}
3434

35-
func (psg *PlanStorageGcp) PlanExists(storedPlanFilePath string) (bool, error) {
35+
func (psg *PlanStorageGcp) PlanExists(artifactName string, storedPlanFilePath string) (bool, error) {
3636
obj := psg.Bucket.Object(storedPlanFilePath)
3737
_, err := obj.Attrs(psg.Context)
3838
if err != nil {
@@ -44,34 +44,20 @@ func (psg *PlanStorageGcp) PlanExists(storedPlanFilePath string) (bool, error) {
4444
return true, nil
4545
}
4646

47-
func (psg *PlanStorageGcp) StorePlan(localPlanFilePath string, storedPlanFilePath string) error {
48-
file, err := os.Open(localPlanFilePath)
49-
if err != nil {
50-
return fmt.Errorf("unable to open file: %v", err)
51-
}
52-
defer file.Close()
53-
54-
obj := psg.Bucket.Object(storedPlanFilePath)
55-
wc := obj.NewWriter(psg.Context)
56-
57-
if _, err = io.Copy(wc, file); err != nil {
58-
wc.Close()
59-
return fmt.Errorf("unable to write data to bucket: %v", err)
60-
}
47+
func (psg *PlanStorageGcp) StorePlanFile(fileContents []byte, artifactName string, fileName string) error {
48+
fullPath := fileName
49+
obj := psg.Bucket.Object(fullPath)
50+
writer := obj.NewWriter(context.Background())
51+
defer writer.Close()
6152

62-
if err := wc.Close(); err != nil {
63-
return fmt.Errorf("unable to close writer: %v", err)
53+
if _, err := writer.Write(fileContents); err != nil {
54+
log.Printf("Failed to write file to bucket: %v", err)
55+
return err
6456
}
65-
66-
return nil
67-
}
68-
69-
func (psg *PlanStorageGcp) StorePlanFile(fileContents []byte, artifactName string, fileName string) error {
70-
// TODO: implement me
7157
return nil
7258
}
7359

74-
func (psg *PlanStorageGcp) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) {
60+
func (psg *PlanStorageGcp) RetrievePlan(localPlanFilePath string, artifactName string, storedPlanFilePath string) (*string, error) {
7561
obj := psg.Bucket.Object(storedPlanFilePath)
7662
rc, err := obj.NewReader(psg.Context)
7763
if err != nil {
@@ -95,7 +81,7 @@ func (psg *PlanStorageGcp) RetrievePlan(localPlanFilePath string, storedPlanFile
9581
return &fileName, nil
9682
}
9783

98-
func (psg *PlanStorageGcp) DeleteStoredPlan(storedPlanFilePath string) error {
84+
func (psg *PlanStorageGcp) DeleteStoredPlan(artifactName string, storedPlanFilePath string) error {
9985
obj := psg.Bucket.Object(storedPlanFilePath)
10086
err := obj.Delete(psg.Context)
10187

@@ -105,12 +91,7 @@ func (psg *PlanStorageGcp) DeleteStoredPlan(storedPlanFilePath string) error {
10591
return nil
10692
}
10793

108-
func (gps *GithubPlanStorage) StorePlan(localPlanFilePath string, storedPlanFilePath string) error {
109-
_ = fmt.Sprintf("Skipping storing plan %s. It should be achieved using actions/upload-artifact@v3", localPlanFilePath)
110-
return nil
111-
}
112-
113-
func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, fileName string) error {
94+
func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, storedPlanFilePath string) error {
11495
actionsRuntimeToken := os.Getenv("ACTIONS_RUNTIME_TOKEN")
11596
actionsRuntimeURL := os.Getenv("ACTIONS_RUNTIME_URL")
11697
githubRunID := os.Getenv("GITHUB_RUN_ID")
@@ -139,7 +120,7 @@ func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName st
139120
resourceURL := createArtifactResponseMap["fileContainerResourceUrl"].(string)
140121

141122
// Upload Data
142-
uploadURL := fmt.Sprintf("%s?itemPath=%s/%s", resourceURL, artifactName, fileName)
123+
uploadURL := fmt.Sprintf("%s?itemPath=%s/%s", resourceURL, artifactName, storedPlanFilePath)
143124
uploadData := fileContents
144125
dataLen := len(uploadData)
145126
headers["Content-Type"] = "application/octet-stream"
@@ -191,8 +172,8 @@ func doRequest(method, url string, headers map[string]string, body []byte) (*htt
191172
return resp, nil
192173
}
193174

194-
func (gps *GithubPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) {
195-
plansFilename, err := gps.DownloadLatestPlans(storedPlanFilePath)
175+
func (gps *GithubPlanStorage) RetrievePlan(localPlanFilePath string, artifactName string, storedPlanFilePath string) (*string, error) {
176+
plansFilename, err := gps.DownloadLatestPlans(artifactName)
196177

197178
if err != nil {
198179
return nil, fmt.Errorf("error downloading plan: %v", err)
@@ -210,7 +191,7 @@ func (gps *GithubPlanStorage) RetrievePlan(localPlanFilePath string, storedPlanF
210191
return &plansFilename, nil
211192
}
212193

213-
func (gps *GithubPlanStorage) PlanExists(storedPlanFilePath string) (bool, error) {
194+
func (gps *GithubPlanStorage) PlanExists(artifactName string, storedPlanFilePath string) (bool, error) {
214195
artifacts, _, err := gps.Client.Actions.ListArtifacts(context.Background(), gps.Owner, gps.RepoName, &github.ListOptions{
215196
PerPage: 100,
216197
})
@@ -219,15 +200,15 @@ func (gps *GithubPlanStorage) PlanExists(storedPlanFilePath string) (bool, error
219200
return false, err
220201
}
221202

222-
latestPlans := getLatestArtifactWithName(artifacts.Artifacts, storedPlanFilePath)
203+
latestPlans := getLatestArtifactWithName(artifacts.Artifacts, artifactName)
223204

224205
if latestPlans == nil {
225206
return false, nil
226207
}
227208
return true, nil
228209
}
229210

230-
func (gps *GithubPlanStorage) DeleteStoredPlan(storedPlanFilePath string) error {
211+
func (gps *GithubPlanStorage) DeleteStoredPlan(artifactName string, storedPlanFilePath string) error {
231212
return nil
232213
}
233214

0 commit comments

Comments
 (0)