CLI sync S3 end to end integration test (#3001)

Co-authored-by: Nick Z <[email protected]>

.github/workflows/go.yml

@@ -80,6 +80,11 @@ jobs:
   integration-tests:
     name: Integration Tests
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write # required for requesting JWT for use with retrieving AWS creds
+      contents: read # required for actions/checkout
+    env:
+      AWS_REGION: "us-west-2"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -90,11 +95,21 @@ jobs:
           go-version-file: go.mod
           cache-dependency-path: go.sum
 
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ vars.INTEGRATION_TEST_AWS_ROLE_ARN }}
+          role-session-name: NeosyncCiIntegrationTests
+
       - name: Run Integration Tests
         run: |
           go test -race -timeout 1800s -coverprofile=integration-coverage.out -covermode=atomic -run TestIntegrationTestSuite ./...
         env:
           INTEGRATION_TESTS_ENABLED: 1
+          S3_INTEGRATION_TESTS_ENABLED: 1
+          TEST_S3_REGION: ${{ env.AWS_REGION }}
+          TEST_S3_BUCKET: ${{ vars.INTEGRATION_TEST_BUCKET_NAME }}
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5

backend/pkg/integration-test/integration-test-util.go

@@ -76,6 +76,36 @@ func CreateMysqlConnection(
 	return resp.Msg.GetConnection()
 }
 
+func CreateS3Connection(
+	ctx context.Context,
+	t *testing.T,
+	connclient mgmtv1alpha1connect.ConnectionServiceClient,
+	accountId, name string,
+	bucket string,
+	region *string,
+) *mgmtv1alpha1.Connection {
+	resp, err := connclient.CreateConnection(
+		ctx,
+		connect.NewRequest(&mgmtv1alpha1.CreateConnectionRequest{
+			AccountId: accountId,
+			Name:      name,
+			ConnectionConfig: &mgmtv1alpha1.ConnectionConfig{
+				Config: &mgmtv1alpha1.ConnectionConfig_AwsS3Config{
+					AwsS3Config: &mgmtv1alpha1.AwsS3ConnectionConfig{
+						Bucket:      bucket,
+						PathPrefix:  nil,
+						Region:      region,
+						Endpoint:    nil,
+						Credentials: nil,
+					},
+				},
+			},
+		}),
+	)
+	RequireNoErrResp(t, resp, err)
+	return resp.Msg.GetConnection()
+}
+
 func SetUser(ctx context.Context, t *testing.T, client mgmtv1alpha1connect.UserAccountServiceClient) string {
 	resp, err := client.SetUser(ctx, connect.NewRequest(&mgmtv1alpha1.SetUserRequest{}))
 	RequireNoErrResp(t, resp, err)

backend/pkg/integration-test/integration-test.go

@@ -40,6 +40,7 @@ import (
 	promapiv1mock "github.com/nucleuscloud/neosync/internal/mocks/github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/nucleuscloud/neosync/internal/testutil"
 	tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres"
+	"github.com/stretchr/testify/mock"
 )
 
 var (
@@ -399,6 +400,27 @@ func (s *NeosyncApiTestClient) Setup(ctx context.Context, t testing.TB) error {
 	return nil
 }
 
+func (s *NeosyncApiTestClient) MockTemporalForCreateJob(returnId string) {
+	s.Mocks.TemporalClientManager.
+		On(
+			"DoesAccountHaveNamespace", mock.Anything, mock.Anything, mock.Anything,
+		).
+		Return(true, nil).
+		Once()
+	s.Mocks.TemporalClientManager.
+		On(
+			"GetSyncJobTaskQueue", mock.Anything, mock.Anything, mock.Anything,
+		).
+		Return("sync-job", nil).
+		Once()
+	s.Mocks.TemporalClientManager.
+		On(
+			"CreateSchedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
+		).
+		Return(returnId, nil).
+		Once()
+}
+
 func (s *NeosyncApiTestClient) InitializeTest(ctx context.Context, t testing.TB) error {
 	err := neomigrate.Up(ctx, s.Pgcontainer.URL, s.migrationsDir, testutil.GetTestLogger(t))
 	if err != nil {

backend/services/mgmt/v1alpha1/integration_tests/jobs-service_integration_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/google/uuid"
 	mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1"
 	"github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
@@ -29,7 +28,7 @@ func (s *IntegrationTestSuite) Test_CreateJob_Ok() {
 	srcconn := s.createPostgresConnection(s.UnauthdClients.Connections, accountId, "source", "test")
 	destconn := s.createPostgresConnection(s.UnauthdClients.Connections, accountId, "dest", "test2")
 
-	s.mockTemporalForCreateJob("test-id")
+	s.MockTemporalForCreateJob("test-id")
 
 	resp, err := s.UnauthdClients.Jobs.CreateJob(s.ctx, connect.NewRequest(&mgmtv1alpha1.CreateJobRequest{
 		AccountId: accountId,
@@ -59,27 +58,6 @@ func (s *IntegrationTestSuite) Test_CreateJob_Ok() {
 	require.NotNil(s.T(), resp.Msg.GetJob())
 }
 
-func (s *IntegrationTestSuite) mockTemporalForCreateJob(returnId string) {
-	s.Mocks.TemporalClientManager.
-		On(
-			"DoesAccountHaveNamespace", mock.Anything, mock.Anything, mock.Anything,
-		).
-		Return(true, nil).
-		Once()
-	s.Mocks.TemporalClientManager.
-		On(
-			"GetSyncJobTaskQueue", mock.Anything, mock.Anything, mock.Anything,
-		).
-		Return("sync-job", nil).
-		Once()
-	s.Mocks.TemporalClientManager.
-		On(
-			"CreateSchedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
-		).
-		Return(returnId, nil).
-		Once()
-}
-
 func (s *IntegrationTestSuite) Test_JobService_JobHooks() {
 	t := s.T()
 	ctx := s.ctx
@@ -131,7 +109,7 @@ func (s *IntegrationTestSuite) Test_JobService_JobHooks() {
 		srcconn := s.createPostgresConnection(s.NeosyncCloudClients.GetConnectionClient(testAuthUserId), accountId, "source", "test")
 		destconn := s.createPostgresConnection(s.NeosyncCloudClients.GetConnectionClient(testAuthUserId), accountId, "dest", "test2")
 
-		s.mockTemporalForCreateJob("test-id")
+		s.MockTemporalForCreateJob("test-id")
 		jobResp, err := client.CreateJob(ctx, connect.NewRequest(&mgmtv1alpha1.CreateJobRequest{
 			JobName:   "cloud-testjob-1",
 			AccountId: accountId,