Skip to content

Commit

Permalink
🔄 Merge remote-tracking branch 'origin' into release/upstream-v0.13.6-17
Browse files Browse the repository at this point in the history
  • Loading branch information
Danieloni1 committed Feb 26, 2024
2 parents 87f028d + 0aaf181 commit 6a3772d
Show file tree
Hide file tree
Showing 161 changed files with 2,841 additions and 6,446 deletions.
14 changes: 14 additions & 0 deletions .cloudbuild/dev.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '--build-arg', 'BUILDPLATFORM=${_BUILD_PLATFORM}', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA', '-f', 'Dockerfile', '.']
- name: gcr.io/cloud-builders/kubectl
args: ['-n', 'dendrite', 'set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA']
env:
- CLOUDSDK_CORE_PROJECT=globekeeper-development
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
- CLOUDSDK_CONTAINER_CLUSTER=synapse
substitutions:
_BUILD_PLATFORM: linux/amd64 # default
images:
- gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA
timeout: 480s
14 changes: 14 additions & 0 deletions .cloudbuild/prod.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '--build-arg', 'BUILDPLATFORM=${_BUILD_PLATFORM}', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME', '-f', 'Dockerfile', '.']
- name: gcr.io/cloud-builders/kubectl
args: ['set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME']
env:
- CLOUDSDK_CORE_PROJECT=globekeeper-production
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
- CLOUDSDK_CONTAINER_CLUSTER=synapse-production
substitutions:
_BUILD_PLATFORM: linux/amd64 # default
images:
- gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME
timeout: 480s
43 changes: 13 additions & 30 deletions .github/workflows/dendrite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3

# run go test with different go versions
# run go test with go 1.19
test:
timeout-minutes: 10
timeout-minutes: 15
name: Unit tests
runs-on: ubuntu-latest
# Service containers to run with `container-job`
Expand Down Expand Up @@ -130,7 +130,7 @@ jobs:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite

# build Dendrite for linux with different architectures and go versions
# build Dendrite for linux amd64 with go 1.18
build:
name: Build for Linux
timeout-minutes: 10
Expand All @@ -139,7 +139,7 @@ jobs:
fail-fast: false
matrix:
goos: ["linux"]
goarch: ["amd64", "386"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup go
Expand Down Expand Up @@ -199,7 +199,7 @@ jobs:
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [lint, test, build, build_windows]
needs: [lint, test, build]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
Expand Down Expand Up @@ -265,7 +265,7 @@ jobs:
uses: codecov/codecov-action@v3
with:
flags: unittests
fail_ci_if_error: true
fail_ci_if_error: false

# run database upgrade tests
upgrade_test:
Expand All @@ -285,9 +285,7 @@ jobs:
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests --sqlite --head .
run: ./dendrite-upgrade-tests .

# run database upgrade tests, skipping over one version
upgrade_test_direct:
Expand All @@ -307,9 +305,7 @@ jobs:
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
run: ./dendrite-upgrade-tests -direct -from HEAD-2 .

# run Sytest in different variations
sytest:
Expand All @@ -321,11 +317,6 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite native

- label: SQLite Cgo
cgo: 1

- label: PostgreSQL
postgres: postgres

Expand Down Expand Up @@ -364,13 +355,12 @@ jobs:
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/logs/results.tap
/logs/**/*.log*
/logs
# run Complement
complement:
Expand All @@ -382,12 +372,6 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite native
cgo: 0

- label: SQLite Cgo
cgo: 1

- label: PostgreSQL
postgres: Postgres
cgo: 0
Expand Down Expand Up @@ -429,7 +413,8 @@ jobs:
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
(wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
Expand All @@ -453,8 +438,6 @@ jobs:
needs:
[
initial-tests-done,
upgrade_test,
upgrade_test_direct,
sytest,
complement,
integration
Expand All @@ -477,4 +460,4 @@ jobs:
needs: [integration-tests-done]
uses: matrix-org/dendrite/.github/workflows/docker.yml@main
secrets:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
97 changes: 49 additions & 48 deletions .github/workflows/k8s.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,52 +40,53 @@ jobs:
run: ct lint --config helm/ct.yaml

# only bother to run if lint step reports a change to the helm chart
install:
needs:
- lint
if: ${{ needs.lint.outputs.changed == 'true' }}
name: Install Helm charts
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Install Kubernetes tools
uses: yokawasa/[email protected]
with:
setup-tools: |
helmv3
helm: "3.10.3"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Set up chart-testing
uses: helm/[email protected]
- name: Create k3d cluster
uses: nolar/setup-k3d-k3s@v1
with:
version: v1.28
- name: Remove node taints
run: |
kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
- name: Run chart-testing (install)
run: ct install --config helm/ct.yaml
# GlobeKeeper: Skipping this as it not needed for now and it is failing due to env.
# install:
# needs:
# - lint
# if: ${{ needs.lint.outputs.changed == 'true' }}
# name: Install Helm charts
# runs-on: ubuntu-latest
# steps:
# - name: Checkout
# uses: actions/checkout@v3
# with:
# fetch-depth: 0
# ref: ${{ inputs.checkoutCommit }}
# - name: Install Kubernetes tools
# uses: yokawasa/[email protected]
# with:
# setup-tools: |
# helmv3
# helm: "3.10.3"
# - uses: actions/setup-python@v4
# with:
# python-version: "3.10"
# - name: Set up chart-testing
# uses: helm/[email protected]
# - name: Create k3d cluster
# uses: nolar/setup-k3d-k3s@v1
# with:
# version: v1.21
# - name: Remove node taints
# run: |
# kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
# - name: Run chart-testing (install)
# run: ct install --config helm/ct.yaml

# Install the chart using helm directly and test with create-account
- name: Install chart
run: |
helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
- name: Wait for Postgres and Dendrite to be up
run: |
kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
kubectl get pods -A
kubectl get services
kubectl get ingress
kubectl logs -l app.kubernetes.io/name=dendrite
- name: Run create account
run: |
podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword
# # Install the chart using helm directly and test with create-account
# - name: Install chart
# run: |
# helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
# - name: Wait for Postgres and Dendrite to be up
# run: |
# kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
# kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
# kubectl get pods -A
# kubectl get services
# kubectl get ingress
# kubectl logs -l app.kubernetes.io/name=dendrite
# - name: Run create account
# run: |
# podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
# kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword
10 changes: 8 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

# Hidden files
.*
!.vscode
!.cloudbuild

# Allow GitHub config
!.github
Expand Down Expand Up @@ -74,7 +76,11 @@ complement/
docs/_site

media_store/
build

# golang workspaces
go.work*
go.work*

__debug_bin*

cmd/dendrite-monolith-server/dendrite-monolith-server
build
16 changes: 16 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/cmd/dendrite",
"args": [
"-really-enable-open-registration",
"-config",
"../../../adminas/.ci/config/dendrite-local/dendrite.yaml"
],
}
]
}
9 changes: 9 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"go.lintTool": "golangci-lint",
"go.testEnvVars": {
"POSTGRES_HOST": "localhost",
"POSTGRES_USER": "postgres",
"POSTGRES_PASSWORD": "foobar",
"POSTGRES_DB": "postgres"
}
}
32 changes: 19 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,32 @@
# base installs required dependencies and runs go mod download to cache dependencies
#
# Pinned to alpine3.18 until https://github.com/mattn/go-sqlite3/issues/1164 is solved
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine3.18 AS base
ARG BUILDPLATFORM=${BUILDPLATFORM}
FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 AS base
RUN apk --update --no-cache add bash build-base curl git

#
# build creates all needed binaries
#
FROM --platform=${BUILDPLATFORM} base AS build
FROM --platform=$BUILDPLATFORM base AS build
WORKDIR /src
ARG TARGETOS
ARG TARGETARCH
RUN --mount=target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
USERARCH=`go env GOARCH` \
GOARCH="$TARGETARCH" \
GOOS="linux" \
CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
go build -v -trimpath -o /out/ ./cmd/...

ARG FLAGS

# Not mounting volumes using the --mount flag to avoid requiring BuildKit which is not easily supported using cloudbuild.
COPY . .
RUN mkdir -p /root/.cache/go-build && \
mkdir -p /go/pkg/mod
VOLUME /root/.cache/go-build
VOLUME /go/pkg/mod

# Run the build command in multiple RUN commands
RUN USERARCH=`go env GOARCH`
RUN GOARCH="$TARGETARCH"
RUN GOOS="linux"
RUN CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0")
RUN go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...

#
# Builds the Dendrite image containing all required binaries
Expand All @@ -45,5 +52,4 @@ VOLUME /etc/dendrite
WORKDIR /etc/dendrite

ENTRYPOINT ["/usr/bin/dendrite"]
EXPOSE 8008 8448

EXPOSE 8008 8448
Loading

0 comments on commit 6a3772d

Please sign in to comment.