diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index e6755444b2f4f..26be0b76315ff 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -46,17 +46,22 @@ runs: with: use-uv: ${{ inputs.use-uv }} id: breeze + - name: Check free space + run: df -H + shell: bash + - name: Make /mnt/ directory writeable + run: sudo chown -R ${USER} /mnt + shell: bash - name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }} - path: "/tmp/" + path: "/mnt/" - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" env: PLATFORM: ${{ inputs.platform }} PYTHON: ${{ inputs.python }} IMAGE_TYPE: ${{ inputs.image-type }} run: > - breeze ${IMAGE_TYPE}-image load - --platform ${PLATFORM} --python ${PYTHON} + breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt" shell: bash diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml index 3dde30033aa15..ecae9f802c966 100644 --- a/.github/actions/prepare_single_ci_image/action.yml +++ b/.github/actions/prepare_single_ci_image/action.yml @@ -35,16 +35,22 @@ inputs: runs: using: "composite" steps: + - name: Check free space + run: df -H + shell: bash + - name: Make /mnt/ directory writeable + run: sudo chown -R ${USER} /mnt + shell: bash - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}" uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: key: ci-image-save-${{ inputs.platform }}-${{ inputs.python }} - path: "/tmp/" + path: "/mnt/" if: contains(inputs.python-versions-list-as-string, inputs.python) - name: "Load CI image ${{ inputs.platform }}:${{ inputs.python }}" env: PLATFORM: ${{ inputs.platform }} PYTHON: ${{ inputs.python }} - run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" + run: breeze ci-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt/" shell: bash if: contains(inputs.python-versions-list-as-string, inputs.python) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 56cee1697620c..a6b7bdafcb5af 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -84,6 +84,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv to build the image (true/false)" required: true type: string +permissions: + contents: read jobs: # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the # tests to complete - it is run very early in the build process for "main" merges in order to refresh diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index bca5e3a592713..7b55121571471 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -60,6 +60,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv" required: true type: string +permissions: + contents: read jobs: prod-image-extra-checks-main: name: PROD image extra checks (main) diff --git a/.github/workflows/automatic-backport.yml b/.github/workflows/automatic-backport.yml index b5b22b7491a9c..4c72401a5d317 100644 --- a/.github/workflows/automatic-backport.yml +++ b/.github/workflows/automatic-backport.yml @@ -21,7 +21,8 @@ on: # yamllint disable-line rule:truthy push: branches: - main - +permissions: + contents: read jobs: get-pr-info: name: "Get PR information" diff --git a/.github/workflows/backport-cli.yml b/.github/workflows/backport-cli.yml index 3706cd65bb01e..53243006137a6 100644 --- a/.github/workflows/backport-cli.yml +++ b/.github/workflows/backport-cli.yml @@ -41,6 +41,9 @@ on: # yamllint disable-line rule:truthy type: string permissions: + # Those permissions are only active for workflow dispatch (only committers can trigger it) and workflow call + # Which is triggered automatically by "automatic-backport" push workflow (only when merging by committer) + # Branch protection prevents from pushing to the "code" branches contents: write pull-requests: write jobs: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 353f65d9a6c9c..5cb71cb7f5c1f 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -60,6 +60,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv in the image" required: true type: string +permissions: + contents: read jobs: run-breeze-tests: timeout-minutes: 10 diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index d15c297d82a00..9283dc06b936f 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -96,6 +96,8 @@ on: # yamllint disable-line rule:truthy description: "Disable airflow repo cache read from main." required: true type: string +permissions: + contents: read jobs: build-ci-images: strategy: @@ -173,16 +175,22 @@ jobs: PUSH: ${{ inputs.push-image }} VERBOSE: "true" PLATFORM: ${{ inputs.platform }} + - name: Check free space + run: df -H + shell: bash + - name: Make /mnt/ directory writeable + run: sudo chown -R ${USER} /mnt + shell: bash - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" env: PLATFORM: ${{ inputs.platform }} - run: breeze ci-image save --platform "${PLATFORM}" + run: breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt" if: inputs.upload-image-artifact == 'true' - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: key: ci-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} - path: "/tmp/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" + path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 21c857e7bd710..c6784042cec2c 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -108,7 +108,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv to build the image (true/false)" required: true type: string - +permissions: + contents: read jobs: install-pre-commit: timeout-minutes: 5 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index be0d690799550..1fcf81a84fd5b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,46 +33,12 @@ concurrency: cancel-in-progress: true jobs: - selective-checks: - name: Selective checks - runs-on: ["ubuntu-22.04"] - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - outputs: - needs-python-scans: ${{ steps.selective-checks.outputs.needs-python-scans }} - needs-javascript-scans: ${{ steps.selective-checks.outputs.needs-javascript-scans }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "true" - - name: "Get information about the Workflow" - id: source-run-info - run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} - env: - SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" - - name: Selective checks - id: selective-checks - env: - PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" - COMMIT_REF: "${{ github.sha }}" - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - analyze: name: Analyze runs-on: ["ubuntu-22.04"] - needs: [selective-checks] strategy: fail-fast: false matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ['python', 'javascript', 'actions'] permissions: actions: read @@ -84,37 +50,14 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - if: | - matrix.language == 'actions' || - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - if: | - matrix.language == 'actions' || - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 - if: | - matrix.language == 'actions' || - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 - if: | - matrix.language == 'actions' || - matrix.language == 'python' && needs.selective-checks.outputs.needs-python-scans == 'true' || - matrix.language == 'javascript' && needs.selective-checks.outputs.needs-javascript-scans == 'true' diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 1d0ac8a600c1d..ac13089caf656 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -76,6 +76,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to debug resources or not (true/false)" required: true type: string +permissions: + contents: read jobs: update-constraints: runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 9dc300c61c0a1..1b4aa19cbe595 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -40,6 +40,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uvloop (true/false)" required: true type: string +permissions: + contents: read jobs: tests-helm: timeout-minutes: 80 diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index f992b726e30df..7c3916d9d19c9 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -64,6 +64,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv" required: true type: string +permissions: + contents: read jobs: tests-core-integration: timeout-minutes: 130 diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 6f867af65e9cd..40f73e3c59c66 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -48,6 +48,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to debug resources" required: true type: string +permissions: + contents: read jobs: tests-kubernetes: timeout-minutes: 60 diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml index 73e58a0193711..46cb294d7a5b9 100644 --- a/.github/workflows/news-fragment.yml +++ b/.github/workflows/news-fragment.yml @@ -21,7 +21,8 @@ name: CI on: # yamllint disable-line rule:truthy pull_request: types: [labeled, unlabeled, opened, reopened, synchronize] - +permissions: + contents: read jobs: check-news-fragment: name: Check News Fragment diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index d90d1910f9336..5784c7c58ba60 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -116,8 +116,9 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a prod-image build (true/false)" required: true type: string +permissions: + contents: read jobs: - build-prod-packages: name: "Build Airflow and provider packages" timeout-minutes: 10 @@ -282,17 +283,23 @@ jobs: if: inputs.build-provider-packages != 'true' - name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" run: breeze prod-image verify + - name: Check free space + run: df -H + shell: bash + - name: Make /mnt/ directory writeable + run: sudo chown -R ${USER} /mnt + shell: bash - name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" env: PLATFORM: ${{ inputs.platform }} run: > - breeze prod-image save --platform "${PLATFORM}" + breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt" if: inputs.upload-image-artifact == 'true' - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: apache/infrastructure-actions/stash/save@c94b890bbedc2fc61466d28e6bd9966bc6c6643c with: key: prod-image-save-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} - path: "/tmp/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" + path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" if-no-files-found: 'error' retention-days: '2' if: inputs.upload-image-artifact == 'true' diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index f5a4b771436a7..56fa4b2b1a28d 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -64,6 +64,8 @@ on: # yamllint disable-line rule:truthy description: "Disable airflow repo cache read from main." required: true type: string +permissions: + contents: read jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index b1c9d12754206..7698fc88e5388 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -88,6 +88,9 @@ jobs: # instead of an array of strings. # yamllint disable-line rule:line-length runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: @@ -161,6 +164,9 @@ jobs: # instead of an array of strings. # yamllint disable-line rule:line-length runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 1c24e659d0979..e67d59ee08d37 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -116,6 +116,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv" required: true type: string +permissions: + contents: read jobs: tests: timeout-minutes: 120 diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 36ccbf871cca9..8507294e535c6 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -80,7 +80,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv or not (true/false)" required: true type: string - +permissions: + contents: read jobs: tests-min-sqlalchemy: name: "Min SQLAlchemy test" diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml index 501e880fd3be0..b8ecf0eb798c6 100644 --- a/.github/workflows/task-sdk-tests.yml +++ b/.github/workflows/task-sdk-tests.yml @@ -44,7 +44,8 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a canary run (true/false)" required: true type: string - +permissions: + contents: read jobs: task-sdk-tests: timeout-minutes: 80 diff --git a/.github/workflows/test-provider-packages.yml b/.github/workflows/test-provider-packages.yml index 877ff1f1b23c9..b0912fa6dfe37 100644 --- a/.github/workflows/test-provider-packages.yml +++ b/.github/workflows/test-provider-packages.yml @@ -62,6 +62,8 @@ on: # yamllint disable-line rule:truthy description: "Whether to use uv" required: true type: string +permissions: + contents: read jobs: prepare-install-verify-provider-packages: timeout-minutes: 80 diff --git a/airflow/api/common/mark_tasks.py b/airflow/api/common/mark_tasks.py index 7c55c3527d154..3bcfdeaa9657d 100644 --- a/airflow/api/common/mark_tasks.py +++ b/airflow/api/common/mark_tasks.py @@ -239,15 +239,18 @@ def set_dag_run_state_to_success( return [] if not run_id: raise ValueError(f"Invalid dag_run_id: {run_id}") + + # Mark all task instances of the dag run to success - except for teardown as they need to complete work. + normal_tasks = [task for task in dag.tasks if not task.is_teardown] + # Mark the dag run to success. - if commit: + if commit and len(normal_tasks) == len(dag.tasks): _set_dag_run_state(dag.dag_id, run_id, DagRunState.SUCCESS, session) - # Mark all task instances of the dag run to success. - for task in dag.tasks: + for task in normal_tasks: task.dag = dag return set_state( - tasks=dag.tasks, + tasks=normal_tasks, run_id=run_id, state=TaskInstanceState.SUCCESS, commit=commit, @@ -280,10 +283,6 @@ def set_dag_run_state_to_failed( if not run_id: raise ValueError(f"Invalid dag_run_id: {run_id}") - # Mark the dag run to failed. - if commit: - _set_dag_run_state(dag.dag_id, run_id, DagRunState.FAILED, session) - running_states = ( TaskInstanceState.RUNNING, TaskInstanceState.DEFERRED, @@ -292,25 +291,26 @@ def set_dag_run_state_to_failed( # Mark only RUNNING task instances. task_ids = [task.task_id for task in dag.tasks] - tis = session.scalars( + running_tis: list[TaskInstance] = session.scalars( select(TaskInstance).where( TaskInstance.dag_id == dag.dag_id, TaskInstance.run_id == run_id, TaskInstance.task_id.in_(task_ids), TaskInstance.state.in_(running_states), ) - ) + ).all() - task_ids_of_running_tis = [task_instance.task_id for task_instance in tis] + # Do not kill teardown tasks + task_ids_of_running_tis = [ti.task_id for ti in running_tis if not dag.task_dict[ti.task_id].is_teardown] - tasks = [] + running_tasks = [] for task in dag.tasks: if task.task_id in task_ids_of_running_tis: task.dag = dag - tasks.append(task) + running_tasks.append(task) # Mark non-finished tasks as SKIPPED. - tis = session.scalars( + pending_tis: list[TaskInstance] = session.scalars( select(TaskInstance).filter( TaskInstance.dag_id == dag.dag_id, TaskInstance.run_id == run_id, @@ -324,12 +324,19 @@ def set_dag_run_state_to_failed( ) ).all() + # Do not skip teardown tasks + pending_normal_tis = [ti for ti in pending_tis if not dag.task_dict[ti.task_id].is_teardown] + if commit: - for ti in tis: + for ti in pending_normal_tis: ti.set_state(TaskInstanceState.SKIPPED) - return tis + set_state( - tasks=tasks, + # Mark the dag run to failed if there is no pending teardown (else this would not be scheduled later). + if not any(dag.task_dict[ti.task_id].is_teardown for ti in (running_tis + pending_tis)): + _set_dag_run_state(dag.dag_id, run_id, DagRunState.FAILED, session) + + return pending_normal_tis + set_state( + tasks=running_tasks, run_id=run_id, state=TaskInstanceState.FAILED, commit=commit, diff --git a/airflow/api_fastapi/core_api/datamodels/task_instances.py b/airflow/api_fastapi/core_api/datamodels/task_instances.py index 07a1f77ad5421..3d191b96828a8 100644 --- a/airflow/api_fastapi/core_api/datamodels/task_instances.py +++ b/airflow/api_fastapi/core_api/datamodels/task_instances.py @@ -220,18 +220,3 @@ def validate_new_state(cls, ns: str | None) -> str: if ns not in valid_states: raise ValueError(f"'{ns}' is not one of {valid_states}") return ns - - -class TaskInstanceReferenceResponse(BaseModel): - """Task Instance Reference serializer for responses.""" - - task_id: str - dag_run_id: str = Field(validation_alias="run_id") - dag_id: str - - -class TaskInstanceReferenceCollectionResponse(BaseModel): - """Task Instance Reference collection serializer for responses.""" - - task_instances: list[TaskInstanceReferenceResponse] - total_entries: int diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index b5decab386769..92cd7219a6770 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -5555,7 +5555,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/TaskInstanceReferenceCollectionResponse' + $ref: '#/components/schemas/TaskInstanceCollectionResponse' '401': content: application/json: @@ -9139,40 +9139,6 @@ components: - executor_config title: TaskInstanceHistoryResponse description: TaskInstanceHistory serializer for responses. - TaskInstanceReferenceCollectionResponse: - properties: - task_instances: - items: - $ref: '#/components/schemas/TaskInstanceReferenceResponse' - type: array - title: Task Instances - total_entries: - type: integer - title: Total Entries - type: object - required: - - task_instances - - total_entries - title: TaskInstanceReferenceCollectionResponse - description: Task Instance Reference collection serializer for responses. - TaskInstanceReferenceResponse: - properties: - task_id: - type: string - title: Task Id - dag_run_id: - type: string - title: Dag Run Id - dag_id: - type: string - title: Dag Id - type: object - required: - - task_id - - dag_run_id - - dag_id - title: TaskInstanceReferenceResponse - description: Task Instance Reference serializer for responses. TaskInstanceResponse: properties: id: diff --git a/airflow/api_fastapi/core_api/routes/public/task_instances.py b/airflow/api_fastapi/core_api/routes/public/task_instances.py index 9eaf191374746..91e4cb0ddcc2a 100644 --- a/airflow/api_fastapi/core_api/routes/public/task_instances.py +++ b/airflow/api_fastapi/core_api/routes/public/task_instances.py @@ -55,8 +55,6 @@ TaskInstanceCollectionResponse, TaskInstanceHistoryCollectionResponse, TaskInstanceHistoryResponse, - TaskInstanceReferenceCollectionResponse, - TaskInstanceReferenceResponse, TaskInstanceResponse, TaskInstancesBatchBody, ) @@ -550,7 +548,7 @@ def post_clear_task_instances( request: Request, body: ClearTaskInstancesBody, session: SessionDep, -) -> TaskInstanceReferenceCollectionResponse: +) -> TaskInstanceCollectionResponse: """Clear task instances.""" dag = request.app.state.dag_bag.get_dag(dag_id) if not dag: @@ -597,6 +595,7 @@ def post_clear_task_instances( dry_run=True, task_ids=task_ids, dag_bag=request.app.state.dag_bag, + session=session, **body.model_dump( include={ "start_date", @@ -615,9 +614,9 @@ def post_clear_task_instances( DagRunState.QUEUED if reset_dag_runs else False, ) - return TaskInstanceReferenceCollectionResponse( + return TaskInstanceCollectionResponse( task_instances=[ - TaskInstanceReferenceResponse.model_validate( + TaskInstanceResponse.model_validate( ti, from_attributes=True, ) diff --git a/airflow/dag_processing/bundles/manager.py b/airflow/dag_processing/bundles/manager.py index 0a17ab3f8c5f5..1ae751f8d3304 100644 --- a/airflow/dag_processing/bundles/manager.py +++ b/airflow/dag_processing/bundles/manager.py @@ -64,6 +64,29 @@ def parse_config(self) -> None: "Bundle config is not a list. Check config value" " for section `dag_bundles` and key `backends`." ) + + if any(b["name"] == "example_dags" for b in backends): + raise AirflowConfigException( + "Bundle name 'example_dags' is a reserved name. Please choose another name for your bundle." + " Example DAGs can be enabled with the '[core] load_examples' config." + ) + + # example dags! + if conf.getboolean("core", "LOAD_EXAMPLES"): + from airflow import example_dags + + example_dag_folder = next(iter(example_dags.__path__)) + backends.append( + { + "name": "example_dags", + "classpath": "airflow.dag_processing.bundles.local.LocalDagBundle", + "kwargs": { + "local_folder": example_dag_folder, + "refresh_interval": conf.getint("scheduler", "dag_dir_list_interval"), + }, + } + ) + seen = set() for cfg in backends: name = cfg["name"] diff --git a/airflow/dag_processing/processor.py b/airflow/dag_processing/processor.py index 8d48b5ab6aeb3..f5623418c917d 100644 --- a/airflow/dag_processing/processor.py +++ b/airflow/dag_processing/processor.py @@ -19,7 +19,6 @@ import os import sys import traceback -from collections.abc import Generator from typing import TYPE_CHECKING, Annotated, Callable, Literal, Union import attrs @@ -181,7 +180,7 @@ class DagFileParsingResult(BaseModel): ] -@attrs.define() +@attrs.define(kw_only=True) class DagFileProcessorProcess(WatchedSubprocess): """ Parses dags with Task SDK API. @@ -194,6 +193,7 @@ class DagFileProcessorProcess(WatchedSubprocess): """ parsing_result: DagFileParsingResult | None = None + decoder: TypeAdapter[ToParent] = TypeAdapter[ToParent](ToParent) @classmethod def start( # type: ignore[override] @@ -203,39 +203,30 @@ def start( # type: ignore[override] target: Callable[[], None] = _parse_file_entrypoint, **kwargs, ) -> Self: - return super().start(path, callbacks, target=target, client=None, **kwargs) # type:ignore[arg-type] + proc: Self = super().start(target=target, **kwargs) + proc._on_child_started(callbacks, path) + return proc - def _on_child_started( # type: ignore[override] - self, callbacks: list[CallbackRequest], path: str | os.PathLike[str], child_comms_fd: int - ) -> None: + def _on_child_started(self, callbacks: list[CallbackRequest], path: str | os.PathLike[str]) -> None: msg = DagFileParseRequest( file=os.fspath(path), - requests_fd=child_comms_fd, + requests_fd=self._requests_fd, callback_requests=callbacks, ) self.stdin.write(msg.model_dump_json().encode() + b"\n") - def handle_requests(self, log: FilteringBoundLogger) -> Generator[None, bytes, None]: - # TODO: Make decoder an instance variable, then this can live in the base class - decoder = TypeAdapter[ToParent](ToParent) - - while True: - line = yield - - try: - msg = decoder.validate_json(line) - except Exception: - log.exception("Unable to decode message", line=line) - continue - - self._handle_request(msg, log) # type: ignore[arg-type] - def _handle_request(self, msg: ToParent, log: FilteringBoundLogger) -> None: # type: ignore[override] + # TODO: GetVariable etc -- parsing a dag can run top level code that asks for an Airflow Variable + resp = None if isinstance(msg, DagFileParsingResult): self.parsing_result = msg return - # GetVariable etc -- parsing a dag can run top level code that asks for an Airflow Variable - super()._handle_request(msg, log) + else: + log.error("Unhandled request", msg=msg) + return + + if resp: + self.stdin.write(resp + b"\n") @property def is_ready(self) -> bool: diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 6e4455ea021cf..801387eebb811 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -3016,7 +3016,7 @@ export const useTaskInstanceServiceGetTaskInstancesBatch = < * @param data The data for the request. * @param data.dagId * @param data.requestBody - * @returns TaskInstanceReferenceCollectionResponse Successful Response + * @returns TaskInstanceCollectionResponse Successful Response * @throws ApiError */ export const useTaskInstanceServicePostClearTaskInstances = < diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 0c8876c93a47c..443aef97a332b 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -4337,47 +4337,6 @@ export const $TaskInstanceHistoryResponse = { description: "TaskInstanceHistory serializer for responses.", } as const; -export const $TaskInstanceReferenceCollectionResponse = { - properties: { - task_instances: { - items: { - $ref: "#/components/schemas/TaskInstanceReferenceResponse", - }, - type: "array", - title: "Task Instances", - }, - total_entries: { - type: "integer", - title: "Total Entries", - }, - }, - type: "object", - required: ["task_instances", "total_entries"], - title: "TaskInstanceReferenceCollectionResponse", - description: "Task Instance Reference collection serializer for responses.", -} as const; - -export const $TaskInstanceReferenceResponse = { - properties: { - task_id: { - type: "string", - title: "Task Id", - }, - dag_run_id: { - type: "string", - title: "Dag Run Id", - }, - dag_id: { - type: "string", - title: "Dag Id", - }, - }, - type: "object", - required: ["task_id", "dag_run_id", "dag_id"], - title: "TaskInstanceReferenceResponse", - description: "Task Instance Reference serializer for responses.", -} as const; - export const $TaskInstanceResponse = { properties: { id: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index b5836e76ca7e3..c2a5a3ce6ece1 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -2421,7 +2421,7 @@ export class TaskInstanceService { * @param data The data for the request. * @param data.dagId * @param data.requestBody - * @returns TaskInstanceReferenceCollectionResponse Successful Response + * @returns TaskInstanceCollectionResponse Successful Response * @throws ApiError */ public static postClearTaskInstances( diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 568d1e5245467..f2f709b02db0a 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1065,23 +1065,6 @@ export type TaskInstanceHistoryResponse = { executor_config: string; }; -/** - * Task Instance Reference collection serializer for responses. - */ -export type TaskInstanceReferenceCollectionResponse = { - task_instances: Array; - total_entries: number; -}; - -/** - * Task Instance Reference serializer for responses. - */ -export type TaskInstanceReferenceResponse = { - task_id: string; - dag_run_id: string; - dag_id: string; -}; - /** * TaskInstance serializer for responses. */ @@ -2005,7 +1988,7 @@ export type PostClearTaskInstancesData = { requestBody: ClearTaskInstancesBody; }; -export type PostClearTaskInstancesResponse = TaskInstanceReferenceCollectionResponse; +export type PostClearTaskInstancesResponse = TaskInstanceCollectionResponse; export type GetLogData = { accept?: "application/json" | "text/plain" | "*/*"; @@ -4073,7 +4056,7 @@ export type $OpenApiTs = { /** * Successful Response */ - 200: TaskInstanceReferenceCollectionResponse; + 200: TaskInstanceCollectionResponse; /** * Unauthorized */ diff --git a/airflow/ui/src/components/DataTable/DataTable.tsx b/airflow/ui/src/components/DataTable/DataTable.tsx index 3a2086a040e96..4f02b57cb8a6e 100644 --- a/airflow/ui/src/components/DataTable/DataTable.tsx +++ b/airflow/ui/src/components/DataTable/DataTable.tsx @@ -115,33 +115,34 @@ export const DataTable = ({ const { rows } = table.getRowModel(); const display = displayMode === "card" && Boolean(cardDef) ? "card" : "table"; + const hasRows = rows.length > 0; return ( <> {errorMessage} - {display === "table" && } - {display === "card" && cardDef !== undefined && ( + {hasRows && display === "table" ? : undefined} + {hasRows && display === "card" && cardDef !== undefined ? ( - )} - {!Boolean(isLoading) && !rows.length && ( - {noRowsMessage ?? `No ${modelName}s found.`} - )} - table.setPageIndex(page.page - 1)} - page={table.getState().pagination.pageIndex + 1} - pageSize={table.getState().pagination.pageSize} - siblingCount={1} - > - - - - - - + ) : undefined} + {!hasRows && !Boolean(isLoading) && {noRowsMessage ?? `No ${modelName}s found.`}} + {hasRows ? ( + table.setPageIndex(page.page - 1)} + page={table.getState().pagination.pageIndex + 1} + pageSize={table.getState().pagination.pageSize} + siblingCount={1} + > + + + + + + + ) : undefined} ); }; diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index 0651820ce78d7..9a916da184c13 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -95,7 +95,7 @@ def wrapper(*args, **kwargs): user_display = get_auth_manager().get_user_display_name() isAPIRequest = request.blueprint == "/api/v1" - hasJsonBody = request.headers.get("content-type") == "application/json" and request.json + hasJsonBody = "application/json" in request.headers.get("content-type", "") and request.json fields_skip_logging = { "csrf_token", diff --git a/chart/dockerfiles/pgbouncer/build_and_push.sh b/chart/dockerfiles/pgbouncer/build_and_push.sh index cede5ab8b0ab3..e4345cc44d018 100755 --- a/chart/dockerfiles/pgbouncer/build_and_push.sh +++ b/chart/dockerfiles/pgbouncer/build_and_push.sh @@ -22,13 +22,13 @@ readonly DOCKERHUB_USER DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} readonly DOCKERHUB_REPO -PGBOUNCER_VERSION="1.22.1" +PGBOUNCER_VERSION="1.24.0" readonly PGBOUNCER_VERSION -PGBOUNCER_SHA256="2b018aa6ce7f592c9892bb9e0fd90262484eb73937fd2af929770a45373ba215" +PGBOUNCER_SHA256="e76adf941a3191a416e223c0b2cdbf73159eef80a2a32314af6fbd82e41a1d41" readonly PGBOUNCER_SHA256 -AIRFLOW_PGBOUNCER_VERSION="2024.09.19" +AIRFLOW_PGBOUNCER_VERSION="2025.01.10" readonly AIRFLOW_PGBOUNCER_VERSION COMMIT_SHA=$(git rev-parse HEAD) diff --git a/chart/values.schema.json b/chart/values.schema.json index fedae1557ab6b..e8c1d1f2cfe54 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -887,7 +887,7 @@ "tag": { "description": "The PgBouncer image tag.", "type": "string", - "default": "airflow-pgbouncer-2024.09.19-1.22.1" + "default": "airflow-pgbouncer-2025.01.10-1.24.0" }, "pullPolicy": { "description": "The PgBouncer image pull policy.", diff --git a/chart/values.yaml b/chart/values.yaml index c0764fb54a0d7..73e00ed4281d8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -115,7 +115,7 @@ images: pullPolicy: IfNotPresent pgbouncer: repository: apache/airflow - tag: airflow-pgbouncer-2024.09.19-1.22.1 + tag: airflow-pgbouncer-2025.01.10-1.24.0 pullPolicy: IfNotPresent pgbouncerExporter: repository: apache/airflow diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 261e463589c05..3fa033b4a5952 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -892,7 +892,7 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo # and finally open a PR ``` -The `--run-in-parallell` switch allows to speed up SBOM generation significantly, but it might take a lot +The `--run-in-parallel` switch allows to speed up SBOM generation significantly, but it might take a lot of memory - if you are running into memory issues you can limit parallelism by setting `--parallelism N` where N is a number of parallel `cdxgen` servers that should be started. diff --git a/dev/breeze/doc/images/output_ci-image_load.svg b/dev/breeze/doc/images/output_ci-image_load.svg index 962b9cb52c9ed..cb7036af1c946 100644 --- a/dev/breeze/doc/images/output_ci-image_load.svg +++ b/dev/breeze/doc/images/output_ci-image_load.svg @@ -1,4 +1,4 @@ - +