flink-conf.yaml

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# To use this config, FLINK_CONF_DIR env. var should be set to the parent dir.

# This is needed to prevent an "Insufficient number of network buffers"
# exceptions when running the merger on large input with many workers.
taskmanager.memory.network.max: 5gb

# This is needed to be able to process large resources, otherwise in JDBC
# mode we may get the following exception:
# "The record exceeds the maximum size of a sort buffer ..."
taskmanager.memory.managed.size: 2gb

# This is to make pipeline.run() non-blocking with FlinkRunner; unfortunately
# this is overwritten in `local` mode: https://stackoverflow.com/a/74416240
execution.attached: false