Skip to content

Commit

Permalink
Fix default user name label
Browse files Browse the repository at this point in the history
  • Loading branch information
chi-yelp committed Feb 21, 2024
1 parent c8f0180 commit 536bfa2
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion service_configuration_lib/spark_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,14 +270,16 @@ def _get_k8s_spark_env(
service_account_name: Optional[str] = None,
include_self_managed_configs: bool = True,
k8s_server_address: Optional[str] = None,
user: Optional[str] = None,
) -> Dict[str, str]:
# RFC 1123: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
# technically only paasta instance can be longer than 63 chars. But we apply the normalization regardless.
# NOTE: this affects only k8s labels, not the pod names.
_paasta_cluster = utils.get_k8s_resource_name_limit_size_with_hash(paasta_cluster)
_paasta_service = utils.get_k8s_resource_name_limit_size_with_hash(paasta_service)
_paasta_instance = utils.get_k8s_resource_name_limit_size_with_hash(paasta_instance)
user = os.environ.get('USER', '_unspecified_')
if not user:
user = os.environ.get('USER', 'UNSPECIFIED')

spark_env = {
'spark.master': f'k8s://https://k8s.{paasta_cluster}.paasta:6443',
Expand Down Expand Up @@ -1017,6 +1019,7 @@ def get_spark_conf(
aws_region: Optional[str] = None,
service_account_name: Optional[str] = None,
force_spark_resource_configs: bool = True,
user: Optional[str] = None,
) -> Dict[str, str]:
"""Build spark config dict to run with spark on paasta
Expand Down Expand Up @@ -1129,6 +1132,7 @@ def get_spark_conf(
service_account_name=service_account_name,
include_self_managed_configs=not use_eks,
k8s_server_address=k8s_server_address,
user=user,
))
elif cluster_manager == 'local':
spark_conf.update(_get_local_spark_env(
Expand Down

0 comments on commit 536bfa2

Please sign in to comment.