diff --git a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
index cbf79bb066213..dcd0f09361fa7 100644
--- a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
+++ b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
@@ -1051,6 +1051,8 @@ static int starfive_jh7110_pinconf_set(struct pinctrl_dev *pctldev,
 		value = readl_relaxed(pctl->padctl_base +
 				pin_reg->io_conf_reg);
 		value = value|(configs[i] & 0xFF);
+		if (!(configs[i] & JH7110_PADCFG_IE))
+			value &= ~JH7110_PADCFG_IE;
 		writel_relaxed(value, pctl->padctl_base +
 				pin_reg->io_conf_reg);
 	}