diff --git a/drivers/net/zynq_gem.c b/drivers/net/zynq_gem.c
index e3f9fd72d2f..f570ae9ee73 100644
--- a/drivers/net/zynq_gem.c
+++ b/drivers/net/zynq_gem.c
@@ -631,6 +631,7 @@ static int zynq_gem_init(struct udevice *dev)
 		break;
 	}
 	nwcfg = readl(&regs->nwcfg);
+	nwcfg &= ~(ZYNQ_GEM_NWCFG_SPEED100 | ZYNQ_GEM_NWCFG_SPEED1000);
 	nwcfg |= nwconfig;
 	if (nwcfg)
 		writel(nwcfg, &regs->nwcfg);