diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index f6e0560e898c350d435fb701dcfc82ccc992349d..4f98f7bfeaf2305b1b47caf39b6b1924262da860 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -1423,6 +1423,7 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 
 error_module_put:
 	module_put(d->driver->owner);
+	d->driver = NULL;
 error_put_device:
 	put_device(d);
 	if (ndev_owner != bus->owner)