diff --git a/arch/arm/mach-orion5x/Kconfig b/arch/arm/mach-orion5x/Kconfig
index 2cb2f06c20f5..14f2cae4109c 100644
--- a/arch/arm/mach-orion5x/Kconfig
+++ b/arch/arm/mach-orion5x/Kconfig
@@ -33,7 +33,6 @@ config MACH_KUROBOX_PRO
 config MACH_DNS323
 	bool "D-Link DNS-323"
 	select I2C_BOARDINFO
-	select PHYLIB
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  D-Link DNS-323 platform.
diff --git a/arch/arm/mach-orion5x/dns323-setup.c b/arch/arm/mach-orion5x/dns323-setup.c
index 70974732cbf0..56edeab17b68 100644
--- a/arch/arm/mach-orion5x/dns323-setup.c
+++ b/arch/arm/mach-orion5x/dns323-setup.c
@@ -642,6 +642,8 @@ static void __init dns323_init(void)
 		platform_device_register_simple("dns323c-fan", 0, NULL, 0);
 
 		/* Register fixup for the PHY LEDs */
+		if (!IS_BUILTIN(CONFIG_PHYLIB))
+			break;
 		phy_register_fixup_for_uid(MARVELL_PHY_ID_88E1118,
 					   MARVELL_PHY_ID_MASK,
 					   dns323c_phy_fixup);