diff --git a/arch/mips/Makefile b/arch/mips/Makefile
index 641aa30b3638..d580d46f967b 100644
--- a/arch/mips/Makefile
+++ b/arch/mips/Makefile
@@ -63,7 +63,9 @@ cflags-y		+= -mabi=64
 ifdef CONFIG_BUILD_ELF64
 cflags-y		+= $(call cc-option,-mno-explicit-relocs)
 else
-cflags-y		+= $(call cc-option,-msym32)
+# -msym32 can not be used for modules since they are loaded into XKSEG
+CFLAGS_MODULE		+= $(call cc-option,-mno-explicit-relocs)
+CFLAGS_KERNEL		+= $(call cc-option,-msym32)
 endif
 endif