diff --git a/meta/recipes-graphics/xorg-xserver/xserver-xorg/0001-compiler.h-Do-not-include-sys-io.h-on-ARM-with-glibc.patch b/meta/recipes-graphics/xorg-xserver/xserver-xorg/0001-compiler.h-Do-not-include-sys-io.h-on-ARM-with-glibc.patch
new file mode 100644
index 0000000000..da9a4f2b25
--- /dev/null
+++ b/meta/recipes-graphics/xorg-xserver/xserver-xorg/0001-compiler.h-Do-not-include-sys-io.h-on-ARM-with-glibc.patch
@@ -0,0 +1,63 @@
+From 2ba9510721b7a76cb7fe507449fa0ac997a4cce3 Mon Sep 17 00:00:00 2001
+From: Olivier Fourdan <ofourdan@redhat.com>
+Date: Mon, 1 Jul 2019 13:20:39 +0200
+Subject: [PATCH] compiler.h: Do not include sys/io.h on ARM with glibc
+
+<sys/io.h> on ARM hasn't worked for a long, long time, so it was removed
+it from glibc upstream.
+
+Remove the include to avoid a compilation failure on ARM with glibc.
+
+Signed-off-by: Olivier Fourdan <ofourdan@redhat.com>
+Closes: https://gitlab.freedesktop.org/xorg/xserver/issues/840
+Signed-off-by: Khem Raj <raj.khem@gmail.com>
+
+Upstream-Status: Backport [https://gitlab.freedesktop.org/xorg/xserver/commit/fe4cd0e7f5c58fa94db36326aadc1bd4e6d73eba]
+---
+ hw/xfree86/common/compiler.h | 30 ------------------------------
+ 1 file changed, 30 deletions(-)
+
+diff --git a/hw/xfree86/common/compiler.h b/hw/xfree86/common/compiler.h
+index 7144c6a..2b2008b 100644
+--- a/hw/xfree86/common/compiler.h
++++ b/hw/xfree86/common/compiler.h
+@@ -758,36 +758,6 @@ inl(unsigned short port)
+     return xf86ReadMmio32Le((void *) ioBase, port);
+ }
+ 
+-#elif defined(__arm__) && defined(__linux__)
+-
+-/* for Linux on ARM, we use the LIBC inx/outx routines */
+-/* note that the appropriate setup via "ioperm" needs to be done */
+-/*  *before* any inx/outx is done. */
+-
+-#include <sys/io.h>
+-
+-static __inline__ void
+-xf_outb(unsigned short port, unsigned char val)
+-{
+-    outb(val, port);
+-}
+-
+-static __inline__ void
+-xf_outw(unsigned short port, unsigned short val)
+-{
+-    outw(val, port);
+-}
+-
+-static __inline__ void
+-xf_outl(unsigned short port, unsigned int val)
+-{
+-    outl(val, port);
+-}
+-
+-#define outb xf_outb
+-#define outw xf_outw
+-#define outl xf_outl
+-
+ #elif defined(__nds32__)
+ 
+ /*
+-- 
+2.22.0
+
diff --git a/meta/recipes-graphics/xorg-xserver/xserver-xorg/musl-arm-inb-outb.patch b/meta/recipes-graphics/xorg-xserver/xserver-xorg/musl-arm-inb-outb.patch
deleted file mode 100644
index 4be441fb71..0000000000
--- a/meta/recipes-graphics/xorg-xserver/xserver-xorg/musl-arm-inb-outb.patch
+++ /dev/null
@@ -1,18 +0,0 @@
-inb/outb family for arm is only implemented on glibc
-so assumption across linux is wrong
-
-Signed-off-by: Khem Raj <raj.khem@gmail.com>
-Upstream-Status: Pending
-Index: xorg-server-1.18.0/hw/xfree86/common/compiler.h
-===================================================================
---- xorg-server-1.18.0.orig/hw/xfree86/common/compiler.h
-+++ xorg-server-1.18.0/hw/xfree86/common/compiler.h
-@@ -758,7 +758,7 @@ inl(unsigned short port)
-     return xf86ReadMmio32Le((void *) ioBase, port);
- }
- 
--#elif defined(__arm__) && defined(__linux__)
-+#elif defined(__arm__) && defined(__GLIBC__)
- 
- /* for Linux on ARM, we use the LIBC inx/outx routines */
- /* note that the appropriate setup via "ioperm" needs to be done */
diff --git a/meta/recipes-graphics/xorg-xserver/xserver-xorg_1.20.4.bb b/meta/recipes-graphics/xorg-xserver/xserver-xorg_1.20.4.bb
index ad99d6bece..d7c5e6b355 100644
--- a/meta/recipes-graphics/xorg-xserver/xserver-xorg_1.20.4.bb
+++ b/meta/recipes-graphics/xorg-xserver/xserver-xorg_1.20.4.bb
@@ -1,9 +1,9 @@
 require xserver-xorg.inc
 
-SRC_URI += "file://musl-arm-inb-outb.patch \
-            file://0001-xf86pciBus.c-use-Intel-ddx-only-for-pre-gen4-hardwar.patch \
+SRC_URI += "file://0001-xf86pciBus.c-use-Intel-ddx-only-for-pre-gen4-hardwar.patch \
             file://pkgconfig.patch \
             file://0001-test-xtest-Initialize-array-with-braces.patch \
+            file://0001-compiler.h-Do-not-include-sys-io.h-on-ARM-with-glibc.patch \
             "
 SRC_URI[md5sum] = "c4841cc24b79420205d082fe82e0a650"
 SRC_URI[sha256sum] = "fe0fd493ebe93bfc56bede382fa204458ff5f636ea54d413a5d1bd58e19166ee"
