File CVE-2010-2240-address_space_limit.patch of Package xorg-x11-server

From: Matthieu Herrb <matthieu.herrb@laas.fr>
Date: Mon, 28 Jun 2010 23:54:13 +0200
Subject: [PATCH] Workaround for CVE-2010-2240.

By limiting the address space that the X server can use,
it prevents stack and mmap()ed areas to become so close that
the stack will grow over a mmaped area.

Credits: Rafal Wojtczuk <rafal@invisiblethingslab.com>

diff --git a/doc/Xserver.man.pre b/doc/Xserver.man.pre
index ce3b3a1..91c595f 100644
--- a/doc/Xserver.man.pre
+++ b/doc/Xserver.man.pre
@@ -285,6 +285,13 @@ sets the stack space limit of the server to the specified number of kilobytes.
 A value of zero makes the stack size as large as possible.  The default value
 of \-1 leaves the stack space limit unchanged.
 .TP 8
+.B \-la \fIkilobytes\fP
+sets the address space limit of the server to the specified number of 
+kilobytes.
+A value of zero makes address space as large as possible.
+The default value is 1572864 (1.5GB) on 32 bit architectures and
+10485760 (10GB) on 64 bit architectures.
+.TP 8
 .B \-logo
 turns on the X Window System logo display in the screen-saver.
 There is currently no way to change this from a client.
--- xorg-server-1.6.5/include/opaque.h.orig	2009-10-12 02:52:40.000000000 +0000
+++ xorg-server-1.6.5/include/opaque.h	2010-08-17 08:52:06.000000000 +0000
@@ -67,6 +67,9 @@ extern int limitStackSpace;
 #ifdef RLIMIT_NOFILE
 extern int limitNoFile;
 #endif
+#ifdef RLIMIT_AS
+extern int limitAddressSpace;
+#endif
 extern Bool defeatAccessControl;
 extern long maxBigRequestSize;
 extern Bool party_like_its_1989;
--- xorg-server-1.6.5/os/osinit.c.orig	2009-10-12 02:52:40.000000000 +0000
+++ xorg-server-1.6.5/os/osinit.c	2010-08-17 08:56:57.000000000 +0000
@@ -87,6 +87,14 @@ int limitStackSpace = -1;
 #ifdef RLIMIT_NOFILE
 int limitNoFile = -1;
 #endif
+#ifdef RLIMIT_AS
+#ifdef _XSERVER64
+#define XORG_AS_LIMIT 10737418240LL
+#else
+#define XORG_AS_LIMIT 1610612736
+#endif
+long limitAddressSpace = XORG_AS_LIMIT;
+#endif
 
 void
 OsInit(void)
@@ -187,6 +195,22 @@ OsInit(void)
 	    }
 	}
 #endif
+#ifdef RLIMIT_AS
+	if (limitAddressSpace >= 0)
+	{
+	    struct rlimit	rlim;
+
+	    if (!getrlimit(RLIMIT_AS, &rlim))
+	    {
+		if ((limitAddressSpace > 0) 
+		    && (limitAddressSpace < rlim.rlim_max))
+		    rlim.rlim_cur = limitAddressSpace;
+		else
+		    rlim.rlim_cur = rlim.rlim_max;
+		(void)setrlimit(RLIMIT_AS, &rlim);
+	    }
+	}
+#endif
 #ifdef SERVER_LOCK
 	LockServer();
 #endif
--- xorg-server-1.6.5/os/utils.c.orig	2009-10-12 02:52:40.000000000 +0000
+++ xorg-server-1.6.5/os/utils.c	2010-08-17 08:55:15.000000000 +0000
@@ -761,6 +761,22 @@ ProcessCommandLine(int argc, char *argv[
 		UseMsg();
 	}
 #endif
+#ifdef RLIMIT_AS
+	else if ( strcmp( argv[i], "-la") == 0)
+	{
+	    if (getuid() != geteuid()) {
+		FatalError("The '-la' option can only be used by root.\n");
+	    }
+	    if(++i < argc)
+	    {
+		limitAddressSpace = atol(argv[i]);
+		if (limitAddressSpace > 0)
+		    limitAddressSpace *= 1024;
+	    }
+	    else
+		UseMsg();
+	}
+#endif
 #ifdef SERVER_LOCK
 	else if ( strcmp ( argv[i], "-nolock") == 0)
 	{