File N-VNC-Enable-use-of-all-keyboard-layouts-independent-o.patch of Package xorg-x11-server

From: Matthias Hopf <mhopf@suse.de>
Date: Tue, 6 Sep 2011 08:35:57 +0200
Subject: [PATCH 5/6] VNC: Enable use of all keyboard layouts, independent of remotely set layout
Patch-Mainline: Currently no upstream project.
Git-commit: 6885b927a6065e6379cfaa3ebbf6c51445a015d9
Signed-off: Egbert Eich <eich@suse.de> 
References: bnc #400520, #605015, #660797

Changes:

- Use virtual core keyboard for events and key state lookup:
  Make layout changes work again - see discussion on
  https://defect.opensolaris.org/bz/show_bug.cgi?id=8687
- keycode lookup:
  Don't use any static keyboard layout any more.
- ISO-Level3-Shift handling:
  Enable the use of keyboard layouts that use AltGr for 3rd and 4th level.
- Make keyboard handling more XKB aware:
  Previous code was e.g. not multi-group aware.
- Nuke use of legacy keymap as far as possible:
  Creating legacy keymap takes time, and it has to be freed again afterwards.
- Free index lookup:
  Make XKB aware.
- Ignore calls for NoSymbol:
  This destroys otherwise valid entries.
- Fix analysis for shift/level3 event faking:
  Previous broken version lead to e.g. Shift+PgUp not being recognized.
- Add tons of debug output (disabled).

Signed-off-by: Egbert Eich <eich@freedesktop.org>

Rebased to 1.12.1 by Mike Gorse <mgorse@suse.com>
---
diff -pruN xorg-server-1.12.1.orig/hw/vnc/kbdptr.c xorg-server-1.12.1/hw/vnc/kbdptr.c
--- xorg-server-1.12.1.orig/hw/vnc/kbdptr.c	2012-04-18 14:14:07.436250934 -0500
+++ xorg-server-1.12.1/hw/vnc/kbdptr.c	2012-04-18 14:15:27.656248047 -0500
@@ -34,6 +34,8 @@
 #include "X11/Xproto.h"
 #include "inputstr.h"
 #include "inpututils.h"
+#include "xkbsrv.h"
+#include "xkbstr.h"
 #define XK_CYRILLIC
 #include <X11/keysym.h>
 #include <X11/Xatom.h>
@@ -46,6 +48,7 @@
 #include "dmxinput.h"
 #endif
 
+#if 0
 #if !XFREE86VNC
 
 #define MIN_KEY_CODE		8
@@ -196,21 +199,20 @@ static KeySym map[MAX_KEY_CODE * GLYPHS_
 #define N_PREDEFINED_KEYS (sizeof(map) / (sizeof(KeySym) * GLYPHS_PER_KEY))
 
 #endif
+#endif
 
 #define KEY_IS_PRESSED(keycode) \
-    (kbdDevice->key->down[(keycode) >> 3] & (1 << ((keycode) & 7)))
+    (inputInfo.keyboard->key->down[(keycode) >> 3] & (1 << ((keycode) & 7)))
 
 static void vncXConvertCase(KeySym sym, KeySym *lower, KeySym *upper);
 
-static DeviceIntPtr ptrDevice = NULL, kbdDevice = NULL;
+static DeviceIntPtr ptrDevice = NULL;
 
 
 void
 vncSetKeyboardDevice(DeviceIntPtr kbd)
 {
-   if (kbdDevice && kbd)
-      return; /* set once */
-   kbdDevice = kbd;
+  // obsoleted by inputInfo
 }
 
 
@@ -263,6 +265,29 @@ EnqueueKey(DeviceIntPtr kbdDev, int type
    QueueKeyboardEvents(kbdDev, type, detail, NULL);
 }
 
+/* In-server and highly changed version of XkbKeycodeToKeysym */
+static KeySym
+_XkbKeycodeToKeysym(XkbDescPtr xkb, KeyCode kc, int group, int level)
+{
+    KeySym ks;
+
+    if ((kc<xkb->min_key_code)||(kc>xkb->max_key_code))
+	return NoSymbol;
+    /* Treat single group elements as present in all groups */
+    if (XkbKeyNumGroups (xkb,kc) == 1)
+	group = 0;
+    if ((group<0)||(level<0)||(group>=XkbKeyNumGroups(xkb,kc)))
+	return NoSymbol;
+    if (level < XkbKeyGroupWidth(xkb, kc, group))
+	ks = XkbKeySymEntry(xkb, kc, level, group);
+    else
+	ks = NoSymbol;
+    /* Treat 'K' as 'K K', */
+    if (ks == NoSymbol && (level & 1) && level-1 < XkbKeyGroupWidth(xkb, kc, group))
+	ks = XkbKeySymEntry(xkb, kc, level-1, group);
+    return ks;
+}
+
 /*
  * Called when the rfbserver receives a rfbKeyEvent event from a client.
  * Put an X keyboard event into the event queue.
@@ -271,21 +296,35 @@ void
 KbdAddEvent(Bool down, KeySym keySym, rfbClientPtr cl)
 {
     const int type = down ? KeyPress : KeyRelease;
-    KeySymsPtr keySyms;
-    XkbStateRec *xkb;
-    int i;
+    XkbSrvInfoPtr xkbInfo;
+    int i, group, level;
     int keyCode = 0;
-    int freeIndex = -1;
     Bool fakeShiftPress = FALSE;
     Bool fakeShiftLRelease = FALSE;
     Bool fakeShiftRRelease = FALSE;
     Bool shiftMustBeReleased = FALSE;
     Bool shiftMustBePressed = FALSE;
+    Bool fakeLevel3Press = FALSE;
+    Bool fakeLevel3Release = FALSE;
+    Bool level3MustBeReleased = FALSE;
+    Bool level3MustBePressed = FALSE;
+
+    /* Incomplete maps may create NoSymbol - which lets us
+     * select and/or overwrite otherwise valid entries.
+     * E.g Level3+a in serbian layout creates NoSymbol on os11.4
+     * 2011-05-24 mhopf@suse.de */
+    if (keySym == NoSymbol) {
+	ErrorF("KbdAddEvent: ignoring illegal NoSymbol\n");
+	return;
+    }
 
-    if (!kbdDevice)
-        return;
-
-    keySyms = XkbGetCoreMap(kbdDevice);
+    xkbInfo = inputInfo.keyboard->key->xkbInfo;
+    group   = xkbInfo->state.group;
+    level   = (KEY_IS_PRESSED(ISO_LEVEL3_KEY_CODE) ? 2 : 0) |
+	      (XkbStateFieldFromRec(&xkbInfo->state) & ShiftMask ? 1 : 0);
+#ifdef DEBUG
+    ErrorF ("VNCkbd:\t%s Sym %04x\n", down ? "+":"-", (int)keySym);
+#endif
 
 #ifdef CORBA
     if (cl) {
@@ -303,6 +342,12 @@ KbdAddEvent(Bool down, KeySym keySym, rf
      *
      * Alan.
      */
+    /* Never use predefined keys.
+     * This is inherently incapable of dealing with changing
+     * keyboard layouts. Not being able to work with non-local xmodmaps
+     * is a nuisance at worst, and probably even preferred.
+     * 2011-04-15 mhopf@suse.de */
+#ifdef NOTANYMORE
 #if !XFREE86VNC
     /* First check if it's one of our predefined keys.  If so then we can make
        some attempt at allowing an xmodmap inside a VNC desktop behave
@@ -329,107 +374,227 @@ KbdAddEvent(Bool down, KeySym keySym, rf
 	}
     }
 #endif
+#endif
 
     if (!keyCode) {
 
 	/* not one of our predefined keys - see if it's in the current keyboard
            mapping (i.e. we've already allocated an extra keycode for it) */
 
-	if (keySyms->mapWidth < 2) {
-	    ErrorF("KbdAddEvent: Sanity check failed - Keyboard mapping has "
-		   "less than 2 keysyms per keycode (KeySym 0x%x)\n", (int)keySym);
-	    return;
-	}
+	for (keyCode = MIN_KEY_CODE; keyCode < MIN_KEY_CODE + NO_OF_KEYS; keyCode++) {
+	    /* Check all keycodes, but only continue on those where
+	     * backconversion results in keySym.
+	     * 2011-05-20 mhopf@suse.de */
+
+#ifdef DEBUG
+	    int j;
+	    ErrorF ("   keyCode %3d map# %4d++ level %d of %d:  keySyms",
+		    keyCode, (i / keySyms->mapWidth) * keySyms->mapWidth,
+		    i % keySyms->mapWidth, keySyms->mapWidth);
+	    for (j = 0; j < keySyms->mapWidth; j++)
+		ErrorF (" %02x", (int)keySyms->map[(i / keySyms->mapWidth) * keySyms->mapWidth + j]);
+	    ErrorF ("\n");
+	    ErrorF ("   group %d of %d width %d:  keySyms",
+		    group, XkbKeyNumGroups(xkbInfo->desc, keyCode),
+		    XkbKeyGroupWidth(xkbInfo->desc, keyCode, group));
+	    if (XkbKeyNumGroups(xkbInfo->desc, keyCode) > group)
+		for (j = 0; j < XkbKeyGroupWidth(xkbInfo->desc, keyCode, group); j++)
+		    ErrorF (" %02x", (int) XkbKeySymEntry(xkbInfo->desc, keyCode, j, group));
+	    ErrorF ("\n");
+#endif
 
-	for (i = 0; i < NO_OF_KEYS * keySyms->mapWidth; i++) {
-	    if (keySym == keySyms->map[i]) {
-		keyCode = MIN_KEY_CODE + i / keySyms->mapWidth;
-
-		if (keySyms->map[(i / keySyms->mapWidth)
-					* keySyms->mapWidth + 1] != NoSymbol) {
-
-		    /* this keycode has more than one symbol associated with
-		       it, so shift state is important */
-
-		    if ((i % keySyms->mapWidth) == 0)
-			shiftMustBeReleased = TRUE;
-		    else
-			shiftMustBePressed = TRUE;
-		}
+	    /* Check whether keySym is reachable in current group
+	     * by any shift/Level3_shift state (preferrable w/o change).
+	     * This doesn't do real modifyer analysis, only Shift and Level3_Shift.
+	     * 2011-05-23 mhopf@suse.de */
+	    if (_XkbKeycodeToKeysym(xkbInfo->desc, keyCode, group, level) == keySym)
+		break;
+	    if (_XkbKeycodeToKeysym(xkbInfo->desc, keyCode, group, level ^ 2) == keySym) {
+		if (level & 2)
+		    level3MustBeReleased = TRUE;
+		else
+		    level3MustBePressed = TRUE;
 		break;
 	    }
-	    if ((freeIndex == -1) && (keySyms->map[i] == NoSymbol)
-		&& (i % keySyms->mapWidth) == 0)
-	    {
-		freeIndex = i;
+	    if (_XkbKeycodeToKeysym(xkbInfo->desc, keyCode, group, level ^ 1) == keySym) {
+		if (level & 1)
+		    shiftMustBeReleased = TRUE;
+		else
+		    shiftMustBePressed = TRUE;
+		break;
+	    }
+	    if (_XkbKeycodeToKeysym(xkbInfo->desc, keyCode, group, level ^ 3) == keySym) {
+		if (level & 2)
+		    level3MustBeReleased = TRUE;
+		else
+		    level3MustBePressed = TRUE;
+		if (level & 1)
+		    shiftMustBeReleased = TRUE;
+		else
+		    shiftMustBePressed = TRUE;
+		break;
 	    }
 	}
+	if (keyCode == MIN_KEY_CODE + NO_OF_KEYS)
+	    keyCode = 0;
     }
 
     if (!keyCode) {
 	KeySym lower, upper;
+	KeySymsPtr keySyms = XkbGetCoreMap(inputInfo.keyboard);
 
 	/* we don't have an existing keycode - make one up on the fly and add
 	   it to the keyboard mapping.  Thanks to Vlad Harchev for pointing
 	   out problems with non-ascii capitalisation. */
 
-	if (freeIndex == -1) {
+	/* Find free index for current group. */
+	for (keyCode = MIN_KEY_CODE; keyCode < MIN_KEY_CODE + NO_OF_KEYS; keyCode++) {
+	    /* A keyCode is free if no groups are assigned at all */
+	    if (XkbKeyNumGroups(xkbInfo->desc, keyCode) == 0)
+		break;
+#ifdef NOTANYMORE
+	    /* We can use exact map positions for group 1+2, but only partially
+	     * filling out xkb legacy maps may suddenly change the # of groups.
+	     * Reason for that is unknown yet. Might be related to (fixed) NoSymbol issue.
+	     * 2011-05-24 mhopf@suse.de */
+	    /* For primary groups: A keyCode is free if current group is empty */
+	    if (XkbKeyGroupWidth(xkbInfo->desc, keyCode, group) < 1 && group < 2)
+		break;
+	    /* Never touch groups that have a single level only (weird group?!?) */
+	    if (XkbKeyGroupWidth(xkbInfo->desc, keyCode, group) < 2)
+		continue;
+	    /* For primary groups: A keyCode is free if only NoSymbol is assigned
+	     * to available levels (only validating levels 0-3) */
+	    if (group < 2 &&
+		XkbKeySymEntry(xkbInfo->desc, keyCode, 0, group) == NoSymbol &&
+		XkbKeySymEntry(xkbInfo->desc, keyCode, 1, group) == NoSymbol &&
+		(XkbKeyGroupWidth(xkbInfo->desc, keyCode, group) < 3 ||
+		 (XkbKeySymEntry(xkbInfo->desc, keyCode, 2, group) == NoSymbol &&
+		  (XkbKeyGroupWidth(xkbInfo->desc, keyCode, group) < 4 ||
+		   XkbKeySymEntry(xkbInfo->desc, keyCode, 3, group) == NoSymbol))))
+		break;
+#endif
+	}
+
+	if (keyCode == MIN_KEY_CODE + NO_OF_KEYS) {
 	    ErrorF("KbdAddEvent: ignoring KeySym 0x%x - no free KeyCodes\n",
 		   (int)keySym);
+	    free (keySyms->map);
+	    free (keySyms);
 	    return;
 	}
 
-	keyCode = MIN_KEY_CODE + freeIndex / keySyms->mapWidth;
-
 	vncXConvertCase(keySym, &lower, &upper);
 
-	if (lower == upper) {
-	    keySyms->map[freeIndex] = keySym;
-
-	} else {
-	    keySyms->map[freeIndex] = lower;
-	    keySyms->map[freeIndex+1] = upper;
-
+	/* Adding keys is not using xkb mechanisms yet, but relying on support
+	 * for changing keys in the legacy map. Should be changed, eventually.
+	 * 2011-05-19 mhopf@suse.de */
+#ifdef NOTANYMORE
+	if (group < 2) {
+	    /* Only set mapping for active group. Will only work with dual layouts.
+	     * 2011-05-23 mhopf@suse.de */
+	    int active_group_offset = group ? 2 : 0;
+
+	    if (lower == upper) {
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + active_group_offset] = keySym;
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + active_group_offset + 1] = NoSymbol;
+	    } else {
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + active_group_offset] = lower;
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + active_group_offset + 1] = upper;
+	    }
+	}
+#endif
+	/* Generic layouts needs to set the full map width.
+	 * Weird enough, mapWidth seems too big...
+	 * 2011-05-23 mhopf@suse.de */
+	for (i = 0; i < (keySyms->mapWidth & ~1); i += 2) {
+	    if (lower == upper) {
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + i]     = keySym;
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + i + 1] = NoSymbol;
+	    } else {
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + i]     = lower;
+		keySyms->map[(keyCode - MIN_KEY_CODE) * keySyms->mapWidth + i + 1] = upper;
+	    }
+	}
+	if (lower != upper) {
 	    if (keySym == lower)
 		shiftMustBeReleased = TRUE;
 	    else
 		shiftMustBePressed = TRUE;
 	}
+	level3MustBeReleased = TRUE;
 
-	XkbApplyMappingChange(kbdDevice, keySyms, keyCode, 1, NULL, serverClient);
+	XkbApplyMappingChange(inputInfo.keyboard, keySyms, keyCode, 1, NULL, serverClient);
 
 	ErrorF("KbdAddEvent: unknown KeySym 0x%x - allocating KeyCode %d\n",
 	       (int)keySym, keyCode);
+	free (keySyms->map);
+	free (keySyms);
     }
 
-    xkb = &kbdDevice->key->xkbInfo->state;
+#ifdef DEBUG
+    ErrorF ("\t%s Sym %04x Code%3d\tState x%02x %s%s%s\tSh %s%s\tL3 %s%s\n",
+	    down ? "+":"-", (int)keySym, keyCode, XkbStateFieldFromRec(&xkbInfo->state),
+	    KEY_IS_PRESSED(SHIFT_L_KEY_CODE) ? "Sl":"",
+	    KEY_IS_PRESSED(SHIFT_R_KEY_CODE) ? "Sr":"",
+	    KEY_IS_PRESSED(ISO_LEVEL3_KEY_CODE) ? "L3":"",
+	    shiftMustBePressed ? "+":"", shiftMustBeReleased ? "-":"",
+	    level3MustBePressed ? "+":"", level3MustBeReleased ? "-":"");
+#endif
+#ifdef NOTANYMORE
+    int back = _XkbKeycodeToKeysym (xkbInfo->desc, keyCode, group,
+				    ((level3MustBePressed || (!level3MustBeReleased && (level & 2))) ? 2 : 0) |
+				    ((shiftMustBePressed || (!shiftMustBeReleased && (level & 1))) ? 1 : 0));
+    ErrorF ("\tvalidate code %d %-2s%-2s -> sym %04x %s\n\n", keyCode,
+	    (shiftMustBePressed || (!shiftMustBeReleased && (level & 1)))   ? "Sh" : "",
+	    (level3MustBePressed || (!level3MustBeReleased && (level & 2))) ? "L3" : "",
+	    back, (back == keySym ? "ok" : "FAILED"));
+#endif
+
     if (down) {
-	if (shiftMustBePressed && !(XkbStateFieldFromRec(xkb) & ShiftMask)) {
+	/* TODO: would require to check which keycodes are actually
+	 * bound to ISO_Level3_Shift and/or Shift_L.
+	 * 2011-04-18 mhopf@suse.de */
+	if (level3MustBePressed && !(level & 2)) {
+	    fakeLevel3Press = TRUE;
+	    EnqueueKey(inputInfo.keyboard, KeyPress, ISO_LEVEL3_KEY_CODE);
+	}
+	if (level3MustBeReleased && (level & 2)) {
+	    fakeLevel3Release = TRUE;
+	    EnqueueKey(inputInfo.keyboard, KeyRelease, ISO_LEVEL3_KEY_CODE);
+	}
+	if (shiftMustBePressed && !(level & 1)) {
 	    fakeShiftPress = TRUE;
-            EnqueueKey(kbdDevice, KeyPress, SHIFT_L_KEY_CODE);
+            EnqueueKey(inputInfo.keyboard, KeyPress, SHIFT_L_KEY_CODE);
 	}
-	if (shiftMustBeReleased && (XkbStateFieldFromRec(xkb) & ShiftMask)) {
+	if (shiftMustBeReleased && (level & 1)) {
 	    if (KEY_IS_PRESSED(SHIFT_L_KEY_CODE)) {
 		fakeShiftLRelease = TRUE;
-                EnqueueKey(kbdDevice, KeyRelease, SHIFT_L_KEY_CODE);
+                EnqueueKey(inputInfo.keyboard, KeyRelease, SHIFT_L_KEY_CODE);
 	    }
 	    if (KEY_IS_PRESSED(SHIFT_R_KEY_CODE)) {
 		fakeShiftRRelease = TRUE;
-                EnqueueKey(kbdDevice, KeyRelease, SHIFT_R_KEY_CODE);
+                EnqueueKey(inputInfo.keyboard, KeyRelease, SHIFT_R_KEY_CODE);
 	    }
 	}
     }
 
-    EnqueueKey(kbdDevice, type, keyCode);
+    EnqueueKey(inputInfo.keyboard, type, keyCode);
 
     if (fakeShiftPress) {
-        EnqueueKey(kbdDevice, KeyRelease, SHIFT_L_KEY_CODE);
+        EnqueueKey(inputInfo.keyboard, KeyRelease, SHIFT_L_KEY_CODE);
     }
     if (fakeShiftLRelease) {
-        EnqueueKey(kbdDevice, KeyPress, SHIFT_L_KEY_CODE);
+        EnqueueKey(inputInfo.keyboard, KeyPress, SHIFT_L_KEY_CODE);
     }
     if (fakeShiftRRelease) {
-        EnqueueKey(kbdDevice, KeyPress, SHIFT_R_KEY_CODE);
+        EnqueueKey(inputInfo.keyboard, KeyPress, SHIFT_R_KEY_CODE);
+    }
+    if (fakeLevel3Press) {
+        EnqueueKey(inputInfo.keyboard, KeyRelease, ISO_LEVEL3_KEY_CODE);
+    }
+    if (fakeLevel3Release) {
+        EnqueueKey(inputInfo.keyboard, KeyPress, ISO_LEVEL3_KEY_CODE);
     }
 }
 
@@ -480,15 +645,15 @@ KbdReleaseAllKeys(void)
 {
     int i, j;
 
-    if (!kbdDevice) 
+    if (!inputInfo.keyboard) 
 	return;
 
     for (i = 0; i < DOWN_LENGTH; i++) {
-	if (kbdDevice->key->down[i] != 0) {
+	if (inputInfo.keyboard->key->down[i] != 0) {
 	    for (j = 0; j < 8; j++) {
-		if (kbdDevice->key->down[i] & (1 << j)) {
+		if (inputInfo.keyboard->key->down[i] & (1 << j)) {
                     int detail = (i << 3) | j;
-                    EnqueueKey(kbdDevice, KeyRelease, detail);
+                    EnqueueKey(inputInfo.keyboard, KeyRelease, detail);
 		}
 	    }
 	}
diff -pruN xorg-server-1.12.1.orig/hw/vnc/keyboard.h xorg-server-1.12.1/hw/vnc/keyboard.h
--- xorg-server-1.12.1.orig/hw/vnc/keyboard.h	2012-04-18 14:14:07.437250922 -0500
+++ xorg-server-1.12.1/hw/vnc/keyboard.h	2012-04-18 14:15:27.657248035 -0500
@@ -32,3 +32,4 @@
 #define META_R_KEY_CODE		(MIN_KEY_CODE + 108)
 #define ALT_L_KEY_CODE		(MIN_KEY_CODE + 56)
 #define ALT_R_KEY_CODE		(MIN_KEY_CODE + 105)
+#define ISO_LEVEL3_KEY_CODE    ALT_R_KEY_CODE