diff --git a/arch/sim/src/sim/up_x11eventloop.c b/arch/sim/src/sim/up_x11eventloop.c
index 8cd8d25968..12c290548b 100644
--- a/arch/sim/src/sim/up_x11eventloop.c
+++ b/arch/sim/src/sim/up_x11eventloop.c
@@ -58,11 +58,13 @@ extern Display *g_display;
  * Name: up_buttonmap
  ****************************************************************************/
 
-static int up_buttonmap(int state)
+static int up_buttonmap(int state, int button)
 {
   int buttons = 0;
 
-  /* Remove any X11 dependencies.  Just maps ButtonNMask to bit N. */
+  /* "state" is a bitmask representing the state prior to the event, so
+   * translate that first to our button bitmap
+   */
 
   if ((state & Button1Mask) != 0)
     {
@@ -79,6 +81,23 @@ static int up_buttonmap(int state)
       buttons |= 4;
     }
 
+  /* button represents the button which changed state, so change the
+   * corresponding bit now
+   */
+
+  switch(button)
+    {
+      case Button1:
+        buttons ^= 1;
+        break;
+      case Button2:
+        buttons ^= 2;
+        break;
+      case Button3:
+        buttons ^= 4;
+        break;
+    }
+
   return buttons;
 }
 
@@ -115,7 +134,7 @@ void up_x11events(void)
           case MotionNotify : /* Enabled by ButtonMotionMask */
             {
               up_buttonevent(event.xmotion.x, event.xmotion.y,
-                             up_buttonmap(event.xmotion.state));
+                             up_buttonmap(event.xmotion.state, 0));
             }
             break;
 
@@ -123,7 +142,8 @@ void up_x11events(void)
           case ButtonRelease: /* Enabled by ButtonReleaseMask */
             {
               up_buttonevent(event.xbutton.x, event.xbutton.y,
-                             up_buttonmap(event.xbutton.state));
+                             up_buttonmap(event.xbutton.state,
+                                          event.xbutton.button));
             }
             break;