Fix(VIM-3615): Escape closes dialog while waiting for more keys

src/main/java/com/maddyhome/idea/vim/handler/VimEnterHandler.kt

@@ -218,13 +218,17 @@ internal class VimEnterHandler(nextHandler: EditorActionHandler?) : VimKeyHandle
 internal class VimEscHandler(nextHandler: EditorActionHandler) : VimKeyHandler(nextHandler) {
   override val key: String = "<Esc>"
 
-  override fun isHandlerEnabled(editor: Editor, dataContext: DataContext?): Boolean {
-    val ideaVimSupportDialog =
-      injector.globalIjOptions().ideavimsupport.contains(IjOptionConstants.ideavimsupport_dialog)
+  private val ideaVimSupportDialog
+    get() = injector.globalIjOptions().ideavimsupport.contains(IjOptionConstants.ideavimsupport_dialog)
   
+  override fun isHandlerEnabled(editor: Editor, dataContext: DataContext?): Boolean {
     return editor.isPrimaryEditor() ||
-      EditorHelper.isFileEditor(editor) && !editor.vim.mode.inNormalMode ||
-      ideaVimSupportDialog && !editor.vim.mode.inNormalMode
+      EditorHelper.isFileEditor(editor) && vimStateNeedsToHandleEscape(editor) ||
+      ideaVimSupportDialog && vimStateNeedsToHandleEscape(editor)
+  }
+  
+  private fun vimStateNeedsToHandleEscape(editor: Editor): Boolean {
+    return !editor.vim.mode.inNormalMode || KeyHandler.getInstance().keyHandlerState.mappingState.hasKeys
   }
 }
 

vim-engine/src/main/kotlin/com/maddyhome/idea/vim/command/MappingState.kt

@@ -38,6 +38,9 @@ class MappingState: Cloneable {
   val keys: Iterable<KeyStroke>
     get() = keyList
   
+  val hasKeys
+    get() = keyList.isNotEmpty()
+
   private var timer = VimTimer(injector.globalOptions().timeoutlen)
   private var keyList = mutableListOf<KeyStroke>()