Specialize bytecode for pairs()/next() iterator. Speedup: 3.5x.

Parser predict pairs/next and emits specialized bytecode.
Bytecode is descpecialized at runtime if the prediction was wrong.
Store slot index in hidden control var to avoid key lookups.

src/buildvm_x86.dasc

@@ -4576,6 +4576,109 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop, int cmov, int sse)
     |  ins_call
     break;
 
+  case BC_ITERN:
+    |  ins_A	// RA = base, (RB = nresults+1 (2+1), RC = nargs+1 (2+1))
+#if LJ_HASJIT
+    |  // NYI: add hotloop, record BC_ITERN.
+#endif
+    |  mov TMP1, KBASE			// Need two more free registers.
+    |  mov TMP2, DISPATCH
+    |  mov TAB:RB, [BASE+RA*8-16]
+    |  mov RC, [BASE+RA*8-8]		// Get index from control var.
+    |  mov DISPATCH, TAB:RB->asize
+    |  add PC, 4
+    |  mov KBASE, TAB:RB->array
+    |1:  // Traverse array part.
+    |  cmp RC, DISPATCH; jae >5		// Index points after array part?
+    |  cmp dword [KBASE+RC*8+4], LJ_TNIL; je >4
+    if (sse) {
+      |  cvtsi2sd xmm0, RC
+    } else {
+      |  fild dword [BASE+RA*8-8]
+    }
+    |  // Copy array slot to returned value.
+    |.if X64
+    |  mov RBa, [KBASE+RC*8]
+    |  mov [BASE+RA*8+8], RBa
+    |.else
+    |  mov RB, [KBASE+RC*8+4]
+    |  mov [BASE+RA*8+12], RB
+    |  mov RB, [KBASE+RC*8]
+    |  mov [BASE+RA*8+8], RB
+    |.endif
+    |  add RC, 1
+    |  // Return array index as a numeric key.
+    if (sse) {
+      |  movsd qword [BASE+RA*8], xmm0
+    } else {
+      |  fstp qword [BASE+RA*8]
+    }
+    |  mov [BASE+RA*8-8], RC		// Update control var.
+    |2:
+    |  movzx RD, PC_RD			// Get target from ITERL.
+    |  branchPC RD
+    |3:
+    |  mov DISPATCH, TMP2
+    |  mov KBASE, TMP1
+    |  ins_next
+    |
+    |4:  // Skip holes in array part.
+    |  add RC, 1
+    if (!sse) {
+      |  mov [BASE+RA*8-8], RC
+    }
+    |  jmp <1
+    |
+    |5:  // Traverse hash part.
+    |  sub RC, DISPATCH
+    |6:
+    |  cmp RC, TAB:RB->hmask; ja <3	// End of iteration? Branch to ITERL+1.
+    |  imul KBASE, RC, #NODE
+    |  add NODE:KBASE, TAB:RB->node
+    |  cmp dword NODE:KBASE->val.it, LJ_TNIL; je >7
+    |  lea DISPATCH, [RC+DISPATCH+1]
+    |  // Copy key and value from hash slot.
+    |.if X64
+    |  mov RBa, NODE:KBASE->key
+    |  mov RCa, NODE:KBASE->val
+    |  mov [BASE+RA*8], RBa
+    |  mov [BASE+RA*8+8], RCa
+    |.else
+    |  mov RB, NODE:KBASE->key.gcr
+    |  mov RC, NODE:KBASE->key.it
+    |  mov [BASE+RA*8], RB
+    |  mov [BASE+RA*8+4], RC
+    |  mov RB, NODE:KBASE->val.gcr
+    |  mov RC, NODE:KBASE->val.it
+    |  mov [BASE+RA*8+8], RB
+    |  mov [BASE+RA*8+12], RC
+    |.endif
+    |  mov [BASE+RA*8-8], DISPATCH
+    |  jmp <2
+    |
+    |7:  // Skip holes in hash part.
+    |  add RC, 1
+    |  jmp <6
+    break;
+
+  case BC_ISNEXT:
+    |  ins_AD	// RA = base, RD = target (points to ITERN)
+    |  cmp dword [BASE+RA*8-20], LJ_TFUNC; jne >5
+    |  mov CFUNC:RB, [BASE+RA*8-24]
+    |  cmp dword [BASE+RA*8-12], LJ_TTAB; jne >5
+    |  cmp dword [BASE+RA*8-4], LJ_TNIL; jne >5
+    |  cmp byte CFUNC:RB->ffid, FF_next_N; jne >5
+    |  branchPC RD
+    |  mov dword [BASE+RA*8-8], 0	// Initialize control var.
+    |1:
+    |  ins_next
+    |5:  // Despecialize bytecode if any of the checks fail.
+    |  mov PC_OP, BC_JMP
+    |  branchPC RD
+    |  mov byte [PC], BC_ITERC
+    |  jmp <1
+    break;
+
   case BC_VARG:
     |  ins_ABC	// RA = base, RB = nresults+1, RC = numparams
     |  mov TMP1, KBASE			// Need one more free register.