8289697: buffer overflow in MTLVertexCache.m: MTLVertexCache_AddGlyphQuad

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m

@@ -65,6 +65,10 @@
         MTLVC_ADD_VERTEX(TX1, TY1, DX1, DY1, 0); \
     } while (0)
 
+// Next define should exactly match to the amount
+// of MTLVC_ADD_VERTEX in MTLVC_ADD_TRIANGLES
+#define VERTS_FOR_A_QUAD 6
+
 jboolean
 MTLVertexCache_InitVertexCache()
 {
@@ -219,7 +223,11 @@ void MTLVertexCache_FreeVertexCache()
     J2dTraceLn1(J2D_TRACE_INFO, "MTLVertexCache_AddMaskQuad: %d",
                 maskCacheIndex);
 
-    if (maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX)
+    // MTLVC_ADD_TRIANGLES at the end of this function
+    // will place VERTS_FOR_A_QUAD vertexes to the vertex cache
+    // check free space and flush if needed.
+    if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) ||
+         ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX))
     {
         J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex);
         MTLVertexCache_FlushVertexCache(mtlc);
@@ -305,7 +313,9 @@ void MTLVertexCache_FreeVertexCache()
 {
     J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad");
 
-    if (vertexCacheIndex >= MTLVC_MAX_INDEX)
+    // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache
+    // so need to check space for VERTS_FOR_A_QUAD elements
+    if ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)
     {
         J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex);
         MTLVertexCache_FlushGlyphVertexCache();