How to quickly create asynchronous inks into an InkEdit window based on screen mouse coordinates ?

R FranCam 0 Reputation points
2026-11-05T06:26:10.44+00:00

How to quickly create asynchronous inks into an ActiveX InkEdit window based on screen mouse coordinates ?

I am looking for a way to convert the handwriting data coordinates into text and send it to inputs. I managed to capture the coordinates during writing. The problem is that I cannot find a way to quickly convert the captured coordinates into inks and inject them into InkEdit to perform recognition (conversion to text). I use autohotkey v2 on PC windows 10 64-bit.

The autohotkey code is like this :

#SingleInstance Force
Persistent()

#Include Gdip_all.ahk

CoordMode("Mouse", "Screen")

global drawingMode := false
global drawing := false
global currentShape := {}
global allShapes := []
global activeMonitor := { width: A_ScreenWidth, height: A_ScreenHeight }

global ui := { overlay: "" }
global gdi := { token: 0, G_Mem: 0, G_Baked: 0, hdcMem: 0, hbmBuffer: 0, hdcBaked: 0, hbmBaked: 0 }
global recGui := unset, inkObj := unset

GdiStartup()
OnExit((*) => ExitDrawingMode(true))

; Event Listeners for drawing
OnMessage(0x201, WM_LBUTTONDOWN)
OnMessage(0x202, WM_LBUTTONUP)
OnMessage(0x200, WM_MOUSEMOVE)
OnMessage(0x0F, WM_PAINT_Handler)

; Hotkeys
^F9::ToggleDrawingMode()
#HotIf drawingMode
F3::ProcessHandwritingRecognition()
Esc::ClearDrawing()
#HotIf

ToggleDrawingMode(*) {
    global drawingMode, ui, gdi, allShapes, currentShape, drawing
    if (drawingMode) {
        ExitDrawingMode(false)
        return
    }

    hdcScreen := DllCall("GetDC", "Ptr", 0, "Ptr")
    gdi.hdcMem := DllCall("CreateCompatibleDC", "Ptr", hdcScreen, "Ptr")
    gdi.hbmBuffer := DllCall("CreateCompatibleBitmap", "Ptr", hdcScreen, "Int", A_ScreenWidth, "Int", A_ScreenHeight, "Ptr")
    gdi.hdcBaked := DllCall("CreateCompatibleDC", "Ptr", hdcScreen, "Ptr")
    gdi.hbmBaked := DllCall("CreateCompatibleBitmap", "Ptr", hdcScreen, "Int", A_ScreenWidth, "Int", A_ScreenHeight, "Ptr")
    DllCall("ReleaseDC", "Ptr", 0, "Ptr", hdcScreen)

    DllCall("SelectObject", "Ptr", gdi.hdcMem, "Ptr", gdi.hbmBuffer)
    DllCall("SelectObject", "Ptr", gdi.hdcBaked, "Ptr", gdi.hbmBaked)

    gdi.G_Mem := Gdip_GraphicsFromHDC(gdi.hdcMem)
    gdi.G_Baked := Gdip_GraphicsFromHDC(gdi.hdcBaked)

    drawing := false
    drawingMode := true
    allShapes := []
    currentShape := {}

    Gdip_SetSmoothingMode(gdi.G_Mem, 4)
    Gdip_SetSmoothingMode(gdi.G_Baked, 4)

    ui.overlay := Gui("+AlwaysOnTop -DPIScale -Caption +ToolWindow +E0x20")
    ui.overlay.Show("NA x0 y0 w" A_ScreenWidth " h" A_ScreenHeight)
}

ExitDrawingMode(UserInitiated := false) {
    global drawingMode, drawing, ui, gdi
    drawingMode := false
    drawing := false
    if (IsObject(ui.overlay)) {
        ui.overlay.Destroy()
        ui.overlay := ""
    }
    if (gdi.G_Mem) {
        Gdip_DeleteGraphics(gdi.G_Mem)
        gdi.G_Mem := 0
    }
    if (gdi.G_Baked) {
        Gdip_DeleteGraphics(gdi.G_Baked)
        gdi.G_Baked := 0
    }
    if (UserInitiated && gdi.token) {
        Gdip_Shutdown(gdi.token)
        gdi.token := 0
    }
}

WM_PAINT_Handler(wParam, lParam, msg, hwnd) {
    global ui, gdi
    if (!drawingMode || !IsObject(ui.overlay) || hwnd != ui.overlay.Hwnd || !gdi.hdcMem)
        return
    ps := Buffer(A_PtrSize == 8 ? 72 : 64)
    hDC := DllCall("BeginPaint", "Ptr", hwnd, "Ptr", ps.Ptr, "Ptr")
    DllCall("BitBlt", "Ptr", hDC, "Int", 0, "Int", 0, "Int", A_ScreenWidth, "Int", A_ScreenHeight, "Ptr", gdi.hdcMem, "Int", 0, "Int", 0, "UInt", 0x00CC0020)
    DllCall("EndPaint", "Ptr", hwnd, "Ptr", ps.Ptr)
    return 0
}

WM_LBUTTONDOWN(wParam, lParam, msg, hwnd) {
    global drawing, currentShape, ui
    if (!drawingMode || hwnd != ui.overlay.Hwnd)
        return
    x := lParam & 0xFFFF, y := (lParam >> 16) & 0xFFFF
    drawing := true
    currentShape := { type: "free", points: [[x, y]], pointsStr: x "," y }
    SetTimer(UpdateBuffer, 16)
    DllCall("SetCapture", "Ptr", hwnd)
}

WM_MOUSEMOVE(wParam, lParam, msg, hwnd) {
    global drawing, currentShape
    if (!drawing || !drawingMode)
        return
    x := lParam & 0xFFFF, y := (lParam >> 16) & 0xFFFF
    currentShape.points.Push([x, y])
    currentShape.pointsStr .= "|" x "," y
}

WM_LBUTTONUP(wParam, lParam, msg, hwnd) {
    global drawing, currentShape, allShapes
    if (!drawingMode || hwnd != ui.overlay.Hwnd)
        return
    DllCall("ReleaseCapture")
    drawing := false
    SetTimer(UpdateBuffer, 0)
    if (currentShape.points.Length >= 2) {
        allShapes.Push(currentShape)
        pPen := Gdip_CreatePen(0xFF0000FF, 2)
        Gdip_DrawLines(gdi.G_Baked, pPen, currentShape.pointsStr)
        Gdip_DeletePen(pPen)
    }
    currentShape := {}
    UpdateBuffer()
}

ClearDrawing(*) {
    global allShapes, gdi
    allShapes := []
    Gdip_GraphicsClear(gdi.G_Baked, 0x00000000)
    UpdateBuffer()
}

UpdateBuffer() {
    global gdi, currentShape, drawing, ui
    if (!gdi.G_Mem || !gdi.hdcBaked)
        return
    DllCall("BitBlt", "Ptr", gdi.hdcMem, "Int", 0, "Int", 0, "Int", A_ScreenWidth, "Int", A_ScreenHeight, "Ptr", gdi.hdcBaked, "Int", 0, "Int", 0, "UInt", 0x00CC0020)
    if (drawing && currentShape.HasProp("points")) {
        pPen := Gdip_CreatePen(0xFF0000FF, 2)
        Gdip_DrawLines(gdi.G_Mem, pPen, currentShape.pointsStr)
        Gdip_DeletePen(pPen)
    }
    DllCall("InvalidateRect", "Ptr", ui.overlay.Hwnd, "Ptr", 0, "Int", 0)
}

GdiStartup() {
    global gdi
    gdi.token := Gdip_Startup()
}

; =============================================
; RECOGNITION & EXACT EMULATION LOGIC (F3)
; =============================================

ProcessHandwritingRecognition(*) {
    global drawingMode, allShapes
    if (!drawingMode)
        return

    capturedShapes := allShapes.Clone()
    ExitDrawingMode(false)

    if (capturedShapes.Length = 0)
        return

    ActiverInkRecognitionWindow(capturedShapes)
}

ActiverInkRecognitionWindow(shapesToPlay) {
    global recGui, inkObj

    recGui := Gui("+AlwaysOnTop -Caption +ToolWindow", "InkFullscreen")
    recGui.BackColor := "FFFFFF" 

    inkControl := recGui.Add("ActiveX", "w" . A_ScreenWidth . " h" . A_ScreenHeight, "InkEd.InkEdit.1")
    inkObj := inkControl.Value

    try {
        inkObj.UseMouseForInput := 1
        inkObj.InkInsertMode := 0
        inkObj.BackColor := 0xFFFFFF
        inkObj.SelColor := 0xFF0000
        inkObj.RecognitionTimeout := 50000 
    } catch as err {
        MsgBox("InkEdit initialization error: " . err.Message, "Error", "Icon!")
        FermerInkWindow()
        return
    }

    recGui.Show("w" . A_ScreenWidth . " h" . A_ScreenHeight)
    
    ; Wait for the ActiveX control to be fully initialized
    Sleep(250)

    ; 4. Ultra-fast sequential emulation of strokes
    CoordMode("Mouse", "Screen")
    try {
        for shape in shapesToPlay {
            if (shape.type = "free" && shape.HasProp("points") && shape.points.Length >= 2) {
                pts := shape.points
                
                ; Position the mouse at the starting point without delay
                MouseMove(pts[1][1], pts[1][2], 0)
                
                ; Immediate mouse down
                Click("down")
                
                ; Traverse points at maximum speed (without Sleep(1))
                Loop pts.Length - 1 {
                    pt := pts[A_Index + 1]
                    MouseMove(pt[1], pt[2], 0)
                }
                
                ; Immediate button release
                Click("up")
            }
        }
    } catch as err {
        OutputDebug("Ink sequential replay error: " err.Message "`n")
    }
    
    ; 5. Immediate launch of text recognition and cleanup
    Sleep(100)
    try {
        inkObj.Recognize()
        
        ; Safety: Active wait for the recognition engine to finish (Status = 0)
        loop 50 {
            if (inkObj.Status == 0)
                break
            Sleep(10)
        }
        
        recognizedText := Trim(inkObj.Text)
        FermerInkWindow()
        
        if (recognizedText != "") {
            ; Save text to clipboard
            A_Clipboard := recognizedText
            
            Sleep(150)
            SendText(recognizedText)
        }
    } catch as err {
        FermerInkWindow()
    }
}

FermerInkWindow() {
    global recGui, inkObj
    if (IsSet(recGui) && recGui) {
        try recGui.Destroy()
    }
    recGui := unset
    inkObj := unset
}

If you want to understand the code quickly, here is a summary:

  1. First, we use the file Gdip_all.ahk available on https://github.com/mesutakcan/On-Screen-Drawing-Tool ;
  2. by pressing Ctrl+F9, we open a transparent window to draw. At this stage, there is no handwriting recognition yet; we simply record the coordinates of the strokes;
  3. At the end of the writing, by pressing F3, we inject the recorded stroke coordinates into the ActiverInkRecognitionWindow(shapesToPlay) function. ActiverInkRecognitionWindow() replays the strokes and performs handwriting recognition. The result is saved to the clipboard and sent to the texte input.

The writing and display of the strokes work well. What is causing a problem is the slowness of the ActiverInkRecognitionWindow() function, which replays all the strokes at the end, especially in the part:

    try {
        for shape in shapesToPlay {
            if (shape.type = "free" && shape.HasProp("points") && shape.points.Length >= 2) {
                pts := shape.points
                
                ; Position the mouse at the starting point without delay
                MouseMove(pts[1][1], pts[1][2], 0)
                
                ; Immediate mouse down
                Click("down")
                
                ; Traverse points at maximum speed (without Sleep(1))
                Loop pts.Length - 1 {
                    pt := pts[A_Index + 1]
                    MouseMove(pt[1], pt[2], 0)
                }
                
                ; Immediate button release
                Click("up")
            }
        }
    } catch as err {
        OutputDebug("Ink sequential replay error: " err.Message)
    }

where it replays all the strokes. ould there be a way to automatically and quickly inject the inks into the ActiveX InkEdit window from these recorded coordinates (capturedShapes)?
I search a way to inject them quickly like this :

    try {
        inkDisp := inkObj.Ink
        for shape in shapesToPlay {
            if (shape.HasProp("points") && shape.points.Length >= 2) {
                ptCount := shape.points.Length
                arr := ComObjArray(3, ptCount * 2) ; VT_I4 = 3
                for i, pt in shape.points {
                    arr[(i - 1) * 2]     := Integer(pt[1])
                    arr[(i - 1) * 2 + 1] := Integer(pt[2])
                }
                inkDisp.CreateStroke(arr)
            }
        }
    } catch as err {
        OutputDebug("Erreur injection Ink : " err.Message)
    }

However, this last example show no output text, no error message. There is no handwritten stroke appears in the ActiveX window. It seems that the injection of the writing or strokes didn't work on this last example.

Windows development
Windows development

A set of tools and technologies that enable developers to build, optimize, and deploy applications and services within the Windows ecosystem.

0 comments

Your answer

Answers can be marked as 'Accepted' by the question author and 'Recommended' by moderators, which helps users know the answer solved the author's problem.