Насколько я понимаю, это рамка самого окна?
Можно попробовать задать один лишь стиль $WS_POPUP:
Код:

#include <WindowsConstants.au3>
$hWindow = WinGetHandle("Title")
_WinSetStyle($hWindow, $WS_POPUP)
Func _WinSetStyle($hWnd, $nStyle = -1, $nExStyle = 0)
Local Const $GWL_STYLE = -16, $GWL_EXSTYLE = -20
Local Const $SWP_NOMOVE = 0x2, $SWP_NOSIZE = 0x1, $SWP_SHOWWINDOW = 0x40, $SWP_NOZORDER = 0x4
Local $iFlags = BitOR($SWP_SHOWWINDOW, $SWP_NOSIZE, $SWP_NOMOVE, $SWP_NOZORDER)
If $nStyle = -1 Then $nStyle = BitOR($WS_MINIMIZEBOX, $WS_CAPTION, $WS_POPUP, $WS_SYSMENU)
DllCall("User32.dll", "int", "SetWindowLong", "hwnd", $hWnd, "int", $GWL_STYLE, "int", $nStyle)
DllCall("User32.dll", "int", "SetWindowLong", "hwnd", $hWnd, "int", $GWL_EXSTYLE, "int", $nExStyle)
DllCall("User32.dll", "int", "SetWindowPos", "hwnd", $hWnd, "hwnd", 0, "int", 0, "int", 0, "int", 0, "int", 0, "int", $iFlags)
EndFunc