Use SetWindowModality instead of WindowStayOnTopHint, and both modal modes (Qt::WindowModal and Qt::ApplicationModal) permit other applications to be on top of your modal window.
LE: You can read more about the difference between ApplicationModal and WindowModal in the QDialog's documentation page, here
LE 2: The problem is that you don't set a parent, so to solve this set a parent for every child window (everything except your main window) and everything will work as you expected (the child windows will be on top of the parent, but won't be on top of any other application windows):
int main(int argc, char** argv)
{
QApplication a(argc, argv);
QWidget w;
QVBoxLayout* layout = new QVBoxLayout(&w);
QPushButton* btn = new QPushButton("Show a non-modal window");
layout->addWidget(btn);
QWidget* mainWindow = &w;
QObject::connect(btn, &QPushButton::clicked, [mainWindow]()
{
QWidget* dlg = new QWidget(mainWindow);
QVBoxLayout* dlgLayout = new QVBoxLayout(dlg);
dlg->setWindowFlags(Qt::Window);
QLabel* lbl = new QLabel("Non-modal window...", dlg);
dlgLayout->addWidget(lbl);
dlg->show();
});
w.show();
return a.exec();
}