I've done this same sort of thing with a listener on the beforesubmit event of the Dialog (see beforesubmit for CQ.Dialog at http://dev.day.com/docs/en/cq/current/widgets-api/index.html - search for "Dialog"):
<listeners
jcr:primaryType="nt:unstructured"
beforesubmit="function(dialog){return myNamespace.myCustomFunction(dialog);}"/>
Then the custom JavaScript function, included on the page via a client library, could be something like this:
myNamespace = {};
myNamespace.myCustomFunction = function (dialog) {
var isValid = function () {
var valStatus = true;
... custom JavaScript/jQuery to check if 3 items exist ...
return valStatus;
};
if (!isValid()) {
CQ.Ext.Msg.show({title: 'Validation Error', msg: 'Must contain at least 3 items!', buttons: CQ.Ext.MessageBox.OK, icon: CQ.Ext.MessageBox.ERROR});
return false;
} else {
return true;
}
}