I know this is over a year old, but this was the first hit on my Google search and was unanswered, so I'm posting my answer for sake of those still looking for a solution.
For my app, I used Core Animation with a custom NSView for the NSMenuItem view. I created a new layer-backed view, set the background color, and added it to my custom view. I then animated the layer (the flashing part). Then in the -(void) animationDidStop:(CAAnimation *)anim finished:(BOOL)flag
callback, I removed the overlay and closed the menu. This doesn't perfectly match the default NSMenu's flash, but I wanted a 37Signals/Stack Overflow Yellow Fade Technique, so it works for me. Here it is in code:
-(void) mouseUp:(NSEvent *)theEvent {
CALayer *layer = [CALayer layer];
[layer setDelegate:self];
[layer setBackgroundColor:CGColorCreateGenericRGB(0.0, 0.0, 1.0, 1.0)];
selectionOverlayView = [[NSView alloc] init];
[selectionOverlayView setWantsLayer:YES];
[selectionOverlayView setFrame:self.frame];
[selectionOverlayView setLayer:layer];
[[selectionOverlayView layer] setNeedsDisplay];
[selectionOverlayView setAlphaValue:0.0];
[self addSubview:selectionOverlayView];
CABasicAnimation *alphaAnimation1 = [CABasicAnimation animationWithKeyPath: @"alphaValue"];
alphaAnimation1.beginTime = 0.0;
alphaAnimation1.fromValue = [NSNumber numberWithFloat: 0.0];
alphaAnimation1.toValue = [NSNumber numberWithFloat: 1.0];
alphaAnimation1.duration = 0.07;
CABasicAnimation *alphaAnimation2 = [CABasicAnimation animationWithKeyPath: @"alphaValue"];
alphaAnimation2.beginTime = 0.07;
alphaAnimation2.fromValue = [NSNumber numberWithFloat: 1.0];
alphaAnimation2.toValue = [NSNumber numberWithFloat: 0.0];
alphaAnimation2.duration = 0.07;
CAAnimationGroup *selectionAnimation = [CAAnimationGroup animation];
selectionAnimation.delegate = self;
selectionAnimation.animations = [NSArray arrayWithObjects:alphaAnimation1, alphaAnimation2, nil];
selectionAnimation.duration = 0.14;
[selectionOverlayView setAnimations:[NSDictionary dictionaryWithObject:selectionAnimation forKey:@"frameOrigin"]];
[[selectionOverlayView animator] setFrame:[selectionOverlayView frame]];
}
-(void) animationDidStop:(CAAnimation *)anim finished:(BOOL)flag {
[selectionOverlayView removeFromSuperview];
NSMenuItem *enclosingMenuItem = [self enclosingMenuItem];
NSMenu *enclosingMenu = [enclosingMenuItem menu];
[enclosingMenu cancelTracking];
[enclosingMenu performActionForItemAtIndex:[enclosingMenu indexOfItem:enclosingMenuItem]];
}