You can actually do this without relying on private API's, at least if your willing to require Mac OS X 10.4 or better.
Put the following in your cell subclass:
- (NSColor *)highlightColorWithFrame:(NSRect)cellFrame inView:(NSView *)controlView
{
// Returning nil circumvents the standard row highlighting.
return nil;
}
And then subclass the NSOutlineView and re-implement the method, - (void)highlightSelectionInClipRect:(NSRect)clipRect;
Here's an example that draws one color for non-group rows and another for group rows
- (void)highlightSelectionInClipRect:(NSRect)clipRect
{
NSIndexSet *selectedRowIndexes = [self selectedRowIndexes];
NSRange visibleRows = [self rowsInRect:clipRect];
NSUInteger selectedRow = [selectedRowIndexes firstIndex];
while (selectedRow != NSNotFound)
{
if (selectedRow == -1 || !NSLocationInRange(selectedRow, visibleRows))
{
selectedRow = [selectedRowIndexes indexGreaterThanIndex:selectedRow];
continue;
}
// determine if this is a group row or not
id delegate = [self delegate];
BOOL isGroupRow = NO;
if ([delegate respondsToSelector:@selector(outlineView:isGroupItem:)])
{
id item = [self itemAtRow:selectedRow];
isGroupRow = [delegate outlineView:self isGroupItem:item];
}
if (isGroupRow)
{
[[NSColor alternateSelectedControlColor] set];
} else {
[[NSColor secondarySelectedControlColor] set];
}
NSRectFill([self rectOfRow:selectedRow]);
selectedRow = [selectedRowIndexes indexGreaterThanIndex:selectedRow];
}
}