I've managed to implement this in Swift 3 (with the help of this great tip and this and this SO answer):

Make sure the table view cell in the NSTableView
has a delegate connection to your subclass/view controller which adopts the NSTextFieldDelegate
protocol.
Also give it these constraints to make sure it resizes according to the height of the row:

In the delegate use this code:
var editedString: String? = nil
var textCellForHeight: NSTextFieldCell = NSTextFieldCell.init()
func control(_ control: NSControl, textShouldBeginEditing fieldEditor: NSText) -> Bool {
editedString = fieldEditor.string ?? ""
return true
}
func control(_ control: NSControl, textShouldEndEditing fieldEditor: NSText) -> Bool {
editedString = nil
return true
}
func control(_ control: NSControl, textView: NSTextView, doCommandBy commandSelector: Selector) -> Bool {
if commandSelector == #selector(insertNewline(_:)) {
textView.insertNewlineIgnoringFieldEditor(self)
editedString = textView.string ?? ""
//The NSAnimationContext lines get rid of the animation that normally happens when the height changes
NSAnimationContext.beginGrouping()
NSAnimationContext.current().duration = 0
myTableView.noteHeightOfRows(withIndexesChanged: IndexSet.init(integer: selected))
NSAnimationContext.endGrouping()
myTable.needsDisplay = true
return true
}
return false
}
func tableView(_ tableView: NSTableView, heightOfRow row: Int) -> CGFloat {
if let temp = editedString { //we know it’s currently being edited if it’s not nil
textCellForHeight.stringValue = temp
//in my case, there was a specific table column that I knew would be edited
//you might need to decide on the column width another way.
let column = myTable.tableColumns[myTable.column(withIdentifier: “TheColumnThatsBeingEdited”)]
let frame: NSRect = NSMakeRect(0, 0, column.width, CGFloat.greatestFiniteMagnitude)
return textCellForHeight.cellSize(forBounds: frame).height
}
return yourStandardHeightCGFloat
}