Bravo's code works except it fails with an error when no layers are selected, so I tweaked it to remove the error and simply return an empty array:
function getSelectedLayers(){
var resultLayers=new Array();
try{
var idGrp = stringIDToTypeID( "groupLayersEvent" );
var descGrp = new ActionDescriptor();
var refGrp = new ActionReference();
refGrp.putEnumerated(charIDToTypeID( "Lyr " ),charIDToTypeID( "Ordn" ),charIDToTypeID( "Trgt" ));
descGrp.putReference(charIDToTypeID( "null" ), refGrp );
executeAction( idGrp, descGrp, DialogModes.NO );
for (var ix=0;ix<app.activeDocument.activeLayer.layers.length;ix++){resultLayers.push(app.activeDocument.activeLayer.layers[ix])}
var id8 = charIDToTypeID( "slct" );
var desc5 = new ActionDescriptor();
var id9 = charIDToTypeID( "null" );
var ref2 = new ActionReference();
var id10 = charIDToTypeID( "HstS" );
var id11 = charIDToTypeID( "Ordn" );
var id12 = charIDToTypeID( "Prvs" );
ref2.putEnumerated( id10, id11, id12 );
desc5.putReference( id9, ref2 );
executeAction( id8, desc5, DialogModes.NO );
} catch (err) { }
return resultLayers;
}
$.writeln(getSelectedLayers());`
Simply wrapping the code in a try/catch block didnt work, so I also changed:
executeAction( idGrp, descGrp, DialogModes.ALL );
to
executeAction( idGrp, descGrp, DialogModes.NO );
and that made the runtime error go away.