My goal is to play a raw H264 stream being fed through a tcp/ip port on an Android device (a Samsung S10) using the MediaCodec class. It isn't working, I just see a blank screen.
I have a few guesses to the issue: 1) Do I need to group together NAL units? Right now I feed each unit individually to the MediaCodec. 2) Do I need to make changes to the server? are there variants of H264 that the MediaCodec class cannot handle?
I was able to port FFmpeg to Android studio and got this solution to work. However, it is slow as it uses a software codec. I decided to use MediaCodec to try and use the hardware codec. The code below shows my effort. The codec is initialized in asynchronous mode. I have a separate thread to read and queue up the NAL frames from the tcp socket. Frames are stored in a buffer and if the buffer overflows, then some frames will be discarded. The onInputBufferAvailable codec feeds one NAL unit at a time to the MediaCodec class.
public void initializePlaybackCodec()
{
mWidth = 1536;
mHeight = 864;
MediaFormat decoderFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
try {
codec = MediaCodec.createDecoderByType(MIME_TYPE);
} catch (IOException e) {
Log.e(TAG, "CODEC INIT: Failed to initialize media codec", e);
Toast.makeText(this, "Failed to initialize media codec",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"HERE CODEC INITIALIZED");
final int videoQueueSize = 10;
final Semaphore mutex = new Semaphore(1);
final Semaphore queueData = new Semaphore(0);
final ArrayBlockingQueue<ByteBuffer> queue = new ArrayBlockingQueue<ByteBuffer>(videoQueueSize);
codec.setCallback(new MediaCodec.Callback() {
long reference_epoch = System.currentTimeMillis();
long current_epoch = reference_epoch;
byte[] buffer = new byte[blockSize];
int nextStart = 0;
@Override
public void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
current_epoch = System.currentTimeMillis();
ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId);
android.media.MediaCodecInfo info = codec.getCodecInfo();
//Log.i(TAG,"CODEC CALLBACK: info "+info.getName()+" Encoder: "+info.isEncoder()+" ");
//String[] types = info.getSupportedTypes();
//for (int j = 0; j < types.length; j++) {
// Log.i(TAG,"CODEC CALLBACK: supportedType "+types[j]);
//}
// Read data from the Queue
ByteBuffer b = null;
Log.i(TAG,"CODEC CALLBACK: input");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: queueData acquire interrupted");
codec.stop();
finish();
return;
}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: mutex acquire interrupted");
codec.stop();
finish();
return;
}
try {
b = queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: take interrupted");
codec.stop();
finish();
return;
}
byte[] bb = b.array();
//Log.i(TAG,"CODEC CALLBACK: Contents being sent "+bb[4]/32+" "+bb[4]%32+" "+bb.length);
Log.i(TAG,"CODEC CALLBACK: Contents being sent "+Integer.toHexString(bb[0])+" "+Integer.toHexString(bb[1])+" "+Integer.toHexString(bb[2])+" "+Integer.toHexString(bb[3])+" "+Integer.toHexString(bb[4])+" ");
int ref_idc = bb[4]/32;
int unit_type = bb[4]%32;
//for (int i = 0; i < bb.length && i < 5; ++i) {
// Log.i(TAG, "CODEC CALLBACK: bb["+i+"]="+bb[i]);
//}
mutex.release();
// fill inputBuffer with valid data
//Log.i(TAG,"CODEC CALLBACK: put "+b.remaining()+" "+b.capacity());
inputBuffer.clear();
//Log.i(TAG,"CODEC CALLBACK: before put "+inputBuffer.remaining()+" "+b.position());
b.position(0);
inputBuffer.put(b);
//Log.i(TAG,"CODEC CALLBACK: after put "+inputBuffer.remaining());
//Log.i(TAG,"CODEC CALLBACK: queue "+(current_epoch-reference_epoch)*1000+" "+inputBuffer.capacity()+" "+inputBuffer.remaining());
codec.queueInputBuffer(inputBufferId,0, b.remaining(), (current_epoch-reference_epoch)*1000, 0);
}
@Override
public void onOutputBufferAvailable(MediaCodec mc, int outputBufferId,
MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
// bufferFormat is equivalent to mOutputFormat
// outputBuffer is ready to be processed or rendered.
Log.i(TAG,"CODEC CALLBACK: output");
codec.releaseOutputBuffer(outputBufferId, true);
Log.i(TAG,"CODEC CALLBACK: output done");
}
@Override
public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
// Subsequent data will conform to new format.
// Can ignore if using getOutputFormat(outputBufferId)
//mOutputFormat = format; // option B
Log.i(TAG,"CODEC CALLBACK: output format changed");
}
@Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG,"CODEC CALLBACK: Media Codec Error");
}
});
codec.configure(decoderFormat, m_surface.getHolder().getSurface(), null, 0);
Thread thread = new Thread(){
public void run(){
Socket socket;
InputStream input;
try {
socket = new Socket(mServerAddr, Integer.parseInt(mServerPort));
input = socket.getInputStream();
} catch (IOException e) {
Log.e(TAG, "RLOOP: Failed to open video socket", e);
Toast.makeText(ARActivity.this, "Failed to open video socket",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE SOCKET OPENED");
System.out.println("Socket opened");
byte[] buffer = new byte[blockSize];
java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.allocate(blockSize);
int nextStart = 0;
while (true) {
int size = 1;
try {
size = input.read(buffer,nextStart,blockSize-nextStart);
Log.i(TAG,"RLOOP: Read from video stream "+size+" bytes start="+nextStart);
Log.i(TAG, "RLOOP: First bytes "+buffer[nextStart]+" "+buffer[nextStart+1]+" "+
buffer[nextStart+2]+" "+buffer[nextStart+3]+" "+buffer[nextStart+4]);
if (size==0) {
Log.e(TAG, "RLOOP: Video stream finished");
Toast.makeText(ARActivity.this, "Video stream finished",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
int endPos = 2;
while (endPos > 0) {
endPos = -1;
int zeroCount = 0;
for (int i = nextStart; (i < size+nextStart && endPos < 1); ++i) {
//Log.i(TAG,"Zero count pos "+i+" "+zeroCount);
if (buffer[i]==0) {
++zeroCount;
} else if (buffer[i]==1 && zeroCount > 1) {
if (zeroCount > 3) {
zeroCount = 3;
}
endPos = i-zeroCount;
Log.i(TAG,"RLOOP: Found marker at pos "+(i-zeroCount));
zeroCount = 0;
} else {
zeroCount = 0;
}
}
Log.i(TAG,"RLOOP: State nextStart="+nextStart+" endPos="+endPos+" size="+size);
if (endPos < 0) {
if (size + nextStart == blockSize) {
Log.e(TAG, "RLOOP: Error reading video stream2");
//Toast.makeText(ARActivity.this, "Error reading video stream2",
// Toast.LENGTH_LONG).show();
//finish();
endPos = blockSize;
nextStart = 0;
Log.i(TAG, "RLOOP: BLOCK OVERFLOW " + endPos);
} else {
nextStart = size + nextStart;
}
} else if (endPos==0) {
Log.i(TAG, "RLOOP: BLOCK NOT COMPLETE " + endPos);
//nextStart = size+nextStart;
} else {
Log.i(TAG, "RLOOP: PROCESSING BLOCK " + endPos);
//Log.i(TAG,"BUFFER REMAINING "+byteBuffer.remaining());
//Log.i(TAG,"BUFFER POSITION "+byteBuffer.position());
//System.arraycopy(buffer, 4, buffer, 0, size + nextStart - 4);
//nextStart = nextStart - 4;
//if (nextStart < 0) {
// size = size + nextStart;
// nextStart = 0;
//}
//endPos = endPos-4;
byteBuffer = java.nio.ByteBuffer.allocate(endPos+3);
byteBuffer.put(buffer, 0, endPos);
//byteBuffer = java.nio.ByteBuffer.wrap(buffer, 0, endPos);
//byteBuffer.put(buffer,0, endPos);
Log.i(TAG, "RLOOP: BUFFER REMAINING2 " + byteBuffer.remaining());
Log.i(TAG, "RLOOP: BUFFER POSITION2 " + byteBuffer.position());
Log.i(TAG, "RLOOP: First send bytes " + buffer[0] + " " + buffer[1] + " " +
buffer[2] + " " + buffer[3] + " " + buffer[4]);
//byte[] bb = byteBuffer.array();
Log.i(TAG,"RLOOP: Contents being sent");
//for (int i = 0; i < bb.length && i < 10; ++i) {
// Log.i(TAG, "RLOOP: bb["+i+"]="+bb[i]);
//}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: Mutex interrupted");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE1");
if (queue.size() == videoQueueSize) {
try {
queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue.take interrupted 2");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE2");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queueData.acquire() interrupted 2");
codec.stop();
finish();
return;
}
}
Log.i(TAG,"RLOOP: HERE3");
try {
queue.put(byteBuffer);
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue put interrupted");
codec.stop();
finish();
return;
}
queueData.release();
mutex.release();
if (endPos < size+nextStart) {
System.arraycopy(buffer, endPos, buffer, 0, size + nextStart - endPos);
nextStart = nextStart - endPos;
if (nextStart < 0) {
size = size + nextStart;
nextStart = 0;
}
}
}
}
nextStart = nextStart + size;
} catch (IOException e) {
Log.e(TAG, "RLOOP: Error reading from video stream");
Toast.makeText(ARActivity.this, "Error reading from video stream",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
}
}
};
thread.start();
codec.start();
return;
}
My expected result is to see a video on the android device. My actual result is that the onOutputBufferAvailable function is never called.
I am including a sample debugging output to show some of the NAL units being sent to the MediaCodec class.
2019-06-19 12:22:38.229 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.249 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.251 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.266 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.268 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.281 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.282 3325-3539/com.example.unrealar I/MediaCodec: setCodecState state : 0
2019-06-19 12:22:38.282 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
