How to stream consecutive DYNAMIC audio files from NodeJS to HTML5 audio tag- as one stream

Question

From an android device, the node server is receiving chunks of raw data file in arrayBuffer format. The server is converting them individually (let's say 10 seconds worth of playable audio) into WAV format in order to stream to an HTML5 audio file. The idea being that as soon as the android presses "send", the browser can immediately begin to hear the streamed audio file.

How can I stream these files consecutively to the HTML audio tag so that it sounds like one big file?

Currently I can stream one 10 second file to the tag and it plays. I also tried the audio tags "onended" attribute to try and fetch the next 10 second file, but you can hear that it is not connected as one file.

   const audioStream = fs.createReadStream(audioPath);

   audioStream.pipe(res);

Ideally I need to pipe all the files consecutively AS they come in from the android client, meaning I cannot first concatenate them into one file because the server is only receiving them as the browser is playing them (a latency of up to a few seconds is ok).

What can be done?

Edit:

Here is what I've tried:

  var audio = document.getElementById('my');
  var mediaSource = new MediaSource();
  var SEGMENTS = 5;
  var oReq = new XMLHttpRequest();

  mediaSource.addEventListener('sourceopen', function() {

    var sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');

    sourceBuffer.addEventListener('updateend', function() {
        //GET('6.mp3', function(data) { onAudioLoaded(data, index); });
      oReq.open("GET", '6.mp3');
      oReq.responseType = "arraybuffer";
        oReq.send();
    });

    function onAudioLoaded(data, index) {

      sourceBuffer.appendBuffer(data);
    }

    function h(){
      var arraybuffer = oReq.response; // not responseText
      onAudioLoaded(arraybuffer, 0)
     // console.log(arraybuffer);
    }

currently, I am fetching the same mp3 file from the server with a desire to string them together to see if it works. As I mentioned below, I get an "source buffer full" error.

in the long run i'd move this to a websocket so the each time the server receives a chunk, it will send to here, thus stringing the result into one "live stream". Any thoughts? Thank you

EDIT:

As it seems to have turned out, my whole approach here is wrong; Because I am converting each received chunk of raw audio into a unique mp3 file (it cannot be WAV because of no "Media Source Extension" support)-- even though I am able to string them together using MSE, a gap between each file is heard. I am assuming this is because of "padding" in each mp3 file.

Does anyone have any alternatives so that I can seamlessly play my chunks of raw audio in the browser? Unless I am missing something in this "implementation". Thank you.

Try using Media Source Extensions. Google search for Media Source Extensions play mp3 and after research, ask here if anything you tried (your own code) didn't work and we'll try help you to fix it. — VC.One
After trying it out, it seems like it has potential, however, the sourceBuffer has a limit to how much it can hold, then I get this error: (index):118 Uncaught DOMException: Failed to execute 'appendBuffer' on 'SourceBuffer': The SourceBuffer is full, and cannot free space to append additional buffers. It appears there is a method to remove from the buffer but this can only be done if the buffer is not updating. Can you help? — A S
I've seen that error before but I can't remember the cause. I think I had wrong formatting of mime string. Show me some testable code... How do you load the data and append? Use the edit mode option to update your post with some code we can check for problems. — VC.One
you can also see in this example, from where you referred me, in the inspector the same exception is thrown: 72lions.github.io/PlayingChunkedMP3MediaSourceAPI — A S

VC.One VC.One · Accepted Answer · 2021-01-22T03:35:13

Regarding the question "How to play consecutive dynamic audio files as one stream"...

Try a code setup like this below. Tested on Chrome.
(some names are changed, like your var arraybuffer is now input_MP3_Bytes) :

<!DOCTYPE html>
<html>
<body>

<audio id="audioMSE" controls></audio>

<script>

var input_MP3_Bytes = []; var mediaSource; var sourceBuffer;
var is_FirstAppend = true; //# for setting up MSE if first append.

var mimeCodec = 'audio/mpeg;'; //# tested on Chrome browser only
//var mimeCodec = 'audio/mpeg; codecs="mp3"';  //# untested.. maybe needed by other browsers?

//# for audio tag reference
var myAudio = document.getElementById('audioMSE'); 
myAudio.play(); //# start conveyor belt (to receive incoming bytes)

//# load some input MP3 bytes for MSE
var fileURL = "file_01.mp3"; //"6.mp3"
load_MP3( fileURL ); 

function load_MP3( filename ) 
{
    var oReq = new XMLHttpRequest();
    oReq.open("GET", filename, true);
    oReq.responseType = "arraybuffer";
    oReq.addEventListener('loadend', on_MP3isLoaded  ); 
    oReq.send(null);
}

function on_MP3isLoaded() 
{
    input_MP3_Bytes = new Uint8Array( this.response ); //# fill the byte array
    
    if (is_FirstAppend == true) 
    {
        console.log( "MSE support : " + MediaSource.isTypeSupported(mimeCodec) );
        makeMSE(); is_FirstAppend = false;
    }
    else { sourceBuffer.appendBuffer( input_MP3_Bytes ); }
 
}

function makeMSE ( ) 
{
    if ('MediaSource' in window && MediaSource.isTypeSupported(mimeCodec) ) 
    {
        mediaSource = new MediaSource;
        myAudio.src = URL.createObjectURL( mediaSource );
        mediaSource.addEventListener( "sourceopen", mse_sourceOpen );
    } 
    else { console.log("## Unsupported MIME type or codec: " + mimeCodec); }
}

function mse_sourceOpen() 
{
    sourceBuffer = mediaSource.addSourceBuffer( mimeCodec );
    sourceBuffer.addEventListener( "updateend", mse_updateEnd );
    
    //sourceBuffer.mode = "sequence";
    sourceBuffer.appendBuffer( input_MP3_Bytes );
    
};

function mse_updateEnd() 
{
    //# what to do after feeding-in the bytes...?
    //# 1) Here you could update (overwrite) same "input_MP3_Bytes" array with new GET request 
    //# 2) Or below is just a quick test... endless looping
    
    //# 1) load another MP3 file 
    fileURL = "file_25.mp3";
    load_MP3( fileURL ); 
    
    //# 2) re-feed same bytes for endless looping
    //sourceBuffer.appendBuffer( input_MP3_Bytes );
    
    //mediaSource.endOfStream(); //# close stream to close audio
    //console.log("mediaSource.readyState : " + mediaSource.readyState); //# is ended
}

</script>

</body>
</html>

How to stream consecutive DYNAMIC audio files from NodeJS to HTML5 audio tag- as one stream

1 Answers