Unable to stream video over a websocket to Firefox

Question

I have written some code stream video over a websocket so a sourcebuffer which works in Chrome and Edge.

However, when I run this in Firefox, the video never plays back, just a spinning wheel animation is displayed. When I check the <video> statistics, It reads HAVE_METADATA as the ready state and NETWORK_LOADING as the network state.

The code looks follows:

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
  </head>
  <body>
    <video controls></video>
    <script>
      var mime = 'video/mp4; codecs="avc1.4D401E,mp4a.40.2"';
      var address = 'ws://localhost:54132'

      /* Media Source */

      var source = new MediaSource();
      var video = document.querySelector('video');
      video.src = URL.createObjectURL(source);
      source.addEventListener('sourceopen', sourceOpen);

      /* Buffer */

      var buffer;
      var socket;
      var queue = [];
      var offset = -1;
      var timescale;

      // When the media source opens:
      function sourceOpen() {
        buffer = source.addSourceBuffer(mime);
        buffer.addEventListener('updateend', processQueue);

        socket = new WebSocket(address);
        socket.binaryType = 'arraybuffer';
        socket.onmessage = onMessage; 
      }

      // When more data is received.
      function onMessage(event) {
        queue.push(event.data);
        processQueue();
      }

      // Process queue if possible.
      function processQueue() {
        if ((queue.length == 0) || (buffer.updating)) {
          return;
        }

        var data = queue.shift();
        if (offset === -1) {
          var parsed = parseMP4(data);
          if (parsed.hasOwnProperty('moov')) {
            timescale = parsed.moov.mvhd.timescale;
          } else if (parsed.hasOwnProperty('moof')) {
            offset = 0 - (parsed.moof.traf[0].tfdt.baseMediaDecodeTime / this.timescale - 0.4);
            buffer.timestampOffset = offset;
          }
        }

        // console.log('appending ' + data.byteLength + ' bytes');
        buffer.appendBuffer(data);
      }

      // Parse out the offset.
      function parseMP4(data) {
        // SNIP for brevity
      }
    </script>
  </body>
</html>

do you get error event on video element or source buffer ? what happens when you directly use the video URL as "src" (without media source) — sbr
No error event. The data is generated on the fly by the server (from an IP stream for example), so there is no URL I can point the video element to. — Hans
No, the video is fed from a custom c# server application. All the code (except for the bit that parses the MP4 container) is up there. — Hans
i don't see the part wheere you call .endOfStream() or .play(), so it won't start playing. — dandavis

guest271314 guest271314 · Accepted Answer · 2016-07-02T18:11:14

Could not reproduce <video> element not playing at firefox 47.

Merged approaches at Mocking Websocket Message Events to create mock WebSocket events; bufferAll.html demo at Let's Make a Netflix An Intro to Streaming Media on the Web for MediaSource usage pattern.

Included <progress> and progress event to notify user of media loading status.

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
  </head>
  <body>
    <progress min="0" value="0"></progress><br><label></label><br>
    <video controls></video>
    <script>
        // http://nickdesaulniers.github.io/netfix/demo/bufferAll.html
        // http://jsfiddle.net/adamboduch/JVfkt/
        // The global web socket.    
        var sock, sourceBuffer;
        sock = new WebSocket( "ws://mock" );
        sock.onerror = function(e) {
          console.log("sock error", e)
        }
        // This is unchanging production code that doesn"t know
        // we"re mocking the web socket.
        sock.onmessage = function( e ) {
          console.log("socket message", e.data);
          sourceBuffer.appendBuffer(e.data);
        }; 
      var video = document.querySelector("video");
      var progress = document.querySelector("progress");
      var label = document.querySelector("label");
      var assetURL = "http://nickdesaulniers.github.io/netfix/"
                     + "demo/frag_bunny.mp4";
      // Need to be specific for Blink regarding codecs
      // ./mp4info frag_bunny.mp4 | grep Codec
      var mimeCodec = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"';

      if ("MediaSource" in window 
          && MediaSource.isTypeSupported(mimeCodec)) {
        var mediaSource = new MediaSource;
        //console.log(mediaSource.readyState); // closed
        video.src = URL.createObjectURL(mediaSource);
        mediaSource.addEventListener("sourceopen", sourceOpen);
      } else {
        console.error("Unsupported MIME type or codec: ", mimeCodec);
      }
      video.addEventListener("canplay", function() {
        alert("video canplay")
      })
      function sourceOpen (_) {
        //console.log(this.readyState); // open
        var mediaSource = this;
        sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
        fetchAB(assetURL, function (buf) {
          sourceBuffer.addEventListener("updateend", function (event) {
          console.log("sourceBuffer updateend event;"
                      + "mediaSource.readyState:"
                     , mediaSource.readyState);
            // mediaSource.endOfStream();
            // video.play();
            // console.log(mediaSource.readyState); // ended
          });
          
        });
      };
      // mock `WebSocket` message
      function fetchAB (url, cb) {
        var xhr = new XMLHttpRequest;
        xhr.open("get", url);
        var file = url.split("/").pop();
        xhr.responseType = "arraybuffer";
        xhr.onload = function () {
          // mock `WebSocket` message
          sock.dispatchEvent( new MessageEvent( "message", {
            data: xhr.response
        }));
        console.log("video sent to sock", sock);
        cb();
        };
        xhr.onprogress = function(e) {
           progress.max = e.total;
           progress.value = e.loaded;
           label.innerHTML = "loading " + file + " ...<br>"
                             + e.loaded + " of " 
                             + e.total + " bytes loaded";
        }
        xhr.send();
      };  
    </script>
  </body>
  </html>

plnkr http://plnkr.co/edit/RCIqDXTB2BL3lec9bhfz

Unable to stream video over a websocket to Firefox

2 Answers