Using Bing Speech Recognition API with node.js Bot Framework on Skype

Question

I would like to use the Bing Speech Recognition API to convert speech to text when sending audio attachments in Skype to my node.js chatbot. I have tried using the code from BotBuilder-Samples intelligence-SpeechToText, however the speech recognition only works in the Emulator. When sending an audio/wave file in Skype, the bot does not respond at all instead of "You said: What’s the weather like?".

I suspected that the issue might be due to the fact that a JWT Token is required to access attachments in Skype. Hence, I have tried accessing the audio attachment in Skype using the code from BotBuilder-Samples core-ReceiveAttachment which uses request-promise instead of needle to make the HTTP request. However, the result from request-promise is not a stream and cannot be processed by the function getTextFromAudioStream().

I there would like to ask how to get speech recognition to work with audio attachments in Skype.

Thanks and best regards!

// Add your requirements
var restify = require("restify");
var builder = require("botbuilder");
var fs = require("fs");
var needle = require("needle");
var request = require("request");
var speechService = require("./speech-service.js");
var Promise = require('bluebird');
var request = require('request-promise').defaults({ encoding: null });

//=========================================================
// Bot Setup
//=========================================================

// Setup Restify Server
var server = restify.createServer();
server.listen(process.env.PORT || 3000, function() {
   console.log("%s listening to %s", server.name, server.url); 
});

// Create chat bot
var connector = new builder.ChatConnector ({
    appId: process.env.MICROSOFT_APP_ID,
    appPassword: process.env.MICROSOFT_APP_PASSWORD
});

server.post("/api/messages", connector.listen());

var bot = new builder.UniversalBot(connector);

//=========================================================
// Bots Middleware
//=========================================================

// Anytime the major version is incremented any existing conversations will be restarted.
bot.use(builder.Middleware.dialogVersion({ version: 1.0, resetCommand: /^reset/i }));

//=========================================================
// Bots Dialogs
//=========================================================

bot.dialog("/", [
    function (session, results, next) {
        var msg = session.message;

        if (hasAudioAttachment(msg)) {
            // Message with attachment, proceed to download it.
            // Skype attachment URLs are secured by a JwtToken, so we need to pass the token from our bot.
            var attachment = msg.attachments[0];
            var fileDownload = isSkypeMessage(msg)
                ? requestWithToken(attachment.contentUrl)
                : request(attachment.contentUrl);

            fileDownload.then(
                function (response) {
                    // Send reply with attachment type & size
                    var reply = new builder.Message(session)
                        .text('Attachment from %s of %s type and size of %s bytes received.', msg.source, attachment.contentType, response.length);
                    session.send(reply);
                }).catch(function (err) {
                    console.log('Error downloading attachment:', { statusCode: err.statusCode, message: err.response.statusMessage });
            });

            var stream = isSkypeMessage(msg)
                ? getAudioStreamWithToken(attachment)
                : getAudioStream(attachment);

            speechService.getTextFromAudioStream(stream)
                .then(text => {
                    session.send("You said: " + text);
                })
                .catch(error => {
                    session.send("Oops! Something went wrong. Try again later.");
                    console.error(error);
                });
        }
        else {
            session.send("Did you upload an audio file? I'm more of an audible person. Try sending me a wav file");
        }
    }
]);

function getAudioStream(attachment) {
    return needle.get(attachment.contentUrl, { headers: {'Content-Type': "audio/wav"} });
}

function getAudioStreamWithToken(attachment) {
    var headers = {};

    connector.getAccessToken((error, token) => {
        headers['Authorization'] = 'Bearer ' + token;
    });

    headers['Content-Type'] = attachment.contentType;

    return needle.get(attachment.contentUrl, { headers: headers });
}

// Request file with Authentication Header
function requestWithToken(url) {
    return obtainToken().then(function (token) {
        return request({
            url: url,
            headers: {
                'Authorization': 'Bearer ' + token,
                'Content-Type': 'application/octet-stream'
            }
        });
    });
};

// Promise for obtaining JWT Token (requested once)
var obtainToken = Promise.promisify(connector.getAccessToken.bind(connector));

function isSkypeMessage(message) {
    return message.source === "skype";
};

Ezequiel Jadib Ezequiel Jadib · Accepted Answer · 2017-01-13T15:59:39

The code in the sample is already considering Skype when accessing to the attachments (see here). I think the problem you were hitting is because the key in the sample exceeded the quota. Yesterday a new Bing Speech Key was added to the sample, so I would suggest you to try again.

Also, an updated version of the sample is going to be added soon. The code is currently under code review.

Using Bing Speech Recognition API with node.js Bot Framework on Skype

1 Answers