Extract faces from video using AWS Rekognition

4
votes

With AWS Rekognition I was able to get faces detected in a mp4 video with the following nodejs,

var AWS = require("aws-sdk");                                                                                              
AWS.config.update({                                                                                                        
  region: "us-east-1"                                                                                                      
});                                                                                                                        

var rekognition = new AWS.Rekognition();                                                                                   



var params = {                                                                                                             
  Video: { /* required */                                                                                                  
    S3Object: {                                                                                                            
      Bucket: 'videobucket',                                                                                          
      Name: 'testvideo.mp4'                                                                                                
    }                                                                                                                      
  },                                                                                                                       
  FaceAttributes: "ALL",                                                                                                   
  NotificationChannel: {                                                                                                   
    RoleArn: 'arn:aws:iam::xxx:role/xxx', /* required */                                                     
    SNSTopicArn: 'arn:aws:sns:us-east-1:xxx:alerts' /* required */                                                
  }                                                                                                                        
};                                                                                                                         
rekognition.startFaceDetection(params, function(err, data) {                                                               
  if (err) console.log(err, err.stack); // an error occurred                                                               
  else     console.log(data);           // successful response                                                             
});

And was able to get the results with the following cli,

aws rekognition get-face-detection --job-id xxxxxxx

and outputs the faces in the following json format,

{                                                                                                                          
    "Faces": [                                                                                                             
        {                                                                                                                  
            "Timestamp": 0,                                                                                                
            "Face": {                                                                                                      
                "BoundingBox": {                                                                                           
                    "Width": 0.029999999329447746,                                                                         
                    "Top": 0.2588889002799988,                                                                             
                    "Left": 0.29374998807907104,                                                                           
                    "Height": 0.052222222089767456                                                                         
                },                                                                                                         
                "Landmarks": [                                                                                             
                    {                                                                                                      
                        "Y": 0.28277161717414856,                                                                          
                        "X": 0.3052537739276886,                                                                           
                        "Type": "eyeLeft"                                                                                  
                    },                                                                                                     
                    {                                                                                                      
                        "Y": 0.27957838773727417,                                                                          
                        "X": 0.3085327744483948,                                                                           
                        "Type": "eyeRight"

How to extract those faces as images and dump them in an s3 bucket?

Thanks

2
amazon-web-servicesaws-cliamazon-rekognition
How did you solve your problem? – Jiūtas Brólickaitis
Not yet solved. – Kannaiyan
Why do not you just take the frame at the timestamp provided by aws, crop the region (again, provided by aws) and dump faces one by one? – Jiūtas Brólickaitis
How to select a frame and crop the image out of it ? – Kannaiyan

2 Answers

2
votes

to your question, you can get the faces alone by cropping the sections of the face from the original image using the bounding boxes returned from aws rekognition. I have done with python

     widtho = 717 #width of the given image
     heighto = 562 #height of the given image
     counter = 0
     facecount = 1
     s3 = boto3.resource('s3')
     bucket = s3.Bucket('rek')
     if __name__ == "__main__":
     #Choosing the file in s3 bucket
         photo = 'sl.jpg'
         bucket = 'rek'
     #Intilization of rekognition and performing detect_faces 
     client = boto3.client('rekognition', region_name='eu-west-1')
     response = client.detect_faces(
     Image={'S3Object': {'Bucket': bucket, 'Name': photo}}, Attributes=['ALL'])
     print('Detected faces for ' + photo)
     print('The faces are detected and labled from left to right')
     for faceDetail in response['FaceDetails']:
         print('Face Detected= ', i)
         #To mark a bounding box of the image using coordinates
         print('Bounding Box')
         bboxlen = len(faceDetail['BoundingBox'])
         print(bboxlen)
         width = faceDetail['BoundingBox'].get('Width')
         height = faceDetail['BoundingBox'].get('Height')
         left = faceDetail['BoundingBox'].get('Left')
         top = faceDetail['BoundingBox'].get('Top')
         w = int(width * widtho)
         h = int(height * heighto)
         x = int(left * widtho)
         y = int(top * heighto)
         img2 = image_np[y:h, x:w]
         #now you can push the img2 which is one of the face in the single frame
1
votes

Here are the steps followed to extract the faces. With the output from AWS rekognition, create a shell script output to extract the faces using ffmpeg. Added extra padding to the face to get hair, chin and side ears. You can adjust based on your needs.

var faces = require('./output1.json');                                                                                     

var oldtimestamp = -1;                                                                                                     
var sequence = 1;                                                                                                          



faces.Faces.forEach(function(element) {                                                                                    
    var size = element.Face.BoundingBox;                                                                                   
    var vheight = 576;                                                                                                     
    var vwidth = 720;                                                                                                      
    size.Width = parseInt(size.Width * 1.25 *  vwidth);                                                                    
    size.Left = parseInt(size.Left * vwidth);                                                                              
    size.Top = parseInt(size.Top * 0.5  * vheight)  ;                                                                      
    size.Height = parseInt(size.Height * 1.75 * vheight);                                                                  

  var newtimestamp = element.Timestamp;                                                                                    
  if( oldtimestamp != newtimestamp ) {                                                                                     
        console.log('ffmpeg  -i f1.vob  -ss 00:00:' + (element.Timestamp/1000) + '   -qmin 1 -q:v 1 -frames:v 1 ./pictures/frame' + element.Timestamp + '.png ');                                                                                     
     oldtimestamp = newtimestamp;                                                                                          
     sequence = 1;                                                                                                         
        }                                                                                                                  
  console.log('ffmpeg -i ./pictures/frame' + element.Timestamp + '.png -vf  "crop=' + size.Width + ':' + size.Height + ':' + size.Left + ':' + size.Top +  '"  ./faces/frame' + element.Timestamp + '' + sequence + '.png');                          
     sequence++;                                                                                                           
});

Used docker for the ffmpeg, zero installation, pick and run.

docker run -it --rm --entrypoint='bash' -v ${PWD}:/tmp/workdir jrottenberg/ffmpeg

The timestamp you need to divide by 1000 to convert milliseconds to seconds. added the #!/bin/sh manually.

#!/bin/sh                                                                                                                  
ffmpeg  -i f1.vob  -ss 00:00:0   -qmin 1 -q:v 1 -frames:v 1 ./pictures/frame0.png                                          
ffmpeg -i ./pictures/frame0.png -vf  "crop=171:200:446:70"  ./faces/frame01.png                                            
ffmpeg -i ./pictures/frame0.png -vf  "crop=170:198:168:43"  ./faces/frame02.png                                            
ffmpeg -i ./pictures/frame0.png -vf  "crop=192:224:532:23"  ./faces/frame03.png                                            
ffmpeg -i ./pictures/frame0.png -vf  "crop=151:176:82:85"  ./faces/frame04.png                                             
ffmpeg  -i f1.vob  -ss 00:00:0.2   -qmin 1 -q:v 1 -frames:v 1 ./pictures/frame200.png                                      
ffmpeg -i ./pictures/frame200.png -vf  "crop=170:198:446:71"  ./faces/frame2001.png                                        
ffmpeg -i ./pictures/frame200.png -vf  "crop=174:203:167:43"  ./faces/frame2002.png                                        
ffmpeg -i ./pictures/frame200.png -vf  "crop=194:226:531:22"  ./faces/frame2003.png                                        
ffmpeg -i ./pictures/frame200.png -vf  "crop=152:176:81:86"  ./faces/frame2004.png                                         
ffmpeg  -i f1.vob  -ss 00:00:0.4   -qmin 1 -q:v 1 -frames:v 1 ./pictures/frame400.png                                      
ffmpeg -i ./pictures/frame400.png -vf  "crop=170:197:446:71"  ./faces/frame4001.png                                        
ffmpeg -i ./pictures/frame400.png -vf  "crop=174:203:167:43"  ./faces/frame4002.png                                        
ffmpeg -i ./pictures/frame400.png -vf  "crop=194:226:531:22"  ./faces/frame4003.png                                        
ffmpeg -i ./pictures/frame400.png -vf  "crop=152:176:81:86"  ./faces/frame4004.png                                         
ffmpeg  -i f1.vob  -ss 00:00:0.6   -qmin 1 -q:v 1 -frames:v 1 ./pictures/frame600.png

Copy output to S3:

aws s3 cp faces s3://outputbucket/faces/ --recursive

Hope it helps.