Convert a CameraImage to ui.Image in Flutter

Question

[edit] The problem seems to be that the code I'm trying to use, doesn't work inside an isolate. When I try the same ode outside of an isole, that works. So probably should check issues on flutter github or submut a nex one. [edit2] didn't find an issue for this, just submitted one : https://github.com/flutter/flutter/issues/55654

I can convert a CameraImage to png with this piece of code ( https://gist.github.com/Alby-o/fe87e35bc21d534c8220aed7df028e03 ), and then display the png with an Image widget, but the png conversion is very slow. I would like to convert a CameraImage to a ui.Image and then use a RawImage widget to display it. That sounds possible. But when I try this piece of code, I get an error on the ui.decodeImageFromPixels line :

Future<ui.Image> makeUiImage(List<int> pixels,int width,int height) {
  final c = Completer<ui.Image>();
  ui.decodeImageFromPixels(
    pixels,
    width,
    height,
    ui.PixelFormat.rgba8888,
    c.complete,
  );
  return c.future;
}
Future<ui.Image> convertCameraImageToUiImage(CameraImage image) async {
  int startTime = DateTime.now().millisecondsSinceEpoch;
  int time;
  imglib.Image img = convertCameraImage(image);
  time = DateTime.now().millisecondsSinceEpoch;
  print("Converted in "+(time-startTime).toString()+"ms img "+img.toString()+" ("+img.width.toString()+","+img.height.toString()+")");
  startTime=time;

  ui.Image ret = await makeUiImage(img.getBytes(), image.width, image.height);
  return ret;
}

E/flutter ( 1194): [ERROR:flutter/runtime/dart_isolate.cc(915)]

Unhandled exception: E/flutter ( 1194): error: native function 'instantiateImageCodec' (5 arguments) cannot be found

E/flutter ( 1194): #0 decodeImageFromPixels.

(dart:ui/painting.dart:1740:36) E/flutter ( 1194): #1 _futurize

(dart:ui/painting.dart:4296:34) E/flutter ( 1194): #2

decodeImageFromPixels (dart:ui/painting.dart:1739:37) E/flutter (

1194): #3 makeUiImage

(package:flutterapp/utils/image_converter.dart:37:3) E/flutter (

1194): #4 convertCameraImageToUiImage

(package:flutterapp/utils/image_converter.dart:54:24) E/flutter (

1194): #5 processCameraImageIsolate.

(package:flutterapp/cameraoverlay2.dart:448:5) E/flutter ( 1194): #6

_RootZone.runUnaryGuarded (dart:async/zone.dart:1316:10) E/flutter ( 1194): #7
_BufferingStreamSubscription._sendData

(dart:async/stream_impl.dart:338:11) E/flutter ( 1194): #8

_BufferingStreamSubscription._add (dart:async/stream_impl.dart:265:7) E/flutter ( 1194): #9 _SyncStreamControllerDispatch._sendData

(dart:async/stream_controller.dart:766:19) E/flutter ( 1194): #10

_StreamController._add (dart:async/stream_controller.dart:642:7) E/flutter ( 1194): #11 _StreamController.add

(dart:async/stream_controller.dart:588:5) E/flutter ( 1194): #12
_RawReceivePortImpl._handleMessage (dart:isolate-patch/isolate_patch.dart:174:12)

Answer 1

the following is a code snippet from this link and works for me: https://github.com/flutter/flutter/issues/26348#issuecomment-694092314

Make a class or file with the following methods in it:

/// imgLib -> Image package from https://pub.dartlang.org/packages/image
  static Future<List<int>> convertImagetoPng(CameraImage image) async {
    try {
      imglib.Image img;
      if (image.format.group == ImageFormatGroup.yuv420) {
        img = _convertYUV420(image);
      } else if (image.format.group == ImageFormatGroup.bgra8888) {
        img = _convertBGRA8888(image);
      }
      imglib.PngEncoder pngEncoder = new imglib.PngEncoder();

      // Convert to png
      List<int> png = pngEncoder.encodeImage(img);
      return png;
    } catch (e) {
      print(">>>>>>>>>>>> ERROR:" + e.toString());
    }
    return null;
  }

  /// CameraImage BGRA8888 -> PNG
  /// Color
  static imglib.Image _convertBGRA8888(CameraImage image) {
    return imglib.Image.fromBytes(
      (image.planes[0].bytesPerRow / 4).round(),
      image.height,
      image.planes[0].bytes,
      format: imglib.Format.bgra,
    );
  }

  /// CameraImage YUV420_888 -> PNG -> Image (compresion:0, filter: none)
  /// Black
  static imglib.Image _convertYUV420(CameraImage image) {
    var img = imglib.Image(image.width, image.height); // Create Image buffer

    Plane plane = image.planes[0];
    const int shift = (0xFF << 24);

    // Fill image buffer with plane[0] from YUV420_888
    for (int x = 0; x < image.width; x++) {
      for (int planeOffset = 0;
          planeOffset < image.height * image.width;
          planeOffset += image.width) {
        final pixelColor = plane.bytes[planeOffset + x];
        // color: 0x FF  FF  FF  FF
        //           A   B   G   R
        // Calculate pixel color
        var newVal =
            shift | (pixelColor << 16) | (pixelColor << 8) | pixelColor;

        img.data[planeOffset + x] = newVal;
      }
    }

    return img;
  }

Then use this inside your method to get the ui.Image Use the comp.future to get the ui.Image

Completer<ui.Image> comp = Completer();
ui.decodeImageFromList(
    await convertImagetoPng(cameraImage),
    (result) {
  comp.complete(result);
});