Flutter Objectbox performance issue

Question

Please help me if someone has a solution.

Now I'm in a trouble with the performance issue of ObjectBox in Flutter. I have models of Event Recurrence EventPlace which have the relationship in each.

The problem is that the data insertion takes too long time and this cause the app crash since the operation stacks at the main thread until they finish it.

It takes approximately 300 milliseconds to finish 1 loop...

I have no idea how can I fix this. Should I not use the relationship between models? Or am I taking the wrong approach to use the Objectbox?

[✓] Flutter (Channel stable, 1.22.5, on Mac OS X 10.15.7 19H524 darwin-x64, locale
    en-GB)
 
[✓] Android toolchain - develop for Android devices (Android SDK version 29.0.3)
[✓] Xcode - develop for iOS and macOS (Xcode 12.4)
[✓] Android Studio (version 4.1)
[✓] VS Code (version 1.54.3)
[✓] Connected device (1 available)

@Entity(uid: EVENT_UID)
class ObEvent {
  @Id(assignable: true)
  int id;

  String deviceEventId;
  String eventTitle;
  String eventDesc;
  String eventType;
  List<String> eventLabels;
  String timezone;
  bool isAllDay;
  String link;
  int recurrId;
  int originalRecurrId;
  bool toDateUnfixed;
  bool isOwned;
  bool editable;

  @Property(type: PropertyType.byteVector)
  List<int> remainders;

  @Index()
  int calendarId;

  @Index()
  int parentId;

  @Index()
  String usersId;

  @Index()
  @Property(type: PropertyType.date)
  DateTime eventPublishDate;

  @Index()
  @Property(type: PropertyType.date)
  DateTime eventStart;

  @Index()
  @Property(type: PropertyType.date)
  DateTime eventEnd;

  @Index()
  @Property(type: PropertyType.date)
  DateTime createdAt;

  @Index()
  @Property(type: PropertyType.date)
  DateTime updatedAt;

  @Index()
  bool deleted;

  @Index()
  bool notSynced;

  final recurrence = ToOne<ObRecurrence>();
  final place = ToOne<ObEventPlace>();

  ObEvent({
    this.id,
    this.usersId,
    this.calendarId,
    this.eventTitle,
    this.eventStart,
    this.eventEnd,
    this.deviceEventId,
    this.editable,
    this.eventType,
    this.isAllDay,
    this.toDateUnfixed,
    this.eventDesc,
    this.parentId,
    this.eventLabels,
    this.eventPublishDate,
    this.link,
    this.remainders,
    this.recurrId,
    this.originalRecurrId,
    this.createdAt,
    this.updatedAt,
    this.isOwned = true,
    this.deleted = false,
    this.notSynced = false,
    timezone,
  }) : timezone = getIt<Vars>().defaultTimezone;

  ObEvent.fromEvent(Event event)
      : this.id = event.id,
        this.usersId = event.userId,
        this.calendarId = event.calendar.calId,
        this.eventTitle = event.eventTitle,
        this.eventStart = event.eventStart,
        this.eventEnd = event.eventEnd,
        this.deviceEventId = event.deviceEventId,
        this.editable = event.editable,
        this.eventType = EnumHandler.enumToString(event.eventType),
        this.isAllDay = event.isAllDay,
        this.toDateUnfixed = event.toDateUnfixed,
        this.eventDesc = event.eventDesc,
        this.parentId = event.parentId,
        this.eventLabels = event.eventLabels,
        this.eventPublishDate = event.eventPublishDate,
        this.link = event.link,
        this.remainders = event.remainders?.toList(),
        this.recurrId = event.recurrId,
        this.originalRecurrId = event.originalRecurrId,
        this.isOwned = event.isOwned,
        this.createdAt = event.createdAt,
        this.updatedAt = event.updatedAt,
        this.timezone = event.timezone;

  ObEvent merge(ObEvent event) {
    return ObEvent(
      id: event.id ?? this.id,
      usersId: event.usersId ?? this.usersId,
      calendarId: event.calendarId ?? this.calendarId,
      eventTitle: event.eventTitle ?? this.eventTitle,
      eventStart: event.eventStart ?? this.eventStart,
      eventEnd: event.eventEnd ?? this.eventEnd,
      deviceEventId: event.deviceEventId ?? this.deviceEventId,
      editable: event.editable ?? this.editable,
      eventType: event.eventType ?? this.eventType,
      isAllDay: event.isAllDay ?? this.isAllDay,
      toDateUnfixed: event.toDateUnfixed ?? this.toDateUnfixed,
      eventDesc: event.eventDesc ?? this.eventDesc,
      parentId: event.parentId ?? this.parentId,
      eventLabels: event.eventLabels ?? this.eventLabels,
      eventPublishDate: event.eventPublishDate ?? this.eventPublishDate,
      link: event.link ?? this.link,
      remainders: event.remainders ?? this.remainders,
      recurrId: event.recurrId ?? this.recurrId,
      originalRecurrId: event.originalRecurrId ?? this.originalRecurrId,
      isOwned: event.isOwned ?? this.isOwned,
      createdAt: event.createdAt ?? this.createdAt,
      updatedAt: event.updatedAt ?? this.updatedAt,
      timezone: event.timezone ?? this.timezone,
      deleted: event.deleted ?? this.deleted,
      notSynced: event.notSynced ?? this.notSynced,
    );
  }

  // For event type
  EventType get type => eventType.toEnum(EventType.values);
  set type(EventType value) => eventType = EnumHandler.enumToString(value);
}

@Entity(uid: EVENT_PLACE_UID)
class ObEventPlace {
  @Id()
  int id;

  String address;
  double lat;
  double lon;

  ObEventPlace({
    this.id,
    this.address,
    this.lat,
    this.lon,
  });

  ObEventPlace.fromEventPlace(EventPlace place)
      : this.address = place.address,
        this.lat = place.lat,
        this.lon = place.lon;

  ObEventPlace.fromMap(Map data)
      : this.address = data['address'],
        this.lat = data['lat'],
        this.lon = data['lon'];

  ObEventPlace merge(ObEventPlace ob) {
    return ObEventPlace(
      id: ob.id ?? this.id,
      address: ob.address ?? this.address,
      lat: ob.lat ?? this.lat,
      lon: ob.lon ?? this.lon,
    );
  }
}

@Entity(uid: RECURRENCE_UID)
class ObRecurrence {
  @Id(assignable: true)
  int id;

  String freq;
  int intval;
  List<String> byDay;
  String timezone;

  @Property(type: PropertyType.date)
  DateTime dtStart;
  @Property(type: PropertyType.date)
  DateTime dtEnd;
  @Property(type: PropertyType.byteVector)
  List<int> byMonth;
  @Property(type: PropertyType.byteVector)
  List<int> byMonthDay;
  @Property(type: PropertyType.byteVector)
  List<int> setPositions;
  @Property(type: PropertyType.byteVector)
  List<int> exDates;

  ObRecurrence({
    this.id,
    this.freq,
    this.intval,
    this.dtStart,
    this.dtEnd,
    this.byDay,
    this.byMonth,
    this.byMonthDay,
    this.setPositions,
    this.timezone,
    this.exDates,
  });

  ObRecurrence.fromRecurr(Recurrence recurr)
      : this.id = recurr.id,
        this.freq = EnumHandler.enumToString(recurr.freq),
        this.intval = recurr.intval,
        this.dtStart = recurr.dtStart,
        this.dtEnd = recurr.dtEnd,
        this.byDay = recurr.byDay?.map((day) => EnumHandler.enumToString(day))?.toList(),
        this.byMonth = recurr.byMonth?.toList(),
        this.byMonthDay = recurr.byMonthDay?.toList(),
        this.setPositions = recurr.setPositions?.toList(),
        this.timezone = recurr.timezone,
        this.exDates = recurr.exDates?.map((date) => date.millisecondsSinceEpoch)?.toList();

  ObRecurrence merge(ObRecurrence rec) {
    return ObRecurrence(
      id: rec.id ?? this.id,
      freq: rec.freq ?? this.freq,
      intval: rec.intval ?? this.intval,
      dtStart: rec.dtStart ?? this.dtStart,
      dtEnd: rec.dtEnd ?? this.dtEnd,
      byDay: rec.byDay ?? this.byDay,
      byMonth: rec.byMonth ?? this.byMonth,
      byMonthDay: rec.byMonthDay ?? this.byMonthDay,
      setPositions: rec.setPositions ?? this.setPositions,
      timezone: rec.timezone ?? this.timezone,
      exDates: rec.exDates ?? this.exDates,
    );
  }

  // For freq
  Freq get frequency => freq.toEnum(Freq.values);
  set frequency(Freq value) => EnumHandler.enumToString(value);

  // For byDay field
  Set<Day> get bydayList => byDay?.map((day) => day.toEnum(Day.values))?.toSet() ?? Set<Day>();
  set bydayList(Set<Day> value) => byDay = value.map((day) => EnumHandler.enumToString(day)).toList();

  // For exdates field
  Set<DateTime> get exdates =>
      exDates?.map((date) => DateTime.fromMillisecondsSinceEpoch(date))?.toSet() ?? Set<DateTime>();
  set exdates(Set<DateTime> value) => exDates = value.map((date) => date.millisecondsSinceEpoch).toList();

  MonthlyType get monthlyType => frequency == Freq.MONTHLY
      ? setPositions.isEmpty
          ? MonthlyType.BY_DATE
          : MonthlyType.BY_WEEK_DAY
      : MonthlyType.BY_DATE;
}

class Repository {
  Store store;
  Box<ObCalendar> _calBox;
  Box<ObEvent> _eventBox;
  Box<ObCalendarLink> _calLinkBox;
  Box<ObTag> _tagBox;
  Box<ObEventPlace> _placeBox;

  CalendarRepository({
    @required this.store,
  })  : this._calBox = store.box<ObCalendar>(),
        this._eventBox = store.box<ObEvent>(),
        this._calLinkBox = store.box<ObCalendarLink>(),
        this._tagBox = store.box<ObTag>(),
        this._placeBox = store.box<ObEventPlace>(),
        this._recBox = store.box<ObObRecurrence>();

  cacheEvents(data, user) async {
    final _obEvents = data.fold(<ObEvent>[], (prev, event) {
      // This is the modal used in application and properties are the same with ObEvent class
      final Event _event = Event.fromMap(user?.id, event);
      final ObEvent _obOldEvent = _eventBox.get(_event.id) ?? ObEvent();
      final ObEvent _obNewEvent = _obOldEvent.merge(ObEvent.fromEvent(_event))
        ..deleted = false
        ..notSynced = false;

      // No update
      if (_obOldEvent.id != null && (_event.updatedAt?.compareTo(_obNewEvent.updatedAt) ?? 0) == 0) {
        return prev;
      }

      // Update place cache of event
      if (_event.place != null) {
        final ObEventPlace _oldPlace = _obNewEvent.place.target ?? ObEventPlace();
        final ObEventPlace _newPlace = _oldPlace.merge(_oldPlace);

        _placeBox.put(_newPlace);
        _obNewEvent.place.target = _newPlace;
      }

      // Cache reucrrence if exists
      if (_event.recurrence != null) {
        final ObRecurrence _obOldRecurr = _obNewEvent.recurrence.target ?? ObRecurrence();
        final ObRecurrence _obNewRecurr = _obOldRecurr.merge(ObRecurrence.fromRecurr(_event.recurrence));

        _recBox.put(_obNewRecurr);
        _obNewEvent..recurrence.target = _obNewRecurr;
      }

      prev.add(_obNewEvent);

      return prev;
    });

    _eventBox.putMany(_obEvents);
  }
}

Temporary I do this operation in other thread with compute to prevent the crash but I don't need to do that if I could really get the performance like the benchmark document says.

vaind vaind · Accepted Answer · 2021-04-09T08:27:47

The biggest issue I can see is not using a Transaction while doing many write operations (put()) in a loop. To do that, use Store.runInTransaction(). Also, there is more information in ObjectBox-Java documentation and it applies to Dart as well.

In short, you can wrap the whole cacheEvents() in a single write transaction, something like:

cacheEvents(data, user) => store.runInTransaction(TxMode.write, () {
  // existing code
});

In that case, you don't really need to use putMany(), it's just a utility function that does the transaction for you if you insert only one "Entity". You can use plain put() in a transaction. So your code could look like this (not tested, just out of the top of my head):

cacheEvents(data, user) =>
    store.runInTransaction(TxMode.write, () =>
        data.forEach((event) {
          // This is the modal used in application and properties are the same with ObEvent class
          final Event _event = Event.fromMap(user?.id, event);
          final ObEvent _obOldEvent = _eventBox.get(_event.id) ?? ObEvent();
          final ObEvent _obNewEvent = _obOldEvent.merge(ObEvent.fromEvent(_event))
            ..deleted = false
            ..notSynced = false;

          // No update
          if (_obOldEvent.id != null && (_event.updatedAt?.compareTo(_obNewEvent.updatedAt) ?? 0) == 0) {
            return;
          }

          // Update place cache of event
          if (_event.place != null) {
            final ObEventPlace _oldPlace = _obNewEvent.place.target ?? ObEventPlace();
            final ObEventPlace _newPlace = _oldPlace.merge(_oldPlace);

            _placeBox.put(_newPlace);
            _obNewEvent.place.target = _newPlace;
          }

          // Cache reucrrence if exists
          if (_event.recurrence != null) {
            final ObRecurrence _obOldRecurr = _obNewEvent.recurrence.target ?? ObRecurrence();
            final ObRecurrence _obNewRecurr = _obOldRecurr.merge(ObRecurrence.fromRecurr(_event.recurrence));

            _recBox.put(_obNewRecurr);
            _obNewEvent..recurrence.target = _obNewRecurr;
          }
          _eventBox.put(_obNewEvent);
        }));

Also a couple of other suggestions/questions, though they're not going to have such a big influence:

consider changing .merge() to actually update this object instead of creating a new one
consider if you need all those indexes: Indexing, while good for queries, also increases time to insert/update data, because each index also needs to be updated. Do you use queries with conditions on all the indexed properties?
you could also use eventBox.getMany() to first get all events you're going to touch - though that's probably not going to affect performance in this case too much

BTW why do you have all those "bridge" classes, like ObEvent instead of defining Event as an @Entity()? Would be nice to know if there's something that could be done on ObjectBox's side so you wouldn't have to.

Flutter Objectbox performance issue

1 Answers