Merge branch 'develop' into feature/db_perf_research

matrix-sdk-android/build.gradle

@@ -9,7 +9,7 @@ buildscript {
         jcenter()
     }
     dependencies {
-        classpath "io.realm:realm-gradle-plugin:6.1.0"
+        classpath "io.realm:realm-gradle-plugin:10.0.0"
     }
 }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/CryptoModule.kt

@@ -123,6 +123,7 @@ internal abstract class CryptoModule {
                     }
                     .name("crypto_store.realm")
                     .modules(RealmCryptoStoreModule())
+                    .allowWritesOnUiThread(true)
                     .schemaVersion(RealmCryptoStoreMigration.CRYPTO_STORE_SCHEMA_VERSION)
                     .migration(realmCryptoStoreMigration)
                     .build()

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/DatabaseCleaner.kt

@@ -51,12 +51,12 @@ internal class DatabaseCleaner @Inject constructor(@SessionDatabase private val
             awaitTransaction(realmConfiguration) { realm ->
                 val allRooms = realm.where(RoomEntity::class.java).findAll()
                 Timber.v("There are ${allRooms.size} rooms in this session")
-                cleanUp(realm, MAX_NUMBER_OF_EVENTS_IN_DB / 2L)
+                //cleanUp(realm, MAX_NUMBER_OF_EVENTS_IN_DB / 2L)
             }
         }
     }
 
-    private suspend fun cleanUp(realm: Realm, threshold: Long) {
+    private fun cleanUp(realm: Realm, threshold: Long) {
         val numberOfEvents = realm.where(EventEntity::class.java).findAll().size
         val numberOfTimelineEvents = realm.where(TimelineEventEntity::class.java).findAll().size
         Timber.v("Number of events in db: $numberOfEvents | Number of timeline events in db: $numberOfTimelineEvents")

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/EventInsertLiveObserver.kt

@@ -36,7 +36,7 @@ internal class EventInsertLiveObserver @Inject constructor(@SessionDatabase real
                                                            private val eventDecryptor: EventDecryptor)
     : RealmLiveEntityObserver<EventInsertEntity>(realmConfiguration) {
 
-    override val query = Monarchy.Query<EventInsertEntity> {
+    override val query = Monarchy.Query {
         it.where(EventInsertEntity::class.java)
     }
 

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/RealmKeysUtils.kt

@@ -18,6 +18,7 @@ package org.matrix.android.sdk.internal.database
 import android.content.Context
 import android.util.Base64
 import androidx.core.content.edit
+import io.realm.Realm
 import org.matrix.android.sdk.BuildConfig
 import org.matrix.android.sdk.internal.session.securestorage.SecretStoringUtils
 import io.realm.RealmConfiguration
@@ -46,7 +47,7 @@ internal class RealmKeysUtils @Inject constructor(context: Context,
     private val sharedPreferences = context.getSharedPreferences("im.vector.matrix.android.keys", Context.MODE_PRIVATE)
 
     private fun generateKeyForRealm(): ByteArray {
-        val keyForRealm = ByteArray(RealmConfiguration.KEY_LENGTH)
+        val keyForRealm = ByteArray(Realm.ENCRYPTION_KEY_LENGTH)
         rng.nextBytes(keyForRealm)
         return keyForRealm
     }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/SessionRealmConfigurationFactory.kt

@@ -69,6 +69,7 @@ internal class SessionRealmConfigurationFactory @Inject constructor(
                 .apply {
                     realmKeysUtils.configureEncryption(this, SessionModule.getKeyAlias(userMd5))
                 }
+                .allowWritesOnUiThread(true)
                 .modules(SessionRealmModule())
                 .schemaVersion(RealmSessionStoreMigration.SESSION_STORE_SCHEMA_VERSION)
                 .migration(migration)

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EditAggregatedSummaryEntity.kt

@@ -29,5 +29,16 @@ internal open class EditAggregatedSummaryEntity(
         var lastEditTs: Long = 0
 ) : RealmObject() {
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is EditAggregatedSummaryEntity) return false
+        if (aggregatedContent != other.aggregatedContent) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return aggregatedContent?.hashCode() ?: 0
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventAnnotationsSummaryEntity.kt

@@ -29,5 +29,26 @@ internal open class EventAnnotationsSummaryEntity(
         var pollResponseSummary: PollResponseAggregatedSummaryEntity? = null
 ) : RealmObject() {
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is EventAnnotationsSummaryEntity) return false
+        if (eventId != other.eventId) return false
+        if (reactionsSummary != other.reactionsSummary) return false
+        if (editSummary != other.editSummary) return false
+        if (referencesSummaryEntity != other.referencesSummaryEntity) return false
+        if (pollResponseSummary != other.pollResponseSummary) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = eventId.hashCode()
+        result = 31 * result + reactionsSummary.hashCode()
+        result = 31 * result + (editSummary?.hashCode() ?: 0)
+        result = 31 * result + (referencesSummaryEntity?.hashCode() ?: 0)
+        result = 31 * result + (pollResponseSummary?.hashCode() ?: 0)
+        return result
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventEntity.kt

@@ -65,4 +65,33 @@ internal open class EventEntity(@Index var eventId: String = "",
         decryptionErrorCode = null
         decryptionErrorReason = null
     }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is EventEntity) return false
+        if (eventId != other.eventId) return false
+        if (content != other.content) return false
+        if (prevContent != other.prevContent) return false
+        if (unsignedData != other.unsignedData) return false
+        if (redacts != other.redacts) return false
+        if (decryptionResultJson != other.decryptionResultJson) return false
+        if (decryptionErrorCode != other.decryptionErrorCode) return false
+        if (decryptionErrorReason != other.decryptionErrorReason) return false
+        if (sendStateStr != other.sendStateStr) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = eventId.hashCode()
+        result = 31 * result + (content?.hashCode() ?: 0)
+        result = 31 * result + (prevContent?.hashCode() ?: 0)
+        result = 31 * result + (unsignedData?.hashCode() ?: 0)
+        result = 31 * result + (redacts?.hashCode() ?: 0)
+        result = 31 * result + (decryptionResultJson?.hashCode() ?: 0)
+        result = 31 * result + (decryptionErrorCode?.hashCode() ?: 0)
+        result = 31 * result + (decryptionErrorReason?.hashCode() ?: 0)
+        result = 31 * result + sendStateStr.hashCode()
+        return result
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/PollResponseAggregatedSummaryEntity.kt

@@ -36,5 +36,21 @@ internal open class PollResponseAggregatedSummaryEntity(
         var sourceLocalEchoEvents: RealmList<String> = RealmList()
 ) : RealmObject() {
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is PollResponseAggregatedSummaryEntity) return false
+        if (aggregatedContent != other.aggregatedContent) return false
+        if (closedTime != other.closedTime) return false
+        if (nbOptions != other.nbOptions) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = aggregatedContent?.hashCode() ?: 0
+        result = 31 * result + (closedTime?.hashCode() ?: 0)
+        result = 31 * result + nbOptions
+        return result
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/ReactionAggregatedSummaryEntity.kt

@@ -37,5 +37,22 @@ internal open class ReactionAggregatedSummaryEntity(
         var sourceLocalEcho: RealmList<String> = RealmList()
 ) : RealmObject() {
 
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ReactionAggregatedSummaryEntity) return false
+        if (key != other.key) return false
+        if (count != other.count) return false
+        if (addedByMe != other.addedByMe) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = key.hashCode()
+        result = 31 * result + count
+        result = 31 * result + addedByMe.hashCode()
+        return result
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/ReadReceiptEntity.kt

@@ -31,4 +31,18 @@ internal open class ReadReceiptEntity(@PrimaryKey var primaryKey: String = "",
 
     @LinkingObjects("readReceipts")
     val summary: RealmResults<ReadReceiptsSummaryEntity>? = null
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ReadReceiptEntity) return false
+        if (primaryKey != other.primaryKey) return false
+        if (originServerTs != other.originServerTs) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = primaryKey.hashCode()
+        result = 31 * result + originServerTs.hashCode()
+        return result
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/ReadReceiptsSummaryEntity.kt

@@ -32,5 +32,23 @@ internal open class ReadReceiptsSummaryEntity(
     @LinkingObjects("readReceipts")
     val timelineEvent: RealmResults<TimelineEventEntity>? = null
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ReadReceiptsSummaryEntity) return false
+
+        if (eventId != other.eventId) return false
+        if (roomId != other.roomId) return false
+        if (readReceipts != other.readReceipts) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = eventId.hashCode()
+        result = 31 * result + roomId.hashCode()
+        result = 31 * result + readReceipts.hashCode()
+        return result
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/ReferencesAggregatedSummaryEntity.kt

@@ -27,5 +27,19 @@ internal open class ReferencesAggregatedSummaryEntity(
         var sourceLocalEcho: RealmList<String> = RealmList()
 ) : RealmObject() {
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is ReferencesAggregatedSummaryEntity) return false
+        if (eventId != other.eventId) return false
+        if (content != other.content) return false
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = eventId.hashCode()
+        result = 31 * result + (content?.hashCode() ?: 0)
+        return result
+    }
+
     companion object
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/TimelineEventEntity.kt

@@ -37,5 +37,29 @@ internal open class TimelineEventEntity(var localId: Long = 0,
     @LinkingObjects("timelineEvents")
     val chunk: RealmResults<ChunkEntity>? = null
 
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is TimelineEventEntity) return false
+        if (localId != other.localId) return false
+        if (eventId != other.eventId) return false
+        if (displayIndex != other.displayIndex) return false
+        if (root != other.root) return false
+        if (annotations != other.annotations) return false
+        if (readReceipts != other.readReceipts) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        var result = localId.hashCode()
+        result = 31 * result + eventId.hashCode()
+        result = 31 * result + displayIndex
+        result = 31 * result + (root?.hashCode() ?: 0)
+        result = 31 * result + (annotations?.hashCode() ?: 0)
+        result = 31 * result + (readReceipts?.hashCode() ?: 0)
+        return result
+    }
+
     companion object
+
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/filter/FilterUtil.kt

@@ -98,7 +98,8 @@ internal object FilterUtil {
                             state = filter.room.state?.copy(lazyLoadMembers = true)
                                     ?: RoomEventFilter(lazyLoadMembers = true)
                     )
-                            ?: RoomFilter(state = RoomEventFilter(lazyLoadMembers = true))
+                            ?: RoomFilter(state = RoomEventFilter(lazyLoadMembers = true),
+                            timeline = RoomEventFilter(limit = 1))
             )
         } else {
             val newRoomEventFilter = filter.room?.state?.copy(lazyLoadMembers = null)?.takeIf { it.hasData() }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/identity/IdentityModule.kt

@@ -66,6 +66,7 @@ internal abstract class IdentityModule {
                     .apply {
                         realmKeysUtils.configureEncryption(this, SessionModule.getKeyAlias(userMd5))
                     }
+                    .allowWritesOnUiThread(true)
                     .modules(IdentityRealmModule())
                     .build()
         }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/DefaultTimeline.kt

@@ -16,8 +16,6 @@
 
 package org.matrix.android.sdk.internal.session.room.timeline
 
-import io.realm.OrderedCollectionChangeSet
-import io.realm.OrderedRealmCollectionChangeListener
 import io.realm.Realm
 import io.realm.RealmConfiguration
 import io.realm.RealmQuery
@@ -58,6 +56,8 @@ import org.matrix.android.sdk.internal.task.configureWith
 import org.matrix.android.sdk.internal.util.Debouncer
 import org.matrix.android.sdk.internal.util.createBackgroundHandler
 import org.matrix.android.sdk.internal.util.createUIHandler
+import org.matrix.android.sdk.internal.util.diff.DiffRealmChangeListener
+import org.matrix.android.sdk.internal.util.diff.ListUpdateCallbackAdapter
 import timber.log.Timber
 import java.util.Collections
 import java.util.UUID
@@ -88,14 +88,12 @@ internal class DefaultTimeline(
     data class OnLocalEchoCreated(val roomId: String, val timelineEvent: TimelineEvent)
     data class OnLocalEchoUpdated(val roomId: String, val eventId: String, val sendState: SendState)
 
-    companion object {
-        val BACKGROUND_HANDLER = createBackgroundHandler("TIMELINE_DB_THREAD")
-    }
 
     private val listeners = CopyOnWriteArrayList<Timeline.Listener>()
     private val isStarted = AtomicBoolean(false)
     private val isReady = AtomicBoolean(false)
     private val mainHandler = createUIHandler()
+    private val backgroundHandler = createBackgroundHandler("TIMELINE_DB_THREAD")
     private val backgroundRealm = AtomicReference<Realm>()
     private val cancelableBag = CancelableBag()
     private val debouncer = Debouncer(mainHandler)
@@ -119,22 +117,14 @@ internal class DefaultTimeline(
     override val isLive
         get() = !hasMoreToLoad(Timeline.Direction.FORWARDS)
 
-    private val eventsChangeListener = OrderedRealmCollectionChangeListener<RealmResults<TimelineEventEntity>> { results, changeSet ->
-        if (!results.isLoaded || !results.isValid) {
-            return@OrderedRealmCollectionChangeListener
-        }
-        Timber.v("## SendEvent: [${System.currentTimeMillis()}] DB update for room $roomId")
-        handleUpdates(results, changeSet)
-    }
-
     // Public methods ******************************************************************************
 
     override fun paginate(direction: Timeline.Direction, count: Int) {
-        BACKGROUND_HANDLER.post {
+        backgroundHandler.post {
             if (!canPaginate(direction)) {
                 return@post
             }
-            Timber.v("Paginate $direction of $count items")
+            Timber.v("Paginate $direction of $count items for room $roomId")
             val startDisplayIndex = if (direction == Timeline.Direction.BACKWARDS) prevDisplayIndex else nextDisplayIndex
             val shouldPostSnapshot = paginateInternal(startDisplayIndex, direction, count)
             if (shouldPostSnapshot) {
@@ -159,7 +149,7 @@ internal class DefaultTimeline(
         if (isStarted.compareAndSet(false, true)) {
             Timber.v("Start timeline for roomId: $roomId and eventId: $initialEventId")
             eventBus.register(this)
-            BACKGROUND_HANDLER.post {
+            backgroundHandler.post {
                 eventDecryptor.start()
                 val realm = Realm.getInstance(realmConfiguration)
                 backgroundRealm.set(realm)
@@ -179,7 +169,8 @@ internal class DefaultTimeline(
                 filteredEvents = nonFilteredEvents.where()
                         .filterEventsWithSettings()
                         .findAll()
-                nonFilteredEvents.addChangeListener(eventsChangeListener)
+                val changeListener = createChangeListener(nonFilteredEvents.freeze())
+                nonFilteredEvents.addChangeListener(changeListener)
                 handleInitialLoad()
                 if (settings.shouldHandleHiddenReadReceipts()) {
                     hiddenReadReceipts.start(realm, filteredEvents, nonFilteredEvents, this)
@@ -199,8 +190,7 @@ internal class DefaultTimeline(
             eventBus.unregister(this)
             Timber.v("Dispose timeline for roomId: $roomId and eventId: $initialEventId")
             cancelableBag.cancel()
-            BACKGROUND_HANDLER.removeCallbacksAndMessages(null)
-            BACKGROUND_HANDLER.post {
+            backgroundHandler.post {
                 if (this::sendingEvents.isInitialized) {
                     sendingEvents.removeAllChangeListeners()
                 }
@@ -301,7 +291,7 @@ internal class DefaultTimeline(
         listeners.clear()
     }
 
-// TimelineHiddenReadReceipts.Delegate
+    // TimelineHiddenReadReceipts.Delegate
 
     override fun rebuildEvent(eventId: String, readReceipts: List<ReadReceipt>): Boolean {
         return rebuildEvent(eventId) { te ->
@@ -338,6 +328,17 @@ internal class DefaultTimeline(
 
 // Private methods *****************************************************************************
 
+    private fun createChangeListener(list: List<TimelineEventEntity>) = object : DiffRealmChangeListener<TimelineEventEntity>(list) {
+        override fun areSameItems(old: TimelineEventEntity?, new: TimelineEventEntity?): Boolean {
+            return old?.localId == new?.localId
+        }
+
+        override fun handleDiffResults(listUpdateCallbackAdapter: ListUpdateCallbackAdapter) {
+            Timber.v("Diff results for room $roomId: $listUpdateCallbackAdapter")
+            handleTimelineEventListUpdates(listUpdateCallbackAdapter)
+        }
+    }
+
     private fun rebuildEvent(eventId: String, builder: (TimelineEvent) -> TimelineEvent?): Boolean {
         return tryOrNull {
             builtEventsIdMap[eventId]?.let { builtIndex ->
@@ -433,14 +434,14 @@ internal class DefaultTimeline(
 
     private fun getState(direction: Timeline.Direction): State {
         return when (direction) {
-            Timeline.Direction.FORWARDS  -> forwardsState.get()
+            Timeline.Direction.FORWARDS -> forwardsState.get()
             Timeline.Direction.BACKWARDS -> backwardsState.get()
         }
     }
 
     private fun updateState(direction: Timeline.Direction, update: (State) -> State) {
         val stateReference = when (direction) {
-            Timeline.Direction.FORWARDS  -> forwardsState
+            Timeline.Direction.FORWARDS -> forwardsState
             Timeline.Direction.BACKWARDS -> backwardsState
         }
         val currentValue = stateReference.get()
@@ -483,17 +484,20 @@ internal class DefaultTimeline(
     /**
      * This has to be called on TimelineThread as it accesses realm live results
      */
-    private fun handleUpdates(results: RealmResults<TimelineEventEntity>, changeSet: OrderedCollectionChangeSet) {
+    private fun handleTimelineEventListUpdates(updateCallbackAdapter: ListUpdateCallbackAdapter) {
         // If changeSet has deletion we are having a gap, so we clear everything
-        if (changeSet.deletionRanges.isNotEmpty()) {
+        if (updateCallbackAdapter.deletions.isNotEmpty()) {
             clearAllValues()
         }
         var postSnapshot = false
-        changeSet.insertionRanges.forEach { range ->
-            val (startDisplayIndex, direction) = if (range.startIndex == 0) {
-                Pair(results[range.length - 1]!!.displayIndex, Timeline.Direction.FORWARDS)
+
+        val numberOfInsertions = updateCallbackAdapter.insertions.size
+        if (numberOfInsertions > 0) {
+            val startIndex = updateCallbackAdapter.insertions.first()
+            val (startDisplayIndex, direction) = if (startIndex == 0) {
+                Pair(nonFilteredEvents[numberOfInsertions - 1]!!.displayIndex, Timeline.Direction.FORWARDS)
             } else {
-                Pair(results[range.startIndex]!!.displayIndex, Timeline.Direction.BACKWARDS)
+                Pair(nonFilteredEvents[startIndex]!!.displayIndex, Timeline.Direction.BACKWARDS)
             }
             val state = getState(direction)
             if (state.isPaginating) {
@@ -501,12 +505,12 @@ internal class DefaultTimeline(
                 postSnapshot = paginateInternal(startDisplayIndex, direction, state.requestedPaginationCount)
             } else {
                 // We are getting new items from sync
-                buildTimelineEvents(startDisplayIndex, direction, range.length.toLong())
+                buildTimelineEvents(startDisplayIndex, direction, numberOfInsertions.toLong())
                 postSnapshot = true
             }
         }
-        changeSet.changes.forEach { index ->
-            val eventEntity = results[index]
+        updateCallbackAdapter.changes.forEach { index ->
+            val eventEntity = nonFilteredEvents[index]
             eventEntity?.eventId?.let { eventId ->
                 postSnapshot = rebuildEvent(eventId) {
                     val builtEvent = buildTimelineEvent(eventEntity)
@@ -560,7 +564,7 @@ internal class DefaultTimeline(
                     direction = direction.toPaginationDirection(),
                     limit = limit
             )
-            Timber.v("Should fetch $limit items $direction")
+            Timber.v("Should fetch $limit items $direction for room $roomId")
             cancelableBag += paginationTask
                     .configureWith(params) {
                         this.callback = createPaginationCallback(limit, direction)
@@ -646,7 +650,7 @@ internal class DefaultTimeline(
             builtEventsIdMap[eventEntity.eventId] = position
         }
         val time = System.currentTimeMillis() - start
-        Timber.v("Built ${offsetResults.size} items from db in $time ms")
+        Timber.v("Built ${offsetResults.size} items from db for room $roomId: in $time ms")
         // For the case where wo reach the lastForward chunk
         updateLoadingStates(filteredEvents)
         return offsetResults.size
@@ -711,7 +715,7 @@ internal class DefaultTimeline(
     }
 
     private fun postSnapshot() {
-        BACKGROUND_HANDLER.post {
+        backgroundHandler.post {
             if (isReady.get().not()) {
                 return@post
             }
@@ -751,15 +755,15 @@ internal class DefaultTimeline(
         return object : MatrixCallback<TokenChunkEventPersistor.Result> {
             override fun onSuccess(data: TokenChunkEventPersistor.Result) {
                 when (data) {
-                    TokenChunkEventPersistor.Result.SUCCESS           -> {
-                        Timber.v("Success fetching $limit items $direction from pagination request")
+                    TokenChunkEventPersistor.Result.SUCCESS -> {
+                        Timber.v("Success fetching $limit items $direction from pagination request for room $roomId")
                     }
-                    TokenChunkEventPersistor.Result.REACHED_END       -> {
+                    TokenChunkEventPersistor.Result.REACHED_END -> {
                         postSnapshot()
                     }
                     TokenChunkEventPersistor.Result.SHOULD_FETCH_MORE ->
                         // Database won't be updated, so we force pagination request
-                        BACKGROUND_HANDLER.post {
+                        backgroundHandler.post {
                             executePaginationTask(direction, limit)
                         }
                 }
@@ -768,7 +772,7 @@ internal class DefaultTimeline(
             override fun onFailure(failure: Throwable) {
                 updateState(direction) { it.copy(isPaginating = false, requestedPaginationCount = 0) }
                 postSnapshot()
-                Timber.v("Failure fetching $limit items $direction from pagination request")
+                Timber.v("Failure fetching $limit items $direction from pagination request for room $roomId")
             }
         }
     }
@@ -866,7 +870,7 @@ internal class DefaultTimeline(
                 when (onLocalEchoCreated.timelineEvent.root.getClearType()) {
                     EventType.REDACTION -> {
                     }
-                    EventType.REACTION  -> {
+                    EventType.REACTION -> {
                         val content = onLocalEchoCreated.timelineEvent.root.content?.toModel<ReactionContent>()
                         if (RelationType.ANNOTATION == content?.relatesTo?.type) {
                             val reaction = content.relatesTo.key
@@ -891,7 +895,7 @@ internal class DefaultTimeline(
                     listeners.forEach {
                         it.onNewTimelineEvents(listOf(onLocalEchoCreated.timelineEvent.eventId))
                     }
-                    Timber.v("On local echo created: ${onLocalEchoCreated.timelineEvent.eventId}")
+                    Timber.v("On local echo created: ${onLocalEchoCreated.timelineEvent.eventId} for room $roomId")
                     inMemorySendingEvents.add(0, onLocalEchoCreated.timelineEvent)
                     postSnapshot = true
                 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineHiddenReadReceipts.kt

@@ -17,7 +17,6 @@
 package org.matrix.android.sdk.internal.session.room.timeline
 
 import android.util.SparseArray
-import io.realm.OrderedRealmCollectionChangeListener
 import io.realm.Realm
 import io.realm.RealmQuery
 import io.realm.RealmResults
@@ -30,6 +29,8 @@ import org.matrix.android.sdk.internal.database.model.TimelineEventEntity
 import org.matrix.android.sdk.internal.database.model.TimelineEventEntityFields
 import org.matrix.android.sdk.internal.database.query.TimelineEventFilter
 import org.matrix.android.sdk.internal.database.query.whereInRoom
+import org.matrix.android.sdk.internal.util.diff.DiffRealmChangeListener
+import org.matrix.android.sdk.internal.util.diff.ListUpdateCallbackAdapter
 
 /**
  * This class is responsible for handling the read receipts for hidden events (check [TimelineSettings] to see filtering).
@@ -53,13 +54,10 @@ internal class TimelineHiddenReadReceipts constructor(private val readReceiptsSu
     private lateinit var filteredEvents: RealmResults<TimelineEventEntity>
     private lateinit var delegate: Delegate
 
-    private val hiddenReadReceiptsListener = OrderedRealmCollectionChangeListener<RealmResults<ReadReceiptsSummaryEntity>> { collection, changeSet ->
-        if (!collection.isLoaded || !collection.isValid) {
-            return@OrderedRealmCollectionChangeListener
-        }
+    private fun handleChanges(listUpdateCallbackAdapter: ListUpdateCallbackAdapter) {
         var hasChange = false
         // Deletion here means we don't have any readReceipts for the given hidden events
-        changeSet.deletions.forEach {
+        listUpdateCallbackAdapter.deletions.forEach {
             val eventId = correctedReadReceiptsEventByIndex.get(it, "")
             val timelineEvent = filteredEvents.where()
                     .equalTo(TimelineEventEntityFields.EVENT_ID, eventId)
@@ -125,7 +123,7 @@ internal class TimelineHiddenReadReceipts constructor(private val readReceiptsSu
                 .isNotEmpty(ReadReceiptsSummaryEntityFields.READ_RECEIPTS.`$`)
                 .filterReceiptsWithSettings()
                 .findAllAsync()
-                .also { it.addChangeListener(hiddenReadReceiptsListener) }
+                .also { it.addChangeListener(createChangeListener()) }
     }
 
     /**
@@ -144,6 +142,16 @@ internal class TimelineHiddenReadReceipts constructor(private val readReceiptsSu
         return correctedReadReceiptsByEvent[eventId]
     }
 
+    private fun createChangeListener() = object : DiffRealmChangeListener<ReadReceiptsSummaryEntity>(emptyList()) {
+        override fun areSameItems(old: ReadReceiptsSummaryEntity?, new: ReadReceiptsSummaryEntity?): Boolean {
+            return old?.eventId == new?.eventId
+        }
+
+        override fun handleDiffResults(listUpdateCallbackAdapter: ListUpdateCallbackAdapter) {
+            handleChanges(listUpdateCallbackAdapter)
+        }
+    }
+
     /**
      * We are looking for receipts related to filtered events. So, it's the opposite of [DefaultTimeline.filterEventsWithSettings] method.
      */

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt

@@ -16,13 +16,27 @@
 
 package org.matrix.android.sdk.internal.session.sync
 
+import io.realm.Realm
+import io.realm.RealmConfiguration
 import org.greenrobot.eventbus.EventBus
 import org.matrix.android.sdk.R
+import org.matrix.android.sdk.api.session.events.model.EventType
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.room.model.RoomHistoryVisibility
+import org.matrix.android.sdk.api.session.room.model.RoomHistoryVisibilityContent
+import org.matrix.android.sdk.internal.database.mapper.asDomain
+import org.matrix.android.sdk.internal.database.model.ChunkEntity
+import org.matrix.android.sdk.internal.database.model.CurrentStateEventEntity
+import org.matrix.android.sdk.internal.database.query.findLastForwardChunkOfRoom
+import org.matrix.android.sdk.internal.database.query.whereType
+import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.UserId
 import org.matrix.android.sdk.internal.network.executeRequest
 import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService
 import org.matrix.android.sdk.internal.session.filter.FilterRepository
 import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask
+import org.matrix.android.sdk.internal.session.room.timeline.PaginationDirection
+import org.matrix.android.sdk.internal.session.room.timeline.PaginationTask
 import org.matrix.android.sdk.internal.session.sync.model.SyncResponse
 import org.matrix.android.sdk.internal.session.user.UserStore
 import org.matrix.android.sdk.internal.task.Task
@@ -47,6 +61,9 @@ internal class DefaultSyncTask @Inject constructor(
         private val getHomeServerCapabilitiesTask: GetHomeServerCapabilitiesTask,
         private val userStore: UserStore,
         private val syncTaskSequencer: SyncTaskSequencer,
+        private val paginationTask: PaginationTask,
+        @SessionDatabase
+        private val realmConfiguration: RealmConfiguration,
         private val eventBus: EventBus
 ) : SyncTask {
 
@@ -84,7 +101,42 @@ internal class DefaultSyncTask @Inject constructor(
         syncResponseHandler.handleResponse(syncResponse, token)
         if (isInitialSync) {
             initialSyncProgressService.endAll()
+            paginateBackwardJoinedRooms(syncResponse)
         }
         Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}")
     }
+
+    private suspend fun paginateBackwardJoinedRooms(syncResponse: SyncResponse) {
+        val roomIdsToPaginate = syncResponse.rooms?.join?.keys.orEmpty()
+        roomIdsToPaginate.forEach { roomId ->
+            val paginate = Realm.getInstance(realmConfiguration).use { realm ->
+                val visibilityStateEvent = CurrentStateEventEntity.whereType(realm, roomId, EventType.STATE_ROOM_HISTORY_VISIBILITY).findFirst()
+                visibilityStateEvent?.root?.asDomain()?.content?.toModel<RoomHistoryVisibilityContent>()?.historyVisibility != RoomHistoryVisibility.WORLD_READABLE
+            }
+            if (paginate) {
+                repeat(10) { paginateRoom(roomId) }
+            }
+        }
+    }
+
+    private suspend fun paginateRoom(roomId: String) {
+        val prevToken = Realm.getInstance(realmConfiguration).use { realm ->
+            val liveChunk = ChunkEntity.findLastForwardChunkOfRoom(realm, roomId) ?: return@use null
+            if (liveChunk.isLastBackward) {
+                return@use null
+            }
+            return@use liveChunk.prevToken
+        } ?: return
+
+        val paginationParams = PaginationTask.Params(
+                roomId = roomId,
+                from = prevToken,
+                direction = PaginationDirection.BACKWARDS,
+                limit = 100)
+        try {
+            paginationTask.execute(paginationParams)
+        } catch (failure: Throwable) {
+            Timber.v("Failure: $failure")
+        }
+    }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/diff/DiffRealmChangeListener.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.util.diff
+
+import android.os.Handler
+import androidx.recyclerview.widget.DiffUtil
+import io.realm.RealmChangeListener
+import io.realm.RealmObject
+import io.realm.RealmResults
+
+internal abstract class DiffRealmChangeListener<T : RealmObject>(
+        private var previousResults: List<T> = emptyList()
+) : RealmChangeListener<RealmResults<T>> {
+
+    override fun onChange(results: RealmResults<T>) {
+        if (!results.isLoaded || !results.isValid) {
+            return
+        }
+        val snapshotResults = results.freeze()
+        val diffCallback = SimpleDiffUtilCallback<T>(previousResults, snapshotResults) { old, new ->
+            areSameItems(old, new)
+        }
+        previousResults = snapshotResults
+        val diffResult = DiffUtil.calculateDiff(diffCallback, false)
+        val listUpdateCallbackAdapter = ListUpdateCallbackAdapter()
+        diffResult.dispatchUpdatesTo(listUpdateCallbackAdapter)
+        handleDiffResults(listUpdateCallbackAdapter)
+    }
+
+    abstract fun areSameItems(old: T?, new: T?): Boolean
+
+    internal abstract fun handleDiffResults(listUpdateCallbackAdapter: ListUpdateCallbackAdapter)
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/diff/ListUpdateCallbackAdapter.kt

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.util.diff
+
+import androidx.recyclerview.widget.ListUpdateCallback
+
+/**
+ * Used to translate results of DiffUtil into 3 separate arrays.
+ *
+ */
+internal class ListUpdateCallbackAdapter : ListUpdateCallback {
+
+    var insertions: IntArray = IntArray(0)
+        private set
+    var deletions: IntArray = IntArray(0)
+        private set
+    var changes: IntArray = IntArray(0)
+        private set
+
+    override fun onInserted(position: Int, count: Int) {
+        insertions = (position until position + count).toList().toIntArray()
+    }
+
+    override fun onRemoved(position: Int, count: Int) {
+        deletions = (position until position + count).toList().toIntArray()
+    }
+
+    override fun onMoved(fromPosition: Int, toPosition: Int) {
+        //Noop
+    }
+
+    override fun onChanged(position: Int, count: Int, payload: Any?) {
+        changes = (position until position + count).toList().toIntArray()
+    }
+
+    override fun toString(): String {
+        return "ListUpdateCallbackAdapter(insertions=${insertions.size}, deletions=${deletions.size}, changes=${changes.size})"
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/diff/SimpleDiffUtilCallback.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.util.diff
+
+import androidx.recyclerview.widget.DiffUtil
+
+/*
+    Simple implementation of DiffUtilCallback.
+    It relies on equals method for checking content.
+ */
+internal class SimpleDiffUtilCallback<T>(
+        private val oldList: List<T>,
+        private val newList: List<T>,
+        private val areItemsTheSame: (T?, T?) -> Boolean
+) : DiffUtil.Callback() {
+
+    override fun getOldListSize(): Int {
+        return oldList.size
+    }
+
+    override fun getNewListSize(): Int {
+        return newList.size
+    }
+
+    override fun areItemsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
+        val oldItem = oldList.getOrNull(oldItemPosition)
+        val newItem = newList.getOrNull(newItemPosition)
+        return areItemsTheSame(oldItem, newItem)
+    }
+
+    override fun areContentsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
+        return oldList.getOrNull(oldItemPosition) == newList.getOrNull(newItemPosition)
+    }
+}

vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt

@@ -145,7 +145,7 @@ class RoomDetailViewModel @AssistedInject constructor(
 
     companion object : MvRxViewModelFactory<RoomDetailViewModel, RoomDetailViewState> {
 
-        const val PAGINATION_COUNT = 50
+        const val PAGINATION_COUNT = 100
 
         @JvmStatic
         override fun create(viewModelContext: ViewModelContext, state: RoomDetailViewState): RoomDetailViewModel? {