Clean some code about chunk and fix merging chunk

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt

@@ -1,13 +1,41 @@
 package im.vector.matrix.android.internal.database.helper
 
 import im.vector.matrix.android.api.session.events.model.Event
+import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.mapper.asEntity
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.query.fastContains
+import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
 
-internal fun ChunkEntity.add(event: Event, stateIndex: Int, paginationDirection: PaginationDirection) {
+
-    if (!this.isManaged) {
+internal fun ChunkEntity.merge(chunkEntity: ChunkEntity,
+                               direction: PaginationDirection) {
+
+    val events = chunkEntity.events.map { it.asDomain() }
+    addAll(events, direction)
+    if (direction == PaginationDirection.FORWARDS) {
+        nextToken = chunkEntity.nextToken
+    } else {
+        prevToken = chunkEntity.prevToken
+    }
+}
+
+internal fun ChunkEntity.addAll(events: List<Event>,
+                                direction: PaginationDirection,
+                                updateStateIndex: Boolean = true) {
+
+    events.forEach { event ->
+        if (updateStateIndex && event.isStateEvent()) {
+            updateStateIndex(direction)
+        }
+        addOrUpdate(event, direction)
+    }
+}
+
+internal fun ChunkEntity.addOrUpdate(event: Event,
+                                     direction: PaginationDirection) {
+    if (!isManaged) {
         throw IllegalStateException("Chunk entity should be managed to use fast contains")
     }
 
@@ -15,11 +43,14 @@ internal fun ChunkEntity.add(event: Event, stateIndex: Int, paginationDirection:
         return
     }
 
+    val currentStateIndex = stateIndex(direction)
+    if (!events.fastContains(event.eventId)) {
         val eventEntity = event.asEntity()
-    eventEntity.stateIndex = stateIndex
+        eventEntity.stateIndex = currentStateIndex
-
+        val position = if (direction == PaginationDirection.FORWARDS) 0 else this.events.size
-    if (!this.events.fastContains(eventEntity)) {
+        events.add(position, eventEntity)
-        val position = if (paginationDirection == PaginationDirection.FORWARDS) 0 else this.events.size
+    } else {
-        this.events.add(position, eventEntity)
+        val eventEntity = events.find(event.eventId)
+        eventEntity?.stateIndex = currentStateIndex
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/RoomEntityHelper.kt

@@ -0,0 +1,11 @@
+package im.vector.matrix.android.internal.database.helper
+
+import im.vector.matrix.android.internal.database.model.ChunkEntity
+import im.vector.matrix.android.internal.database.model.RoomEntity
+
+
+internal fun RoomEntity.deleteOnCascade(chunkEntity: ChunkEntity) {
+    chunks.remove(chunkEntity)
+    chunkEntity.events.deleteAllFromRealm()
+    chunkEntity.deleteFromRealm()
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/mapper/EventMapper.kt

@@ -13,16 +13,7 @@ internal object EventMapper {
 
     fun map(event: Event): EventEntity {
         val eventEntity = EventEntity()
-        eventEntity.eventId = event.eventId ?: ""
+        fill(eventEntity, with = event)
-        eventEntity.content = adapter.toJson(event.content)
-        val resolvedPrevContent = event.prevContent ?: event.unsignedData?.prevContent
-        eventEntity.prevContent = adapter.toJson(resolvedPrevContent)
-        eventEntity.stateKey = event.stateKey
-        eventEntity.type = event.type
-        eventEntity.sender = event.sender
-        eventEntity.originServerTs = event.originServerTs
-        eventEntity.redacts = event.redacts
-        eventEntity.age = event.unsignedData?.age ?: event.originServerTs
         return eventEntity
     }
 
@@ -40,6 +31,20 @@ internal object EventMapper {
                 redacts = eventEntity.redacts
         )
     }
+
+    fun fill(eventEntity: EventEntity, with: Event) {
+        eventEntity.eventId = with.eventId ?: ""
+        eventEntity.content = adapter.toJson(with.content)
+        val resolvedPrevContent = with.prevContent ?: with.unsignedData?.prevContent
+        eventEntity.prevContent = adapter.toJson(resolvedPrevContent)
+        eventEntity.stateKey = with.stateKey
+        eventEntity.type = with.type
+        eventEntity.sender = with.sender
+        eventEntity.originServerTs = with.originServerTs
+        eventEntity.redacts = with.redacts
+        eventEntity.age = with.unsignedData?.age ?: with.originServerTs
+    }
+
 }
 
 internal fun EventEntity.asDomain(): Event {
@@ -49,3 +54,7 @@ internal fun EventEntity.asDomain(): Event {
 internal fun Event.asEntity(): EventEntity {
     return EventMapper.map(this)
 }
+
+internal fun EventEntity.fillWith(event: Event) {
+    EventMapper.fill(this, with = event)
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt

@@ -26,12 +26,11 @@ internal open class ChunkEntity(var prevToken: String? = null,
         }
     }
 
-    fun updateStateIndex(stateIndex: Int, direction: PaginationDirection){
+    fun updateStateIndex(direction: PaginationDirection) {
         when (direction) {
-            PaginationDirection.FORWARDS  -> nextStateIndex = stateIndex
+            PaginationDirection.FORWARDS  -> nextStateIndex += 1
-            PaginationDirection.BACKWARDS -> prevStateIndex = stateIndex
+            PaginationDirection.BACKWARDS -> prevStateIndex -= 1
         }
     }
 
-
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/ChunkEntityQueries.kt

@@ -14,24 +14,15 @@ internal fun ChunkEntity.Companion.where(realm: Realm, roomId: String): RealmQue
             .equalTo("${ChunkEntityFields.ROOM}.${RoomEntityFields.ROOM_ID}", roomId)
 }
 
-internal fun ChunkEntity.Companion.findWithPrevToken(realm: Realm, roomId: String, prevToken: String?): ChunkEntity? {
+internal fun ChunkEntity.Companion.find(realm: Realm, roomId: String, prevToken: String? = null, nextToken: String? = null): ChunkEntity? {
-    if (prevToken == null) {
+    val query = where(realm, roomId)
-        return null
+    if (prevToken != null) {
+        query.equalTo(ChunkEntityFields.PREV_TOKEN, prevToken)
     }
-    return where(realm, roomId)
+    if (nextToken != null) {
-            .and()
+        query.equalTo(ChunkEntityFields.NEXT_TOKEN, nextToken)
-            .equalTo(ChunkEntityFields.PREV_TOKEN, prevToken)
-            .findFirst()
-}
-
-internal fun ChunkEntity.Companion.findWithNextToken(realm: Realm, roomId: String, nextToken: String?): ChunkEntity? {
-    if (nextToken == null) {
-        return null
     }
-    return where(realm, roomId)
+    return query.findFirst()
-            .and()
-            .equalTo(ChunkEntityFields.NEXT_TOKEN, nextToken)
-            .findFirst()
 }
 
 internal fun ChunkEntity.Companion.findLastLiveChunkFromRoom(realm: Realm, roomId: String): ChunkEntity? {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/EventEntityQueries.kt

@@ -46,6 +46,10 @@ internal fun RealmQuery<EventEntity>.last(): EventEntity? {
             .findFirst()
 }
 
-internal fun RealmList<EventEntity>.fastContains(eventEntity: EventEntity): Boolean {
+internal fun RealmList<EventEntity>.find(eventId: String): EventEntity? {
-    return this.where().equalTo(EventEntityFields.EVENT_ID, eventEntity.eventId).findFirst() != null
+    return this.where().equalTo(EventEntityFields.EVENT_ID, eventId).findFirst()
+}
+
+internal fun RealmList<EventEntity>.fastContains(eventId: String): Boolean {
+    return this.find(eventId) != null
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/members/RoomMemberExtractor.kt

@@ -7,8 +7,8 @@ import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
+import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findMostSuitableStateEvent
-import im.vector.matrix.android.internal.database.query.findWithNextToken
 import im.vector.matrix.android.internal.database.query.last
 import io.realm.Realm
 import io.realm.RealmQuery
@@ -32,7 +32,7 @@ internal class RoomMemberExtractor(private val realm: Realm,
         }
 
         // If the content is null, we try get the last state event coming from a state events chunk
-        val stateChunkEntity = ChunkEntity.findWithNextToken(realm, roomId, DBConstants.STATE_EVENTS_CHUNK_TOKEN)
+        val stateChunkEntity = ChunkEntity.find(realm, roomId, nextToken = DBConstants.STATE_EVENTS_CHUNK_TOKEN)
                                ?: return null
 
         return buildQuery(stateChunkEntity, event.sender)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/PaginationDirection.kt

@@ -13,11 +13,4 @@ internal enum class PaginationDirection(val value: String) {
      */
     BACKWARDS("b");
 
-    val incrementStateIndex: Int by lazy {
-        when (this) {
-            FORWARDS  -> 1
-            BACKWARDS -> -1
-        }
-    }
-
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/PaginationRequest.kt

@@ -4,15 +4,14 @@ import arrow.core.Try
 import arrow.core.failure
 import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.api.MatrixCallback
-import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.util.Cancelable
-import im.vector.matrix.android.internal.database.helper.add
+import im.vector.matrix.android.internal.database.helper.addAll
+import im.vector.matrix.android.internal.database.helper.deleteOnCascade
+import im.vector.matrix.android.internal.database.helper.merge
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
-import im.vector.matrix.android.internal.database.query.fastContains
+import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findAllIncludingEvents
-import im.vector.matrix.android.internal.database.query.findWithNextToken
-import im.vector.matrix.android.internal.database.query.findWithPrevToken
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.legacy.util.FilterUtil
 import im.vector.matrix.android.internal.network.executeRequest
@@ -29,7 +28,8 @@ import kotlinx.coroutines.withContext
 internal class PaginationRequest(private val roomAPI: RoomAPI,
                                  private val monarchy: Monarchy,
                                  private val coroutineDispatchers: MatrixCoroutineDispatchers,
-                                 private val stateEventsChunkHandler: StateEventsChunkHandler) {
+                                 private val stateEventsChunkHandler: StateEventsChunkHandler
+) {
 
     fun execute(roomId: String,
                 from: String?,
@@ -67,43 +67,26 @@ internal class PaginationRequest(private val roomAPI: RoomAPI,
                     val roomEntity = RoomEntity.where(realm, roomId).findFirst()
                                      ?: throw IllegalStateException("You shouldn't use this method without a room")
 
-                    val currentChunk = ChunkEntity.findWithPrevToken(realm, roomId, receivedChunk.nextToken)
+                    val currentChunk = ChunkEntity.find(realm, roomId, prevToken = receivedChunk.nextToken)
                                        ?: realm.createObject()
 
                     currentChunk.prevToken = receivedChunk.prevToken
+                    currentChunk.addAll(receivedChunk.events, direction)
 
-                    val prevChunk = ChunkEntity.findWithNextToken(realm, roomId, receivedChunk.prevToken)
+                    // Now, handles chunk merge
-
-                    val eventIds = receivedChunk.events.filter { it.eventId != null }.map { it.eventId!! }
-                    val chunksOverlapped = realm.copyFromRealm(ChunkEntity.findAllIncludingEvents(realm, eventIds))
-                    val hasOverlapped = chunksOverlapped.isNotEmpty()
-
-                    var currentStateIndex = currentChunk.stateIndex(direction)
-                    val incrementStateIndex = direction.incrementStateIndex
-
-                    receivedChunk.events.forEach { event ->
-                        if (EventType.isStateEvent(event.type)) {
-                            currentStateIndex += incrementStateIndex
-                        }
-                        currentChunk.add(event, currentStateIndex, direction)
-                    }
 
+                    val prevChunk = ChunkEntity.find(realm, roomId, nextToken = receivedChunk.prevToken)
                     if (prevChunk != null) {
-                        currentChunk.events.addAll(prevChunk.events)
+                        currentChunk.merge(prevChunk, direction)
-                        roomEntity.chunks.remove(prevChunk)
+                        roomEntity.deleteOnCascade(prevChunk)
-
+                    } else {
-                    } else if (hasOverlapped) {
+                        val eventIds = receivedChunk.events.mapNotNull { it.eventId }
-                        chunksOverlapped.forEach { overlapped ->
+                        ChunkEntity
-                            overlapped.events.forEach { event ->
+                                .findAllIncludingEvents(realm, eventIds)
-                                if (!currentChunk.events.fastContains(event)) {
+                                .filter { it != currentChunk }
-                                    currentChunk.events.add(event)
+                                .forEach { overlapped ->
-                                }
+                                    currentChunk.merge(overlapped, direction)
-                                if (EventType.isStateEvent(event.type)) {
+                                    roomEntity.deleteOnCascade(overlapped)
-                                    currentStateIndex += incrementStateIndex
-                                }
-                            }
-                            currentChunk.prevToken = overlapped.prevToken
-                            roomEntity.chunks.remove(overlapped)
                                 }
                     }
 
@@ -111,8 +94,6 @@ internal class PaginationRequest(private val roomAPI: RoomAPI,
                         roomEntity.chunks.add(currentChunk)
                     }
 
-                    currentChunk.updateStateIndex(currentStateIndex, direction)
-
                     // TODO : there is an issue with the pagination sending unwanted room member events
                     val stateEventsChunk = stateEventsChunkHandler.handle(realm, roomId, receivedChunk.stateEvents)
                     if (!roomEntity.chunks.contains(stateEventsChunk)) {
@@ -122,4 +103,5 @@ internal class PaginationRequest(private val roomAPI: RoomAPI,
                 .map { receivedChunk }
     }
 
+
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt

@@ -4,7 +4,7 @@ import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.api.session.room.model.MyMembership
-import im.vector.matrix.android.internal.database.helper.add
+import im.vector.matrix.android.internal.database.helper.addAll
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
@@ -151,14 +151,7 @@ internal class RoomSyncHandler(private val monarchy: Monarchy,
         chunkEntity.isLast = true
         chunkEntity.nextToken = nextToken
 
-        var currentStateIndex = chunkEntity.nextStateIndex
+        chunkEntity.addAll(eventList, PaginationDirection.FORWARDS)
-        eventList.forEach { event ->
-            if (event.isStateEvent()) {
-                currentStateIndex += PaginationDirection.FORWARDS.incrementStateIndex
-            }
-            chunkEntity.add(event, currentStateIndex, PaginationDirection.FORWARDS)
-        }
-        chunkEntity.nextStateIndex = currentStateIndex
         return chunkEntity
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/StateEventsChunkHandler.kt

@@ -2,9 +2,9 @@ package im.vector.matrix.android.internal.session.sync
 
 import im.vector.matrix.android.api.session.events.model.Event
 import im.vector.matrix.android.internal.database.DBConstants
-import im.vector.matrix.android.internal.database.helper.add
+import im.vector.matrix.android.internal.database.helper.addAll
 import im.vector.matrix.android.internal.database.model.ChunkEntity
-import im.vector.matrix.android.internal.database.query.findWithNextToken
+import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
 import io.realm.Realm
 import io.realm.kotlin.createObject
@@ -12,7 +12,7 @@ import io.realm.kotlin.createObject
 internal class StateEventsChunkHandler {
 
     fun handle(realm: Realm, roomId: String, stateEvents: List<Event>): ChunkEntity {
-        val chunkEntity = ChunkEntity.findWithNextToken(realm, roomId, DBConstants.STATE_EVENTS_CHUNK_TOKEN)
+        val chunkEntity = ChunkEntity.find(realm, roomId, nextToken = DBConstants.STATE_EVENTS_CHUNK_TOKEN)
                           ?: realm.createObject<ChunkEntity>()
                                   .apply {
                                       prevToken = DBConstants.STATE_EVENTS_CHUNK_TOKEN
@@ -22,12 +22,7 @@ internal class StateEventsChunkHandler {
                                   }
 
         // We always consider going forwards as data from server are the most recent
-        val direction = PaginationDirection.FORWARDS
+        chunkEntity.addAll(stateEvents, direction = PaginationDirection.FORWARDS, updateStateIndex = false)
-        val stateIndex = chunkEntity.stateIndex(direction) + direction.incrementStateIndex
-        stateEvents.forEach { event ->
-            chunkEntity.add(event, stateIndex, direction)
-        }
-        chunkEntity.updateStateIndex(stateIndex, direction)
         return chunkEntity
     }