WIP: Start to make permalink works

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt

@@ -16,8 +16,9 @@ internal fun ChunkEntity.deleteOnCascade() {
     this.deleteFromRealm()
 }
 
+// By default if a chunk is empty we consider it unlinked
 internal fun ChunkEntity.isUnlinked(): Boolean {
-    return events.where().equalTo(EventEntityFields.IS_UNLINKED, true).findAll().isNotEmpty()
+    return events.where().equalTo(EventEntityFields.IS_UNLINKED, false).findAll().isEmpty()
 }
 
 internal fun ChunkEntity.merge(chunkToMerge: ChunkEntity,
@@ -89,7 +90,7 @@ internal fun ChunkEntity.add(event: Event,
 
 internal fun ChunkEntity.lastStateIndex(direction: PaginationDirection, defaultValue: Int = 0): Int {
     return when (direction) {
-               PaginationDirection.FORWARDS  -> events.where().sort(EventEntityFields.STATE_INDEX, Sort.DESCENDING).findFirst()?.stateIndex
+        PaginationDirection.FORWARDS -> events.where().sort(EventEntityFields.STATE_INDEX, Sort.DESCENDING).findFirst()?.stateIndex
-               PaginationDirection.BACKWARDS -> events.where().sort(EventEntityFields.STATE_INDEX, Sort.ASCENDING).findFirst()?.stateIndex
+        PaginationDirection.BACKWARDS -> events.where().sort(EventEntityFields.STATE_INDEX, Sort.ASCENDING).findFirst()?.stateIndex
-           } ?: defaultValue
+    } ?: defaultValue
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/EventEntity.kt

@@ -18,6 +18,12 @@ internal open class EventEntity(var eventId: String = "",
                                 var isUnlinked: Boolean = false
 ) : RealmObject() {
 
+    enum class LinkFilterMode {
+        LINKED_ONLY,
+        UNLINKED_ONLY,
+        BOTH
+    }
+
     companion object {
         const val DEFAULT_STATE_INDEX = Int.MIN_VALUE
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/ChunkEntityQueries.kt

@@ -6,6 +6,7 @@ import im.vector.matrix.android.internal.database.model.RoomEntityFields
 import io.realm.Realm
 import io.realm.RealmQuery
 import io.realm.RealmResults
+import io.realm.kotlin.createObject
 import io.realm.kotlin.where
 
 internal fun ChunkEntity.Companion.where(realm: Realm, roomId: String): RealmQuery<ChunkEntity> {
@@ -34,4 +35,11 @@ internal fun ChunkEntity.Companion.findAllIncludingEvents(realm: Realm, eventIds
     return realm.where<ChunkEntity>()
             .`in`(ChunkEntityFields.EVENTS.EVENT_ID, eventIds.toTypedArray())
             .findAll()
+}
+
+internal fun ChunkEntity.Companion.create(realm: Realm, prevToken: String?, nextToken: String?): ChunkEntity {
+    return realm.createObject<ChunkEntity>().apply {
+        this.prevToken = prevToken
+        this.nextToken = nextToken
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/EventEntityQueries.kt

@@ -2,6 +2,7 @@ package im.vector.matrix.android.internal.database.query
 
 import im.vector.matrix.android.internal.database.model.ChunkEntityFields
 import im.vector.matrix.android.internal.database.model.EventEntity
+import im.vector.matrix.android.internal.database.model.EventEntity.LinkFilterMode.*
 import im.vector.matrix.android.internal.database.model.EventEntityFields
 import im.vector.matrix.android.internal.database.model.RoomEntityFields
 import io.realm.Realm
@@ -15,7 +16,10 @@ internal fun EventEntity.Companion.where(realm: Realm, eventId: String): RealmQu
             .equalTo(EventEntityFields.EVENT_ID, eventId)
 }
 
-internal fun EventEntity.Companion.where(realm: Realm, roomId: String? = null, type: String? = null): RealmQuery<EventEntity> {
+internal fun EventEntity.Companion.where(realm: Realm,
+                                         roomId: String? = null,
+                                         type: String? = null,
+                                         linkFilterMode: EventEntity.LinkFilterMode = LINKED_ONLY): RealmQuery<EventEntity> {
     val query = realm.where<EventEntity>()
     if (roomId != null) {
         query.beginGroup()
@@ -27,8 +31,11 @@ internal fun EventEntity.Companion.where(realm: Realm, roomId: String? = null, t
     if (type != null) {
         query.equalTo(EventEntityFields.TYPE, type)
     }
-    query.notEqualTo(EventEntityFields.IS_UNLINKED, true)
+    return when (linkFilterMode) {
-    return query
+        LINKED_ONLY -> query.equalTo(EventEntityFields.IS_UNLINKED, false)
+        UNLINKED_ONLY -> query.equalTo(EventEntityFields.IS_UNLINKED, true)
+        BOTH -> query
+    }
 }
 
 internal fun RealmQuery<EventEntity>.next(from: Int? = null, strict: Boolean = true): EventEntity? {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/members/RoomMemberExtractor.kt

@@ -16,23 +16,26 @@ internal class RoomMemberExtractor(private val realm: Realm,
 
     fun extractFrom(event: EventEntity): RoomMember? {
         val sender = event.sender ?: return null
+        // If the event is unlinked we want to fetch unlinked state events
+        val unlinked = event.isUnlinked
         // When stateIndex is negative, we try to get the next stateEvent prevContent()
         // If prevContent is null we fallback to the Int.MIN state events content()
-        val roomMember: RoomMember? = if (event.stateIndex <= 0) {
+        return if (event.stateIndex <= 0) {
-            baseQuery(realm, roomId, sender).next(from = event.stateIndex)?.asDomain()?.prevContent()
+            baseQuery(realm, roomId, sender, unlinked).next(from = event.stateIndex)?.asDomain()?.prevContent()
-            ?: baseQuery(realm, roomId, sender).last(since = event.stateIndex)?.asDomain()?.content()
+                    ?: baseQuery(realm, roomId, sender, unlinked).last(since = event.stateIndex)?.asDomain()?.content()
         } else {
-            baseQuery(realm, roomId, sender).last(since = event.stateIndex)?.asDomain()?.content()
+            baseQuery(realm, roomId, sender, unlinked).last(since = event.stateIndex)?.asDomain()?.content()
         }
-        return roomMember
     }
 
     private fun baseQuery(realm: Realm,
                           roomId: String,
-                          sender: String): RealmQuery<EventEntity> {
+                          sender: String,
+                          isUnlinked: Boolean): RealmQuery<EventEntity> {
+        val filterMode = if (isUnlinked) EventEntity.LinkFilterMode.UNLINKED_ONLY else EventEntity.LinkFilterMode.LINKED_ONLY
 
         return EventEntity
-                .where(realm, roomId = roomId, type = EventType.STATE_ROOM_MEMBER)
+                .where(realm, roomId = roomId, type = EventType.STATE_ROOM_MEMBER, linkFilterMode = filterMode)
                 .equalTo(EventEntityFields.STATE_KEY, sender)
 
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultTimelineHolder.kt

@@ -14,6 +14,7 @@ import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.session.events.interceptor.MessageEventInterceptor
+import im.vector.matrix.android.internal.util.tryTransactionSync
 import io.realm.Realm
 import io.realm.RealmQuery
 
@@ -33,6 +34,7 @@ internal class DefaultTimelineHolder(private val roomId: String,
     }
 
     override fun timeline(eventId: String?): LiveData<PagedList<EnrichedEvent>> {
+        clearUnlinkedEvents()
         if (eventId != null) {
             fetchEventIfNeeded(eventId)
         }
@@ -62,6 +64,16 @@ internal class DefaultTimelineHolder(private val roomId: String,
         return monarchy.findAllPagedWithChanges(realmDataSourceFactory, livePagedListBuilder)
     }
 
+    private fun clearUnlinkedEvents() {
+        monarchy.tryTransactionSync { realm ->
+            val unlinkedEvents = EventEntity
+                    .where(realm, roomId = roomId)
+                    .equalTo(EventEntityFields.IS_UNLINKED, true)
+                    .findAll()
+            unlinkedEvents.deleteAllFromRealm()
+        }
+    }
+
     private fun fetchEventIfNeeded(eventId: String) {
         if (!isEventPersisted(eventId)) {
             contextOfEventRequest.execute(roomId, eventId, object : MatrixCallback<EventContextResponse> {})
@@ -79,11 +91,11 @@ internal class DefaultTimelineHolder(private val roomId: String,
     private fun buildDataSourceFactoryQuery(realm: Realm, eventId: String?): RealmQuery<EventEntity> {
         val query = if (eventId == null) {
             EventEntity
-                    .where(realm, roomId = roomId)
+                    .where(realm, roomId = roomId, linkFilterMode = EventEntity.LinkFilterMode.LINKED_ONLY)
                     .equalTo("${EventEntityFields.CHUNK}.${ChunkEntityFields.IS_LAST}", true)
         } else {
             EventEntity
-                    .where(realm, roomId = roomId)
+                    .where(realm, roomId = roomId, linkFilterMode = EventEntity.LinkFilterMode.BOTH)
                     .`in`("${EventEntityFields.CHUNK}.${ChunkEntityFields.EVENTS.EVENT_ID}", arrayOf(eventId))
         }
         return query.sort(EventEntityFields.DISPLAY_INDEX)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/EventContextResponse.kt

@@ -7,10 +7,10 @@ import im.vector.matrix.android.api.session.events.model.Event
 @JsonClass(generateAdapter = true)
 data class EventContextResponse(
         @Json(name = "event") val event: Event,
-        @Json(name = "start") override val prevToken: String? = null,
+        @Json(name = "start") override val start: String? = null,
         @Json(name = "events_before") val eventsBefore: List<Event> = emptyList(),
         @Json(name = "events_after") val eventsAfter: List<Event> = emptyList(),
-        @Json(name = "end") override val nextToken: String? = null,
+        @Json(name = "end") override val end: String? = null,
         @Json(name = "state") override val stateEvents: List<Event> = emptyList()
 ) : TokenChunkEvent {
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/PaginationResponse.kt

@@ -6,8 +6,8 @@ import im.vector.matrix.android.api.session.events.model.Event
 
 @JsonClass(generateAdapter = true)
 internal data class PaginationResponse(
-        @Json(name = "start") override val nextToken: String? = null,
+        @Json(name = "start") override val start: String? = null,
-        @Json(name = "end") override val prevToken: String? = null,
+        @Json(name = "end") override val end: String? = null,
         @Json(name = "chunk") override val events: List<Event> = emptyList(),
         @Json(name = "state") override val stateEvents: List<Event> = emptyList()
 ) : TokenChunkEvent

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TokenChunkEvent.kt

@@ -3,8 +3,8 @@ package im.vector.matrix.android.internal.session.room.timeline
 import im.vector.matrix.android.api.session.events.model.Event
 
 internal interface TokenChunkEvent {
-    val nextToken: String?
+    val start: String?
-    val prevToken: String?
+    val end: String?
     val events: List<Event>
     val stateEvents: List<Event>
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TokenChunkEventPersistor.kt

@@ -5,11 +5,11 @@ import com.zhuinden.monarchy.Monarchy
 import im.vector.matrix.android.internal.database.helper.*
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.RoomEntity
+import im.vector.matrix.android.internal.database.query.create
 import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findAllIncludingEvents
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.util.tryTransactionSync
-import io.realm.kotlin.createObject
 
 
 internal class TokenChunkEventPersistor(private val monarchy: Monarchy) {
@@ -23,51 +23,67 @@ internal class TokenChunkEventPersistor(private val monarchy: Monarchy) {
                     val roomEntity = RoomEntity.where(realm, roomId).findFirst()
                             ?: throw IllegalStateException("You shouldn't use this method without a room")
 
-                    // We create a new chunk with prev and next token as a base
+                    val nextToken: String?
-                    // In case of permalink, we may not encounter other chunks, so it can be added
+                    val prevToken: String?
-                    // By default, it's an unlinked chunk
+                    if (direction == PaginationDirection.FORWARDS) {
-                    val newChunk = realm.createObject<ChunkEntity>().apply {
+                        nextToken = receivedChunk.end
-                        prevToken = receivedChunk.prevToken
+                        prevToken = receivedChunk.start
-                        nextToken = receivedChunk.nextToken
+                    } else {
+                        nextToken = receivedChunk.start
+                        prevToken = receivedChunk.end
                     }
-                    newChunk.addAll(receivedChunk.events, direction, isUnlinked = true)
+                    val prevChunk = ChunkEntity.find(realm, roomId, nextToken = prevToken)
+                    val nextChunk = ChunkEntity.find(realm, roomId, prevToken = nextToken)
 
                     // The current chunk is the one we will keep all along the merge process.
-                    var currentChunk = newChunk
+                    // We try to look for a chunk next to the token,
-                    val prevChunk = ChunkEntity.find(realm, roomId, nextToken = receivedChunk.prevToken)
+                    // otherwise we create a whole new one
-                    val nextChunk = ChunkEntity.find(realm, roomId, prevToken = receivedChunk.nextToken)
 
-                    // We always merge the bottom chunk into top chunk, so we are always merging backwards
+                    var currentChunk = if (direction == PaginationDirection.FORWARDS) {
-                    if (prevChunk != null) {
+                        prevChunk?.apply { this.nextToken = nextToken }
-                        newChunk.merge(prevChunk, PaginationDirection.BACKWARDS)
+                                ?: ChunkEntity.create(realm, prevToken, nextToken)
-                        roomEntity.deleteOnCascade(prevChunk)
+                    } else {
+                        nextChunk?.apply { this.prevToken = prevToken }
+                                ?: ChunkEntity.create(realm, prevToken, nextToken)
                     }
-                    if (nextChunk != null) {
-                        nextChunk.merge(newChunk, PaginationDirection.BACKWARDS)
-                        roomEntity.deleteOnCascade(newChunk)
-                        currentChunk = nextChunk
-                    }
-                    val newEventIds = receivedChunk.events.mapNotNull { it.eventId }
-                    ChunkEntity
-                            .findAllIncludingEvents(realm, newEventIds)
-                            .filter { it != currentChunk }
-                            .forEach { overlapped ->
-                                if (direction == PaginationDirection.BACKWARDS) {
-                                    currentChunk.merge(overlapped, PaginationDirection.BACKWARDS)
-                                    roomEntity.deleteOnCascade(overlapped)
-                                } else {
-                                    overlapped.merge(currentChunk, PaginationDirection.BACKWARDS)
-                                    roomEntity.deleteOnCascade(currentChunk)
-                                    currentChunk = overlapped
-                                }
-                            }
-                    roomEntity.addOrUpdate(currentChunk)
 
-                    // TODO : there is an issue with the pagination sending unwanted room member events
                     val isUnlinked = currentChunk.isUnlinked()
+                    currentChunk.addAll(receivedChunk.events, direction, isUnlinked = isUnlinked)
+
+                    // Then we merge chunks if needed
+                    if (currentChunk != prevChunk && prevChunk != null) {
+                        currentChunk = handleMerge(roomEntity, direction, currentChunk, prevChunk)
+                    } else if (currentChunk != nextChunk && nextChunk != null) {
+                        currentChunk = handleMerge(roomEntity, direction, currentChunk, nextChunk)
+                    } else {
+                        val newEventIds = receivedChunk.events.mapNotNull { it.eventId }
+                        ChunkEntity
+                                .findAllIncludingEvents(realm, newEventIds)
+                                .filter { it != currentChunk }
+                                .forEach { overlapped ->
+                                    currentChunk = handleMerge(roomEntity, direction, currentChunk, overlapped)
+                                }
+                    }
+                    roomEntity.addOrUpdate(currentChunk)
                     roomEntity.addStateEvents(receivedChunk.stateEvents, isUnlinked = isUnlinked)
                 }
     }
 
+    private fun handleMerge(roomEntity: RoomEntity,
+                            direction: PaginationDirection,
+                            currentChunk: ChunkEntity,
+                            otherChunk: ChunkEntity): ChunkEntity {
+
+        // We always merge the bottom chunk into top chunk, so we are always merging backwards
+        return if (direction == PaginationDirection.BACKWARDS) {
+            currentChunk.merge(otherChunk, PaginationDirection.BACKWARDS)
+            roomEntity.deleteOnCascade(otherChunk)
+            currentChunk
+        } else {
+            otherChunk.merge(currentChunk, PaginationDirection.BACKWARDS)
+            roomEntity.deleteOnCascade(currentChunk)
+            otherChunk
+        }
+    }
 
 }