diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentAttachmentData.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentAttachmentData.kt index 82a4a762..c8dca869 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentAttachmentData.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentAttachmentData.kt @@ -27,8 +27,8 @@ data class ContentAttachmentData( val height: Long? = 0, val width: Long? = 0, val name: String? = null, - val path: String? = null, - val mimeType: String? = null, + val path: String, + val mimeType: String, val type: Type ) : Parcelable { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentUploadStateTracker.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentUploadStateTracker.kt index 0d88e5fa..a08060e6 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentUploadStateTracker.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/content/ContentUploadStateTracker.kt @@ -18,9 +18,15 @@ package im.vector.matrix.android.api.session.content interface ContentUploadStateTracker { - fun track(eventId: String, updateListener: UpdateListener) + fun track(key: String, updateListener: UpdateListener) - fun untrack(eventId: String, updateListener: UpdateListener) + fun untrack(key: String, updateListener: UpdateListener) + + fun setFailure(key: String) + + fun setSuccess(key: String) + + fun setProgress(key: String, current: Long, total: Long) interface UpdateListener { fun onUpdate(state: State) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentModule.kt index b149cd38..67ff2d7b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentModule.kt @@ -31,7 +31,7 @@ internal class ContentModule { } scope(DefaultSession.SCOPE) { - ContentUploader(get(), get(), get() as DefaultContentUploadStateTracker) + FileUploader(get(), get()) } scope(DefaultSession.SCOPE) { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/DefaultContentUploadStateTracker.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/DefaultContentUploadStateTracker.kt index 1e6ca3c1..66bd5a82 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/DefaultContentUploadStateTracker.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/DefaultContentUploadStateTracker.kt @@ -23,42 +23,42 @@ import im.vector.matrix.android.api.session.content.ContentUploadStateTracker internal class DefaultContentUploadStateTracker : ContentUploadStateTracker { private val mainHandler = Handler(Looper.getMainLooper()) - private val progressByEvent = mutableMapOf() - private val listenersByEvent = mutableMapOf>() + private val states = mutableMapOf() + private val listeners = mutableMapOf>() - override fun track(eventId: String, updateListener: ContentUploadStateTracker.UpdateListener) { - val listeners = listenersByEvent[eventId] ?: ArrayList() + override fun track(key: String, updateListener: ContentUploadStateTracker.UpdateListener) { + val listeners = listeners[key] ?: ArrayList() listeners.add(updateListener) - listenersByEvent[eventId] = listeners - val currentState = progressByEvent[eventId] ?: ContentUploadStateTracker.State.Idle + this.listeners[key] = listeners + val currentState = states[key] ?: ContentUploadStateTracker.State.Idle mainHandler.post { updateListener.onUpdate(currentState) } } - override fun untrack(eventId: String, updateListener: ContentUploadStateTracker.UpdateListener) { - listenersByEvent[eventId]?.apply { + override fun untrack(key: String, updateListener: ContentUploadStateTracker.UpdateListener) { + listeners[key]?.apply { remove(updateListener) } } - internal fun setFailure(eventId: String) { + override fun setFailure(key: String) { val failure = ContentUploadStateTracker.State.Failure - updateState(eventId, failure) + updateState(key, failure) } - internal fun setSuccess(eventId: String) { + override fun setSuccess(key: String) { val success = ContentUploadStateTracker.State.Success - updateState(eventId, success) + updateState(key, success) } - internal fun setProgress(eventId: String, current: Long, total: Long) { + override fun setProgress(key: String, current: Long, total: Long) { val progressData = ContentUploadStateTracker.State.ProgressData(current, total) - updateState(eventId, progressData) + updateState(key, progressData) } - private fun updateState(eventId: String, state: ContentUploadStateTracker.State) { - progressByEvent[eventId] = state + private fun updateState(key: String, state: ContentUploadStateTracker.State) { + states[key] = state mainHandler.post { - listenersByEvent[eventId]?.also { listeners -> + listeners[key]?.also { listeners -> listeners.forEach { it.onUpdate(state) } } } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentUploader.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/FileUploader.kt similarity index 51% rename from matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentUploader.kt rename to matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/FileUploader.kt index 42dff4f0..1ec18127 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ContentUploader.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/FileUploader.kt @@ -19,7 +19,6 @@ package im.vector.matrix.android.internal.session.content import arrow.core.Try import arrow.core.Try.Companion.raise import im.vector.matrix.android.api.auth.data.SessionParams -import im.vector.matrix.android.api.session.content.ContentAttachmentData import im.vector.matrix.android.internal.di.MoshiProvider import im.vector.matrix.android.internal.network.ProgressRequestBody import okhttp3.HttpUrl @@ -31,44 +30,51 @@ import java.io.File import java.io.IOException -internal class ContentUploader(private val okHttpClient: OkHttpClient, - private val sessionParams: SessionParams, - private val contentUploadProgressTracker: DefaultContentUploadStateTracker) { +internal class FileUploader(private val okHttpClient: OkHttpClient, + private val sessionParams: SessionParams) { + + private val uploadUrl = sessionParams.homeServerConnectionConfig.homeServerUri.toString() + URI_PREFIX_CONTENT_API + "upload" private val moshi = MoshiProvider.providesMoshi() private val responseAdapter = moshi.adapter(ContentUploadResponse::class.java) - fun uploadFile(eventId: String, attachment: ContentAttachmentData): Try { - if (attachment.path == null || attachment.mimeType == null) { - return raise(RuntimeException()) - } - val file = File(attachment.path) - val urlString = sessionParams.homeServerConnectionConfig.homeServerUri.toString() + URI_PREFIX_CONTENT_API + "upload" - val urlBuilder = HttpUrl.parse(urlString)?.newBuilder() - ?: return raise(RuntimeException()) + fun uploadFile(file: File, + filename: String?, + mimeType: String, + progressListener: ProgressRequestBody.Listener? = null): Try { + + val uploadBody = RequestBody.create(MediaType.parse(mimeType), file) + return upload(uploadBody, filename, progressListener) + + } + + fun uploadByteArray(byteArray: ByteArray, + filename: String?, + mimeType: String, + progressListener: ProgressRequestBody.Listener? = null): Try { + + val uploadBody = RequestBody.create(MediaType.parse(mimeType), byteArray) + return upload(uploadBody, filename, progressListener) + + } + + + private fun upload(uploadBody: RequestBody, filename: String?, progressListener: ProgressRequestBody.Listener?): Try { + val urlBuilder = HttpUrl.parse(uploadUrl)?.newBuilder() ?: return raise(RuntimeException()) val httpUrl = urlBuilder - .addQueryParameter( - "filename", attachment.name - ).build() + .addQueryParameter("filename", filename) + .build() - val requestBody = RequestBody.create( - MediaType.parse(attachment.mimeType), - file - ) - val progressRequestBody = ProgressRequestBody(requestBody, object : ProgressRequestBody.Listener { - override fun onProgress(current: Long, total: Long) { - contentUploadProgressTracker.setProgress(eventId, current, total) - } - }) + val requestBody = if (progressListener != null) ProgressRequestBody(uploadBody, progressListener) else uploadBody val request = Request.Builder() .url(httpUrl) - .post(progressRequestBody) + .post(requestBody) .build() - val result = Try { + return Try { okHttpClient.newCall(request).execute().use { response -> if (!response.isSuccessful) { throw IOException() @@ -80,11 +86,7 @@ internal class ContentUploader(private val okHttpClient: OkHttpClient, } } } - if (result.isFailure()) { - contentUploadProgressTracker.setFailure(eventId) - } else { - contentUploadProgressTracker.setSuccess(eventId) - } - return result + } + } \ No newline at end of file diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ThumbnailExtractor.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ThumbnailExtractor.kt new file mode 100644 index 00000000..8fc6f5f9 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/ThumbnailExtractor.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2019 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.content + +import android.graphics.Bitmap +import android.media.ThumbnailUtils +import android.provider.MediaStore +import im.vector.matrix.android.api.session.content.ContentAttachmentData +import java.io.ByteArrayOutputStream +import java.io.File + +internal object ThumbnailExtractor { + + class ThumbnailData( + val width: Int, + val height: Int, + val size: Long, + val bytes: ByteArray, + val mimeType: String + ) + + fun extractThumbnail(attachment: ContentAttachmentData): ThumbnailData? { + val file = File(attachment.path) + if (!file.exists() || !file.isFile) { + return null + } + return if (attachment.type == ContentAttachmentData.Type.VIDEO) { + extractVideoThumbnail(attachment) + } else { + null + } + } + + private fun extractVideoThumbnail(attachment: ContentAttachmentData): ThumbnailData? { + val thumbnail = ThumbnailUtils.createVideoThumbnail(attachment.path, MediaStore.Video.Thumbnails.MINI_KIND) + val outputStream = ByteArrayOutputStream() + thumbnail.compress(Bitmap.CompressFormat.JPEG, 100, outputStream) + val thumbnailWidth = thumbnail.width + val thumbnailHeight = thumbnail.height + val thumbnailSize = outputStream.size() + val thumbnailData = ThumbnailData( + width = thumbnailWidth, + height = thumbnailHeight, + size = thumbnailSize.toLong(), + bytes = outputStream.toByteArray(), + mimeType = "image/jpeg" + ) + thumbnail.recycle() + outputStream.reset() + return thumbnailData + } + + +} \ No newline at end of file diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/UploadContentWorker.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/UploadContentWorker.kt index 84adddd5..4eeb124d 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/UploadContentWorker.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/content/UploadContentWorker.kt @@ -21,6 +21,7 @@ import androidx.work.CoroutineWorker import androidx.work.WorkerParameters import com.squareup.moshi.JsonClass import im.vector.matrix.android.api.session.content.ContentAttachmentData +import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.events.model.Event import im.vector.matrix.android.api.session.events.model.toContent import im.vector.matrix.android.api.session.events.model.toModel @@ -30,14 +31,19 @@ import im.vector.matrix.android.api.session.room.model.message.MessageFileConten import im.vector.matrix.android.api.session.room.model.message.MessageImageContent import im.vector.matrix.android.api.session.room.model.message.MessageVideoContent import im.vector.matrix.android.internal.di.MatrixKoinComponent +import im.vector.matrix.android.internal.network.ProgressRequestBody import im.vector.matrix.android.internal.session.room.send.SendEventWorker import im.vector.matrix.android.internal.util.WorkerParamsFactory import org.koin.standalone.inject +import timber.log.Timber +import java.io.File + internal class UploadContentWorker(context: Context, params: WorkerParameters) : CoroutineWorker(context, params), MatrixKoinComponent { - private val mediaUploader by inject() + private val fileUploader by inject() + private val contentUploadProgressTracker by inject() @JsonClass(generateAdapter = true) internal data class Params( @@ -50,29 +56,63 @@ internal class UploadContentWorker(context: Context, params: WorkerParameters) val params = WorkerParamsFactory.fromData(inputData) ?: return Result.failure() - if (params.event.eventId == null) { - return Result.failure() + val eventId = params.event.eventId ?: return Result.failure() + val attachment = params.attachment + + val thumbnailData = ThumbnailExtractor.extractThumbnail(params.attachment) + val attachmentFile = createAttachmentFile(attachment) ?: return Result.failure() + var uploadedThumbnailUrl: String? = null + + if (thumbnailData != null) { + fileUploader + .uploadByteArray(thumbnailData.bytes, "thumb_${attachment.name}", thumbnailData.mimeType) + .fold( + { Timber.e(it) }, + { uploadedThumbnailUrl = it.contentUri } + ) } - return mediaUploader - .uploadFile(params.event.eventId, params.attachment) - .fold({ handleFailure() }, { handleSuccess(params, it) }) + + val progressListener = object : ProgressRequestBody.Listener { + override fun onProgress(current: Long, total: Long) { + contentUploadProgressTracker.setProgress(eventId, current, total) + } + } + return fileUploader + .uploadFile(attachmentFile, attachment.name, attachment.mimeType, progressListener) + .fold( + { handleFailure(params) }, + { handleSuccess(params, it.contentUri, uploadedThumbnailUrl) } + ) } - private fun handleFailure(): Result { - return Result.retry() + private fun createAttachmentFile(attachment: ContentAttachmentData): File? { + return try { + File(attachment.path) + } catch (e: Exception) { + Timber.e(e) + null + } } - private fun handleSuccess(params: Params, contentUploadResponse: ContentUploadResponse): Result { - val event = updateEvent(params.event, contentUploadResponse.contentUri) + private fun handleFailure(params: Params): Result { + contentUploadProgressTracker.setFailure(params.event.eventId!!) + return Result.failure() + } + + private fun handleSuccess(params: Params, + attachmentUrl: String, + thumbnailUrl: String?): Result { + contentUploadProgressTracker.setFailure(params.event.eventId!!) + val event = updateEvent(params.event, attachmentUrl, thumbnailUrl) val sendParams = SendEventWorker.Params(params.roomId, event) return Result.success(WorkerParamsFactory.toData(sendParams)) } - private fun updateEvent(event: Event, url: String): Event { + private fun updateEvent(event: Event, url: String, thumbnailUrl: String? = null): Event { val messageContent: MessageContent = event.content.toModel() ?: return event val updatedContent = when (messageContent) { is MessageImageContent -> messageContent.update(url) - is MessageVideoContent -> messageContent.update(url) + is MessageVideoContent -> messageContent.update(url, thumbnailUrl) is MessageFileContent -> messageContent.update(url) is MessageAudioContent -> messageContent.update(url) else -> messageContent @@ -84,8 +124,8 @@ internal class UploadContentWorker(context: Context, params: WorkerParameters) return copy(url = url) } - private fun MessageVideoContent.update(url: String): MessageVideoContent { - return copy(url = url) + private fun MessageVideoContent.update(url: String, thumbnailUrl: String?): MessageVideoContent { + return copy(url = url, info = info?.copy(thumbnailUrl = thumbnailUrl)) } private fun MessageFileContent.update(url: String): MessageFileContent { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt index 56051503..778a49ec 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/LocalEchoEventFactory.kt @@ -16,6 +16,7 @@ package im.vector.matrix.android.internal.session.room.send +import android.media.MediaMetadataRetriever import im.vector.matrix.android.api.auth.data.Credentials import im.vector.matrix.android.api.session.content.ContentAttachmentData import im.vector.matrix.android.api.session.events.model.Event @@ -30,7 +31,9 @@ import im.vector.matrix.android.api.session.room.model.message.MessageImageConte import im.vector.matrix.android.api.session.room.model.message.MessageTextContent import im.vector.matrix.android.api.session.room.model.message.MessageType import im.vector.matrix.android.api.session.room.model.message.MessageVideoContent +import im.vector.matrix.android.api.session.room.model.message.ThumbnailInfo import im.vector.matrix.android.api.session.room.model.message.VideoInfo +import im.vector.matrix.android.internal.session.content.ThumbnailExtractor internal class LocalEchoEventFactory(private val credentials: Credentials) { @@ -53,7 +56,7 @@ internal class LocalEchoEventFactory(private val credentials: Credentials) { type = MessageType.MSGTYPE_IMAGE, body = attachment.name ?: "image", info = ImageInfo( - mimeType = attachment.mimeType ?: "image/png", + mimeType = attachment.mimeType, width = attachment.width?.toInt() ?: 0, height = attachment.height?.toInt() ?: 0, size = attachment.size.toInt() @@ -64,15 +67,35 @@ internal class LocalEchoEventFactory(private val credentials: Credentials) { } private fun createVideoEvent(roomId: String, attachment: ContentAttachmentData): Event { + val mediaDataRetriever = MediaMetadataRetriever() + mediaDataRetriever.setDataSource(attachment.path) + + // Use frame to calculate height and width as we are sure to get the right ones + val firstFrame = mediaDataRetriever.frameAtTime + val height = firstFrame.height + val width = firstFrame.width + mediaDataRetriever.release() + + val thumbnailInfo = ThumbnailExtractor.extractThumbnail(attachment)?.let { + ThumbnailInfo( + width = it.width, + height = it.height, + size = it.size, + mimeType = it.mimeType + ) + } val content = MessageVideoContent( type = MessageType.MSGTYPE_VIDEO, body = attachment.name ?: "video", info = VideoInfo( - mimeType = attachment.mimeType ?: "video/mpeg", - width = attachment.width?.toInt() ?: 0, - height = attachment.height?.toInt() ?: 0, + mimeType = attachment.mimeType, + width = width, + height = height, size = attachment.size, - duration = attachment.duration?.toInt() ?: 0 + duration = attachment.duration?.toInt() ?: 0, + // Glide will be able to use the local path and extract a thumbnail. + thumbnailUrl = attachment.path, + thumbnailInfo = thumbnailInfo ), url = attachment.path )