Handoff media upload (#2947)

* handoff media upload to SendStatusService

* fix bugd

* improve code

* don't check processing state when upload returned 200
This commit is contained in:
Konrad Pozniak 2022-12-29 19:58:23 +01:00 committed by GitHub
parent 9cf949fc2e
commit abca91a420
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 204 additions and 118 deletions

View file

@ -135,8 +135,6 @@ class ComposeActivity :
private lateinit var emojiBehavior: BottomSheetBehavior<*>
private lateinit var scheduleBehavior: BottomSheetBehavior<*>
// this only exists when a status is trying to be sent, but uploads are still occurring
private var finishingUploadDialog: ProgressDialog? = null
private var photoUploadUri: Uri? = null
private val preferences by lazy { PreferenceManager.getDefaultSharedPreferences(this) }
@ -957,16 +955,9 @@ class ComposeActivity :
binding.composeEditField.error = getString(R.string.error_empty)
enableButtons(true, viewModel.editing)
} else if (characterCount <= maximumTootCharacters) {
if (viewModel.media.value.isNotEmpty()) {
finishingUploadDialog = ProgressDialog.show(
this, getString(R.string.dialog_title_finishing_media_upload),
getString(R.string.dialog_message_uploading_media), true, true
)
}
lifecycleScope.launch {
viewModel.sendStatus(contentText, spoilerText)
finishingUploadDialog?.dismiss()
deleteDraftAndFinish()
}
} else {
@ -1133,11 +1124,16 @@ class ComposeActivity :
AlertDialog.Builder(this)
.setMessage(warning)
.setPositiveButton(R.string.action_save) { _, _ ->
viewModel.stopUploads()
saveDraftAndFinish(contentText, contentWarning)
}
.setNegativeButton(R.string.action_delete) { _, _ -> deleteDraftAndFinish() }
.setNegativeButton(R.string.action_delete) { _, _ ->
viewModel.stopUploads()
deleteDraftAndFinish()
}
.show()
} else {
viewModel.stopUploads()
finishWithoutSlideOutAnimation()
}
}
@ -1188,11 +1184,14 @@ class ComposeActivity :
val id: String? = null,
val description: String? = null,
val focus: Attachment.Focus? = null,
val processed: Boolean = false,
val state: State
) {
enum class Type {
IMAGE, VIDEO, AUDIO;
}
enum class State {
UPLOADING, UNPROCESSED, PROCESSED, PUBLISHED
}
}
override fun onTimeSet(time: String) {

View file

@ -33,20 +33,18 @@ import com.keylesspalace.tusky.entity.Emoji
import com.keylesspalace.tusky.entity.NewPoll
import com.keylesspalace.tusky.entity.Status
import com.keylesspalace.tusky.network.MastodonApi
import com.keylesspalace.tusky.service.MediaToSend
import com.keylesspalace.tusky.service.ServiceClient
import com.keylesspalace.tusky.service.StatusToSend
import com.keylesspalace.tusky.util.randomAlphanumericString
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.FlowPreview
import kotlinx.coroutines.Job
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.catch
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.shareIn
import kotlinx.coroutines.flow.update
@ -97,8 +95,6 @@ class ComposeViewModel @Inject constructor(
val media: MutableStateFlow<List<QueuedMedia>> = MutableStateFlow(emptyList())
val uploadError = MutableSharedFlow<Throwable>(replay = 0, extraBufferCapacity = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST)
private val mediaToJob = mutableMapOf<Int, Job>()
// Used in ComposeActivity to pass state to result function when cropImage contract inflight
var cropImageItemOld: QueuedMedia? = null
@ -134,17 +130,18 @@ class ComposeViewModel @Inject constructor(
media.updateAndGet { mediaValue ->
val mediaItem = QueuedMedia(
localId = (mediaValue.maxOfOrNull { it.localId } ?: 0) + 1,
localId = mediaUploader.getNewLocalMediaId(),
uri = uri,
type = type,
mediaSize = mediaSize,
description = description,
focus = focus
focus = focus,
state = QueuedMedia.State.UPLOADING
)
stashMediaItem = mediaItem
if (replaceItem != null) {
mediaToJob[replaceItem.localId]?.cancel()
mediaUploader.cancelUploadScope(replaceItem.localId)
mediaValue.map {
if (it.localId == replaceItem.localId) mediaItem else it
}
@ -154,13 +151,9 @@ class ComposeViewModel @Inject constructor(
}
val mediaItem = stashMediaItem!! // stashMediaItem is always non-null and uncaptured at this point, but Kotlin doesn't know that
mediaToJob[mediaItem.localId] = viewModelScope.launch {
viewModelScope.launch {
mediaUploader
.uploadMedia(mediaItem, instanceInfo.first())
.catch { error ->
media.update { mediaValue -> mediaValue.filter { it.localId != mediaItem.localId } }
uploadError.emit(error)
}
.collect { event ->
val item = media.value.find { it.localId == mediaItem.localId }
?: return@collect
@ -168,7 +161,16 @@ class ComposeViewModel @Inject constructor(
is UploadEvent.ProgressEvent ->
item.copy(uploadPercent = event.percentage)
is UploadEvent.FinishedEvent ->
item.copy(id = event.mediaId, uploadPercent = -1)
item.copy(
id = event.mediaId,
uploadPercent = -1,
state = if (event.processed) { QueuedMedia.State.PROCESSED } else { QueuedMedia.State.UNPROCESSED }
)
is UploadEvent.ErrorEvent -> {
media.update { mediaValue -> mediaValue.filter { it.localId != mediaItem.localId } }
uploadError.emit(event.error)
return@collect
}
}
media.update { mediaValue ->
mediaValue.map { mediaItem ->
@ -187,7 +189,7 @@ class ComposeViewModel @Inject constructor(
private fun addUploadedMedia(id: String, type: QueuedMedia.Type, uri: Uri, description: String?, focus: Attachment.Focus?) {
media.update { mediaValue ->
val mediaItem = QueuedMedia(
localId = (mediaValue.maxOfOrNull { it.localId } ?: 0) + 1,
localId = mediaUploader.getNewLocalMediaId(),
uri = uri,
type = type,
mediaSize = 0,
@ -195,14 +197,14 @@ class ComposeViewModel @Inject constructor(
id = id,
description = description,
focus = focus,
processed = true,
state = QueuedMedia.State.PUBLISHED
)
mediaValue + mediaItem
}
}
fun removeMediaFromQueue(item: QueuedMedia) {
mediaToJob[item.localId]?.cancel()
mediaUploader.cancelUploadScope(item.localId)
media.update { mediaValue -> mediaValue.filter { it.localId != item.localId } }
}
@ -240,6 +242,10 @@ class ComposeViewModel @Inject constructor(
}
}
fun stopUploads() {
mediaUploader.cancelUploadScope(*media.value.map { it.localId }.toIntArray())
}
fun shouldShowSaveDraftDialog(): Boolean {
// if any of the media files need to be downloaded first it could take a while, so show a loading dialog
return media.value.any { mediaValue ->
@ -289,30 +295,22 @@ class ComposeViewModel @Inject constructor(
api.deleteScheduledStatus(scheduledTootId!!)
}
media
.filter { items -> items.all { it.uploadPercent == -1 } }
.first {
val mediaIds: MutableList<String> = mutableListOf()
val mediaUris: MutableList<Uri> = mutableListOf()
val mediaDescriptions: MutableList<String> = mutableListOf()
val mediaFocus: MutableList<Attachment.Focus?> = mutableListOf()
val mediaProcessed: MutableList<Boolean> = mutableListOf()
media.value.forEach { item ->
mediaIds.add(item.id!!)
mediaUris.add(item.uri)
mediaDescriptions.add(item.description ?: "")
mediaFocus.add(item.focus)
mediaProcessed.add(item.processed)
val attachedMedia = media.value.map { item ->
MediaToSend(
localId = item.localId,
id = item.id,
uri = item.uri.toString(),
description = item.description,
focus = item.focus,
processed = item.state == QueuedMedia.State.PROCESSED || item.state == QueuedMedia.State.PUBLISHED
)
}
val tootToSend = StatusToSend(
text = content,
warningText = spoilerText,
visibility = statusVisibility.value.serverString(),
sensitive = mediaUris.isNotEmpty() && (markMediaAsSensitive.value || showContentWarning.value),
mediaIds = mediaIds,
mediaUris = mediaUris.map { it.toString() },
mediaDescriptions = mediaDescriptions,
mediaFocus = mediaFocus,
sensitive = attachedMedia.isNotEmpty() && (markMediaAsSensitive.value || showContentWarning.value),
media = attachedMedia,
scheduledAt = scheduledAt.value,
inReplyToId = inReplyToId,
poll = poll.value,
@ -322,14 +320,11 @@ class ComposeViewModel @Inject constructor(
draftId = draftId,
idempotencyKey = randomAlphanumericString(16),
retries = 0,
mediaProcessed = mediaProcessed,
language = postLanguage,
statusId = originalStatusId,
statusId = originalStatusId
)
serviceClient.sendToot(tootToSend)
true
}
}
// Updates a QueuedMedia item arbitrarily, then sends description and focus to server
@ -360,15 +355,15 @@ class ComposeViewModel @Inject constructor(
}
suspend fun updateDescription(localId: Int, description: String): Boolean {
return updateMediaItem(localId, { mediaItem ->
return updateMediaItem(localId) { mediaItem ->
mediaItem.copy(description = description)
})
}
}
suspend fun updateFocus(localId: Int, focus: Attachment.Focus): Boolean {
return updateMediaItem(localId, { mediaItem ->
return updateMediaItem(localId) { mediaItem ->
mediaItem.copy(focus = focus)
})
}
}
fun searchAutocompleteSuggestions(token: String): List<AutocompleteResult> {

View file

@ -48,7 +48,7 @@ class MediaPreviewAdapter(
val addFocusId = 2
val editImageId = 3
val removeId = 4
if (!item.processed) {
if (item.state != ComposeActivity.QueuedMedia.State.PUBLISHED) {
// Already-published items can't have their metadata edited
popup.menu.add(0, addCaptionId, 0, R.string.action_set_caption)
if (item.type == ComposeActivity.QueuedMedia.Type.IMAGE) {

View file

@ -23,7 +23,6 @@ import android.util.Log
import android.webkit.MimeTypeMap
import androidx.core.content.FileProvider
import androidx.core.net.toUri
import at.connyduck.calladapter.networkresult.fold
import com.keylesspalace.tusky.BuildConfig
import com.keylesspalace.tusky.R
import com.keylesspalace.tusky.components.compose.ComposeActivity.QueuedMedia
@ -35,28 +34,44 @@ import com.keylesspalace.tusky.util.getImageSquarePixels
import com.keylesspalace.tusky.util.getMediaSize
import com.keylesspalace.tusky.util.getServerErrorMessage
import com.keylesspalace.tusky.util.randomAlphanumericString
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.cancel
import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.callbackFlow
import kotlinx.coroutines.flow.catch
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.flatMapLatest
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.flow.shareIn
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.MultipartBody
import retrofit2.HttpException
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.util.Date
import javax.inject.Inject
import javax.inject.Singleton
sealed interface FinalUploadEvent
sealed class UploadEvent {
data class ProgressEvent(val percentage: Int) : UploadEvent()
data class FinishedEvent(val mediaId: String) : UploadEvent()
data class FinishedEvent(val mediaId: String, val processed: Boolean) : UploadEvent(), FinalUploadEvent
data class ErrorEvent(val error: Throwable) : UploadEvent(), FinalUploadEvent
}
data class UploadData(
val flow: Flow<UploadEvent>,
val scope: CoroutineScope
)
fun createNewImageFile(context: Context, suffix: String = ".jpg"): File {
// Create an image file name
val randomId = randomAlphanumericString(12)
@ -76,14 +91,38 @@ class MediaTypeException : Exception()
class CouldNotOpenFileException : Exception()
class UploadServerError(val errorMessage: String) : Exception()
@Singleton
class MediaUploader @Inject constructor(
private val context: Context,
private val mediaUploadApi: MediaUploadApi
) {
private val uploads = mutableMapOf<Int, UploadData>()
private var mostRecentId: Int = 0
fun getNewLocalMediaId(): Int {
return mostRecentId++
}
suspend fun getMediaUploadState(localId: Int): FinalUploadEvent {
return uploads[localId]?.flow
?.filterIsInstance<FinalUploadEvent>()
?.first()
?: UploadEvent.ErrorEvent(IllegalStateException("media upload with id $localId not found"))
}
/**
* Uploads media.
* @param media the media to upload
* @param instanceInfo info about the current media to make sure the media gets resized correctly
* @return A Flow emitting upload events.
* The Flow is hot, in order to cancel upload or clear resources call [cancelUploadScope].
*/
@OptIn(ExperimentalCoroutinesApi::class)
fun uploadMedia(media: QueuedMedia, instanceInfo: InstanceInfo): Flow<UploadEvent> {
return flow {
val uploadScope = CoroutineScope(Dispatchers.IO)
val uploadFlow = flow {
if (shouldResizeMedia(media, instanceInfo)) {
emit(downsize(media, instanceInfo))
} else {
@ -91,7 +130,23 @@ class MediaUploader @Inject constructor(
}
}
.flatMapLatest { upload(it) }
.flowOn(Dispatchers.IO)
.catch { exception ->
emit(UploadEvent.ErrorEvent(exception))
}
.shareIn(uploadScope, SharingStarted.Lazily, 1)
uploads[media.localId] = UploadData(uploadFlow, uploadScope)
return uploadFlow
}
/**
* Cancels the CoroutineScope of a media upload.
* Call this when to abort the upload or to clean up resources after upload info is no longer needed
*/
fun cancelUploadScope(vararg localMediaIds: Int) {
localMediaIds.forEach { localId ->
uploads.remove(localId)?.scope?.cancel()
}
}
fun prepareMedia(inUri: Uri, instanceInfo: InstanceInfo): PreparedMedia {
@ -231,16 +286,20 @@ class MediaUploader @Inject constructor(
null
}
mediaUploadApi.uploadMedia(body, description, focus).fold({ result ->
send(UploadEvent.FinishedEvent(result.id))
}, { throwable ->
val errorMessage = throwable.getServerErrorMessage()
val uploadResponse = mediaUploadApi.uploadMedia(body, description, focus)
val responseBody = uploadResponse.body()
if (uploadResponse.isSuccessful && responseBody != null) {
send(UploadEvent.FinishedEvent(responseBody.id, uploadResponse.code() == 200))
} else {
val error = HttpException(uploadResponse)
val errorMessage = error.getServerErrorMessage()
if (errorMessage == null) {
throw throwable
throw error
} else {
throw UploadServerError(errorMessage)
}
})
}
awaitClose()
}
}

View file

@ -1,8 +1,8 @@
package com.keylesspalace.tusky.network
import at.connyduck.calladapter.networkresult.NetworkResult
import com.keylesspalace.tusky.entity.MediaUploadResult
import okhttp3.MultipartBody
import retrofit2.Response
import retrofit2.http.Multipart
import retrofit2.http.POST
import retrofit2.http.Part
@ -17,5 +17,5 @@ interface MediaUploadApi {
@Part file: MultipartBody.Part,
@Part description: MultipartBody.Part? = null,
@Part focus: MultipartBody.Part? = null
): NetworkResult<MediaUploadResult>
): Response<MediaUploadResult>
}

View file

@ -86,10 +86,7 @@ class SendStatusBroadcastReceiver : BroadcastReceiver() {
warningText = spoiler,
visibility = visibility.serverString(),
sensitive = false,
mediaIds = emptyList(),
mediaUris = emptyList(),
mediaDescriptions = emptyList(),
mediaFocus = emptyList(),
media = emptyList(),
scheduledAt = null,
inReplyToId = citedStatusId,
poll = null,
@ -99,7 +96,6 @@ class SendStatusBroadcastReceiver : BroadcastReceiver() {
draftId = -1,
idempotencyKey = randomAlphanumericString(16),
retries = 0,
mediaProcessed = mutableListOf(),
language = null,
statusId = null,
)

View file

@ -22,6 +22,8 @@ import com.keylesspalace.tusky.R
import com.keylesspalace.tusky.appstore.EventHub
import com.keylesspalace.tusky.appstore.StatusComposedEvent
import com.keylesspalace.tusky.appstore.StatusScheduledEvent
import com.keylesspalace.tusky.components.compose.MediaUploader
import com.keylesspalace.tusky.components.compose.UploadEvent
import com.keylesspalace.tusky.components.drafts.DraftHelper
import com.keylesspalace.tusky.components.notifications.NotificationHelper
import com.keylesspalace.tusky.db.AccountManager
@ -54,6 +56,8 @@ class SendStatusService : Service(), Injectable {
lateinit var eventHub: EventHub
@Inject
lateinit var draftHelper: DraftHelper
@Inject
lateinit var mediaUploader: MediaUploader
private val supervisorJob = SupervisorJob()
private val serviceScope = CoroutineScope(Dispatchers.Main + supervisorJob)
@ -131,14 +135,33 @@ class SendStatusService : Service(), Injectable {
statusToSend.retries++
sendJobs[statusId] = serviceScope.launch {
// first, wait for media uploads to finish
val media = statusToSend.media.map { mediaItem ->
if (mediaItem.id == null) {
when (val uploadState = mediaUploader.getMediaUploadState(mediaItem.localId)) {
is UploadEvent.FinishedEvent -> mediaItem.copy(id = uploadState.mediaId, processed = uploadState.processed)
is UploadEvent.ErrorEvent -> {
Log.w(TAG, "failed uploading media", uploadState.error)
failSending(statusId)
stopSelfWhenDone()
return@launch
}
}
} else {
mediaItem
}
}
// then wait until server finished processing the media
try {
var mediaCheckRetries = 0
while (statusToSend.mediaProcessed.any { !it }) {
while (media.any { mediaItem -> !mediaItem.processed }) {
delay(1000L * mediaCheckRetries)
statusToSend.mediaProcessed.forEachIndexed { index, processed ->
if (!processed) {
when (mastodonApi.getMedia(statusToSend.mediaIds[index]).code()) {
200 -> statusToSend.mediaProcessed[index] = true // success
media.forEach { mediaItem ->
if (!mediaItem.processed) {
when (mastodonApi.getMedia(mediaItem.id!!).code()) {
200 -> mediaItem.processed = true // success
206 -> { } // media is still being processed, continue checking
else -> { // some kind of server error, retrying probably doesn't make sense
failSending(statusId)
@ -156,16 +179,17 @@ class SendStatusService : Service(), Injectable {
return@launch
}
// finally, send the new status
val newStatus = NewStatus(
statusToSend.text,
statusToSend.warningText,
statusToSend.inReplyToId,
statusToSend.visibility,
statusToSend.sensitive,
statusToSend.mediaIds,
statusToSend.scheduledAt,
statusToSend.poll,
statusToSend.language,
status = statusToSend.text,
warningText = statusToSend.warningText,
inReplyToId = statusToSend.inReplyToId,
visibility = statusToSend.visibility,
sensitive = statusToSend.sensitive,
mediaIds = media.map { it.id!! },
scheduledAt = statusToSend.scheduledAt,
poll = statusToSend.poll,
language = statusToSend.language,
)
val sendResult = if (statusToSend.statusId == null) {
@ -192,6 +216,8 @@ class SendStatusService : Service(), Injectable {
draftHelper.deleteDraftAndAttachments(statusToSend.draftId)
}
mediaUploader.cancelUploadScope(*statusToSend.media.map { it.localId }.toIntArray())
val scheduled = !statusToSend.scheduledAt.isNullOrEmpty()
if (scheduled) {
@ -237,6 +263,8 @@ class SendStatusService : Service(), Injectable {
val failedStatus = statusesToSend.remove(statusId)
if (failedStatus != null) {
mediaUploader.cancelUploadScope(*failedStatus.media.map { it.localId }.toIntArray())
saveStatusToDrafts(failedStatus)
val notification = buildDraftNotification(
@ -254,6 +282,9 @@ class SendStatusService : Service(), Injectable {
private fun cancelSending(statusId: Int) = serviceScope.launch {
val statusToCancel = statusesToSend.remove(statusId)
if (statusToCancel != null) {
mediaUploader.cancelUploadScope(*statusToCancel.media.map { it.localId }.toIntArray())
val sendJob = sendJobs.remove(statusId)
sendJob?.cancel()
@ -283,9 +314,9 @@ class SendStatusService : Service(), Injectable {
contentWarning = status.warningText,
sensitive = status.sensitive,
visibility = Status.Visibility.byString(status.visibility),
mediaUris = status.mediaUris,
mediaDescriptions = status.mediaDescriptions,
mediaFocus = status.mediaFocus,
mediaUris = status.media.map { it.uri },
mediaDescriptions = status.media.map { it.description },
mediaFocus = status.media.map { it.focus },
poll = status.poll,
failedToSend = true,
scheduledAt = status.scheduledAt,
@ -358,17 +389,17 @@ class SendStatusService : Service(), Injectable {
val intent = Intent(context, SendStatusService::class.java)
intent.putExtra(KEY_STATUS, statusToSend)
if (statusToSend.mediaUris.isNotEmpty()) {
if (statusToSend.media.isNotEmpty()) {
// forward uri permissions
intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
val uriClip = ClipData(
ClipDescription("Status Media", arrayOf("image/*", "video/*")),
ClipData.Item(statusToSend.mediaUris[0])
ClipData.Item(statusToSend.media[0].uri)
)
statusToSend.mediaUris
statusToSend.media
.drop(1)
.forEach { mediaUri ->
uriClip.addItem(ClipData.Item(mediaUri))
.forEach { mediaItem ->
uriClip.addItem(ClipData.Item(mediaItem.uri))
}
intent.clipData = uriClip
@ -385,10 +416,7 @@ data class StatusToSend(
val warningText: String,
val visibility: String,
val sensitive: Boolean,
val mediaIds: List<String>,
val mediaUris: List<String>,
val mediaDescriptions: List<String>,
val mediaFocus: List<Attachment.Focus?>,
val media: List<MediaToSend>,
val scheduledAt: String?,
val inReplyToId: String?,
val poll: NewPoll?,
@ -398,7 +426,16 @@ data class StatusToSend(
val draftId: Int,
val idempotencyKey: String,
var retries: Int,
val mediaProcessed: MutableList<Boolean>,
val language: String?,
val statusId: String?,
) : Parcelable
@Parcelize
data class MediaToSend(
val localId: Int,
val id: String?, // null if media is not yet completely uploaded
val uri: String,
val description: String?,
val focus: Attachment.Focus?,
var processed: Boolean
) : Parcelable