Caching toots (#809)
* Initial timeline cache implementation * Fix build/DI errors for caching * Rename timeline entities tables. Add migration. Add DB scheme file. * Fix uniqueness problem, change offline strategy, improve mapping * Try to merge in new statuses, fix bottom loading, fix saving spans. * Fix reblogs IDs, fix inserting elements from top * Send one more request to get latest timeline statuses * Give Timeline placeholders string id. Rewrite Either in Kotlin * Initial placeholder implementation for caching * Fix crash on removing overlap statuses * Migrate counters to long * Remove unused counters. Add minimal TimelineDAOTest * Fix bug with placeholder ID * Update cache in response to events. Refactor TimelineCases * Fix crash, reduce number of placeholders * Fix crash, fix filtering, improve placeholder handling * Fix migration, add 8-9 migration test * Fix initial timeline update, remove more placeholders * Add cleanup for old statuses * Fix cleanup * Delete ExampleInstrumentedTest * Improve timeline UX regarding caching * Fix typos * Fix initial timeline update * Cleanup/fix initial timeline update * Workaround for weird behavior of first post on initial tl update. * Change counter types back to int * Clear timeline cache on logout * Fix loading when timeline is completely empty * Fix androidx migration issues * Fix tests * Apply caching feedback * Save account emojis to cache * Fix warnings and bugs
This commit is contained in:
Ivan Kupalov
Konrad Pozniak
parent
75158a3aa0
commit
3ab78a19bc
29 changed files with 1950 additions and 497 deletions
|
|
@ -0,0 +1,404 @@
|
|||
package com.keylesspalace.tusky.repository
|
||||
|
||||
import android.text.SpannedString
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.reflect.TypeToken
|
||||
import com.keylesspalace.tusky.db.*
|
||||
import com.keylesspalace.tusky.entity.Account
|
||||
import com.keylesspalace.tusky.entity.Attachment
|
||||
import com.keylesspalace.tusky.entity.Emoji
|
||||
import com.keylesspalace.tusky.entity.Status
|
||||
import com.keylesspalace.tusky.network.MastodonApi
|
||||
import com.keylesspalace.tusky.repository.TimelineRequestMode.DISK
|
||||
import com.keylesspalace.tusky.repository.TimelineRequestMode.NETWORK
|
||||
import com.keylesspalace.tusky.util.Either
|
||||
import com.keylesspalace.tusky.util.HtmlUtils
|
||||
import io.reactivex.Single
|
||||
import io.reactivex.schedulers.Schedulers
|
||||
import java.io.IOException
|
||||
import java.math.BigInteger
|
||||
import java.util.*
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
data class Placeholder(val id: String)
|
||||
|
||||
typealias TimelineStatus = Either<Placeholder, Status>
|
||||
|
||||
enum class TimelineRequestMode {
|
||||
DISK, NETWORK, ANY
|
||||
}
|
||||
|
||||
interface TimelineRepository {
|
||||
fun getStatuses(maxId: String?, sinceId: String?, limit: Int,
|
||||
requestMode: TimelineRequestMode): Single<out List<TimelineStatus>>
|
||||
|
||||
companion object {
|
||||
val CLEANUP_INTERVAL = TimeUnit.DAYS.toMillis(14)
|
||||
}
|
||||
}
|
||||
|
||||
class TimelineRepositoryImpl(
|
||||
private val timelineDao: TimelineDao,
|
||||
private val mastodonApi: MastodonApi,
|
||||
private val accountManager: AccountManager,
|
||||
private val gson: Gson
|
||||
) : TimelineRepository {
|
||||
|
||||
init {
|
||||
this.cleanup()
|
||||
}
|
||||
|
||||
override fun getStatuses(maxId: String?, sinceId: String?, limit: Int,
|
||||
requestMode: TimelineRequestMode): Single<out List<TimelineStatus>> {
|
||||
val acc = accountManager.activeAccount ?: throw IllegalStateException()
|
||||
val accountId = acc.id
|
||||
val instance = acc.domain
|
||||
|
||||
return if (requestMode == DISK) {
|
||||
this.getStatusesFromDb(accountId, maxId, sinceId, limit)
|
||||
} else {
|
||||
getStatusesFromNetwork(maxId, sinceId, limit, instance, accountId, requestMode)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getStatusesFromNetwork(maxId: String?, sinceId: String?, limit: Int,
|
||||
instance: String, accountId: Long,
|
||||
requestMode: TimelineRequestMode
|
||||
): Single<out List<TimelineStatus>> {
|
||||
val maxIdInc = maxId?.let { this.incId(it, 1) }
|
||||
val sinceIdDec = sinceId?.let { this.incId(it, -1) }
|
||||
return mastodonApi.homeTimelineSingle(maxIdInc, sinceIdDec, limit + 2)
|
||||
.doAfterSuccess { statuses ->
|
||||
this.saveStatusesToDb(instance, accountId, statuses, maxId, sinceId)
|
||||
}
|
||||
.map { statuses -> this.removePlaceholdersAndMap(statuses, maxId, sinceId) }
|
||||
.flatMap { statuses ->
|
||||
this.addFromDbIfNeeded(accountId, statuses, maxId, sinceId, limit, requestMode)
|
||||
}
|
||||
.onErrorResumeNext { error ->
|
||||
if (error is IOException && requestMode != NETWORK) {
|
||||
this.getStatusesFromDb(accountId, maxId, sinceId, limit)
|
||||
} else {
|
||||
Single.error(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun removePlaceholdersAndMap(statuses: List<Status>, maxId: String?,
|
||||
sinceId: String?
|
||||
): List<Either.Right<Placeholder, Status>> {
|
||||
val statusesCopy = statuses.toMutableList()
|
||||
|
||||
// Remove first and last statuses if they were used used just for overlap
|
||||
if (maxId != null && statusesCopy.firstOrNull()?.id == maxId) {
|
||||
statusesCopy.removeAt(0)
|
||||
}
|
||||
if (sinceId != null && statusesCopy.lastOrNull()?.id == sinceId) {
|
||||
statusesCopy.removeAt(statusesCopy.size - 1)
|
||||
}
|
||||
|
||||
return statusesCopy.map { s -> Either.Right<Placeholder, Status>(s) }
|
||||
}
|
||||
|
||||
private fun addFromDbIfNeeded(accountId: Long, statuses: List<Either<Placeholder, Status>>,
|
||||
maxId: String?, sinceId: String?, limit: Int,
|
||||
requestMode: TimelineRequestMode
|
||||
): Single<List<TimelineStatus>>? {
|
||||
return if (requestMode != NETWORK && statuses.size < 2) {
|
||||
val newMaxID = if (statuses.isEmpty()) {
|
||||
maxId
|
||||
} else {
|
||||
// It's statuses from network. They're always Right
|
||||
statuses.last().asRight().id
|
||||
}
|
||||
this.getStatusesFromDb(accountId, newMaxID, sinceId, limit)
|
||||
.map { fromDb ->
|
||||
// If it's just placeholders and less than limit (so we exhausted both
|
||||
// db and server at this point)
|
||||
if (fromDb.size < limit && fromDb.all { !it.isRight() }) {
|
||||
statuses
|
||||
} else {
|
||||
statuses + fromDb
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Single.just(statuses)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getStatusesFromDb(accountId: Long, maxId: String?, sinceId: String?,
|
||||
limit: Int): Single<out List<TimelineStatus>> {
|
||||
return timelineDao.getStatusesForAccount(accountId, maxId, sinceId, limit)
|
||||
.subscribeOn(Schedulers.io())
|
||||
.map { statuses ->
|
||||
statuses.map { it.toStatus() }
|
||||
}
|
||||
}
|
||||
|
||||
private fun saveStatusesToDb(instance: String, accountId: Long, statuses: List<Status>,
|
||||
maxId: String?, sinceId: String?) {
|
||||
Single.fromCallable {
|
||||
val (prepend, append) = calculatePlaceholders(maxId, sinceId, statuses)
|
||||
|
||||
if (prepend != null) {
|
||||
timelineDao.insertStatusIfNotThere(prepend.toEntity(accountId))
|
||||
}
|
||||
|
||||
if (append != null) {
|
||||
timelineDao.insertStatusIfNotThere(append.toEntity(accountId))
|
||||
}
|
||||
|
||||
for (status in statuses) {
|
||||
timelineDao.insertInTransaction(
|
||||
status.toEntity(accountId, instance),
|
||||
status.account.toEntity(instance, accountId),
|
||||
status.reblog?.account?.toEntity(instance, accountId)
|
||||
)
|
||||
}
|
||||
|
||||
// There may be placeholders which we thought could be from our TL but they are not
|
||||
if (statuses.size > 2) {
|
||||
timelineDao.removeAllPlaceholdersBetween(accountId, statuses.first().id,
|
||||
statuses.last().id)
|
||||
} else if (maxId != null && sinceId != null) {
|
||||
timelineDao.removeAllPlaceholdersBetween(accountId, maxId, sinceId)
|
||||
}
|
||||
}
|
||||
.subscribeOn(Schedulers.io())
|
||||
.subscribe()
|
||||
|
||||
}
|
||||
|
||||
private fun calculatePlaceholders(maxId: String?, sinceId: String?,
|
||||
statuses: List<Status>
|
||||
): Pair<Placeholder?, Placeholder?> {
|
||||
if (statuses.isEmpty()) return null to null
|
||||
|
||||
val firstId = statuses.first().id
|
||||
val prepend = if (maxId != null) {
|
||||
if (maxId > firstId) {
|
||||
val decMax = this.incId(maxId, -1)
|
||||
if (decMax != firstId) {
|
||||
Placeholder(decMax)
|
||||
} else null
|
||||
} else null
|
||||
} else {
|
||||
// Placeholders never overwrite real values so it's safe
|
||||
Placeholder(incId(firstId, 1))
|
||||
}
|
||||
|
||||
val lastId = statuses.last().id
|
||||
val append = if (sinceId != null) {
|
||||
if (sinceId < lastId) {
|
||||
val incSince = this.incId(sinceId, 1)
|
||||
if (incSince != lastId) {
|
||||
Placeholder(incSince)
|
||||
} else null
|
||||
} else null
|
||||
} else {
|
||||
// Placeholders never overwrite real values so it's safe
|
||||
Placeholder(incId(lastId, -1))
|
||||
}
|
||||
|
||||
return prepend to append
|
||||
}
|
||||
|
||||
private fun cleanup() {
|
||||
Single.fromCallable {
|
||||
val olderThan = System.currentTimeMillis() - TimelineRepository.CLEANUP_INTERVAL
|
||||
for (account in accountManager.getAllAccountsOrderedByActive()) {
|
||||
timelineDao.cleanup(account.id, account.accountId, olderThan)
|
||||
}
|
||||
}
|
||||
.subscribeOn(Schedulers.io())
|
||||
.subscribe()
|
||||
}
|
||||
|
||||
private fun Account.toEntity(instance: String, accountId: Long): TimelineAccountEntity {
|
||||
return TimelineAccountEntity(
|
||||
serverId = id,
|
||||
timelineUserId = accountId,
|
||||
instance = instance,
|
||||
localUsername = localUsername,
|
||||
username = username,
|
||||
displayName = displayName,
|
||||
url = url,
|
||||
avatar = avatar,
|
||||
emojis = gson.toJson(emojis)
|
||||
)
|
||||
}
|
||||
|
||||
private fun TimelineAccountEntity.toAccount(): Account {
|
||||
return Account(
|
||||
id = serverId,
|
||||
localUsername = localUsername,
|
||||
username = username,
|
||||
displayName = displayName,
|
||||
note = SpannedString(""),
|
||||
url = url,
|
||||
avatar = avatar,
|
||||
header = "",
|
||||
locked = false,
|
||||
followingCount = 0,
|
||||
followersCount = 0,
|
||||
statusesCount = 0,
|
||||
source = null,
|
||||
bot = false,
|
||||
emojis = gson.fromJson(this.emojis, emojisListTypeToken.type),
|
||||
fields = null,
|
||||
moved = null
|
||||
)
|
||||
}
|
||||
|
||||
private fun TimelineStatusWithAccount.toStatus(): TimelineStatus {
|
||||
if (this.status.authorServerId == null) {
|
||||
return Either.Left(Placeholder(this.status.serverId))
|
||||
}
|
||||
|
||||
val attachments: List<Attachment> = gson.fromJson(status.attachments,
|
||||
object : TypeToken<List<Attachment>>() {}.type) ?: listOf()
|
||||
val mentions: Array<Status.Mention> = gson.fromJson(status.mentions,
|
||||
Array<Status.Mention>::class.java) ?: arrayOf()
|
||||
val application = gson.fromJson(status.application, Status.Application::class.java)
|
||||
val emojis: List<Emoji> = gson.fromJson(status.emojis,
|
||||
object : TypeToken<List<Emoji>>() {}.type) ?: listOf()
|
||||
|
||||
val reblog = status.reblogServerId?.let { id ->
|
||||
Status(
|
||||
id = id,
|
||||
url = status.url,
|
||||
account = account.toAccount(),
|
||||
inReplyToId = status.inReplyToId,
|
||||
inReplyToAccountId = status.inReplyToAccountId,
|
||||
reblog = null,
|
||||
content = HtmlUtils.fromHtml(status.content),
|
||||
createdAt = Date(status.createdAt),
|
||||
emojis = emojis,
|
||||
reblogsCount = status.reblogsCount,
|
||||
favouritesCount = status.favouritesCount,
|
||||
reblogged = status.reblogged,
|
||||
favourited = status.favourited,
|
||||
sensitive = status.sensitive,
|
||||
spoilerText = status.spoilerText!!,
|
||||
visibility = status.visibility!!,
|
||||
attachments = attachments,
|
||||
mentions = mentions,
|
||||
application = application,
|
||||
pinned = false
|
||||
|
||||
)
|
||||
}
|
||||
val status = if (reblog != null) {
|
||||
Status(
|
||||
id = status.serverId,
|
||||
url = null, // no url for reblogs
|
||||
account = this.reblogAccount!!.toAccount(),
|
||||
inReplyToId = null,
|
||||
inReplyToAccountId = null,
|
||||
reblog = reblog,
|
||||
content = SpannedString(""),
|
||||
createdAt = Date(status.createdAt), // lie but whatever?
|
||||
emojis = listOf(),
|
||||
reblogsCount = 0,
|
||||
favouritesCount = 0,
|
||||
reblogged = false,
|
||||
favourited = false,
|
||||
sensitive = false,
|
||||
spoilerText = "",
|
||||
visibility = status.visibility!!,
|
||||
attachments = listOf(),
|
||||
mentions = arrayOf(),
|
||||
application = null,
|
||||
pinned = false
|
||||
)
|
||||
} else {
|
||||
Status(
|
||||
id = status.serverId,
|
||||
url = status.url,
|
||||
account = account.toAccount(),
|
||||
inReplyToId = status.inReplyToId,
|
||||
inReplyToAccountId = status.inReplyToAccountId,
|
||||
reblog = null,
|
||||
content = HtmlUtils.fromHtml(status.content),
|
||||
createdAt = Date(status.createdAt),
|
||||
emojis = emojis,
|
||||
reblogsCount = status.reblogsCount,
|
||||
favouritesCount = status.favouritesCount,
|
||||
reblogged = status.reblogged,
|
||||
favourited = status.favourited,
|
||||
sensitive = status.sensitive,
|
||||
spoilerText = status.spoilerText!!,
|
||||
visibility = status.visibility!!,
|
||||
attachments = attachments,
|
||||
mentions = mentions,
|
||||
application = application,
|
||||
pinned = false
|
||||
)
|
||||
}
|
||||
return Either.Right(status)
|
||||
}
|
||||
|
||||
private fun Status.toEntity(timelineUserId: Long, instance: String): TimelineStatusEntity {
|
||||
val actionable = actionableStatus
|
||||
return TimelineStatusEntity(
|
||||
serverId = this.id,
|
||||
url = actionable.url!!,
|
||||
instance = instance,
|
||||
timelineUserId = timelineUserId,
|
||||
authorServerId = actionable.account.id,
|
||||
inReplyToId = actionable.inReplyToId,
|
||||
inReplyToAccountId = actionable.inReplyToAccountId,
|
||||
content = HtmlUtils.toHtml(actionable.content),
|
||||
createdAt = actionable.createdAt.time,
|
||||
emojis = actionable.emojis.let(gson::toJson),
|
||||
reblogsCount = actionable.reblogsCount,
|
||||
favouritesCount = actionable.favouritesCount,
|
||||
reblogged = actionable.reblogged,
|
||||
favourited = actionable.favourited,
|
||||
sensitive = actionable.sensitive,
|
||||
spoilerText = actionable.spoilerText,
|
||||
visibility = actionable.visibility,
|
||||
attachments = actionable.attachments.let(gson::toJson),
|
||||
mentions = actionable.mentions.let(gson::toJson),
|
||||
application = actionable.let(gson::toJson),
|
||||
reblogServerId = reblog?.id,
|
||||
reblogAccountId = reblog?.let { this.account.id }
|
||||
)
|
||||
}
|
||||
|
||||
private fun Placeholder.toEntity(timelineUserId: Long): TimelineStatusEntity {
|
||||
return TimelineStatusEntity(
|
||||
serverId = this.id,
|
||||
url = null,
|
||||
instance = null,
|
||||
timelineUserId = timelineUserId,
|
||||
authorServerId = null,
|
||||
inReplyToId = null,
|
||||
inReplyToAccountId = null,
|
||||
content = null,
|
||||
createdAt = 0L,
|
||||
emojis = null,
|
||||
reblogsCount = 0,
|
||||
favouritesCount = 0,
|
||||
reblogged = false,
|
||||
favourited = false,
|
||||
sensitive = false,
|
||||
spoilerText = null,
|
||||
visibility = null,
|
||||
attachments = null,
|
||||
mentions = null,
|
||||
application = null,
|
||||
reblogServerId = null,
|
||||
reblogAccountId = null
|
||||
|
||||
)
|
||||
}
|
||||
|
||||
private fun incId(id: String, value: Long): String {
|
||||
return BigInteger(id).add(BigInteger.valueOf(value)).toString()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val emojisListTypeToken = object : TypeToken<List<Emoji>>() {}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue