chinwag-android/app/src/main/java/com/keylesspalace/tusky/repository/TimelineRepository.kt
Ivan Kupalov 3ab78a19bc Caching toots (#809)
* Initial timeline cache implementation

* Fix build/DI errors for caching

* Rename timeline entities tables. Add migration. Add DB scheme file.

* Fix uniqueness problem, change offline strategy, improve mapping

* Try to merge in new statuses, fix bottom loading, fix saving spans.

* Fix reblogs IDs, fix inserting elements from top

* Send one more request to get latest timeline statuses

* Give Timeline placeholders string id. Rewrite Either in Kotlin

* Initial placeholder implementation for caching

* Fix crash on removing overlap statuses

* Migrate counters to long

* Remove unused counters. Add minimal TimelineDAOTest

* Fix bug with placeholder ID

* Update cache in response to events. Refactor TimelineCases

* Fix crash, reduce number of placeholders

* Fix crash, fix filtering, improve placeholder handling

* Fix migration, add 8-9 migration test

* Fix initial timeline update, remove more placeholders

* Add cleanup for old statuses

* Fix cleanup

* Delete ExampleInstrumentedTest

* Improve timeline UX regarding caching

* Fix typos

* Fix initial timeline update

* Cleanup/fix initial timeline update

* Workaround for weird behavior of first post on initial tl update.

* Change counter types back to int

* Clear timeline cache on logout

* Fix loading when timeline is completely empty

* Fix androidx migration issues

* Fix tests

* Apply caching feedback

* Save account emojis to cache

* Fix warnings and bugs
2019-01-14 22:05:08 +01:00

404 lines
16 KiB
Kotlin

package com.keylesspalace.tusky.repository
import android.text.SpannedString
import com.google.gson.Gson
import com.google.gson.reflect.TypeToken
import com.keylesspalace.tusky.db.*
import com.keylesspalace.tusky.entity.Account
import com.keylesspalace.tusky.entity.Attachment
import com.keylesspalace.tusky.entity.Emoji
import com.keylesspalace.tusky.entity.Status
import com.keylesspalace.tusky.network.MastodonApi
import com.keylesspalace.tusky.repository.TimelineRequestMode.DISK
import com.keylesspalace.tusky.repository.TimelineRequestMode.NETWORK
import com.keylesspalace.tusky.util.Either
import com.keylesspalace.tusky.util.HtmlUtils
import io.reactivex.Single
import io.reactivex.schedulers.Schedulers
import java.io.IOException
import java.math.BigInteger
import java.util.*
import java.util.concurrent.TimeUnit
data class Placeholder(val id: String)
typealias TimelineStatus = Either<Placeholder, Status>
enum class TimelineRequestMode {
DISK, NETWORK, ANY
}
interface TimelineRepository {
fun getStatuses(maxId: String?, sinceId: String?, limit: Int,
requestMode: TimelineRequestMode): Single<out List<TimelineStatus>>
companion object {
val CLEANUP_INTERVAL = TimeUnit.DAYS.toMillis(14)
}
}
class TimelineRepositoryImpl(
private val timelineDao: TimelineDao,
private val mastodonApi: MastodonApi,
private val accountManager: AccountManager,
private val gson: Gson
) : TimelineRepository {
init {
this.cleanup()
}
override fun getStatuses(maxId: String?, sinceId: String?, limit: Int,
requestMode: TimelineRequestMode): Single<out List<TimelineStatus>> {
val acc = accountManager.activeAccount ?: throw IllegalStateException()
val accountId = acc.id
val instance = acc.domain
return if (requestMode == DISK) {
this.getStatusesFromDb(accountId, maxId, sinceId, limit)
} else {
getStatusesFromNetwork(maxId, sinceId, limit, instance, accountId, requestMode)
}
}
private fun getStatusesFromNetwork(maxId: String?, sinceId: String?, limit: Int,
instance: String, accountId: Long,
requestMode: TimelineRequestMode
): Single<out List<TimelineStatus>> {
val maxIdInc = maxId?.let { this.incId(it, 1) }
val sinceIdDec = sinceId?.let { this.incId(it, -1) }
return mastodonApi.homeTimelineSingle(maxIdInc, sinceIdDec, limit + 2)
.doAfterSuccess { statuses ->
this.saveStatusesToDb(instance, accountId, statuses, maxId, sinceId)
}
.map { statuses -> this.removePlaceholdersAndMap(statuses, maxId, sinceId) }
.flatMap { statuses ->
this.addFromDbIfNeeded(accountId, statuses, maxId, sinceId, limit, requestMode)
}
.onErrorResumeNext { error ->
if (error is IOException && requestMode != NETWORK) {
this.getStatusesFromDb(accountId, maxId, sinceId, limit)
} else {
Single.error(error)
}
}
}
private fun removePlaceholdersAndMap(statuses: List<Status>, maxId: String?,
sinceId: String?
): List<Either.Right<Placeholder, Status>> {
val statusesCopy = statuses.toMutableList()
// Remove first and last statuses if they were used used just for overlap
if (maxId != null && statusesCopy.firstOrNull()?.id == maxId) {
statusesCopy.removeAt(0)
}
if (sinceId != null && statusesCopy.lastOrNull()?.id == sinceId) {
statusesCopy.removeAt(statusesCopy.size - 1)
}
return statusesCopy.map { s -> Either.Right<Placeholder, Status>(s) }
}
private fun addFromDbIfNeeded(accountId: Long, statuses: List<Either<Placeholder, Status>>,
maxId: String?, sinceId: String?, limit: Int,
requestMode: TimelineRequestMode
): Single<List<TimelineStatus>>? {
return if (requestMode != NETWORK && statuses.size < 2) {
val newMaxID = if (statuses.isEmpty()) {
maxId
} else {
// It's statuses from network. They're always Right
statuses.last().asRight().id
}
this.getStatusesFromDb(accountId, newMaxID, sinceId, limit)
.map { fromDb ->
// If it's just placeholders and less than limit (so we exhausted both
// db and server at this point)
if (fromDb.size < limit && fromDb.all { !it.isRight() }) {
statuses
} else {
statuses + fromDb
}
}
} else {
Single.just(statuses)
}
}
private fun getStatusesFromDb(accountId: Long, maxId: String?, sinceId: String?,
limit: Int): Single<out List<TimelineStatus>> {
return timelineDao.getStatusesForAccount(accountId, maxId, sinceId, limit)
.subscribeOn(Schedulers.io())
.map { statuses ->
statuses.map { it.toStatus() }
}
}
private fun saveStatusesToDb(instance: String, accountId: Long, statuses: List<Status>,
maxId: String?, sinceId: String?) {
Single.fromCallable {
val (prepend, append) = calculatePlaceholders(maxId, sinceId, statuses)
if (prepend != null) {
timelineDao.insertStatusIfNotThere(prepend.toEntity(accountId))
}
if (append != null) {
timelineDao.insertStatusIfNotThere(append.toEntity(accountId))
}
for (status in statuses) {
timelineDao.insertInTransaction(
status.toEntity(accountId, instance),
status.account.toEntity(instance, accountId),
status.reblog?.account?.toEntity(instance, accountId)
)
}
// There may be placeholders which we thought could be from our TL but they are not
if (statuses.size > 2) {
timelineDao.removeAllPlaceholdersBetween(accountId, statuses.first().id,
statuses.last().id)
} else if (maxId != null && sinceId != null) {
timelineDao.removeAllPlaceholdersBetween(accountId, maxId, sinceId)
}
}
.subscribeOn(Schedulers.io())
.subscribe()
}
private fun calculatePlaceholders(maxId: String?, sinceId: String?,
statuses: List<Status>
): Pair<Placeholder?, Placeholder?> {
if (statuses.isEmpty()) return null to null
val firstId = statuses.first().id
val prepend = if (maxId != null) {
if (maxId > firstId) {
val decMax = this.incId(maxId, -1)
if (decMax != firstId) {
Placeholder(decMax)
} else null
} else null
} else {
// Placeholders never overwrite real values so it's safe
Placeholder(incId(firstId, 1))
}
val lastId = statuses.last().id
val append = if (sinceId != null) {
if (sinceId < lastId) {
val incSince = this.incId(sinceId, 1)
if (incSince != lastId) {
Placeholder(incSince)
} else null
} else null
} else {
// Placeholders never overwrite real values so it's safe
Placeholder(incId(lastId, -1))
}
return prepend to append
}
private fun cleanup() {
Single.fromCallable {
val olderThan = System.currentTimeMillis() - TimelineRepository.CLEANUP_INTERVAL
for (account in accountManager.getAllAccountsOrderedByActive()) {
timelineDao.cleanup(account.id, account.accountId, olderThan)
}
}
.subscribeOn(Schedulers.io())
.subscribe()
}
private fun Account.toEntity(instance: String, accountId: Long): TimelineAccountEntity {
return TimelineAccountEntity(
serverId = id,
timelineUserId = accountId,
instance = instance,
localUsername = localUsername,
username = username,
displayName = displayName,
url = url,
avatar = avatar,
emojis = gson.toJson(emojis)
)
}
private fun TimelineAccountEntity.toAccount(): Account {
return Account(
id = serverId,
localUsername = localUsername,
username = username,
displayName = displayName,
note = SpannedString(""),
url = url,
avatar = avatar,
header = "",
locked = false,
followingCount = 0,
followersCount = 0,
statusesCount = 0,
source = null,
bot = false,
emojis = gson.fromJson(this.emojis, emojisListTypeToken.type),
fields = null,
moved = null
)
}
private fun TimelineStatusWithAccount.toStatus(): TimelineStatus {
if (this.status.authorServerId == null) {
return Either.Left(Placeholder(this.status.serverId))
}
val attachments: List<Attachment> = gson.fromJson(status.attachments,
object : TypeToken<List<Attachment>>() {}.type) ?: listOf()
val mentions: Array<Status.Mention> = gson.fromJson(status.mentions,
Array<Status.Mention>::class.java) ?: arrayOf()
val application = gson.fromJson(status.application, Status.Application::class.java)
val emojis: List<Emoji> = gson.fromJson(status.emojis,
object : TypeToken<List<Emoji>>() {}.type) ?: listOf()
val reblog = status.reblogServerId?.let { id ->
Status(
id = id,
url = status.url,
account = account.toAccount(),
inReplyToId = status.inReplyToId,
inReplyToAccountId = status.inReplyToAccountId,
reblog = null,
content = HtmlUtils.fromHtml(status.content),
createdAt = Date(status.createdAt),
emojis = emojis,
reblogsCount = status.reblogsCount,
favouritesCount = status.favouritesCount,
reblogged = status.reblogged,
favourited = status.favourited,
sensitive = status.sensitive,
spoilerText = status.spoilerText!!,
visibility = status.visibility!!,
attachments = attachments,
mentions = mentions,
application = application,
pinned = false
)
}
val status = if (reblog != null) {
Status(
id = status.serverId,
url = null, // no url for reblogs
account = this.reblogAccount!!.toAccount(),
inReplyToId = null,
inReplyToAccountId = null,
reblog = reblog,
content = SpannedString(""),
createdAt = Date(status.createdAt), // lie but whatever?
emojis = listOf(),
reblogsCount = 0,
favouritesCount = 0,
reblogged = false,
favourited = false,
sensitive = false,
spoilerText = "",
visibility = status.visibility!!,
attachments = listOf(),
mentions = arrayOf(),
application = null,
pinned = false
)
} else {
Status(
id = status.serverId,
url = status.url,
account = account.toAccount(),
inReplyToId = status.inReplyToId,
inReplyToAccountId = status.inReplyToAccountId,
reblog = null,
content = HtmlUtils.fromHtml(status.content),
createdAt = Date(status.createdAt),
emojis = emojis,
reblogsCount = status.reblogsCount,
favouritesCount = status.favouritesCount,
reblogged = status.reblogged,
favourited = status.favourited,
sensitive = status.sensitive,
spoilerText = status.spoilerText!!,
visibility = status.visibility!!,
attachments = attachments,
mentions = mentions,
application = application,
pinned = false
)
}
return Either.Right(status)
}
private fun Status.toEntity(timelineUserId: Long, instance: String): TimelineStatusEntity {
val actionable = actionableStatus
return TimelineStatusEntity(
serverId = this.id,
url = actionable.url!!,
instance = instance,
timelineUserId = timelineUserId,
authorServerId = actionable.account.id,
inReplyToId = actionable.inReplyToId,
inReplyToAccountId = actionable.inReplyToAccountId,
content = HtmlUtils.toHtml(actionable.content),
createdAt = actionable.createdAt.time,
emojis = actionable.emojis.let(gson::toJson),
reblogsCount = actionable.reblogsCount,
favouritesCount = actionable.favouritesCount,
reblogged = actionable.reblogged,
favourited = actionable.favourited,
sensitive = actionable.sensitive,
spoilerText = actionable.spoilerText,
visibility = actionable.visibility,
attachments = actionable.attachments.let(gson::toJson),
mentions = actionable.mentions.let(gson::toJson),
application = actionable.let(gson::toJson),
reblogServerId = reblog?.id,
reblogAccountId = reblog?.let { this.account.id }
)
}
private fun Placeholder.toEntity(timelineUserId: Long): TimelineStatusEntity {
return TimelineStatusEntity(
serverId = this.id,
url = null,
instance = null,
timelineUserId = timelineUserId,
authorServerId = null,
inReplyToId = null,
inReplyToAccountId = null,
content = null,
createdAt = 0L,
emojis = null,
reblogsCount = 0,
favouritesCount = 0,
reblogged = false,
favourited = false,
sensitive = false,
spoilerText = null,
visibility = null,
attachments = null,
mentions = null,
application = null,
reblogServerId = null,
reblogAccountId = null
)
}
private fun incId(id: String, value: Long): String {
return BigInteger(id).add(BigInteger.valueOf(value)).toString()
}
companion object {
private val emojisListTypeToken = object : TypeToken<List<Emoji>>() {}
}
}