Repo created

This commit is contained in:
Fr4nz D13trich 2025-11-22 13:56:56 +01:00
parent 75dc487a7a
commit 39c29d175b
6317 changed files with 388324 additions and 2 deletions

View file

@ -0,0 +1,24 @@
plugins {
id(ThunderbirdPlugins.Library.jvm)
alias(libs.plugins.android.lint)
}
dependencies {
api(projects.backend.api)
implementation(projects.core.common)
api(projects.core.outcome)
api(projects.feature.mail.account.api)
api(projects.mail.protocols.imap)
api(projects.mail.protocols.smtp)
implementation(projects.feature.mail.folder.api)
implementation(libs.kotlinx.coroutines.core)
testImplementation(projects.core.logging.testing)
testImplementation(projects.mail.testing)
testImplementation(projects.backend.testing)
testImplementation(libs.mime4j.dom)
}

View file

@ -0,0 +1,136 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.IdleRefreshManager
import com.fsck.k9.mail.store.imap.IdleRefreshTimer
private typealias Callback = () -> Unit
private const val MIN_TIMER_DELTA = 1 * 60 * 1000L
private const val NO_TRIGGER_TIME = 0L
/**
* Timer mechanism to refresh IMAP IDLE connections.
*
* Triggers timers early if necessary to reduce the number of times the device has to be woken up.
*/
class BackendIdleRefreshManager(private val alarmManager: SystemAlarmManager) : IdleRefreshManager {
private var timers = mutableSetOf<BackendIdleRefreshTimer>()
private var currentTriggerTime = NO_TRIGGER_TIME
private var minTimeout = Long.MAX_VALUE
private var minTimeoutTimestamp = 0L
@Synchronized
override fun startTimer(timeout: Long, callback: Callback): IdleRefreshTimer {
require(timeout > MIN_TIMER_DELTA) { "Timeout needs to be greater than $MIN_TIMER_DELTA ms" }
val now = alarmManager.now()
val triggerTime = now + timeout
updateMinTimeout(timeout, now)
setOrUpdateAlarm(triggerTime)
return BackendIdleRefreshTimer(triggerTime, callback).also { timer ->
timers.add(timer)
}
}
override fun resetTimers() {
synchronized(this) {
cancelAlarm()
}
onTimeout()
}
private fun updateMinTimeout(timeout: Long, now: Long) {
if (minTimeoutTimestamp + minTimeout * 2 < now) {
minTimeout = Long.MAX_VALUE
}
if (timeout <= minTimeout) {
minTimeout = timeout
minTimeoutTimestamp = now
}
}
private fun setOrUpdateAlarm(triggerTime: Long) {
if (currentTriggerTime == NO_TRIGGER_TIME) {
setAlarm(triggerTime)
} else if (currentTriggerTime - triggerTime > MIN_TIMER_DELTA) {
adjustAlarm(triggerTime)
}
}
private fun setAlarm(triggerTime: Long) {
currentTriggerTime = triggerTime
alarmManager.setAlarm(triggerTime, ::onTimeout)
}
private fun adjustAlarm(triggerTime: Long) {
currentTriggerTime = triggerTime
alarmManager.cancelAlarm()
alarmManager.setAlarm(triggerTime, ::onTimeout)
}
private fun cancelAlarm() {
currentTriggerTime = NO_TRIGGER_TIME
alarmManager.cancelAlarm()
}
private fun onTimeout() {
val triggerTimers = synchronized(this) {
currentTriggerTime = NO_TRIGGER_TIME
if (timers.isEmpty()) return
val now = alarmManager.now()
val minNextTriggerTime = now + minTimeout
val triggerTimers = timers.filter { it.triggerTime < minNextTriggerTime - MIN_TIMER_DELTA }
timers.removeAll(triggerTimers)
timers.minOfOrNull { it.triggerTime }?.let { nextTriggerTime ->
setAlarm(nextTriggerTime)
}
triggerTimers
}
for (timer in triggerTimers) {
timer.onTimeout()
}
}
@Synchronized
private fun removeTimer(timer: BackendIdleRefreshTimer) {
timers.remove(timer)
if (timers.isEmpty()) {
cancelAlarm()
}
}
internal inner class BackendIdleRefreshTimer(
val triggerTime: Long,
val callback: Callback,
) : IdleRefreshTimer {
override var isWaiting: Boolean = true
private set
@Synchronized
override fun cancel() {
if (isWaiting) {
isWaiting = false
removeTimer(this)
}
}
internal fun onTimeout() {
synchronized(this) {
isWaiting = false
}
callback.invoke()
}
}
}

View file

@ -0,0 +1,22 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
import net.thunderbird.core.common.exception.MessagingException
internal class CommandDelete(private val imapStore: ImapStore) {
@Throws(MessagingException::class)
fun deleteMessages(folderServerId: String, messageServerIds: List<String>) {
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
val messages = messageServerIds.map { uid -> remoteFolder.getMessage(uid) }
remoteFolder.deleteMessages(messages)
} finally {
remoteFolder.close()
}
}
}

View file

@ -0,0 +1,20 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
import net.thunderbird.core.common.exception.MessagingException
internal class CommandDeleteAll(private val imapStore: ImapStore) {
@Throws(MessagingException::class)
fun deleteAll(folderServerId: String) {
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
remoteFolder.deleteAllMessages()
} finally {
remoteFolder.close()
}
}
}

View file

@ -0,0 +1,55 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.BackendStorage
import com.fsck.k9.mail.FetchProfile
import com.fsck.k9.mail.FetchProfile.Item.BODY
import com.fsck.k9.mail.FetchProfile.Item.ENVELOPE
import com.fsck.k9.mail.FetchProfile.Item.FLAGS
import com.fsck.k9.mail.FetchProfile.Item.STRUCTURE
import com.fsck.k9.mail.MessageDownloadState
import com.fsck.k9.mail.helper.fetchProfileOf
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapMessage
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandDownloadMessage(private val backendStorage: BackendStorage, private val imapStore: ImapStore) {
fun downloadMessageStructure(folderServerId: String, messageServerId: String) {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_ONLY)
val message = folder.getMessage(messageServerId)
// fun fact: ImapFolder.fetch can't handle getting STRUCTURE at same time as headers
fetchMessage(folder, message, fetchProfileOf(FLAGS, ENVELOPE))
fetchMessage(folder, message, fetchProfileOf(STRUCTURE))
val backendFolder = backendStorage.getFolder(folderServerId)
backendFolder.saveMessage(message, MessageDownloadState.ENVELOPE)
} finally {
folder.close()
}
}
fun downloadCompleteMessage(folderServerId: String, messageServerId: String) {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_ONLY)
val message = folder.getMessage(messageServerId)
fetchMessage(folder, message, fetchProfileOf(FLAGS, BODY))
val backendFolder = backendStorage.getFolder(folderServerId)
backendFolder.saveMessage(message, MessageDownloadState.FULL)
} finally {
folder.close()
}
}
private fun fetchMessage(remoteFolder: ImapFolder, message: ImapMessage, fetchProfile: FetchProfile) {
val maxDownloadSize = 0
remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize)
}
}

View file

@ -0,0 +1,34 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
import net.thunderbird.core.logging.legacy.Log
internal class CommandExpunge(private val imapStore: ImapStore) {
fun expunge(folderServerId: String) {
Log.d("processPendingExpunge: folder = %s", folderServerId)
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
remoteFolder.expunge()
Log.d("processPendingExpunge: complete for folder = %s", folderServerId)
} finally {
remoteFolder.close()
}
}
fun expungeMessages(folderServerId: String, messageServerIds: List<String>) {
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
remoteFolder.expungeUids(messageServerIds)
} finally {
remoteFolder.close()
}
}
}

View file

@ -0,0 +1,21 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.BodyFactory
import com.fsck.k9.mail.Part
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandFetchMessage(private val imapStore: ImapStore) {
fun fetchPart(folderServerId: String, messageServerId: String, part: Part, bodyFactory: BodyFactory) {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_WRITE)
val message = folder.getMessage(messageServerId)
folder.fetchPart(message, part, bodyFactory, -1)
} finally {
folder.close()
}
}
}

View file

@ -0,0 +1,17 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandFindByMessageId(private val imapStore: ImapStore) {
fun findByMessageId(folderServerId: String, messageId: String): String? {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_WRITE)
return folder.getUidFromMessageId(messageId)
} finally {
folder.close()
}
}
}

View file

@ -0,0 +1,19 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandMarkAllAsRead(private val imapStore: ImapStore) {
fun markAllAsRead(folderServerId: String) {
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
remoteFolder.setFlagsForAllMessages(setOf(Flag.SEEN), true)
} finally {
remoteFolder.close()
}
}
}

View file

@ -0,0 +1,66 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
import net.thunderbird.core.logging.legacy.Log
internal class CommandMoveOrCopyMessages(private val imapStore: ImapStore) {
fun moveMessages(
sourceFolderServerId: String,
targetFolderServerId: String,
messageServerIds: List<String>,
): Map<String, String>? {
return moveOrCopyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds, false)
}
fun copyMessages(
sourceFolderServerId: String,
targetFolderServerId: String,
messageServerIds: List<String>,
): Map<String, String>? {
return moveOrCopyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds, true)
}
private fun moveOrCopyMessages(
srcFolder: String,
destFolder: String,
uids: Collection<String>,
isCopy: Boolean,
): Map<String, String>? {
var remoteSrcFolder: ImapFolder? = null
var remoteDestFolder: ImapFolder? = null
return try {
remoteSrcFolder = imapStore.getFolder(srcFolder)
if (uids.isEmpty()) {
Log.i("moveOrCopyMessages: no remote messages to move, skipping")
return null
}
remoteSrcFolder.open(OpenMode.READ_WRITE)
val messages = uids.map { uid -> remoteSrcFolder.getMessage(uid) }
Log.d(
"moveOrCopyMessages: source folder = %s, %d messages, destination folder = %s, isCopy = %s",
srcFolder,
messages.size,
destFolder,
isCopy,
)
remoteDestFolder = imapStore.getFolder(destFolder)
if (isCopy) {
remoteSrcFolder.copyMessages(messages, remoteDestFolder)
} else {
remoteSrcFolder.moveMessages(messages, remoteDestFolder)
}
} finally {
remoteSrcFolder?.close()
remoteDestFolder?.close()
}
}
}

View file

@ -0,0 +1,64 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.BackendStorage
import com.fsck.k9.backend.api.FolderInfo
import com.fsck.k9.backend.api.updateFolders
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.store.imap.FolderListItem
import com.fsck.k9.mail.store.imap.ImapStore
import net.thunderbird.core.logging.Logger
import net.thunderbird.core.logging.legacy.Log
import net.thunderbird.feature.mail.folder.api.FolderPathDelimiter
private const val TAG = "CommandRefreshFolderList"
internal class CommandRefreshFolderList(
private val backendStorage: BackendStorage,
private val imapStore: ImapStore,
private val logger: Logger = Log,
) {
private val LegacyFolderListItem.normalizedServerId: String
get() = imapStore.combinedPrefix?.let {
serverId.removePrefix(prefix = it)
} ?: serverId
fun refreshFolderList(): FolderPathDelimiter? {
logger.verbose(TAG) { "refreshFolderList() called" }
val folders = imapStore.getFolders()
val folderPathDelimiter = folders.firstOrNull { it.folderPathDelimiter != null }?.folderPathDelimiter
val foldersOnServer = folders.toLegacyFolderList()
val oldFolderServerIds = backendStorage.getFolderServerIds()
backendStorage.updateFolders {
val foldersToCreate = mutableListOf<FolderInfo>()
for (folder in foldersOnServer) {
if (folder.normalizedServerId !in oldFolderServerIds) {
foldersToCreate.add(FolderInfo(folder.normalizedServerId, folder.name, folder.type))
} else {
changeFolder(folder.normalizedServerId, folder.name, folder.type)
}
}
logger.verbose(TAG) { "refreshFolderList: foldersToCreate = $foldersToCreate" }
createFolders(foldersToCreate)
val newFolderServerIds = foldersOnServer.map { it.normalizedServerId }
val removedFolderServerIds = oldFolderServerIds - newFolderServerIds
logger.verbose(TAG) { "refreshFolderList: folders to remove = $removedFolderServerIds" }
deleteFolders(removedFolderServerIds)
}
return folderPathDelimiter
}
}
private fun List<FolderListItem>.toLegacyFolderList(): List<LegacyFolderListItem> {
return this
.map { LegacyFolderListItem(it.serverId, it.name, it.type) }
}
private data class LegacyFolderListItem(
val serverId: String,
val name: String,
val type: FolderType,
)

View file

@ -0,0 +1,27 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandSearch(private val imapStore: ImapStore) {
fun search(
folderServerId: String,
query: String?,
requiredFlags: Set<Flag>?,
forbiddenFlags: Set<Flag>?,
performFullTextSearch: Boolean,
): List<String> {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_ONLY)
return folder.search(query, requiredFlags, forbiddenFlags, performFullTextSearch)
.sortedWith(UidReverseComparator())
.map { it.uid }
} finally {
folder.close()
}
}
}

View file

@ -0,0 +1,23 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandSetFlag(private val imapStore: ImapStore) {
fun setFlag(folderServerId: String, messageServerIds: List<String>, flag: Flag, newState: Boolean) {
if (messageServerIds.isEmpty()) return
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_WRITE)
val messages = messageServerIds.map { uid -> remoteFolder.getMessage(uid) }
remoteFolder.setFlags(messages, setOf(flag), newState)
} finally {
remoteFolder.close()
}
}
}

View file

@ -0,0 +1,22 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.Message
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
internal class CommandUploadMessage(private val imapStore: ImapStore) {
fun uploadMessage(folderServerId: String, message: Message): String? {
val folder = imapStore.getFolder(folderServerId)
try {
folder.open(OpenMode.READ_WRITE)
val localUid = message.uid
val uidMap = folder.appendMessages(listOf(message))
return uidMap?.get(localUid)
} finally {
folder.close()
}
}
}

View file

@ -0,0 +1,153 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.Backend
import com.fsck.k9.backend.api.BackendPusher
import com.fsck.k9.backend.api.BackendPusherCallback
import com.fsck.k9.backend.api.BackendStorage
import com.fsck.k9.backend.api.SyncConfig
import com.fsck.k9.backend.api.SyncListener
import com.fsck.k9.mail.BodyFactory
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.Message
import com.fsck.k9.mail.Part
import com.fsck.k9.mail.power.PowerManager
import com.fsck.k9.mail.store.imap.IdleRefreshManager
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.transport.smtp.SmtpTransport
import net.thunderbird.feature.mail.folder.api.FolderPathDelimiter
class ImapBackend(
private val accountName: String,
backendStorage: BackendStorage,
internal val imapStore: ImapStore,
private val powerManager: PowerManager,
private val idleRefreshManager: IdleRefreshManager,
private val pushConfigProvider: ImapPushConfigProvider,
private val smtpTransport: SmtpTransport,
) : Backend {
private val imapSync = ImapSync(accountName, backendStorage, imapStore)
private val commandRefreshFolderList = CommandRefreshFolderList(backendStorage, imapStore)
private val commandSetFlag = CommandSetFlag(imapStore)
private val commandMarkAllAsRead = CommandMarkAllAsRead(imapStore)
private val commandExpunge = CommandExpunge(imapStore)
private val commandMoveOrCopyMessages = CommandMoveOrCopyMessages(imapStore)
private val commandDelete = CommandDelete(imapStore)
private val commandDeleteAll = CommandDeleteAll(imapStore)
private val commandSearch = CommandSearch(imapStore)
private val commandDownloadMessage = CommandDownloadMessage(backendStorage, imapStore)
private val commandFetchMessage = CommandFetchMessage(imapStore)
private val commandFindByMessageId = CommandFindByMessageId(imapStore)
private val commandUploadMessage = CommandUploadMessage(imapStore)
override val supportsFlags = true
override val supportsExpunge = true
override val supportsMove = true
override val supportsCopy = true
override val supportsUpload = true
override val supportsTrashFolder = true
override val supportsSearchByDate = true
override val supportsFolderSubscriptions = true
override val isPushCapable = true
override fun refreshFolderList(): FolderPathDelimiter? {
return commandRefreshFolderList.refreshFolderList()
}
override fun sync(folderServerId: String, syncConfig: SyncConfig, listener: SyncListener) {
imapSync.sync(folderServerId, syncConfig, listener)
}
override fun downloadMessage(syncConfig: SyncConfig, folderServerId: String, messageServerId: String) {
imapSync.downloadMessage(syncConfig, folderServerId, messageServerId)
}
override fun downloadMessageStructure(folderServerId: String, messageServerId: String) {
commandDownloadMessage.downloadMessageStructure(folderServerId, messageServerId)
}
override fun downloadCompleteMessage(folderServerId: String, messageServerId: String) {
commandDownloadMessage.downloadCompleteMessage(folderServerId, messageServerId)
}
override fun setFlag(folderServerId: String, messageServerIds: List<String>, flag: Flag, newState: Boolean) {
commandSetFlag.setFlag(folderServerId, messageServerIds, flag, newState)
}
override fun markAllAsRead(folderServerId: String) {
commandMarkAllAsRead.markAllAsRead(folderServerId)
}
override fun expunge(folderServerId: String) {
commandExpunge.expunge(folderServerId)
}
override fun deleteMessages(folderServerId: String, messageServerIds: List<String>) {
commandDelete.deleteMessages(folderServerId, messageServerIds)
}
override fun deleteAllMessages(folderServerId: String) {
commandDeleteAll.deleteAll(folderServerId)
}
override fun moveMessages(
sourceFolderServerId: String,
targetFolderServerId: String,
messageServerIds: List<String>,
): Map<String, String>? {
return commandMoveOrCopyMessages.moveMessages(sourceFolderServerId, targetFolderServerId, messageServerIds)
}
override fun moveMessagesAndMarkAsRead(
sourceFolderServerId: String,
targetFolderServerId: String,
messageServerIds: List<String>,
): Map<String, String>? {
val uidMapping = commandMoveOrCopyMessages.moveMessages(
sourceFolderServerId,
targetFolderServerId,
messageServerIds,
)
if (uidMapping != null) {
setFlag(targetFolderServerId, uidMapping.values.toList(), Flag.SEEN, true)
}
return uidMapping
}
override fun copyMessages(
sourceFolderServerId: String,
targetFolderServerId: String,
messageServerIds: List<String>,
): Map<String, String>? {
return commandMoveOrCopyMessages.copyMessages(sourceFolderServerId, targetFolderServerId, messageServerIds)
}
override fun search(
folderServerId: String,
query: String?,
requiredFlags: Set<Flag>?,
forbiddenFlags: Set<Flag>?,
performFullTextSearch: Boolean,
): List<String> {
return commandSearch.search(folderServerId, query, requiredFlags, forbiddenFlags, performFullTextSearch)
}
override fun fetchPart(folderServerId: String, messageServerId: String, part: Part, bodyFactory: BodyFactory) {
commandFetchMessage.fetchPart(folderServerId, messageServerId, part, bodyFactory)
}
override fun findByMessageId(folderServerId: String, messageId: String): String? {
return commandFindByMessageId.findByMessageId(folderServerId, messageId)
}
override fun uploadMessage(folderServerId: String, message: Message): String? {
return commandUploadMessage.uploadMessage(folderServerId, message)
}
override fun sendMessage(message: Message) {
smtpTransport.sendMessage(message)
}
override fun createPusher(callback: BackendPusherCallback): BackendPusher {
return ImapBackendPusher(imapStore, powerManager, idleRefreshManager, pushConfigProvider, callback, accountName)
}
}

View file

@ -0,0 +1,251 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.BackendPusher
import com.fsck.k9.backend.api.BackendPusherCallback
import com.fsck.k9.mail.AuthenticationFailedException
import com.fsck.k9.mail.power.PowerManager
import com.fsck.k9.mail.store.imap.IdleRefreshManager
import com.fsck.k9.mail.store.imap.IdleRefreshTimeoutProvider
import com.fsck.k9.mail.store.imap.IdleRefreshTimer
import com.fsck.k9.mail.store.imap.ImapStore
import java.io.IOException
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import net.thunderbird.core.common.exception.MessagingException
import net.thunderbird.core.logging.legacy.Log
private const val IO_ERROR_TIMEOUT = 5 * 60 * 1000L
private const val UNEXPECTED_ERROR_TIMEOUT = 60 * 60 * 1000L
/**
* Manages [ImapFolderPusher] instances that listen for changes to individual folders.
*/
internal class ImapBackendPusher(
private val imapStore: ImapStore,
private val powerManager: PowerManager,
private val idleRefreshManager: IdleRefreshManager,
private val pushConfigProvider: ImapPushConfigProvider,
private val callback: BackendPusherCallback,
private val accountName: String,
backgroundDispatcher: CoroutineDispatcher = Dispatchers.IO,
) : BackendPusher, ImapPusherCallback {
private val coroutineScope = CoroutineScope(backgroundDispatcher)
private val lock = Any()
private val pushFolders = mutableMapOf<String, ImapFolderPusher>()
private var currentFolderServerIds: Collection<String> = emptySet()
private val pushFolderSleeping = mutableMapOf<String, IdleRefreshTimer>()
private val idleRefreshTimeoutProvider = object : IdleRefreshTimeoutProvider {
override val idleRefreshTimeoutMs
get() = currentIdleRefreshMs
}
@Volatile
private var currentMaxPushFolders = 0
@Volatile
private var currentIdleRefreshMs = 15 * 60 * 1000L
override fun start() {
coroutineScope.launch {
pushConfigProvider.maxPushFoldersFlow.collect { maxPushFolders ->
currentMaxPushFolders = maxPushFolders
updateFolders()
}
}
coroutineScope.launch {
pushConfigProvider.idleRefreshMinutesFlow.collect { idleRefreshMinutes ->
currentIdleRefreshMs = idleRefreshMinutes * 60 * 1000L
refreshFolderTimers()
}
}
}
private fun refreshFolderTimers() {
synchronized(lock) {
for (pushFolder in pushFolders.values) {
pushFolder.refresh()
}
}
}
override fun updateFolders(folderServerIds: Collection<String>) {
updateFolders(folderServerIds, currentMaxPushFolders)
}
private fun updateFolders() {
val currentFolderServerIds = synchronized(lock) { currentFolderServerIds }
updateFolders(currentFolderServerIds, currentMaxPushFolders)
}
private fun updateFolders(folderServerIds: Collection<String>, maxPushFolders: Int) {
Log.v("ImapBackendPusher.updateFolders(): %s", folderServerIds)
val pushFolderServerIds = if (folderServerIds.size > maxPushFolders) {
folderServerIds.take(maxPushFolders).also { pushFolderServerIds ->
Log.v("..limiting Push to %d folders: %s", maxPushFolders, pushFolderServerIds)
}
} else {
folderServerIds
}
val stopFolderPushers: List<ImapFolderPusher>
val startFolderPushers: List<ImapFolderPusher>
synchronized(lock) {
currentFolderServerIds = folderServerIds
val oldRunningFolderServerIds = pushFolders.keys
val oldFolderServerIds = oldRunningFolderServerIds + pushFolderSleeping.keys
val removeFolderServerIds = oldFolderServerIds - pushFolderServerIds
stopFolderPushers = removeFolderServerIds
.asSequence()
.onEach { folderServerId -> cancelRetryTimer(folderServerId) }
.map { folderServerId -> pushFolders.remove(folderServerId) }
.filterNotNull()
.toList()
val startFolderServerIds = pushFolderServerIds - oldRunningFolderServerIds
startFolderPushers = startFolderServerIds
.asSequence()
.filterNot { folderServerId -> isWaitingForRetry(folderServerId) }
.onEach { folderServerId -> pushFolderSleeping.remove(folderServerId) }
.map { folderServerId ->
createImapFolderPusher(folderServerId).also { folderPusher ->
pushFolders[folderServerId] = folderPusher
}
}
.toList()
}
for (folderPusher in stopFolderPushers) {
folderPusher.stop()
}
for (folderPusher in startFolderPushers) {
folderPusher.start()
}
}
override fun stop() {
Log.v("ImapBackendPusher.stop()")
coroutineScope.cancel()
synchronized(lock) {
for (pushFolder in pushFolders.values) {
pushFolder.stop()
}
pushFolders.clear()
for (retryTimer in pushFolderSleeping.values) {
retryTimer.cancel()
}
pushFolderSleeping.clear()
currentFolderServerIds = emptySet()
}
}
override fun reconnect() {
Log.v("ImapBackendPusher.reconnect()")
synchronized(lock) {
for (pushFolder in pushFolders.values) {
pushFolder.stop()
}
pushFolders.clear()
for (retryTimer in pushFolderSleeping.values) {
retryTimer.cancel()
}
pushFolderSleeping.clear()
}
imapStore.closeAllConnections()
updateFolders()
}
private fun createImapFolderPusher(folderServerId: String): ImapFolderPusher {
return ImapFolderPusher(
imapStore,
powerManager,
idleRefreshManager,
this,
accountName,
folderServerId,
idleRefreshTimeoutProvider,
)
}
override fun onPushEvent(folderServerId: String) {
callback.onPushEvent(folderServerId)
idleRefreshManager.resetTimers()
}
override fun onPushError(folderServerId: String, exception: Exception) {
synchronized(lock) {
pushFolders.remove(folderServerId)
when (exception) {
is AuthenticationFailedException -> {
Log.v(exception, "Authentication failure when attempting to use IDLE")
// TODO: This could be happening because of too many connections to the host. Ideally we'd want to
// detect this case and use a lower timeout.
startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT)
}
is IOException -> {
Log.v(exception, "I/O error while trying to use IDLE")
startRetryTimer(folderServerId, IO_ERROR_TIMEOUT)
}
is MessagingException -> {
Log.v(exception, "MessagingException")
if (exception.isPermanentFailure) {
startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT)
} else {
startRetryTimer(folderServerId, IO_ERROR_TIMEOUT)
}
}
else -> {
Log.v(exception, "Unexpected error")
startRetryTimer(folderServerId, UNEXPECTED_ERROR_TIMEOUT)
}
}
if (pushFolders.isEmpty()) {
callback.onPushError(exception)
}
}
}
override fun onPushNotSupported() {
callback.onPushNotSupported()
}
private fun startRetryTimer(folderServerId: String, timeout: Long) {
Log.v("ImapBackendPusher for folder %s sleeping for %d ms", folderServerId, timeout)
pushFolderSleeping[folderServerId] = idleRefreshManager.startTimer(timeout, ::restartFolderPushers)
}
private fun cancelRetryTimer(folderServerId: String) {
Log.v("Canceling ImapBackendPusher retry timer for folder %s", folderServerId)
pushFolderSleeping.remove(folderServerId)?.cancel()
}
private fun isWaitingForRetry(folderServerId: String): Boolean {
return pushFolderSleeping[folderServerId]?.isWaiting == true
}
private fun restartFolderPushers() {
Log.v("Refreshing ImapBackendPusher (at least one retry timer has expired)")
updateFolders()
}
}

View file

@ -0,0 +1,101 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.power.PowerManager
import com.fsck.k9.mail.store.imap.IdleRefreshManager
import com.fsck.k9.mail.store.imap.IdleRefreshTimeoutProvider
import com.fsck.k9.mail.store.imap.IdleResult
import com.fsck.k9.mail.store.imap.ImapFolderIdler
import com.fsck.k9.mail.store.imap.ImapStore
import kotlin.concurrent.thread
import net.thunderbird.core.logging.legacy.Log
/**
* Listens for changes to an IMAP folder in a dedicated thread.
*/
class ImapFolderPusher(
private val imapStore: ImapStore,
private val powerManager: PowerManager,
private val idleRefreshManager: IdleRefreshManager,
private val callback: ImapPusherCallback,
private val accountName: String,
private val folderServerId: String,
private val idleRefreshTimeoutProvider: IdleRefreshTimeoutProvider,
) {
@Volatile
private var folderIdler: ImapFolderIdler? = null
@Volatile
private var stopPushing = false
fun start() {
Log.v("Starting ImapFolderPusher for %s / %s", accountName, folderServerId)
thread(name = "ImapFolderPusher-$accountName-$folderServerId") {
Log.v("Starting ImapFolderPusher thread for %s / %s", accountName, folderServerId)
runPushLoop()
Log.v("Exiting ImapFolderPusher thread for %s / %s", accountName, folderServerId)
}
}
fun refresh() {
Log.v("Refreshing ImapFolderPusher for %s / %s", accountName, folderServerId)
folderIdler?.refresh()
}
fun stop() {
Log.v("Stopping ImapFolderPusher for %s / %s", accountName, folderServerId)
stopPushing = true
folderIdler?.stop()
}
private fun runPushLoop() {
val wakeLock = powerManager.newWakeLock("ImapFolderPusher-$accountName-$folderServerId")
wakeLock.acquire()
performInitialSync()
val folderIdler = ImapFolderIdler.create(
idleRefreshManager,
wakeLock,
imapStore,
folderServerId,
idleRefreshTimeoutProvider,
).also {
folderIdler = it
}
try {
while (!stopPushing) {
when (folderIdler.idle()) {
IdleResult.SYNC -> {
callback.onPushEvent(folderServerId)
}
IdleResult.STOPPED -> {
// ImapFolderIdler only stops when we ask it to.
// But it can't hurt to make extra sure we exit the loop.
stopPushing = true
}
IdleResult.NOT_SUPPORTED -> {
stopPushing = true
callback.onPushNotSupported()
}
}
}
} catch (e: Exception) {
Log.v(e, "Exception in ImapFolderPusher")
this.folderIdler = null
callback.onPushError(folderServerId, e)
}
wakeLock.release()
}
private fun performInitialSync() {
callback.onPushEvent(folderServerId)
}
}

View file

@ -0,0 +1,8 @@
package com.fsck.k9.backend.imap
import kotlinx.coroutines.flow.Flow
interface ImapPushConfigProvider {
val maxPushFoldersFlow: Flow<Int>
val idleRefreshMinutesFlow: Flow<Int>
}

View file

@ -0,0 +1,7 @@
package com.fsck.k9.backend.imap
interface ImapPusherCallback {
fun onPushEvent(folderServerId: String)
fun onPushError(folderServerId: String, exception: Exception)
fun onPushNotSupported()
}

View file

@ -0,0 +1,732 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.BackendFolder
import com.fsck.k9.backend.api.BackendFolder.MoreMessages
import com.fsck.k9.backend.api.BackendStorage
import com.fsck.k9.backend.api.SyncConfig
import com.fsck.k9.backend.api.SyncConfig.ExpungePolicy
import com.fsck.k9.backend.api.SyncListener
import com.fsck.k9.helper.ExceptionHelper
import com.fsck.k9.mail.AuthenticationFailedException
import com.fsck.k9.mail.BodyFactory
import com.fsck.k9.mail.DefaultBodyFactory
import com.fsck.k9.mail.FetchProfile
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.MessageDownloadState
import com.fsck.k9.mail.internet.MessageExtractor
import com.fsck.k9.mail.store.imap.FetchListener
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapMessage
import com.fsck.k9.mail.store.imap.ImapStore
import com.fsck.k9.mail.store.imap.OpenMode
import java.util.Collections
import java.util.Date
import java.util.concurrent.atomic.AtomicInteger
import kotlin.math.max
import net.thunderbird.core.logging.legacy.Log
internal class ImapSync(
private val accountName: String,
private val backendStorage: BackendStorage,
private val imapStore: ImapStore,
) {
fun sync(folder: String, syncConfig: SyncConfig, listener: SyncListener) {
synchronizeMailboxSynchronous(folder, syncConfig, listener)
}
private fun synchronizeMailboxSynchronous(folder: String, syncConfig: SyncConfig, listener: SyncListener) {
Log.i("Synchronizing folder %s:%s", accountName, folder)
var remoteFolder: ImapFolder? = null
var backendFolder: BackendFolder? = null
var newHighestKnownUid: Long = 0
try {
Log.v("SYNC: About to get local folder %s", folder)
backendFolder = backendStorage.getFolder(folder)
listener.syncStarted(folder)
Log.v("SYNC: About to get remote folder %s", folder)
remoteFolder = imapStore.getFolder(folder)
/*
* Synchronization process:
*
Open the folder
Upload any local messages that are marked as PENDING_UPLOAD (Drafts, Sent, Trash)
Get the message count
Get the list of the newest K9.DEFAULT_VISIBLE_LIMIT messages
getMessages(messageCount - K9.DEFAULT_VISIBLE_LIMIT, messageCount)
See if we have each message locally, if not fetch it's flags and envelope
Get and update the unread count for the folder
Update the remote flags of any messages we have locally with an internal date newer than the remote message.
Get the current flags for any messages we have locally but did not just download
Update local flags
For any message we have locally but not remotely, delete the local message to keep cache clean.
Download larger parts of any new messages.
(Optional) Download small attachments in the background.
*/
/*
* Open the remote folder. This pre-loads certain metadata like message count.
*/
Log.v("SYNC: About to open remote folder %s", folder)
if (syncConfig.expungePolicy === ExpungePolicy.ON_POLL) {
Log.d("SYNC: Expunging folder %s:%s", accountName, folder)
if (!remoteFolder.isOpen || remoteFolder.mode != OpenMode.READ_WRITE) {
remoteFolder.open(OpenMode.READ_WRITE)
}
remoteFolder.expunge()
}
remoteFolder.open(OpenMode.READ_ONLY)
listener.syncAuthenticationSuccess()
val uidValidity = remoteFolder.getUidValidity()
val oldUidValidity = backendFolder.getFolderExtraNumber(EXTRA_UID_VALIDITY)
if (oldUidValidity == null && uidValidity != null) {
Log.d("SYNC: Saving UIDVALIDITY for %s", folder)
backendFolder.setFolderExtraNumber(EXTRA_UID_VALIDITY, uidValidity)
} else if (oldUidValidity != null && oldUidValidity != uidValidity) {
Log.d("SYNC: UIDVALIDITY for %s changed; clearing local message cache", folder)
backendFolder.clearAllMessages()
backendFolder.setFolderExtraNumber(EXTRA_UID_VALIDITY, uidValidity!!)
backendFolder.setFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID, 0)
}
/*
* Get the message list from the local store and create an index of
* the uids within the list.
*/
val highestKnownUid = backendFolder.getFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID) ?: 0
var localUidMap: Map<String, Long?>? = backendFolder.getAllMessagesAndEffectiveDates()
/*
* Get the remote message count.
*/
val remoteMessageCount = remoteFolder.messageCount
var visibleLimit = backendFolder.visibleLimit
if (visibleLimit < 0) {
visibleLimit = syncConfig.defaultVisibleLimit
}
val remoteMessages = mutableListOf<ImapMessage>()
val remoteUidMap = mutableMapOf<String, ImapMessage>()
Log.v("SYNC: Remote message count for folder %s is %d", folder, remoteMessageCount)
val earliestDate = syncConfig.earliestPollDate
val earliestTimestamp = earliestDate?.time ?: 0L
var remoteStart = 1
if (remoteMessageCount > 0) {
/* Message numbers start at 1. */
remoteStart = if (visibleLimit > 0) {
max(0, remoteMessageCount - visibleLimit) + 1
} else {
1
}
Log.v(
"SYNC: About to get messages %d through %d for folder %s",
remoteStart,
remoteMessageCount,
folder,
)
val headerProgress = AtomicInteger(0)
listener.syncHeadersStarted(folder)
val remoteMessageArray = remoteFolder.getMessages(remoteStart, remoteMessageCount, earliestDate, null)
val messageCount = remoteMessageArray.size
for (thisMess in remoteMessageArray) {
headerProgress.incrementAndGet()
listener.syncHeadersProgress(folder, headerProgress.get(), messageCount)
val uid = thisMess.uid.toLong()
if (uid > highestKnownUid && uid > newHighestKnownUid) {
newHighestKnownUid = uid
}
val localMessageTimestamp = localUidMap!![thisMess.uid]
if (localMessageTimestamp == null || localMessageTimestamp >= earliestTimestamp) {
remoteMessages.add(thisMess)
remoteUidMap[thisMess.uid] = thisMess
}
}
Log.v("SYNC: Got %d messages for folder %s", remoteUidMap.size, folder)
listener.syncHeadersFinished(folder, headerProgress.get(), remoteUidMap.size)
} else if (remoteMessageCount < 0) {
throw Exception("Message count $remoteMessageCount for folder $folder")
}
/*
* Remove any messages that are in the local store but no longer on the remote store or are too old
*/
var moreMessages = backendFolder.getMoreMessages()
if (syncConfig.syncRemoteDeletions) {
val destroyMessageUids = mutableListOf<String>()
for (localMessageUid in localUidMap!!.keys) {
if (remoteUidMap[localMessageUid] == null) {
destroyMessageUids.add(localMessageUid)
}
}
if (destroyMessageUids.isNotEmpty()) {
moreMessages = MoreMessages.UNKNOWN
backendFolder.destroyMessages(destroyMessageUids)
for (uid in destroyMessageUids) {
listener.syncRemovedMessage(folder, uid)
}
}
}
@Suppress("UNUSED_VALUE") // free memory early? (better break up the method!)
localUidMap = null
if (moreMessages === MoreMessages.UNKNOWN) {
updateMoreMessages(remoteFolder, backendFolder, earliestDate, remoteStart)
}
/*
* Now we download the actual content of messages.
*/
downloadMessages(
syncConfig,
remoteFolder,
backendFolder,
remoteMessages,
highestKnownUid,
listener,
)
listener.folderStatusChanged(folder)
/* Notify listeners that we're finally done. */
backendFolder.setLastChecked(System.currentTimeMillis())
backendFolder.setStatus(null)
Log.d("Done synchronizing folder %s:%s @ %tc", accountName, folder, System.currentTimeMillis())
listener.syncFinished(folder)
Log.i("Done synchronizing folder %s:%s", accountName, folder)
} catch (e: AuthenticationFailedException) {
listener.syncFailed(folder, "Authentication failure", e)
} catch (e: Exception) {
Log.e(e, "synchronizeMailbox")
// If we don't set the last checked, it can try too often during
// failure conditions
val rootMessage = ExceptionHelper.getRootCauseMessage(e)
if (backendFolder != null) {
try {
backendFolder.setStatus(rootMessage)
backendFolder.setLastChecked(System.currentTimeMillis())
} catch (e: Exception) {
Log.e(e, "Could not set last checked on folder %s:%s", accountName, folder)
}
}
listener.syncFailed(folder, rootMessage, e)
Log.e(
"Failed synchronizing folder %s:%s @ %tc",
accountName,
folder,
System.currentTimeMillis(),
)
} finally {
if (newHighestKnownUid > 0 && backendFolder != null) {
Log.v("Saving new highest known UID: %d", newHighestKnownUid)
backendFolder.setFolderExtraNumber(EXTRA_HIGHEST_KNOWN_UID, newHighestKnownUid)
}
remoteFolder?.close()
}
}
fun downloadMessage(syncConfig: SyncConfig, folderServerId: String, messageServerId: String) {
val backendFolder = backendStorage.getFolder(folderServerId)
val remoteFolder = imapStore.getFolder(folderServerId)
try {
remoteFolder.open(OpenMode.READ_ONLY)
val remoteMessage = remoteFolder.getMessage(messageServerId)
downloadMessages(
syncConfig,
remoteFolder,
backendFolder,
listOf(remoteMessage),
null,
SimpleSyncListener(),
)
} finally {
remoteFolder.close()
}
}
/**
* Fetches the messages described by inputMessages from the remote store and writes them to local storage.
*
* @param remoteFolder
* The remote folder to download messages from.
* @param backendFolder
* The [BackendFolder] instance corresponding to the remote folder.
* @param inputMessages
* A list of messages objects that store the UIDs of which messages to download.
*/
private fun downloadMessages(
syncConfig: SyncConfig,
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
inputMessages: List<ImapMessage>,
highestKnownUid: Long?,
listener: SyncListener,
) {
val folder = remoteFolder.serverId
val syncFlagMessages = mutableListOf<ImapMessage>()
var unsyncedMessages = mutableListOf<ImapMessage>()
val downloadedMessageCount = AtomicInteger(0)
val messages = inputMessages.toMutableList()
for (message in messages) {
evaluateMessageForDownload(
message,
backendFolder,
unsyncedMessages,
syncFlagMessages,
)
}
val progress = AtomicInteger(0)
val todo = unsyncedMessages.size + syncFlagMessages.size
listener.syncProgress(folder, progress.get(), todo)
Log.d("SYNC: Have %d unsynced messages", unsyncedMessages.size)
messages.clear()
val largeMessages = mutableListOf<ImapMessage>()
val smallMessages = mutableListOf<ImapMessage>()
if (unsyncedMessages.isNotEmpty()) {
Collections.sort(unsyncedMessages, UidReverseComparator())
val visibleLimit = backendFolder.visibleLimit
val listSize = unsyncedMessages.size
if (visibleLimit in 1 until listSize) {
unsyncedMessages = unsyncedMessages.subList(0, visibleLimit)
}
Log.d("SYNC: About to fetch %d unsynced messages for folder %s", unsyncedMessages.size, folder)
fetchUnsyncedMessages(
syncConfig,
remoteFolder,
unsyncedMessages,
smallMessages,
largeMessages,
progress,
todo,
listener,
)
Log.d("SYNC: Synced unsynced messages for folder %s", folder)
}
Log.d(
"SYNC: Have %d large messages and %d small messages out of %d unsynced messages",
largeMessages.size,
smallMessages.size,
unsyncedMessages.size,
)
unsyncedMessages.clear()
/*
* Grab the content of the small messages first. This is going to
* be very fast and at very worst will be a single up of a few bytes and a single
* download of 625k.
*/
val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize
// TODO: Only fetch small and large messages if we have some
downloadSmallMessages(
remoteFolder,
backendFolder,
smallMessages,
progress,
downloadedMessageCount,
todo,
highestKnownUid,
listener,
)
smallMessages.clear()
/*
* Now do the large messages that require more round trips.
*/
downloadLargeMessages(
remoteFolder,
backendFolder,
largeMessages,
progress,
downloadedMessageCount,
todo,
highestKnownUid,
listener,
maxDownloadSize,
)
largeMessages.clear()
/*
* Refresh the flags for any messages in the local store that we didn't just
* download.
*/
refreshLocalMessageFlags(syncConfig, remoteFolder, backendFolder, syncFlagMessages, progress, todo, listener)
Log.d("SYNC: Synced remote messages for folder %s, %d new messages", folder, downloadedMessageCount.get())
}
private fun evaluateMessageForDownload(
message: ImapMessage,
backendFolder: BackendFolder,
unsyncedMessages: MutableList<ImapMessage>,
syncFlagMessages: MutableList<ImapMessage>,
) {
val messageServerId = message.uid
if (message.isSet(Flag.DELETED)) {
Log.v("Message with uid %s is marked as deleted", messageServerId)
syncFlagMessages.add(message)
return
}
val messagePresentLocally = backendFolder.isMessagePresent(messageServerId)
if (!messagePresentLocally) {
Log.v("Message with uid %s has not yet been downloaded", messageServerId)
unsyncedMessages.add(message)
return
}
val messageFlags = backendFolder.getMessageFlags(messageServerId)
if (!messageFlags.contains(Flag.DELETED)) {
Log.v("Message with uid %s is present in the local store", messageServerId)
if (!messageFlags.contains(Flag.X_DOWNLOADED_FULL) && !messageFlags.contains(Flag.X_DOWNLOADED_PARTIAL)) {
Log.v("Message with uid %s is not downloaded, even partially; trying again", messageServerId)
unsyncedMessages.add(message)
} else {
syncFlagMessages.add(message)
}
} else {
Log.v("Local copy of message with uid %s is marked as deleted", messageServerId)
}
}
private fun isOldMessage(messageServerId: String, highestKnownUid: Long?): Boolean {
if (highestKnownUid == null) return false
try {
val messageUid = messageServerId.toLong()
return messageUid <= highestKnownUid
} catch (e: NumberFormatException) {
Log.w(e, "Couldn't parse UID: %s", messageServerId)
}
return false
}
private fun fetchUnsyncedMessages(
syncConfig: SyncConfig,
remoteFolder: ImapFolder,
unsyncedMessages: List<ImapMessage>,
smallMessages: MutableList<ImapMessage>,
largeMessages: MutableList<ImapMessage>,
progress: AtomicInteger,
todo: Int,
listener: SyncListener,
) {
val folder = remoteFolder.serverId
val fetchProfile = FetchProfile().apply {
add(FetchProfile.Item.FLAGS)
add(FetchProfile.Item.ENVELOPE)
}
remoteFolder.fetch(
unsyncedMessages,
fetchProfile,
object : FetchListener {
override fun onFetchResponse(message: ImapMessage, isFirstResponse: Boolean) {
try {
if (message.isSet(Flag.DELETED)) {
Log.v(
"Newly downloaded message %s:%s:%s was marked deleted on server, skipping",
accountName,
folder,
message.uid,
)
if (isFirstResponse) {
progress.incrementAndGet()
}
// TODO: This might be the source of poll count errors in the UI. Is todo always the same as ofTotal
listener.syncProgress(folder, progress.get(), todo)
return
}
if (syncConfig.maximumAutoDownloadMessageSize > 0 &&
message.size > syncConfig.maximumAutoDownloadMessageSize
) {
largeMessages.add(message)
} else {
smallMessages.add(message)
}
} catch (e: Exception) {
Log.e(e, "Error while storing downloaded message.")
}
}
},
syncConfig.maximumAutoDownloadMessageSize,
)
}
private fun downloadSmallMessages(
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
smallMessages: List<ImapMessage>,
progress: AtomicInteger,
downloadedMessageCount: AtomicInteger,
todo: Int,
highestKnownUid: Long?,
listener: SyncListener,
) {
val folder = remoteFolder.serverId
val fetchProfile = FetchProfile().apply {
add(FetchProfile.Item.BODY)
}
Log.d("SYNC: Fetching %d small messages for folder %s", smallMessages.size, folder)
remoteFolder.fetch(
smallMessages,
fetchProfile,
object : FetchListener {
override fun onFetchResponse(message: ImapMessage, isFirstResponse: Boolean) {
try {
// Store the updated message locally
backendFolder.saveMessage(message, MessageDownloadState.FULL)
if (isFirstResponse) {
progress.incrementAndGet()
downloadedMessageCount.incrementAndGet()
}
val messageServerId = message.uid
Log.v(
"About to notify listeners that we got a new small message %s:%s:%s",
accountName,
folder,
messageServerId,
)
// Update the listener with what we've found
listener.syncProgress(folder, progress.get(), todo)
val isOldMessage = isOldMessage(messageServerId, highestKnownUid)
listener.syncNewMessage(folder, messageServerId, isOldMessage)
} catch (e: Exception) {
Log.e(e, "SYNC: fetch small messages")
}
}
},
-1,
)
Log.d("SYNC: Done fetching small messages for folder %s", folder)
}
private fun downloadLargeMessages(
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
largeMessages: List<ImapMessage>,
progress: AtomicInteger,
downloadedMessageCount: AtomicInteger,
todo: Int,
highestKnownUid: Long?,
listener: SyncListener,
maxDownloadSize: Int,
) {
val folder = remoteFolder.serverId
val fetchProfile = FetchProfile().apply {
add(FetchProfile.Item.STRUCTURE)
}
Log.d("SYNC: Fetching large messages for folder %s", folder)
remoteFolder.fetch(largeMessages, fetchProfile, null, maxDownloadSize)
for (message in largeMessages) {
if (message.body == null) {
downloadSaneBody(remoteFolder, backendFolder, message, maxDownloadSize)
} else {
downloadPartial(remoteFolder, backendFolder, message, maxDownloadSize)
}
val messageServerId = message.uid
Log.v(
"About to notify listeners that we got a new large message %s:%s:%s",
accountName,
folder,
messageServerId,
)
// Update the listener with what we've found
progress.incrementAndGet()
downloadedMessageCount.incrementAndGet()
listener.syncProgress(folder, progress.get(), todo)
val isOldMessage = isOldMessage(messageServerId, highestKnownUid)
listener.syncNewMessage(folder, messageServerId, isOldMessage)
}
Log.d("SYNC: Done fetching large messages for folder %s", folder)
}
private fun refreshLocalMessageFlags(
syncConfig: SyncConfig,
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
syncFlagMessages: List<ImapMessage>,
progress: AtomicInteger,
todo: Int,
listener: SyncListener,
) {
val folder = remoteFolder.serverId
Log.d("SYNC: About to sync flags for %d remote messages for folder %s", syncFlagMessages.size, folder)
val fetchProfile = FetchProfile()
fetchProfile.add(FetchProfile.Item.FLAGS)
val undeletedMessages = mutableListOf<ImapMessage>()
for (message in syncFlagMessages) {
if (!message.isSet(Flag.DELETED)) {
undeletedMessages.add(message)
}
}
val maxDownloadSize = syncConfig.maximumAutoDownloadMessageSize
remoteFolder.fetch(undeletedMessages, fetchProfile, null, maxDownloadSize)
for (remoteMessage in syncFlagMessages) {
val messageChanged = syncFlags(syncConfig, backendFolder, remoteMessage)
if (messageChanged) {
listener.syncFlagChanged(folder, remoteMessage.uid)
}
progress.incrementAndGet()
listener.syncProgress(folder, progress.get(), todo)
}
}
private fun downloadSaneBody(
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
message: ImapMessage,
maxDownloadSize: Int,
) {
/*
* The provider was unable to get the structure of the message, so
* we'll download a reasonable portion of the message and mark it as
* incomplete so the entire thing can be downloaded later if the user
* wishes to download it.
*/
val fetchProfile = FetchProfile()
fetchProfile.add(FetchProfile.Item.BODY_SANE)
/*
* TODO a good optimization here would be to make sure that all Stores set
* the proper size after this fetch and compare the before and after size. If
* they equal we can mark this SYNCHRONIZED instead of PARTIALLY_SYNCHRONIZED
*/
remoteFolder.fetch(listOf(message), fetchProfile, null, maxDownloadSize)
// Store the updated message locally
backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
}
private fun downloadPartial(
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
message: ImapMessage,
maxDownloadSize: Int,
) {
/*
* We have a structure to deal with, from which
* we can pull down the parts we want to actually store.
* Build a list of parts we are interested in. Text parts will be downloaded
* right now, attachments will be left for later.
*/
val viewables = MessageExtractor.collectTextParts(message)
/*
* Now download the parts we're interested in storing.
*/
val bodyFactory: BodyFactory = DefaultBodyFactory()
for (part in viewables) {
remoteFolder.fetchPart(message, part, bodyFactory, maxDownloadSize)
}
// Store the updated message locally
backendFolder.saveMessage(message, MessageDownloadState.PARTIAL)
}
private fun syncFlags(syncConfig: SyncConfig, backendFolder: BackendFolder, remoteMessage: ImapMessage): Boolean {
val messageServerId = remoteMessage.uid
if (!backendFolder.isMessagePresent(messageServerId)) return false
val localMessageFlags = backendFolder.getMessageFlags(messageServerId)
if (localMessageFlags.contains(Flag.DELETED)) return false
var messageChanged = false
if (remoteMessage.isSet(Flag.DELETED)) {
if (syncConfig.syncRemoteDeletions) {
backendFolder.setMessageFlag(messageServerId, Flag.DELETED, true)
messageChanged = true
}
} else {
for (flag in syncConfig.syncFlags) {
if (remoteMessage.isSet(flag) != localMessageFlags.contains(flag)) {
backendFolder.setMessageFlag(messageServerId, flag, remoteMessage.isSet(flag))
messageChanged = true
}
}
}
return messageChanged
}
private fun updateMoreMessages(
remoteFolder: ImapFolder,
backendFolder: BackendFolder,
earliestDate: Date?,
remoteStart: Int,
) {
if (remoteStart == 1) {
backendFolder.setMoreMessages(MoreMessages.FALSE)
} else {
val moreMessagesAvailable = remoteFolder.areMoreMessagesAvailable(remoteStart, earliestDate)
val newMoreMessages = if (moreMessagesAvailable) MoreMessages.TRUE else MoreMessages.FALSE
backendFolder.setMoreMessages(newMoreMessages)
}
}
companion object {
private const val EXTRA_UID_VALIDITY = "imapUidValidity"
private const val EXTRA_HIGHEST_KNOWN_UID = "imapHighestKnownUid"
}
}

View file

@ -0,0 +1,18 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.backend.api.SyncListener
class SimpleSyncListener : SyncListener {
override fun syncStarted(folderServerId: String) = Unit
override fun syncAuthenticationSuccess() = Unit
override fun syncHeadersStarted(folderServerId: String) = Unit
override fun syncHeadersProgress(folderServerId: String, completed: Int, total: Int) = Unit
override fun syncHeadersFinished(folderServerId: String, totalMessagesInMailbox: Int, numNewMessages: Int) = Unit
override fun syncProgress(folderServerId: String, completed: Int, total: Int) = Unit
override fun syncNewMessage(folderServerId: String, messageServerId: String, isOldMessage: Boolean) = Unit
override fun syncRemovedMessage(folderServerId: String, messageServerId: String) = Unit
override fun syncFlagChanged(folderServerId: String, messageServerId: String) = Unit
override fun syncFinished(folderServerId: String) = Unit
override fun syncFailed(folderServerId: String, message: String, exception: Exception?) = Unit
override fun folderStatusChanged(folderServerId: String) = Unit
}

View file

@ -0,0 +1,7 @@
package com.fsck.k9.backend.imap
interface SystemAlarmManager {
fun setAlarm(triggerTime: Long, callback: () -> Unit)
fun cancelAlarm()
fun now(): Long
}

View file

@ -0,0 +1,24 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.Message
import java.util.Comparator
internal class UidReverseComparator : Comparator<Message> {
override fun compare(messageLeft: Message, messageRight: Message): Int {
val uidLeft = messageLeft.uidOrNull
val uidRight = messageRight.uidOrNull
if (uidLeft == null && uidRight == null) {
return 0
} else if (uidLeft == null) {
return 1
} else if (uidRight == null) {
return -1
}
// reverse order
return uidRight.compareTo(uidLeft)
}
private val Message.uidOrNull
get() = uid?.toLongOrNull()
}

View file

@ -0,0 +1,75 @@
package net.thunderbird.backend.imap
import com.fsck.k9.backend.imap.ImapBackend
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.folders.FolderServerId
import com.fsck.k9.mail.store.imap.ImapStore
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import net.thunderbird.backend.api.BackendFactory
import net.thunderbird.backend.api.folder.RemoteFolderCreationOutcome
import net.thunderbird.backend.api.folder.RemoteFolderCreator
import net.thunderbird.core.common.exception.MessagingException
import net.thunderbird.core.logging.Logger
import net.thunderbird.core.outcome.Outcome
import net.thunderbird.feature.mail.account.api.BaseAccount
class ImapRemoteFolderCreator(
private val logger: Logger,
private val imapStore: ImapStore,
private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO,
) : RemoteFolderCreator {
override suspend fun create(
folderServerId: FolderServerId,
mustCreate: Boolean,
folderType: FolderType,
): Outcome<RemoteFolderCreationOutcome.Success, RemoteFolderCreationOutcome.Error> = withContext(ioDispatcher) {
val remoteFolder = imapStore.getFolder(name = folderServerId.serverId)
val outcome = try {
val folderExists = remoteFolder.exists()
when {
folderExists && mustCreate -> Outcome.failure(
RemoteFolderCreationOutcome.Error.AlreadyExists,
)
folderExists -> Outcome.success(RemoteFolderCreationOutcome.Success.AlreadyExists)
!folderExists && remoteFolder.create(folderType = folderType) -> Outcome.success(
RemoteFolderCreationOutcome.Success.Created,
)
else -> Outcome.failure(
RemoteFolderCreationOutcome.Error.FailedToCreateRemoteFolder(
reason = "Failed to create folder on remote server.",
),
)
}
} catch (e: MessagingException) {
logger.error(message = { "Failed to create remote folder '${folderServerId.serverId}'" }, throwable = e)
Outcome.failure(
RemoteFolderCreationOutcome.Error.FailedToCreateRemoteFolder(
reason = e.message ?: "Unhandled exception. Please check the logs.",
),
)
} finally {
remoteFolder.close()
}
outcome
}
}
class ImapRemoteFolderCreatorFactory(
private val logger: Logger,
private val backendFactory: BackendFactory<BaseAccount>,
) : RemoteFolderCreator.Factory {
override fun create(account: BaseAccount): RemoteFolderCreator {
val backend = backendFactory.createBackend(account) as ImapBackend
return ImapRemoteFolderCreator(
logger = logger,
imapStore = backend.imapStore,
ioDispatcher = Dispatchers.IO,
)
}
}

View file

@ -0,0 +1,185 @@
package com.fsck.k9.backend.imap
import assertk.assertThat
import assertk.assertions.isEqualTo
import assertk.assertions.isFalse
import assertk.assertions.isTrue
import org.junit.Test
private const val START_TIME = 100_000_000L
class BackendIdleRefreshManagerTest {
val alarmManager = MockSystemAlarmManager(START_TIME)
val idleRefreshManager = BackendIdleRefreshManager(alarmManager)
@Test
fun `single timer`() {
val timeout = 15 * 60 * 1000L
val callback = RecordingCallback()
idleRefreshManager.startTimer(timeout, callback::alarm)
alarmManager.advanceTime(timeout)
assertThat(alarmManager.alarmTimes).isEqualTo(listOf(START_TIME + timeout))
assertThat(callback.wasCalled).isTrue()
}
@Test
fun `starting two timers in quick succession`() {
val timeout = 15 * 60 * 1000L
val callback1 = RecordingCallback()
val callback2 = RecordingCallback()
idleRefreshManager.startTimer(timeout, callback1::alarm)
// Advance clock less than MIN_TIMER_DELTA
alarmManager.advanceTime(100)
idleRefreshManager.startTimer(timeout, callback2::alarm)
alarmManager.advanceTime(timeout)
assertThat(alarmManager.alarmTimes).isEqualTo(listOf(START_TIME + timeout))
assertThat(callback1.wasCalled).isTrue()
assertThat(callback2.wasCalled).isTrue()
}
@Test
fun `starting second timer some time after first should trigger both at initial trigger time`() {
val timeout = 15 * 60 * 1000L
val waitTime = 10 * 60 * 1000L
val callback1 = RecordingCallback()
val callback2 = RecordingCallback()
idleRefreshManager.startTimer(timeout, callback1::alarm)
// Advance clock by more than MIN_TIMER_DELTA but less than 'timeout'
alarmManager.advanceTime(waitTime)
assertThat(callback1.wasCalled).isFalse()
idleRefreshManager.startTimer(timeout, callback2::alarm)
alarmManager.advanceTime(timeout - waitTime)
assertThat(alarmManager.alarmTimes).isEqualTo(listOf(START_TIME + timeout))
assertThat(callback1.wasCalled).isTrue()
assertThat(callback2.wasCalled).isTrue()
}
@Test
fun `second timer with lower timeout should reschedule alarm`() {
val timeout1 = 15 * 60 * 1000L
val timeout2 = 10 * 60 * 1000L
val callback1 = RecordingCallback()
val callback2 = RecordingCallback()
idleRefreshManager.startTimer(timeout1, callback1::alarm)
assertThat(alarmManager.triggerTime).isEqualTo(START_TIME + timeout1)
idleRefreshManager.startTimer(timeout2, callback2::alarm)
alarmManager.advanceTime(timeout2)
assertThat(alarmManager.alarmTimes).isEqualTo(listOf(START_TIME + timeout1, START_TIME + timeout2))
assertThat(callback1.wasCalled).isTrue()
assertThat(callback2.wasCalled).isTrue()
}
@Test
fun `do not trigger timers earlier than necessary`() {
val timeout1 = 10 * 60 * 1000L
val timeout2 = 23 * 60 * 1000L
val callback1 = RecordingCallback()
val callback2 = RecordingCallback()
val callback3 = RecordingCallback()
idleRefreshManager.startTimer(timeout1, callback1::alarm)
idleRefreshManager.startTimer(timeout2, callback2::alarm)
alarmManager.advanceTime(timeout1)
assertThat(callback1.wasCalled).isTrue()
assertThat(callback2.wasCalled).isFalse()
idleRefreshManager.startTimer(timeout1, callback3::alarm)
alarmManager.advanceTime(timeout1)
assertThat(alarmManager.alarmTimes).isEqualTo(
listOf(START_TIME + timeout1, START_TIME + timeout2, START_TIME + timeout1 + timeout1),
)
assertThat(callback2.wasCalled).isTrue()
assertThat(callback3.wasCalled).isTrue()
}
@Test
fun `reset timers`() {
val timeout = 10 * 60 * 1000L
val callback = RecordingCallback()
idleRefreshManager.startTimer(timeout, callback::alarm)
alarmManager.advanceTime(5 * 60 * 1000L)
assertThat(callback.wasCalled).isFalse()
idleRefreshManager.resetTimers()
assertThat(alarmManager.triggerTime).isEqualTo(NO_TRIGGER_TIME)
assertThat(callback.wasCalled).isTrue()
}
@Test
fun `cancel timer`() {
val timeout = 10 * 60 * 1000L
val callback = RecordingCallback()
val timer = idleRefreshManager.startTimer(timeout, callback::alarm)
alarmManager.advanceTime(5 * 60 * 1000L)
timer.cancel()
assertThat(alarmManager.triggerTime).isEqualTo(NO_TRIGGER_TIME)
assertThat(callback.wasCalled).isFalse()
}
}
class RecordingCallback {
var wasCalled = false
private set
fun alarm() {
wasCalled = true
}
}
typealias Callback = () -> Unit
private const val NO_TRIGGER_TIME = -1L
class MockSystemAlarmManager(startTime: Long) : SystemAlarmManager {
var now = startTime
var triggerTime = NO_TRIGGER_TIME
var callback: Callback? = null
val alarmTimes = mutableListOf<Long>()
override fun setAlarm(triggerTime: Long, callback: () -> Unit) {
this.triggerTime = triggerTime
this.callback = callback
alarmTimes.add(triggerTime)
}
override fun cancelAlarm() {
this.triggerTime = NO_TRIGGER_TIME
this.callback = null
}
override fun now(): Long = now
fun advanceTime(delta: Long) {
now += delta
if (now >= triggerTime) {
trigger()
}
}
private fun trigger() {
callback?.invoke().also {
triggerTime = NO_TRIGGER_TIME
callback = null
}
}
}

View file

@ -0,0 +1,338 @@
package com.fsck.k9.backend.imap
import app.k9mail.backend.testing.InMemoryBackendStorage
import assertk.assertThat
import assertk.assertions.containsAtLeast
import assertk.assertions.containsExactlyInAnyOrder
import assertk.assertions.isEmpty
import assertk.assertions.isFalse
import assertk.assertions.isTrue
import com.fsck.k9.backend.api.FolderInfo
import com.fsck.k9.backend.api.SyncConfig
import com.fsck.k9.backend.api.SyncConfig.ExpungePolicy
import com.fsck.k9.backend.api.SyncListener
import com.fsck.k9.mail.FetchProfile
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.Message
import com.fsck.k9.mail.MessageDownloadState
import com.fsck.k9.mail.store.imap.FetchListener
import com.fsck.k9.mail.store.imap.ImapMessage
import com.fsck.k9.mail.testing.message.buildMessage
import java.util.Date
import net.thunderbird.core.logging.legacy.Log
import net.thunderbird.core.logging.testing.TestLogger
import org.apache.james.mime4j.dom.field.DateTimeField
import org.apache.james.mime4j.field.DefaultFieldParser
import org.junit.Before
import org.junit.Test
import org.mockito.Mockito.verify
import org.mockito.kotlin.any
import org.mockito.kotlin.atLeast
import org.mockito.kotlin.eq
import org.mockito.kotlin.mock
import org.mockito.kotlin.never
private const val ACCOUNT_NAME = "Account-1"
private const val FOLDER_SERVER_ID = "FOLDER_ONE"
private const val MAXIMUM_AUTO_DOWNLOAD_MESSAGE_SIZE = 1000
private const val DEFAULT_VISIBLE_LIMIT = 25
private const val DEFAULT_MESSAGE_DATE = "Tue, 04 Jan 2022 10:00:00 +0100"
class ImapSyncTest {
private val backendStorage = createBackendStorage()
private val backendFolder = backendStorage.getFolder(FOLDER_SERVER_ID)
private val imapStore = TestImapStore()
private val imapFolder = imapStore.addFolder(FOLDER_SERVER_ID)
private val imapSync = ImapSync(ACCOUNT_NAME, backendStorage, imapStore)
private val syncListener = mock<SyncListener>()
private val defaultSyncConfig = createSyncConfig()
@Before
fun setUp() {
Log.logger = TestLogger()
}
@Test
fun `sync of empty folder should notify listener`() {
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
verify(syncListener).syncStarted(FOLDER_SERVER_ID)
verify(syncListener).syncAuthenticationSuccess()
verify(syncListener).syncFinished(FOLDER_SERVER_ID)
verify(syncListener, never()).syncFailed(folderServerId = any(), message = any(), exception = any())
}
@Test
fun `sync of folder with negative messageCount should return an error`() {
imapFolder.messageCount = -1
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
verify(syncListener).syncFailed(
folderServerId = eq(FOLDER_SERVER_ID),
message = eq("Exception: Message count -1 for folder $FOLDER_SERVER_ID"),
exception = any(),
)
}
@Test
fun `successful sync should close folder`() {
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
assertThat(imapFolder.isClosed).isTrue()
}
@Test
fun `sync with error should close folder`() {
imapFolder.messageCount = -1
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
assertThat(imapFolder.isClosed).isTrue()
}
@Test
fun `sync with ExpungePolicy ON_POLL should expunge remote folder`() {
val syncConfig = defaultSyncConfig.copy(expungePolicy = ExpungePolicy.ON_POLL)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(imapFolder.wasExpunged).isTrue()
}
@Test
fun `sync with ExpungePolicy MANUALLY should not expunge remote folder`() {
val syncConfig = defaultSyncConfig.copy(expungePolicy = ExpungePolicy.MANUALLY)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(imapFolder.wasExpunged).isFalse()
}
@Test
fun `sync with ExpungePolicy IMMEDIATELY should not expunge remote folder`() {
val syncConfig = defaultSyncConfig.copy(expungePolicy = ExpungePolicy.IMMEDIATELY)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(imapFolder.wasExpunged).isFalse()
}
@Test
fun `sync with syncRemoteDeletions=true should remove local messages`() {
addMessageToBackendFolder(uid = 42)
val syncConfig = defaultSyncConfig.copy(syncRemoteDeletions = true)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).isEmpty()
verify(syncListener).syncStarted(FOLDER_SERVER_ID)
verify(syncListener).syncFinished(FOLDER_SERVER_ID)
}
@Test
fun `sync with syncRemoteDeletions=false should not remove local messages`() {
addMessageToBackendFolder(uid = 23)
val syncConfig = defaultSyncConfig.copy(syncRemoteDeletions = false)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("23")
verify(syncListener).syncStarted(FOLDER_SERVER_ID)
verify(syncListener).syncFinished(FOLDER_SERVER_ID)
}
@Test
fun `sync should remove messages older than earliestPollDate`() {
addMessageToImapAndBackendFolder(uid = 23, date = "Mon, 03 Jan 2022 10:00:00 +0100")
addMessageToImapAndBackendFolder(uid = 42, date = "Wed, 05 Jan 2022 20:00:00 +0100")
val syncConfig = defaultSyncConfig.copy(
syncRemoteDeletions = true,
earliestPollDate = "Tue, 04 Jan 2022 12:00:00 +0100".toDate(),
)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("42")
}
@Test
fun `sync with new messages on server should download messages`() {
addMessageToImapFolder(uid = 9)
addMessageToImapFolder(uid = 13)
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("9", "13")
verify(syncListener).syncNewMessage(FOLDER_SERVER_ID, messageServerId = "9", isOldMessage = false)
verify(syncListener).syncNewMessage(FOLDER_SERVER_ID, messageServerId = "13", isOldMessage = false)
}
@Test
fun `sync downloading old messages should notify listener with isOldMessage=true`() {
addMessageToBackendFolder(uid = 42)
addMessageToImapFolder(uid = 23)
addMessageToImapFolder(uid = 42)
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("23", "42")
verify(syncListener).syncNewMessage(FOLDER_SERVER_ID, messageServerId = "23", isOldMessage = true)
}
@Test
fun `determining the highest UID should use numerical ordering`() {
addMessageToBackendFolder(uid = 9)
addMessageToBackendFolder(uid = 100)
// When text ordering is used: "9" > "100" -> highest UID = 9 (when it should be 100)
// With 80 > 9 the message on the server is considered a new message, but it shouldn't be (80 < 100)
addMessageToImapFolder(uid = 80)
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
verify(syncListener).syncNewMessage(FOLDER_SERVER_ID, messageServerId = "80", isOldMessage = true)
}
@Test
fun `sync should update flags of existing messages`() {
addMessageToBackendFolder(uid = 2)
addMessageToImapFolder(uid = 2, flags = setOf(Flag.SEEN, Flag.ANSWERED))
imapSync.sync(FOLDER_SERVER_ID, defaultSyncConfig, syncListener)
assertThat(backendFolder.getMessageFlags(messageServerId = "2")).containsAtLeast(Flag.SEEN, Flag.ANSWERED)
}
@Test
fun `sync with UIDVALIDITY change should clear all messages`() {
imapFolder.setUidValidity(1)
addMessageToImapFolder(uid = 300)
addMessageToImapFolder(uid = 301)
val syncConfig = defaultSyncConfig.copy(syncRemoteDeletions = false)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("300", "301")
imapFolder.setUidValidity(9000)
imapFolder.removeAllMessages()
addMessageToImapFolder(uid = 1)
imapSync.sync(FOLDER_SERVER_ID, syncConfig, syncListener)
assertThat(backendFolder.getMessageServerIds()).containsExactlyInAnyOrder("1")
verify(syncListener).syncNewMessage(FOLDER_SERVER_ID, messageServerId = "1", isOldMessage = false)
}
@Test
fun `sync with multiple FETCH responses when downloading small message should report correct progress`() {
val folderServerId = "FOLDER_TWO"
backendStorage.createBackendFolder(folderServerId)
val specialImapFolder = object : TestImapFolder(folderServerId) {
override fun fetch(
messages: List<ImapMessage>,
fetchProfile: FetchProfile,
listener: FetchListener?,
maxDownloadSize: Int,
) {
super.fetch(messages, fetchProfile, listener, maxDownloadSize)
// When fetching the body simulate an additional FETCH response
if (FetchProfile.Item.BODY in fetchProfile) {
val message = messages.first()
listener?.onFetchResponse(message, isFirstResponse = false)
}
}
}
specialImapFolder.addMessage(42)
imapStore.addFolder(specialImapFolder)
imapSync.sync(folderServerId, defaultSyncConfig, syncListener)
verify(syncListener, atLeast(1)).syncProgress(folderServerId, completed = 1, total = 1)
verify(syncListener, never()).syncProgress(folderServerId, completed = 2, total = 1)
}
private fun addMessageToBackendFolder(uid: Long, date: String = DEFAULT_MESSAGE_DATE) {
val messageServerId = uid.toString()
val message = createSimpleMessage(messageServerId, date).apply {
setUid(messageServerId)
}
backendFolder.saveMessage(message, MessageDownloadState.FULL)
val highestKnownUid = backendFolder.getFolderExtraNumber("imapHighestKnownUid") ?: 0
if (uid > highestKnownUid) {
backendFolder.setFolderExtraNumber("imapHighestKnownUid", uid)
}
}
private fun addMessageToImapFolder(uid: Long, flags: Set<Flag> = emptySet(), date: String = DEFAULT_MESSAGE_DATE) {
imapFolder.addMessage(uid, flags, date)
}
private fun TestImapFolder.addMessage(
uid: Long,
flags: Set<Flag> = emptySet(),
date: String = DEFAULT_MESSAGE_DATE,
) {
val messageServerId = uid.toString()
val message = createSimpleMessage(messageServerId, date)
addMessage(uid, message)
if (flags.isNotEmpty()) {
val imapMessage = getMessage(messageServerId)
setFlags(listOf(imapMessage), flags, true)
}
}
private fun addMessageToImapAndBackendFolder(uid: Long, date: String) {
addMessageToBackendFolder(uid, date)
addMessageToImapFolder(uid, date = date)
}
private fun createBackendStorage(): InMemoryBackendStorage {
return InMemoryBackendStorage().apply {
createBackendFolder(FOLDER_SERVER_ID)
}
}
private fun InMemoryBackendStorage.createBackendFolder(serverId: String) {
createFolderUpdater().use { updater ->
val folderInfo = FolderInfo(
serverId = serverId,
name = "irrelevant",
type = FolderType.REGULAR,
)
updater.createFolders(listOf(folderInfo))
}
}
private fun createSyncConfig(): SyncConfig {
return SyncConfig(
expungePolicy = ExpungePolicy.MANUALLY,
earliestPollDate = null,
syncRemoteDeletions = true,
maximumAutoDownloadMessageSize = MAXIMUM_AUTO_DOWNLOAD_MESSAGE_SIZE,
defaultVisibleLimit = DEFAULT_VISIBLE_LIMIT,
syncFlags = setOf(Flag.SEEN, Flag.FLAGGED, Flag.ANSWERED, Flag.FORWARDED),
)
}
private fun createSimpleMessage(uid: String, date: String, text: String = "UID: $uid"): Message {
return buildMessage {
header("Subject", "Test Message")
header("From", "alice@domain.example")
header("To", "Bob <bob@domain.example>")
header("Date", date)
header("Message-ID", "<msg-$uid@domain.example>")
textBody(text)
}
}
}
private fun String.toDate(): Date {
val dateTimeField = DefaultFieldParser.parse("Date: $this") as DateTimeField
return dateTimeField.date
}

View file

@ -0,0 +1,195 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.BodyFactory
import com.fsck.k9.mail.FetchProfile
import com.fsck.k9.mail.Flag
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.Message
import com.fsck.k9.mail.MessageRetrievalListener
import com.fsck.k9.mail.Part
import com.fsck.k9.mail.store.imap.FetchListener
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapMessage
import com.fsck.k9.mail.store.imap.OpenMode
import com.fsck.k9.mail.store.imap.createImapMessage
import java.util.Date
open class TestImapFolder(override val serverId: String) : ImapFolder {
override var mode: OpenMode? = null
protected set
override val isOpen: Boolean
get() = mode != null
override var messageCount: Int = 0
var wasExpunged: Boolean = false
private set
val isClosed: Boolean
get() = mode == null
private val messages = mutableMapOf<Long, Message>()
private val messageFlags = mutableMapOf<Long, MutableSet<Flag>>()
private var uidValidity: Long? = null
fun addMessage(uid: Long, message: Message) {
require(!messages.containsKey(uid)) {
"Folder '$serverId' already contains a message with the UID $uid"
}
messages[uid] = message
messageFlags[uid] = mutableSetOf()
messageCount = messages.size
}
fun removeAllMessages() {
messages.clear()
messageFlags.clear()
}
fun setUidValidity(value: Long) {
uidValidity = value
}
override fun open(mode: OpenMode) {
this.mode = mode
}
override fun close() {
mode = null
}
override fun exists(): Boolean {
throw UnsupportedOperationException("not implemented")
}
override fun getUidValidity() = uidValidity
override fun getMessage(uid: String): ImapMessage {
return createImapMessage(uid)
}
override fun getUidFromMessageId(messageId: String): String? {
throw UnsupportedOperationException("not implemented")
}
override fun getMessages(
start: Int,
end: Int,
earliestDate: Date?,
listener: MessageRetrievalListener<ImapMessage>?,
): List<ImapMessage> {
require(start > 0)
require(end >= start)
require(end <= messages.size)
return messages.keys.sortedDescending()
.slice((start - 1) until end)
.map { createImapMessage(uid = it.toString()) }
}
override fun areMoreMessagesAvailable(indexOfOldestMessage: Int, earliestDate: Date?): Boolean {
throw UnsupportedOperationException("not implemented")
}
override fun fetch(
messages: List<ImapMessage>,
fetchProfile: FetchProfile,
listener: FetchListener?,
maxDownloadSize: Int,
) {
if (messages.isEmpty()) return
for (imapMessage in messages) {
val uid = imapMessage.uid.toLong()
val flags = messageFlags[uid].orEmpty().toSet()
imapMessage.setFlags(flags, true)
val storedMessage = this.messages[uid] ?: error("Message $uid not found")
for (header in storedMessage.headers) {
imapMessage.addHeader(header.name, header.value)
}
imapMessage.body = storedMessage.body
listener?.onFetchResponse(imapMessage, isFirstResponse = true)
}
}
override fun fetchPart(
message: ImapMessage,
part: Part,
bodyFactory: BodyFactory,
maxDownloadSize: Int,
) {
throw UnsupportedOperationException("not implemented")
}
override fun search(
queryString: String?,
requiredFlags: Set<Flag>?,
forbiddenFlags: Set<Flag>?,
performFullTextSearch: Boolean,
): List<ImapMessage> {
throw UnsupportedOperationException("not implemented")
}
override fun appendMessages(messages: List<Message>): Map<String, String>? {
throw UnsupportedOperationException("not implemented")
}
override fun setFlagsForAllMessages(flags: Set<Flag>, value: Boolean) {
if (value) {
for (messageFlagSet in messageFlags.values) {
messageFlagSet.addAll(flags)
}
} else {
for (messageFlagSet in messageFlags.values) {
messageFlagSet.removeAll(flags)
}
}
}
override fun setFlags(messages: List<ImapMessage>, flags: Set<Flag>, value: Boolean) {
for (message in messages) {
val uid = message.uid.toLong()
val messageFlagSet = messageFlags[uid] ?: error("Unknown message with UID $uid")
if (value) {
messageFlagSet.addAll(flags)
} else {
messageFlagSet.removeAll(flags)
}
}
}
override fun copyMessages(messages: List<ImapMessage>, folder: ImapFolder): Map<String, String>? {
throw UnsupportedOperationException("not implemented")
}
override fun moveMessages(messages: List<ImapMessage>, folder: ImapFolder): Map<String, String>? {
throw UnsupportedOperationException("not implemented")
}
override fun deleteMessages(messages: List<ImapMessage>) {
setFlags(messages, setOf(Flag.DELETED), true)
}
override fun deleteAllMessages() {
setFlagsForAllMessages(setOf(Flag.DELETED), true)
}
override fun expunge() {
mode = OpenMode.READ_WRITE
wasExpunged = true
}
override fun expungeUids(uids: List<String>) {
throw UnsupportedOperationException("not implemented")
}
override fun create(folderType: FolderType): Boolean {
throw UnsupportedOperationException("not implemented")
}
}

View file

@ -0,0 +1,54 @@
package com.fsck.k9.backend.imap
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.store.imap.FolderListItem
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapStore
class TestImapStore : ImapStore {
private val folders = mutableMapOf<String, ImapFolder>()
override val combinedPrefix: String?
get() = throw UnsupportedOperationException("not implemented")
fun addFolder(serverId: String): TestImapFolder {
require(!folders.containsKey(serverId)) { "Folder '$serverId' already exists" }
return TestImapFolder(serverId).also { folder ->
folders[serverId] = folder
}
}
fun addFolder(folder: ImapFolder) {
val serverId = folder.serverId
require(!folders.containsKey(serverId)) { "Folder '$serverId' already exists" }
folders[serverId] = folder
}
override fun getFolder(name: String): ImapFolder {
return folders[name] ?: error("Folder '$name' not found")
}
override fun getFolders(): List<FolderListItem> {
return folders.values.map { folder ->
FolderListItem(
serverId = folder.serverId,
name = "irrelevant",
type = FolderType.REGULAR,
)
}
}
override fun checkSettings() {
throw UnsupportedOperationException("not implemented")
}
override fun closeAllConnections() {
throw UnsupportedOperationException("not implemented")
}
override fun fetchImapPrefix() {
throw UnsupportedOperationException("not implemented")
}
}

View file

@ -0,0 +1,3 @@
package com.fsck.k9.mail.store.imap
fun createImapMessage(uid: String) = ImapMessage(uid)

View file

@ -0,0 +1,145 @@
package net.thunderbird.backend.imap
import assertk.assertAll
import assertk.assertThat
import assertk.assertions.isEqualTo
import assertk.assertions.isInstanceOf
import assertk.assertions.isTrue
import assertk.assertions.prop
import com.fsck.k9.backend.imap.TestImapFolder
import com.fsck.k9.mail.FolderType
import com.fsck.k9.mail.folders.FolderServerId
import com.fsck.k9.mail.store.imap.FolderListItem
import com.fsck.k9.mail.store.imap.ImapFolder
import com.fsck.k9.mail.store.imap.ImapStore
import kotlinx.coroutines.test.runTest
import net.thunderbird.backend.api.folder.RemoteFolderCreationOutcome
import net.thunderbird.backend.api.folder.RemoteFolderCreationOutcome.Error.FailedToCreateRemoteFolder
import net.thunderbird.core.logging.testing.TestLogger
import net.thunderbird.core.outcome.Outcome
import org.junit.Test
class ImapRemoteFolderCreatorTest {
private val logger = TestLogger()
@Test
fun `when mustCreate true and folder exists, should return Error AlreadyExists`() = runTest {
// Arrange
val folderServerId = FolderServerId("New Folder")
val fakeFolder = object : TestImapFolder(folderServerId.serverId) {
override fun exists(): Boolean = true
}
val imapStore = FakeImapStore(fakeFolder)
val sut = ImapRemoteFolderCreator(logger, imapStore)
// Act
val outcome = sut.create(folderServerId, mustCreate = true)
// Assert
assertAll {
assertThat(outcome.isFailure).isTrue()
assertThat(outcome)
.isInstanceOf<Outcome.Failure<RemoteFolderCreationOutcome.Error>>()
.prop("error") { it.error }
.isEqualTo(RemoteFolderCreationOutcome.Error.AlreadyExists)
}
}
@Test
fun `when mustCreate false and folder exists, should return AlreadyExists`() = runTest {
// Arrange
val folderServerId = FolderServerId("New Folder")
val fakeFolder = object : TestImapFolder(folderServerId.serverId) {
override fun exists(): Boolean = true
}
val imapStore = FakeImapStore(fakeFolder)
val sut = ImapRemoteFolderCreator(logger, imapStore)
// Act
val outcome = sut.create(folderServerId, mustCreate = false)
// Assert
assertAll {
assertThat(outcome.isSuccess).isTrue()
assertThat(outcome)
.isInstanceOf<Outcome.Success<RemoteFolderCreationOutcome.Success>>()
.prop("data") { it.data }
.isEqualTo(RemoteFolderCreationOutcome.Success.AlreadyExists)
}
}
@Test
fun `when folder does not exist and creation succeeds, should return Created`() = runTest {
// Arrange
val folderServerId = FolderServerId("New Folder")
val fakeFolder = object : TestImapFolder(folderServerId.serverId) {
override fun exists(): Boolean = false
override fun create(folderType: FolderType): Boolean = true
}
val imapStore = FakeImapStore(fakeFolder)
val sut = ImapRemoteFolderCreator(logger, imapStore)
// Act
val outcome = sut.create(folderServerId, mustCreate = true)
// Assert
assertAll {
assertThat(outcome.isSuccess).isTrue()
assertThat(outcome)
.isInstanceOf<Outcome.Success<RemoteFolderCreationOutcome.Success>>()
.prop("data") { it.data }
.isEqualTo(RemoteFolderCreationOutcome.Success.Created)
}
}
@Test
fun `when folder does not exist and creation fails, should return FailedToCreateRemoteFolder`() = runTest {
// Arrange
val folderServerId = FolderServerId("New Folder")
val fakeFolder = object : TestImapFolder(folderServerId.serverId) {
override fun exists(): Boolean = false
override fun create(folderType: FolderType): Boolean = false
}
val imapStore = FakeImapStore(fakeFolder)
val sut = ImapRemoteFolderCreator(logger, imapStore)
// Act
val outcome = sut.create(folderServerId, mustCreate = true)
// Assert
assertAll {
assertThat(outcome.isFailure).isTrue()
assertThat(outcome)
.isInstanceOf<Outcome.Failure<FailedToCreateRemoteFolder>>()
.prop("error") { it.error }
.isInstanceOf<FailedToCreateRemoteFolder>()
.prop(FailedToCreateRemoteFolder::reason)
.isEqualTo("Failed to create folder on remote server.")
}
}
}
private class FakeImapStore(
private val folder: TestImapFolder,
) : ImapStore {
override val combinedPrefix: String?
get() = throw NotImplementedError("combinedPrefix not implemented")
override fun checkSettings() {
throw NotImplementedError("checkSettings not implemented")
}
override fun getFolder(name: String): ImapFolder = folder
override fun getFolders(): List<FolderListItem> {
throw NotImplementedError("getFolders not implemented")
}
override fun closeAllConnections() {
throw NotImplementedError("closeAllConnections not implemented")
}
override fun fetchImapPrefix() {
throw NotImplementedError("fetchImapPrefix not implemented")
}
}