Commit 1781d2d0 authored by Christophe Henry's avatar Christophe Henry
Browse files

Fix #100: refresh fails when unread article list is huge

parent 8dc55fc9
Pipeline #4225 passed with stage
in 0 seconds
......@@ -3,6 +3,7 @@
# Bug fixes
* Fix [#99](https://git.feneas.org/christophehenry/freshrss-android/-/issues/99): on first sync, images are not automatically refreshed anymore ([!105](https://git.feneas.org/christophehenry/freshrss-android/-/merge_requests/105))
* Fix [#100](https://git.feneas.org/christophehenry/freshrss-android/-/issues/100): refresh fails when unread article list is huge ([!106](https://git.feneas.org/christophehenry/freshrss-android/-/merge_requests/106))
# 1.3.1
......
......@@ -62,17 +62,18 @@ class RefreshWorker(appContext: Context, workerParams: WorkerParameters): Corout
Store.getTags()
Store.getSubscriptions()
// We don't really care about the result of this operation
GlobalScope.launch {
Store.fetchImages()
}
Store.getUnreadCount()
Store.getStreamContents(ALL_ITEMS_ID)
Store.getUnreadItemIds()
Store.getFavoriteItemIds()
}
// We don't really care about the result of this operation
GlobalScope.launch {
Store.fetchImages()
}
cancellAllNotifications()
val now = DateTime.now(getUserTimeZone())
......
......@@ -92,16 +92,16 @@ object Store {
suspend fun fetchImages(): Unit = withContext(Dispatchers.IO) {
db().getAllSubcriptionsWithImageToUpdate()
.map {
async {
val result = it.fetchImage()
if(result.isFailure) null
else SubscriptionImageUpdate(it.id, result.getOrThrow(), it.iconUrl.trim())
.mapNotNull {
val result = it.fetchImage()
if(result.isFailure) null
else async {
SubscriptionImageUpdate(it.id, result.getOrThrow(), it.iconUrl.trim())
.let(db()::insertSubscriptionImages)
}
}
.awaitAll()
.filterNotNull()
.let(db()::insertSubscriptionImages)
.unit()
}
suspend fun getUnreadCount() {
......
......@@ -9,9 +9,10 @@ import org.joda.time.DateTime
import org.joda.time.LocalDateTime
@Database(
version = 9,
version = 10,
entities = [
Account::class, Article::class, Subscription::class,
Account::class, Article::class, UnreadArticleId::class,
FavoriteArticleId::class, Subscription::class,
FeedTag::class, FeedToTagRelation::class
]
)
......@@ -56,7 +57,7 @@ abstract class FreshRSSDabatabase: RoomDatabase() {
fun getArticlesMostRecentPubDate() = getArticlesDAO().mostRecentPubDate()
suspend fun syncAllArticlesReadStatus(ids: List<String>) = getArticlesDAO().syncAllReadStatus(ids)
suspend fun syncAllArticlesReadStatus(ids: List<String>) = getArticlesDAO().syncAllReadStatuses(ids)
suspend fun syncAllArticlesFavoriteStatus(ids: List<String>) = getArticlesDAO().syncAllFavoriteStatus(ids)
// Subscriptions
......@@ -87,8 +88,8 @@ abstract class FreshRSSDabatabase: RoomDatabase() {
fun incrementSubscriptionCount(id: String) = getSubscriptionsDAO().incrementCount(id)
fun decrementSubscriptionCount(id: String) = getSubscriptionsDAO().decrementCount(id)
fun insertSubscriptionImages(imageUpdates: List<SubscriptionImageUpdate>) =
getSubscriptionsDAO().insertImages(imageUpdates)
fun insertSubscriptionImages(imageUpdate: SubscriptionImageUpdate) =
getSubscriptionsDAO().insertImages(imageUpdate)
fun getSubcriptionsById(id: String) = getSubscriptionsDAO().byId(id)
fun getSubcriptionTitleById(id: String) = getSubscriptionsDAO().titleById(id)
......
{
"formatVersion": 1,
"database": {
"version": 10,
"identityHash": "e2e43918cd53065aa45486a0bacf6cfe",
"entities": [
{
"tableName": "accounts",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `SID` TEXT NOT NULL, `Auth` TEXT NOT NULL, `login` TEXT NOT NULL, `serverInstance` TEXT NOT NULL, `unreadCount` INTEGER NOT NULL, `lastFetchDate` TEXT NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "SID",
"columnName": "SID",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "Auth",
"columnName": "Auth",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "login",
"columnName": "login",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "serverInstance",
"columnName": "serverInstance",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "unreadCount",
"columnName": "unreadCount",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "lastFetchDate",
"columnName": "lastFetchDate",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "articles",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `title` TEXT NOT NULL, `href` TEXT NOT NULL, `author` TEXT NOT NULL, `content` TEXT NOT NULL, `streamId` TEXT NOT NULL, `readStatus` TEXT NOT NULL, `favorite` TEXT NOT NULL, `crawled` INTEGER NOT NULL, `published` INTEGER NOT NULL, `scrollPosition` INTEGER NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "href",
"columnName": "href",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "author",
"columnName": "author",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "content",
"columnName": "content",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "streamId",
"columnName": "streamId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "readStatus",
"columnName": "readStatus",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "favorite",
"columnName": "favorite",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "crawled",
"columnName": "crawled",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "published",
"columnName": "published",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "scrollPosition",
"columnName": "scrollPosition",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "unread_article_ids",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "favorite_article_ids",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "subscriptions",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`iconUrlFlag` TEXT NOT NULL, `imageBitmap` BLOB, `id` TEXT NOT NULL, `title` TEXT NOT NULL, `iconUrl` TEXT NOT NULL, `unreadCount` INTEGER NOT NULL, `newestArticleDate` INTEGER NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "iconUrlFlag",
"columnName": "iconUrlFlag",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "imageBitmap",
"columnName": "imageBitmap",
"affinity": "BLOB",
"notNull": false
},
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "iconUrl",
"columnName": "iconUrl",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "unreadCount",
"columnName": "unreadCount",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "newestArticleDate",
"columnName": "newestArticleDate",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "feed_tags",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `label` TEXT NOT NULL, `type` TEXT NOT NULL, `unreadCount` INTEGER NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "label",
"columnName": "label",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "unreadCount",
"columnName": "unreadCount",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "feed_to_tag_relation",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`feedId` TEXT NOT NULL, `tagId` TEXT NOT NULL, PRIMARY KEY(`feedId`, `tagId`))",
"fields": [
{
"fieldPath": "feedId",
"columnName": "feedId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "tagId",
"columnName": "tagId",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"feedId",
"tagId"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'e2e43918cd53065aa45486a0bacf6cfe')"
]
}
}
\ No newline at end of file
......@@ -16,8 +16,21 @@ import io.reactivex.Flowable
import org.joda.time.LocalDateTime
import java.util.concurrent.atomic.AtomicBoolean
/*
* NOTE: the managment of read/unread and favorite/not favorite here is utterly
* complex. This is because, when the number of unread article is long, it is
* not possible to sync their status in a single SQL query.
* See: https://git.feneas.org/christophehenry/freshrss-android/-/issues/100
*/
typealias Articles = List<Article>
@Entity(tableName = "unread_article_ids")
data class UnreadArticleId(@PrimaryKey val id: String)
@Entity(tableName = "favorite_article_ids")
data class FavoriteArticleId(@PrimaryKey val id: String)
@Entity(tableName = "articles")
data class Article(
@PrimaryKey
......@@ -34,6 +47,7 @@ data class Article(
val scrollPosition: Int = 0
): BaseObservable() {
@Ignore
val url = runCatching {Uri.parse(href)}.getOrNull()
......@@ -158,28 +172,28 @@ interface ArticlesDAO {
@Update(entity = Article::class)
suspend fun updateScrollPosition(articleScrollPositionUpdate: ArticleScrollPositionUpdate)
@Query("UPDATE articles SET readStatus = '${ReadStatus.UNREAD_DB_NAME}' WHERE id IN (:ids)")
suspend fun setAllArticlesReadStatus(ids: List<String>): Int
@Query("UPDATE articles SET readStatus = '${ReadStatus.READ_DB_NAME}' WHERE id NOT IN (:ids)")
suspend fun setExcludeArticlesReadStatus(ids: List<String>): Int
@Query("UPDATE articles SET readStatus = '${ReadStatus.READ_DB_NAME}'")
suspend fun markAllArticlesRead(): Int
@Query("UPDATE articles SET favorite = '${FavoriteStatus.FAVORITE_DB_NAME}' WHERE id IN (:ids)")
suspend fun setAllArticlesfavoriteStatus(ids: List<String>): Int
suspend fun markArticlesFavoriteForIds(ids: List<String>): Int
@Query("UPDATE articles SET favorite = '${FavoriteStatus.NOT_FAVORITE_DB_NAME}' WHERE id NOT IN (:ids)")
suspend fun setExcludeArticlesfavoriteStatus(ids: List<String>): Int
suspend fun markArtclesNotFavoriteExcludingIds(ids: List<String>): Int
@Transaction
suspend fun syncAllReadStatus(ids: List<String>) {
setAllArticlesReadStatus(ids)
setExcludeArticlesReadStatus(ids)
@Query("UPDATE articles SET favorite = '${FavoriteStatus.NOT_FAVORITE_DB_NAME}'")
suspend fun markArtclesNotFavorite(): Int
suspend fun syncAllReadStatuses(ids: List<String>) {
val unreadIds = ids.map(::UnreadArticleId)
emptyUnreadTableAndReload(unreadIds)
syncReadAndUnreadArticles()
}
@Transaction
suspend fun syncAllFavoriteStatus(ids: List<String>) {
setAllArticlesfavoriteStatus(ids)
setExcludeArticlesfavoriteStatus(ids)
val unreadIds = ids.map(::FavoriteArticleId)
emptyFavoriteTableAndReload(unreadIds)
syncFavoriteAndNotFavoriteArticles()
}
@Query("SELECT * FROM articles WHERE streamId = :streamId")
......@@ -196,4 +210,82 @@ interface ArticlesDAO {
@Query("SELECT max(published) FROM articles")
fun mostRecentPubDate(): LiveData<LocalDateTime?>
//region Favorites and unread support
@Query("DELETE FROM unread_article_ids")
suspend fun nukeUnreadTable()
@Insert
suspend fun insertUneadArticleIds(ids: List<UnreadArticleId>)
@Query(
"""
UPDATE articles SET readStatus = '${ReadStatus.UNREAD_DB_NAME}'
WHERE id IN (
SELECT id FROM unread_article_ids
)
"""
)
suspend fun syncUneadArticles(): Int
@Query(
"""
UPDATE articles SET readStatus = '${ReadStatus.READ_DB_NAME}'
WHERE id NOT IN (
SELECT id FROM unread_article_ids
)
"""
)
suspend fun syncReadArticles(): Int
@Transaction
suspend fun emptyUnreadTableAndReload(ids: List<UnreadArticleId>) {
nukeUnreadTable()
insertUneadArticleIds(ids)
}
@Transaction
suspend fun syncReadAndUnreadArticles() {
syncUneadArticles()
syncReadArticles()
}
@Query("DELETE FROM favorite_article_ids")
suspend fun nukeFavoriteTable()
@Insert
suspend fun insertFavoriteArticleIds(ids: List<FavoriteArticleId>)
@Query(
"""
UPDATE articles SET favorite = '${FavoriteStatus.FAVORITE_DB_NAME}'
WHERE id IN (
SELECT id FROM favorite_article_ids
)
"""
)
suspend fun syncFavoriteArticles(): Int
@Query(
"""
UPDATE articles SET favorite = '${FavoriteStatus.NOT_FAVORITE_DB_NAME}'
WHERE id NOT IN (
SELECT id FROM favorite_article_ids
)
"""
)
suspend fun syncNotFavoriteArticles(): Int
@Transaction
suspend fun emptyFavoriteTableAndReload(ids: List<FavoriteArticleId>) {
nukeFavoriteTable()
insertFavoriteArticleIds(ids)
}
@Transaction
suspend fun syncFavoriteAndNotFavoriteArticles() {