Keep cache of messages in database

Quotient/database.cpp

@@ -47,7 +47,8 @@ Database::Database(const QString& userId, const QString& deviceId,
     case 6: migrateTo7(); [[fallthrough]];
     case 7: migrateTo8(); [[fallthrough]];
     case 8: migrateTo9(); [[fallthrough]];
-    case 9: migrateTo10();
+    case 9: migrateTo10(); [[fallthrough]];
+    case 10: migrateTo11();
     }
 }
 
@@ -269,9 +270,17 @@ void Database::migrateTo10()
         execute(updateQuery);
     }
 
-    execute(QStringLiteral("pragma user_version = 10"));
+    execute(QStringLiteral("pragma user_version = 10;"));
     commit();
+}
 
+void Database::migrateTo11()
+{
+    qCDebug(DATABASE) << "Migrating database to version 11";
+    transaction();
+    execute(QStringLiteral("CREATE TABLE events (roomId TEXT, ts INTEGER, json TEXT);"));
+    execute(QStringLiteral("pragma user_version = 11;"));
+    commit();
 }
 
 void Database::storeOlmAccount(const QOlmAccount& olmAccount)

Quotient/database.h

@@ -94,6 +94,7 @@ class QUOTIENT_API Database
     void migrateTo8();
     void migrateTo9();
     void migrateTo10();
+    void migrateTo11();
 
     QString m_userId;
     QString m_deviceId;

Quotient/room.cpp

@@ -147,6 +147,7 @@ class Q_DECL_HIDDEN Room::Private {
     //! Map from event id of the request event to the session object
     QHash<QString, KeyVerificationSession *> keyVerificationSessions;
     QPointer<KeyVerificationSession> pendingKeyVerificationSession;
+    int loadedFromDatabase = 0;
 
     struct FileTransferPrivateInfo {
         FileTransferPrivateInfo() = default;
@@ -247,7 +248,9 @@ class Q_DECL_HIDDEN Room::Private {
     }
 
     Changes addNewMessageEvents(RoomEvents&& events);
-    std::pair<Changes, rev_iter_t> addHistoricalMessageEvents(RoomEvents&& events);
+    std::pair<Changes, rev_iter_t> addHistoricalMessageEvents(RoomEvents&& events, bool fromDb = false);
+    void storeInDb(const RoomEvents &events);
+    int loadFromDb(int limit);
 
     Changes updateStatsFromSyncData(const SyncRoomData &data, bool fromCache);
     void postprocessChanges(Changes changes, bool saveState = true);
@@ -488,7 +491,14 @@ Room::Room(Connection* connection, QString id, JoinState initialJoinState)
         connect(this, &Room::beforeDestruction, this, [id, connection] {
             connection->database()->clearRoomData(id);
         });
+        //TODO: delete from database when leaving
     }
+
+    //TODO how do we deal with encrypted events? store encrypted or unencrypted? if unencrypted, we need to also store encrypted to allow clients to view the encrypted event
+    QMetaObject::invokeMethod(this, [=, this](){
+        d->loadFromDb(50);
+    }, Qt::QueuedConnection);
+
     qCDebug(STATE) << "New" << terse << initialJoinState << "Room:" << id;
 }
 
@@ -2304,6 +2314,9 @@ void Room::hangupCall(const QString& callId)
 
 JobHandle<GetRoomEventsJob> Room::getPreviousContent(int limit, const QString& filter)
 {
+    if (d->loadFromDb(limit) > 0) {
+        return {};
+    }
     return d->getPreviousContent(limit, filter);
 }
 
@@ -2833,6 +2846,11 @@ Room::Changes Room::Private::addNewMessageEvents(RoomEvents&& events)
         }
     }
 
+    storeInDb(events);
+    // To load historic messages from the database from the correct point, we need to adjust for the database size increasing here
+    // This looks like it breaks easily; be warned.
+    loadedFromDatabase -= events.size();
+
     // State changes arrive as a part of timeline; the current room state gets
     // updated before merging events to the timeline because that's what
     // clients historically expect. This may eventually change though if we
@@ -2936,9 +2954,41 @@ Room::Changes Room::Private::addNewMessageEvents(RoomEvents&& events)
     return roomChanges;
 }
 
-std::pair<Room::Changes, Room::rev_iter_t> Room::Private::addHistoricalMessageEvents(RoomEvents&& events)
+void Room::Private::storeInDb(const RoomEvents &events)
 {
+    for (const auto& eptr : events) {
+        const auto& e = *eptr;
+        connection->database()->transaction();
+
+        auto query = connection->database()->prepareQuery(QStringLiteral("INSERT INTO events(roomId, ts, json) VALUES(:roomId, :ts, :json);"));
+        query.bindValue(QStringLiteral(":roomId"), q->id());
+        query.bindValue(QStringLiteral(":ts"), e.originTimestamp().toMSecsSinceEpoch());
+        query.bindValue(QStringLiteral(":json"), QString::fromUtf8(QJsonDocument(e.fullJson()).toJson(QJsonDocument::Compact)));
+        connection->database()->execute(query);
 
+        connection->database()->commit();
+    }
+}
+
+int Room::Private::loadFromDb(int limit)
+{
+    auto query = connection->database()->prepareQuery(QStringLiteral("SELECT json FROM events WHERE roomId=:roomId ORDER BY ts DESC LIMIT :limit OFFSET :offset;"));
+    query.bindValue(QStringLiteral(":roomId"), id);
+    query.bindValue(QStringLiteral(":offset"), loadedFromDatabase);
+    query.bindValue(QStringLiteral(":limit"), limit);
+    connection->database()->execute(query);
+
+    RoomEvents events;
+    while(query.next()) {
+        events.push_back(loadEvent<RoomEvent>(QJsonDocument::fromJson(query.value(0).toString().toUtf8()).object()));
+    }
+    loadedFromDatabase += events.size();
+    addHistoricalMessageEvents(std::move(events), true);
+    return events.size();
+}
+
+std::pair<Room::Changes, Room::rev_iter_t> Room::Private::addHistoricalMessageEvents(RoomEvents&& events, bool fromDb)
+{
     dropExtraneousEvents(events);
     if (events.empty())
         return { Change::None, historyEdge() };
@@ -2961,6 +3011,10 @@ std::pair<Room::Changes, Room::rev_iter_t> Room::Private::addHistoricalMessageEv
         }
     }
 
+    if (!fromDb) {
+        storeInDb(events);
+    }
+
     emit q->aboutToAddHistoricalMessages(events);
     const auto insertedSize = moveEventsToTimeline(events, Older);
     const auto from = historyEdge() - insertedSize;